diff --git a/.dockerignore b/.dockerignore
index f27e7fd6..fada7a9b 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,8 +1,11 @@
.vscode
ollama
app
+macapp
dist
-llm/llama.cpp/gguf
+llm/llama.cpp
.env
.cache
-test_data
\ No newline at end of file
+test_data
+llm/build
+llama/build
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 00000000..932ddcc9
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,12 @@
+llm/ext_server/* linguist-vendored
+llama/**/*.cpp linguist-vendored
+llama/**/*.hpp linguist-vendored
+llama/**/*.h linguist-vendored
+llama/**/*.c linguist-vendored
+llama/**/*.cu linguist-vendored
+llama/**/*.cuh linguist-vendored
+llama/**/*.m linguist-vendored
+llama/**/*.metal linguist-vendored
+
+* text=auto
+*.go text eol=lf
diff --git a/.github/ISSUE_TEMPLATE/10_bug_report.yml b/.github/ISSUE_TEMPLATE/10_bug_report.yml
new file mode 100644
index 00000000..d0c79bca
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/10_bug_report.yml
@@ -0,0 +1,60 @@
+name: Bug report
+labels: [bug]
+description: Something isn't working right.
+body:
+ - type: textarea
+ id: description
+ attributes:
+ label: What is the issue?
+ description: What happened? What did you expect to happen?
+ validations:
+ required: true
+ - type: dropdown
+ id: os
+ attributes:
+ label: OS
+ description: Which operating system are you using?
+ multiple: true
+ options:
+ - Linux
+ - macOS
+ - Windows
+ - Docker
+ - WSL2
+ validations:
+ required: false
+ - type: dropdown
+ id: gpu
+ attributes:
+ label: GPU
+ description: Which GPU are you using?
+ multiple: true
+ options:
+ - Nvidia
+ - AMD
+ - Intel
+ - Apple
+ - Other
+ validations:
+ required: false
+ - type: dropdown
+ id: cpu
+ attributes:
+ label: CPU
+ description: Which CPU are you using?
+ multiple: true
+ options:
+ - Intel
+ - AMD
+ - Apple
+ - Other
+ validations:
+ required: false
+ - type: input
+ id: version
+ attributes:
+ label: Ollama version
+ description: What version of Ollama are you using? (`ollama --version`)
+ placeholder: e.g., 0.1.32
+ validations:
+ required: false
diff --git a/.github/ISSUE_TEMPLATE/20_feature_request.md b/.github/ISSUE_TEMPLATE/20_feature_request.md
new file mode 100644
index 00000000..e8997219
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/20_feature_request.md
@@ -0,0 +1,6 @@
+---
+name: Feature request
+about: Request a new feature
+labels: feature request
+---
+
diff --git a/.github/ISSUE_TEMPLATE/30_model_request.md b/.github/ISSUE_TEMPLATE/30_model_request.md
new file mode 100644
index 00000000..c705a5f3
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/30_model_request.md
@@ -0,0 +1,5 @@
+---
+name: Model request
+about: Request support for a new model to be added to Ollama
+labels: model request
+---
\ No newline at end of file
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
new file mode 100644
index 00000000..70d9aa3a
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+ - name: Help
+ url: https://discord.com/invite/ollama
+ about: Please join our Discord server for help using Ollama
+ - name: Troubleshooting
+ url: https://github.com/ollama/ollama/blob/main/docs/faq.md#faq
+ about: See the FAQ for common issues and solutions
diff --git a/.github/workflows/latest.yaml b/.github/workflows/latest.yaml
new file mode 100644
index 00000000..4d47dd36
--- /dev/null
+++ b/.github/workflows/latest.yaml
@@ -0,0 +1,24 @@
+name: latest
+
+on:
+ release:
+ types: [released]
+
+jobs:
+ update-latest:
+ environment: release
+ runs-on: linux
+ steps:
+ - uses: actions/checkout@v4
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ vars.DOCKER_USER }}
+ password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+ - name: Tag images as latest
+ env:
+ PUSH: "1"
+ shell: bash
+ run: |
+ export "VERSION=${GITHUB_REF_NAME#v}"
+ ./scripts/tag_latest.sh
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
new file mode 100644
index 00000000..ac4c19b0
--- /dev/null
+++ b/.github/workflows/release.yaml
@@ -0,0 +1,775 @@
+name: release
+
+on:
+ push:
+ tags:
+ - 'v*'
+
+jobs:
+ # Full build of the Mac assets
+ build-darwin:
+ runs-on: macos-12
+ environment: release
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set Version
+ shell: bash
+ run: |
+ echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
+ echo "RELEASE_VERSION=$(echo ${GITHUB_REF_NAME} | cut -f1 -d-)" >> $GITHUB_ENV
+ - name: key
+ env:
+ MACOS_SIGNING_KEY: ${{ secrets.MACOS_SIGNING_KEY }}
+ MACOS_SIGNING_KEY_PASSWORD: ${{ secrets.MACOS_SIGNING_KEY_PASSWORD }}
+ run: |
+ echo $MACOS_SIGNING_KEY | base64 --decode > certificate.p12
+ security create-keychain -p password build.keychain
+ security default-keychain -s build.keychain
+ security unlock-keychain -p password build.keychain
+ security import certificate.p12 -k build.keychain -P $MACOS_SIGNING_KEY_PASSWORD -T /usr/bin/codesign
+ security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k password build.keychain
+ security set-keychain-settings -lut 3600 build.keychain
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: true
+ - name: Build Darwin
+ env:
+ APPLE_IDENTITY: ${{ secrets.APPLE_IDENTITY }}
+ APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
+ APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }}
+ APPLE_ID: ${{ vars.APPLE_ID }}
+ SDKROOT: /Applications/Xcode_13.4.1.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
+ DEVELOPER_DIR: /Applications/Xcode_13.4.1.app/Contents/Developer
+ run: |
+ ./scripts/build_darwin.sh
+
+ - uses: actions/upload-artifact@v4
+ with:
+ name: dist-darwin
+ path: |
+ dist/*arwin*
+ !dist/*-cov
+
+ # Windows builds take a long time to both install the dependencies and build, so parallelize
+ # CPU generation step
+ generate-windows-cpu:
+ environment: release
+ runs-on: windows
+ env:
+ KEY_CONTAINER: ${{ vars.KEY_CONTAINER }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set Version
+ shell: bash
+ run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
+ - uses: 'google-github-actions/auth@v2'
+ with:
+ project_id: 'ollama'
+ credentials_json: '${{ secrets.GOOGLE_SIGNING_CREDENTIALS }}'
+ - run: echo "${{ vars.OLLAMA_CERT }}" > ollama_inc.crt
+ - name: install Windows SDK 8.1 to get signtool
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading SDK"
+ Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?LinkId=323507" -OutFile "${env:RUNNER_TEMP}\sdksetup.exe"
+ Start-Process "${env:RUNNER_TEMP}\sdksetup.exe" -ArgumentList @("/q") -NoNewWindow -Wait
+ write-host "Win SDK 8.1 installed"
+ gci -path 'C:\Program Files (x86)\Windows Kits\' -r -fi 'signtool.exe'
+ - name: install signing plugin
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading plugin"
+ Invoke-WebRequest -Uri "https://github.com/GoogleCloudPlatform/kms-integrations/releases/download/cng-v1.0/kmscng-1.0-windows-amd64.zip" -OutFile "${env:RUNNER_TEMP}\plugin.zip"
+ Expand-Archive -Path "${env:RUNNER_TEMP}\plugin.zip" -DestinationPath ${env:RUNNER_TEMP}\plugin\
+ write-host "Installing plugin"
+ & "${env:RUNNER_TEMP}\plugin\*\kmscng.msi" /quiet
+ write-host "plugin installed"
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: true
+ - run: go get ./...
+ - run: |
+ $gopath=(get-command go).source | split-path -parent
+ & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Launch-VsDevShell.ps1"
+ cd $env:GITHUB_WORKSPACE
+ $env:CMAKE_SYSTEM_VERSION="10.0.22621.0"
+ $env:PATH="$gopath;$env:PATH"
+ go generate -x ./...
+ name: go generate
+ - uses: actions/upload-artifact@v4
+ with:
+ name: generate-windows-cpu
+ path: |
+ build/**/*
+ build/**/*.a
+ llm/build/**/*.a
+ dist/windows-amd64/**
+
+ # ROCm generation step
+ generate-windows-rocm:
+ environment: release
+ runs-on: windows
+ env:
+ KEY_CONTAINER: ${{ vars.KEY_CONTAINER }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set Version
+ shell: bash
+ run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
+ - uses: 'google-github-actions/auth@v2'
+ with:
+ project_id: 'ollama'
+ credentials_json: '${{ secrets.GOOGLE_SIGNING_CREDENTIALS }}'
+ - run: echo "${{ vars.OLLAMA_CERT }}" > ollama_inc.crt
+ - name: install Windows SDK 8.1 to get signtool
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading SDK"
+ Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?LinkId=323507" -OutFile "${env:RUNNER_TEMP}\sdksetup.exe"
+ Start-Process "${env:RUNNER_TEMP}\sdksetup.exe" -ArgumentList @("/q") -NoNewWindow -Wait
+ write-host "Win SDK 8.1 installed"
+ gci -path 'C:\Program Files (x86)\Windows Kits\' -r -fi 'signtool.exe'
+ - name: install signing plugin
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading plugin"
+ Invoke-WebRequest -Uri "https://github.com/GoogleCloudPlatform/kms-integrations/releases/download/cng-v1.0/kmscng-1.0-windows-amd64.zip" -OutFile "${env:RUNNER_TEMP}\plugin.zip"
+ Expand-Archive -Path "${env:RUNNER_TEMP}\plugin.zip" -DestinationPath ${env:RUNNER_TEMP}\plugin\
+ write-host "Installing plugin"
+ & "${env:RUNNER_TEMP}\plugin\*\kmscng.msi" /quiet
+ write-host "plugin installed"
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: true
+ - name: 'Install ROCm'
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading AMD HIP Installer"
+ Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe"
+ write-host "Installing AMD HIP"
+ Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -Wait
+ write-host "Completed AMD HIP"
+ - name: 'Verify ROCm'
+ run: |
+ & 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' --version
+ - run: go get ./...
+ - run: |
+ $gopath=(get-command go).source | split-path -parent
+ & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Launch-VsDevShell.ps1"
+ cd $env:GITHUB_WORKSPACE
+ $env:CMAKE_SYSTEM_VERSION="10.0.22621.0"
+ $env:PATH="$gopath;$env:PATH"
+ $env:OLLAMA_SKIP_CPU_GENERATE="1"
+ $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)
+ go generate -x ./...
+ name: go generate
+ - name: 'gather rocm dependencies'
+ run: |
+ $HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)
+ md "dist\deps\bin\rocblas\library"
+ cp "${HIP_PATH}\bin\hipblas.dll" "dist\deps\bin\"
+ cp "${HIP_PATH}\bin\rocblas.dll" "dist\deps\bin\"
+ cp "${HIP_PATH}\bin\rocblas\library\*" "dist\deps\bin\rocblas\library\"
+ - uses: actions/upload-artifact@v4
+ with:
+ name: generate-windows-rocm
+ path: |
+ build/**/*
+ dist/windows-amd64/**
+ - uses: actions/upload-artifact@v4
+ with:
+ name: windows-rocm-deps
+ path: dist/deps/*
+
+ # CUDA generation step
+ generate-windows-cuda:
+ environment: release
+ runs-on: windows
+ strategy:
+ matrix:
+ cuda:
+ - version: "11"
+ url: 'https://developer.download.nvidia.com/compute/cuda/11.3.1/local_installers/cuda_11.3.1_465.89_win10.exe'
+ - version: "12"
+ url: 'https://developer.download.nvidia.com/compute/cuda/12.4.0/local_installers/cuda_12.4.0_551.61_windows.exe'
+ env:
+ KEY_CONTAINER: ${{ vars.KEY_CONTAINER }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set Version
+ shell: bash
+ run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
+ - uses: 'google-github-actions/auth@v2'
+ with:
+ project_id: 'ollama'
+ credentials_json: '${{ secrets.GOOGLE_SIGNING_CREDENTIALS }}'
+ - run: echo "${{ vars.OLLAMA_CERT }}" > ollama_inc.crt
+ - name: install Windows SDK 8.1 to get signtool
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading SDK"
+ Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?LinkId=323507" -OutFile "${env:RUNNER_TEMP}\sdksetup.exe"
+ Start-Process "${env:RUNNER_TEMP}\sdksetup.exe" -ArgumentList @("/q") -NoNewWindow -Wait
+ write-host "Win SDK 8.1 installed"
+ gci -path 'C:\Program Files (x86)\Windows Kits\' -r -fi 'signtool.exe'
+ - name: install signing plugin
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading plugin"
+ Invoke-WebRequest -Uri "https://github.com/GoogleCloudPlatform/kms-integrations/releases/download/cng-v1.0/kmscng-1.0-windows-amd64.zip" -OutFile "${env:RUNNER_TEMP}\plugin.zip"
+ Expand-Archive -Path "${env:RUNNER_TEMP}\plugin.zip" -DestinationPath ${env:RUNNER_TEMP}\plugin\
+ write-host "Installing plugin"
+ & "${env:RUNNER_TEMP}\plugin\*\kmscng.msi" /quiet
+ write-host "plugin installed"
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: true
+ - name: 'Install CUDA ${{ matrix.cuda.version }}'
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading CUDA Installer"
+ Invoke-WebRequest -Uri "${{ matrix.cuda.url }}" -OutFile "${env:RUNNER_TEMP}\cuda-install.exe"
+ write-host "Installing CUDA"
+ Start-Process "${env:RUNNER_TEMP}\cuda-install.exe" -ArgumentList '-s' -NoNewWindow -Wait
+ write-host "Completed CUDA"
+ $cudaPath=((resolve-path "c:\Program Files\NVIDIA*\CUDA\v*\bin\nvcc.exe")[0].path | split-path | split-path)
+ $cudaVer=($cudaPath | split-path -leaf ) -replace 'v(\d+).(\d+)', '$1_$2'
+ echo "$cudaPath\bin" >> $env:GITHUB_PATH
+ echo "CUDA_PATH=$cudaPath" >> $env:GITHUB_ENV
+ echo "CUDA_PATH_V${cudaVer}=$cudaPath" >> $env:GITHUB_ENV
+ echo "CUDA_PATH_VX_Y=CUDA_PATH_V${cudaVer}" >> $env:GITHUB_ENV
+ - name: 'Verify CUDA'
+ run: nvcc -V
+ - run: go get ./...
+ - name: go generate
+ run: |
+ $gopath=(get-command go).source | split-path -parent
+ $cudabin=(get-command nvcc).source | split-path
+ & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Launch-VsDevShell.ps1"
+ cd $env:GITHUB_WORKSPACE
+ $env:CMAKE_SYSTEM_VERSION="10.0.22621.0"
+ $env:PATH="$gopath;$cudabin;$env:PATH"
+ $env:OLLAMA_SKIP_CPU_GENERATE="1"
+ go generate -x ./...
+ - name: 'gather cuda dependencies'
+ run: |
+ $NVIDIA_DIR=(resolve-path 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\*\bin\')[0]
+ md "dist\deps"
+ cp "${NVIDIA_DIR}\cudart64_*.dll" "dist\deps\"
+ cp "${NVIDIA_DIR}\cublas64_*.dll" "dist\deps\"
+ cp "${NVIDIA_DIR}\cublasLt64_*.dll" "dist\deps\"
+ - uses: actions/upload-artifact@v4
+ with:
+ name: generate-windows-cuda-${{ matrix.cuda.version }}
+ path: |
+ build/**/*
+ dist/windows-amd64/**
+ - uses: actions/upload-artifact@v4
+ with:
+ name: windows-cuda-deps-${{ matrix.cuda.version }}
+ path: dist/deps/*
+
+
+ # windows arm64 generate, go build, and zip file (no installer)
+ # Output of this build is aggregated into the final x86 build
+ # for a unified windows installer
+ windows-arm64:
+ runs-on: windows-arm64
+ environment: release
+ env:
+ KEY_CONTAINER: ${{ vars.KEY_CONTAINER }}
+ steps:
+ # The current Windows arm64 beta image has effectively zero dev tools installed...
+ - name: Install git and gzip
+ run: |
+ Set-ExecutionPolicy Bypass -Scope Process -Force
+ [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
+ iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+ choco install -y --no-progress git gzip
+ echo "C:\Program Files\Git\cmd" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+ echo "C:\ProgramData\chocolatey\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+ - name: Install Visual Studio 2022
+ run: |
+ $components = @(
+ "Microsoft.VisualStudio.Component.CoreEditor",
+ "Microsoft.VisualStudio.Workload.CoreEditor",
+ "Microsoft.VisualStudio.Component.Roslyn.Compiler",
+ "Microsoft.Component.MSBuild",
+ "Microsoft.VisualStudio.Component.TextTemplating",
+ "Microsoft.VisualStudio.Component.Debugger.JustInTime",
+ "Microsoft.VisualStudio.Component.VC.CoreIde",
+ "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
+ "Microsoft.VisualStudio.Component.Windows11SDK.22621",
+ "Microsoft.VisualStudio.Component.VC.Tools.ARM64EC",
+ "Microsoft.VisualStudio.Component.VC.Tools.ARM64",
+ "Microsoft.VisualStudio.Component.VC.ATL",
+ "Microsoft.VisualStudio.Component.VC.ATL.ARM64",
+ "Microsoft.VisualStudio.Component.Graphics",
+ "Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
+ "Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
+ "Microsoft.VisualStudio.Component.Windows11Sdk.WindowsPerformanceToolkit",
+ "Microsoft.VisualStudio.Component.CppBuildInsights",
+ "Microsoft.VisualStudio.Component.VC.DiagnosticTools",
+ "Microsoft.VisualStudio.ComponentGroup.WebToolsExtensions.CMake",
+ "Microsoft.VisualStudio.Component.VC.CMake.Project",
+ "Microsoft.VisualStudio.Component.VC.ASAN",
+ "Microsoft.VisualStudio.Component.Vcpkg",
+ "Microsoft.VisualStudio.Workload.NativeDesktop"
+ )
+ $config = @{
+ "version" = "1.0"
+ "components" = $components
+ "extensions" = @()
+ }
+ $configPath = "${env:RUNNER_TEMP}\vsconfig"
+ $config | ConvertTo-Json | Out-File -FilePath $configPath
+ $bootstrapperFilePath = "${env:RUNNER_TEMP}\vs_community.exe"
+ write-host "Downloading Visual Studio 2022"
+ Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_community.exe" -outfile $bootstrapperFilePath
+ $bootstrapperArgumentList = ('/c', $bootstrapperFilePath, '--config', $configPath, '--quiet', '--wait' )
+ write-host "Installing Visual Studio 2022"
+ $process = Start-Process -FilePath cmd.exe -ArgumentList $bootstrapperArgumentList -Wait -PassThru
+ $exitCode = $process.ExitCode
+ write-host $exitCode
+ # pacman in mingw/msys2 is ~broken on windows arm right now - hangs consistently during attempts to install
+ # so we'll use this alternative GCC binary
+ - name: Install llvm-mingw GCC
+ run: |
+ $gcc_url="https://github.com/mstorsjo/llvm-mingw/releases/download/20240619/llvm-mingw-20240619-ucrt-aarch64.zip"
+ write-host "Downloading llvm-mingw"
+ Invoke-WebRequest -Uri "${gcc_url}" -OutFile "${env:RUNNER_TEMP}\gcc.zip"
+ write-host "Unpacking llvm-mingw"
+ expand-archive -path "${env:RUNNER_TEMP}\gcc.zip" -destinationpath "c:\"
+ mv c:\llvm-mingw-* c:\llvm-mingw
+ echo "c:\llvm-mingw\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+ - name: Verify GCC
+ run: |
+ echo $env:PATH
+ gcc --version
+ - uses: actions/checkout@v4
+ - name: Set Version
+ run: |
+ $ver=${env:GITHUB_REF_NAME}.trim("v")
+ echo VERSION=$ver | Out-File -FilePath ${env:GITHUB_ENV} -Encoding utf8 -Append
+ - uses: 'google-github-actions/auth@v2'
+ with:
+ project_id: 'ollama'
+ credentials_json: '${{ secrets.GOOGLE_SIGNING_CREDENTIALS }}'
+ - run: echo "${{ vars.OLLAMA_CERT }}" | Out-File -FilePath ollama_inc.crt -Encoding utf8
+ - name: install Windows SDK 8.1 to get signtool
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading SDK"
+ Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?LinkId=323507" -OutFile "${env:RUNNER_TEMP}\sdksetup.exe"
+ Start-Process "${env:RUNNER_TEMP}\sdksetup.exe" -ArgumentList @("/q") -NoNewWindow -Wait
+ write-host "Win SDK 8.1 installed"
+ gci -path 'C:\Program Files (x86)\Windows Kits\' -r -fi 'signtool.exe'
+ - name: install signing plugin
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading plugin"
+ Invoke-WebRequest -Uri "https://github.com/GoogleCloudPlatform/kms-integrations/releases/download/cng-v1.0/kmscng-1.0-windows-amd64.zip" -OutFile "${env:RUNNER_TEMP}\plugin.zip"
+ Expand-Archive -Path "${env:RUNNER_TEMP}\plugin.zip" -DestinationPath ${env:RUNNER_TEMP}\plugin\
+ write-host "Installing plugin"
+ & "${env:RUNNER_TEMP}\plugin\*\kmscng.msi" /quiet
+ write-host "plugin installed"
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: true
+ - run: go get ./...
+ - run: |
+ $gopath=(get-command go).source | split-path -parent
+ $gccpath=(get-command gcc).source | split-path -parent
+ & "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\Tools\Launch-VsDevShell.ps1"
+ cd $env:GITHUB_WORKSPACE
+ $env:CMAKE_SYSTEM_VERSION="10.0.22621.0"
+ $env:PATH="$gopath;$gccpath;$env:PATH;C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin"
+ echo $env:PATH
+ $env:ARCH="arm64"
+ .\scripts\build_windows.ps1 buildOllama buildApp gatherDependencies distZip
+ name: 'Windows Build'
+ - uses: actions/upload-artifact@v4
+ with:
+ name: windows-arm64
+ path: |
+ dist/windows-arm64/**
+ dist/windows-arm64-app.exe
+ dist/ollama-windows-arm64.zip
+
+ # Import the prior generation steps plus the full arm64 build, and build the final windows assets
+ build-windows:
+ environment: release
+ runs-on: windows
+ needs:
+ - generate-windows-cuda
+ - generate-windows-rocm
+ - generate-windows-cpu
+ - windows-arm64
+ env:
+ KEY_CONTAINER: ${{ vars.KEY_CONTAINER }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ - name: Set Version
+ shell: bash
+ run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
+ - uses: 'google-github-actions/auth@v2'
+ with:
+ project_id: 'ollama'
+ credentials_json: '${{ secrets.GOOGLE_SIGNING_CREDENTIALS }}'
+ - run: echo "${{ vars.OLLAMA_CERT }}" > ollama_inc.crt
+ - name: install Windows SDK 8.1 to get signtool
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading SDK"
+ Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?LinkId=323507" -OutFile "${env:RUNNER_TEMP}\sdksetup.exe"
+ Start-Process "${env:RUNNER_TEMP}\sdksetup.exe" -ArgumentList @("/q") -NoNewWindow -Wait
+ write-host "Win SDK 8.1 installed"
+ gci -path 'C:\Program Files (x86)\Windows Kits\' -r -fi 'signtool.exe'
+ - name: install signing plugin
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading plugin"
+ Invoke-WebRequest -Uri "https://github.com/GoogleCloudPlatform/kms-integrations/releases/download/cng-v1.0/kmscng-1.0-windows-amd64.zip" -OutFile "${env:RUNNER_TEMP}\plugin.zip"
+ Expand-Archive -Path "${env:RUNNER_TEMP}\plugin.zip" -DestinationPath ${env:RUNNER_TEMP}\plugin\
+ write-host "Installing plugin"
+ & "${env:RUNNER_TEMP}\plugin\*\kmscng.msi" /quiet
+ write-host "plugin installed"
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: true
+ - run: go get
+ - uses: actions/download-artifact@v4
+ with:
+ name: generate-windows-cpu
+ - uses: actions/download-artifact@v4
+ with:
+ name: generate-windows-cuda-11
+ - uses: actions/download-artifact@v4
+ with:
+ name: generate-windows-cuda-12
+ - uses: actions/download-artifact@v4
+ with:
+ name: windows-cuda-deps-11
+ - uses: actions/download-artifact@v4
+ with:
+ name: windows-cuda-deps-12
+ - uses: actions/download-artifact@v4
+ with:
+ name: windows-rocm-deps
+ - uses: actions/download-artifact@v4
+ with:
+ name: generate-windows-rocm
+ - uses: actions/download-artifact@v4
+ with:
+ name: windows-arm64
+ path: dist
+ - run: dir build
+ - run: |
+ $gopath=(get-command go).source | split-path -parent
+ & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Launch-VsDevShell.ps1"
+ cd $env:GITHUB_WORKSPACE
+ $env:CMAKE_SYSTEM_VERSION="10.0.22621.0"
+ $env:PATH="$gopath;$env:PATH"
+ $env:OLLAMA_SKIP_GENERATE="1"
+ & .\scripts\build_windows.ps1
+ - uses: actions/upload-artifact@v4
+ with:
+ name: dist-windows
+ path: |
+ dist/OllamaSetup.exe
+ dist/ollama-windows-*.zip
+
+ # Linux x86 assets built using the container based build
+ build-linux-amd64:
+ environment: release
+ runs-on: linux
+ env:
+ PLATFORM: linux/amd64
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ - name: Set Version
+ shell: bash
+ run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
+ - run: |
+ ./scripts/build_linux.sh
+ - uses: actions/upload-artifact@v4
+ with:
+ name: dist-linux-amd64
+ path: |
+ dist/*linux*
+ !dist/*-cov
+
+ # Linux ARM assets built using the container based build
+ # (at present, docker isn't pre-installed on arm ubunutu images)
+ build-linux-arm64:
+ environment: release
+ runs-on: linux-arm64
+ env:
+ PLATFORM: linux/arm64
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ - name: Set Version
+ shell: bash
+ run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
+ - name: 'Install Docker'
+ run: |
+ # Add Docker's official GPG key:
+ env
+ uname -a
+ sudo apt-get update
+ sudo apt-get install -y ca-certificates curl
+ sudo install -m 0755 -d /etc/apt/keyrings
+ sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+ sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+ # Add the repository to Apt sources:
+ echo \
+ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+ sudo apt-get update
+ sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+ sudo usermod -aG docker $USER
+ sudo apt-get install acl
+ sudo setfacl --modify user:$USER:rw /var/run/docker.sock
+ - run: |
+ ./scripts/build_linux.sh
+ - uses: actions/upload-artifact@v4
+ with:
+ name: dist-linux-arm64
+ path: |
+ dist/*linux*
+ !dist/*-cov
+
+ # Container image build
+ build-container-image:
+ environment: release
+ strategy:
+ matrix:
+ runner:
+ - linux
+ - linux-arm64
+ runs-on: ${{ matrix.runner }}
+ env:
+ FINAL_IMAGE_REPO: ollama/ollama
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ - name: 'Install Docker'
+ if: ${{ startsWith(matrix.runner, 'linux-arm64') }}
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y ca-certificates curl
+ sudo install -m 0755 -d /etc/apt/keyrings
+ sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+ sudo chmod a+r /etc/apt/keyrings/docker.asc
+ echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+ sudo apt-get update
+ sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+ sudo usermod -aG docker $USER
+ sudo apt-get install acl
+ sudo setfacl --modify user:$USER:rw /var/run/docker.sock
+ - name: Docker meta
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.FINAL_IMAGE_REPO }}
+ flavor: |
+ latest=false
+ tags: |
+ type=ref,enable=true,priority=600,prefix=0.0.0-pr,suffix=,event=pr
+ type=semver,pattern={{version}}
+ - name: Set Version
+ shell: bash
+ run: |
+ machine=$(uname -m)
+ case ${machine} in
+ x86_64) echo ARCH=amd64; echo PLATFORM_PAIR=linux-amd64 ;;
+ aarch64) echo ARCH=arm64; echo PLATFORM_PAIR=linux-arm64 ;;
+ esac >>$GITHUB_ENV
+ echo GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=${{ env.DOCKER_METADATA_OUTPUT_VERSION }}\" \"-X=github.com/ollama/ollama/server.mode=release\"'" >>$GITHUB_ENV
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ vars.DOCKER_USER }}
+ password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+ - name: Build and push by digest
+ id: build
+ uses: docker/build-push-action@v6
+ with:
+ context: "."
+ platforms: linux/${{ env.ARCH }}
+ build-args: |
+ GOFLAGS
+ outputs: type=image,name=${{ env.FINAL_IMAGE_REPO }},push-by-digest=true,name-canonical=true,push=true
+ - name: Export digest
+ run: |
+ mkdir -p /tmp/digests
+ digest="${{ steps.build.outputs.digest }}"
+ touch "/tmp/digests/${digest#sha256:}"
+ - name: Upload digest
+ uses: actions/upload-artifact@v4
+ with:
+ name: digests-${{ env.PLATFORM_PAIR }}
+ path: /tmp/digests/*
+ if-no-files-found: error
+ retention-days: 1
+ merge:
+ environment: release
+ runs-on: linux
+ needs:
+ - build-container-image
+ env:
+ FINAL_IMAGE_REPO: ollama/ollama
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ - name: Download digests
+ uses: actions/download-artifact@v4
+ with:
+ path: /tmp/digests
+ pattern: digests-*
+ merge-multiple: true
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ - name: Docker meta
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.FINAL_IMAGE_REPO }}
+ flavor: |
+ latest=false
+ tags: |
+ type=ref,enable=true,priority=600,prefix=0.0.0-pr,suffix=,event=pr
+ type=semver,pattern={{version}}
+ - name: Set Version
+ shell: bash
+ run: |
+ machine=$(uname -m)
+ case ${machine} in
+ x86_64) echo ARCH=amd64; echo PLATFORM_PAIR=linux-amd64 ;;
+ aarch64) echo ARCH=arm64; echo PLATFORM_PAIR=linux-arm64 ;;
+ esac >>$GITHUB_ENV
+ echo GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=${{ env.DOCKER_METADATA_OUTPUT_VERSION }}\" \"-X=github.com/ollama/ollama/server.mode=release\"'" >>$GITHUB_ENV
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ vars.DOCKER_USER }}
+ password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+ - name: Create manifest list and push
+ working-directory: /tmp/digests
+ run: |
+ docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+ $(printf '${{ env.FINAL_IMAGE_REPO }}@sha256:%s ' *)
+ - name: Inspect image
+ run: |
+ docker buildx imagetools inspect ${{ env.FINAL_IMAGE_REPO }}:${{ steps.meta.outputs.version }}
+ build-container-image-rocm:
+ environment: release
+ runs-on: linux
+ env:
+ FINAL_IMAGE_REPO: ollama/ollama
+ ARCH: amd64
+ PLATFORM_PAIR: linux-amd64
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ - name: Docker meta
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.FINAL_IMAGE_REPO }}
+ flavor: |
+ latest=false
+ tags: |
+ type=ref,enable=true,priority=600,prefix=0.0.0-pr,suffix=,event=pr
+ type=semver,pattern={{version}}
+ - name: Set Version
+ shell: bash
+ run: |
+ echo GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=${{ env.DOCKER_METADATA_OUTPUT_VERSION }}\" \"-X=github.com/ollama/ollama/server.mode=release\"'" >>$GITHUB_ENV
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ vars.DOCKER_USER }}
+ password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+ - name: Build and push by digest
+ id: build
+ uses: docker/build-push-action@v6
+ with:
+ context: "."
+ target: runtime-rocm
+ build-args: |
+ GOFLAGS
+ tags: ${{ env.FINAL_IMAGE_REPO }}:${{ env.DOCKER_METADATA_OUTPUT_VERSION}}-rocm
+ push: true
+
+ # Aggregate all the assets and ship a release
+ release:
+ needs:
+ - build-darwin
+ - build-windows
+ - build-linux-amd64
+ - build-linux-arm64
+ runs-on: linux
+ environment: release
+ permissions:
+ contents: write
+ env:
+ GH_TOKEN: ${{ github.token }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set Version
+ shell: bash
+ run: |
+ echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
+ echo "RELEASE_VERSION=$(echo ${GITHUB_REF_NAME} | cut -f1 -d-)" >> $GITHUB_ENV
+ - name: Retrieve built artifact
+ uses: actions/download-artifact@v4
+ with:
+ path: dist
+ pattern: dist-*
+ merge-multiple: true
+ - run: |
+ ls -lh dist/
+ (cd dist; find . -type f | xargs sha256sum > ../sha256sum.txt)
+ mv sha256sum.txt dist/
+ cat dist/sha256sum.txt
+ - name: Create or update Release
+ run: |
+ echo "Looking for existing release for ${{ env.RELEASE_VERSION }}"
+ OLD_TAG=$(gh release ls --json name,tagName | jq -r ".[] | select(.name == \"${{ env.RELEASE_VERSION }}\") | .tagName")
+ if [ -n "$OLD_TAG" ]; then
+ echo "Updating release ${{ env.RELEASE_VERSION }} to point to new tag ${GITHUB_REF_NAME}"
+ gh release edit ${OLD_TAG} --tag ${GITHUB_REF_NAME}
+ else
+ echo "Creating new release ${{ env.RELEASE_VERSION }} pointing to tag ${GITHUB_REF_NAME}"
+ gh release create ${GITHUB_REF_NAME} \
+ --title ${{ env.RELEASE_VERSION }} \
+ --draft \
+ --generate-notes \
+ --prerelease
+ fi
+ echo "Uploading artifacts for tag ${GITHUB_REF_NAME}"
+ gh release upload ${GITHUB_REF_NAME} dist/* --clobber
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
new file mode 100644
index 00000000..1f9f9bf0
--- /dev/null
+++ b/.github/workflows/test.yaml
@@ -0,0 +1,336 @@
+name: test
+
+concurrency:
+ # For PRs, later CI runs preempt previous ones. e.g. a force push on a PR
+ # cancels running CI jobs and starts all new ones.
+ #
+ # For non-PR pushes, concurrency.group needs to be unique for every distinct
+ # CI run we want to have happen. Use run_id, which in practice means all
+ # non-PR CI runs will be allowed to run without preempting each other.
+ group: ${{ github.workflow }}-$${{ github.pull_request.number || github.run_id }}
+ cancel-in-progress: true
+
+on:
+ pull_request:
+ paths:
+ - '**/*'
+ - '!docs/**'
+ - '!README.md'
+
+jobs:
+ changes:
+ runs-on: ubuntu-latest
+ outputs:
+ GENERATE: ${{ steps.changes.outputs.GENERATE }}
+ GENERATE_CUDA: ${{ steps.changes.outputs.GENERATE_CUDA }}
+ GENERATE_ROCM: ${{ steps.changes.outputs.GENERATE_ROCM }}
+ RUNNERS: ${{ steps.changes.outputs.RUNNERS }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - id: changes
+ run: |
+ changed() {
+ git diff-tree -r --no-commit-id --name-only \
+ $(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }}) \
+ ${{ github.event.pull_request.head.sha }} \
+ | xargs python3 -c "import sys; from pathlib import Path; print(any(Path(x).match(glob) for x in sys.argv[1:] for glob in '$*'.split(' ')))"
+ }
+
+ {
+ echo GENERATE=$(changed 'llm/llama.cpp' 'llm/patches/**' 'llm/ext_server/**' 'llm/generate/**')
+ echo GENERATE_CUDA=$(changed 'llm/llama.cpp' 'llm/patches/**' 'llm/ext_server/**' 'llm/generate/**')
+ echo GENERATE_ROCM=$(changed 'llm/llama.cpp' 'llm/patches/**' 'llm/ext_server/**' 'llm/generate/**')
+ echo RUNNERS=$(changed 'llama/**')
+ } >>$GITHUB_OUTPUT
+
+ generate:
+ needs: [changes]
+ if: ${{ needs.changes.outputs.GENERATE == 'True' }}
+ strategy:
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-2019]
+ arch: [amd64, arm64]
+ exclude:
+ - os: ubuntu-latest
+ arch: arm64
+ - os: windows-2019
+ arch: arm64
+ runs-on: ${{ matrix.os }}
+ env:
+ GOARCH: ${{ matrix.arch }}
+ CGO_ENABLED: '1'
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: true
+ - run: go get ./...
+ - run: |
+ $gopath=(get-command go).source | split-path -parent
+ $gccpath=(get-command gcc).source | split-path -parent
+ & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Launch-VsDevShell.ps1"
+ cd $env:GITHUB_WORKSPACE
+ $env:CMAKE_SYSTEM_VERSION="10.0.22621.0"
+ $env:PATH="$gopath;$gccpath;$env:PATH"
+ echo $env:PATH
+ go generate -x ./...
+ if: ${{ startsWith(matrix.os, 'windows-') }}
+ name: 'Windows Go Generate'
+ - run: go generate -x ./...
+ if: ${{ ! startsWith(matrix.os, 'windows-') }}
+ name: 'Unix Go Generate'
+ - run: go build .
+ generate-cuda:
+ needs: [changes]
+ if: ${{ needs.changes.outputs.GENERATE_CUDA == 'True' }}
+ strategy:
+ matrix:
+ cuda-version:
+ - '11.8.0'
+ runs-on: linux
+ container: nvidia/cuda:${{ matrix.cuda-version }}-devel-ubuntu20.04
+ steps:
+ - run: |
+ apt-get update && apt-get install -y git build-essential curl
+ curl -fsSL https://github.com/Kitware/CMake/releases/download/v3.28.1/cmake-3.28.1-linux-x86_64.tar.gz \
+ | tar -zx -C /usr --strip-components 1
+ env:
+ DEBIAN_FRONTEND: noninteractive
+ - uses: actions/checkout@v4
+ - uses: actions/setup-go@v4
+ with:
+ go-version-file: go.mod
+ cache: true
+ - run: go get ./...
+ - run: |
+ git config --global --add safe.directory /__w/ollama/ollama
+ go generate -x ./...
+ env:
+ OLLAMA_SKIP_CPU_GENERATE: '1'
+ generate-rocm:
+ needs: [changes]
+ if: ${{ needs.changes.outputs.GENERATE_ROCM == 'True' }}
+ strategy:
+ matrix:
+ rocm-version:
+ - '6.1.2'
+ runs-on: linux
+ container: rocm/dev-ubuntu-20.04:${{ matrix.rocm-version }}
+ steps:
+ - run: |
+ apt-get update && apt-get install -y git build-essential curl rocm-libs
+ curl -fsSL https://github.com/Kitware/CMake/releases/download/v3.28.1/cmake-3.28.1-linux-x86_64.tar.gz \
+ | tar -zx -C /usr --strip-components 1
+ env:
+ DEBIAN_FRONTEND: noninteractive
+ - uses: actions/checkout@v4
+ - uses: actions/setup-go@v4
+ with:
+ go-version-file: go.mod
+ cache: true
+ - run: go get ./...
+ - run: |
+ git config --global --add safe.directory /__w/ollama/ollama
+ go generate -x ./...
+ env:
+ OLLAMA_SKIP_CPU_GENERATE: '1'
+
+ # ROCm generation step
+ generate-windows-rocm:
+ needs: [changes]
+ if: ${{ needs.changes.outputs.GENERATE_ROCM == 'True' }}
+ runs-on: windows
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: true
+ - name: 'Install ROCm'
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading AMD HIP Installer"
+ Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe"
+ write-host "Installing AMD HIP"
+ Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -Wait
+ write-host "Completed AMD HIP"
+ - name: 'Verify ROCm'
+ run: |
+ & 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' --version
+ - run: go get ./...
+ - run: |
+ $gopath=(get-command go).source | split-path -parent
+ & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Launch-VsDevShell.ps1"
+ cd $env:GITHUB_WORKSPACE
+ $env:CMAKE_SYSTEM_VERSION="10.0.22621.0"
+ $env:PATH="$gopath;$env:PATH"
+ $env:OLLAMA_SKIP_CPU_GENERATE="1"
+ $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)
+ go generate -x ./...
+ name: go generate
+ env:
+ OLLAMA_SKIP_CPU_GENERATE: '1'
+
+ # CUDA generation step
+ generate-windows-cuda:
+ needs: [changes]
+ if: ${{ needs.changes.outputs.GENERATE_CUDA == 'True' }}
+ runs-on: windows
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: true
+ - name: 'Install CUDA'
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "downloading CUDA Installer"
+ Invoke-WebRequest -Uri "https://developer.download.nvidia.com/compute/cuda/11.3.1/local_installers/cuda_11.3.1_465.89_win10.exe" -OutFile "${env:RUNNER_TEMP}\cuda-install.exe"
+ write-host "Installing CUDA"
+ Start-Process "${env:RUNNER_TEMP}\cuda-install.exe" -ArgumentList '-s' -NoNewWindow -Wait
+ write-host "Completed CUDA"
+ $cudaPath=((resolve-path "c:\Program Files\NVIDIA*\CUDA\v*\bin\nvcc.exe")[0].path | split-path | split-path)
+ $cudaVer=($cudaPath | split-path -leaf ) -replace 'v(\d+).(\d+)', '$1_$2'
+ echo "$cudaPath\bin" >> $env:GITHUB_PATH
+ echo "CUDA_PATH=$cudaPath" >> $env:GITHUB_ENV
+ echo "CUDA_PATH_V${cudaVer}=$cudaPath" >> $env:GITHUB_ENV
+ echo "CUDA_PATH_VX_Y=CUDA_PATH_V${cudaVer}" >> $env:GITHUB_ENV
+ - name: 'Verify CUDA'
+ run: nvcc -V
+ - run: go get ./...
+ - name: go generate
+ run: |
+ $gopath=(get-command go).source | split-path -parent
+ $cudabin=(get-command nvcc).source | split-path
+ & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Launch-VsDevShell.ps1"
+ cd $env:GITHUB_WORKSPACE
+ $env:CMAKE_SYSTEM_VERSION="10.0.22621.0"
+ $env:PATH="$gopath;$cudabin;$env:PATH"
+ $env:OLLAMA_SKIP_CPU_GENERATE="1"
+ go generate -x ./...
+ env:
+ OLLAMA_SKIP_CPU_GENERATE: '1'
+
+ runners:
+ needs: [changes]
+ if: ${{ needs.changes.outputs.RUNNERS == 'True' }}
+ strategy:
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-2019]
+ arch: [amd64, arm64]
+ exclude:
+ - os: ubuntu-latest
+ arch: arm64
+ - os: windows-2019
+ arch: arm64
+ runs-on: ${{ matrix.os }}
+ env:
+ GOARCH: ${{ matrix.arch }}
+ ARCH: ${{ matrix.arch }}
+ CGO_ENABLED: '1'
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: true
+ - run: go get ./...
+ - name: 'Build Windows Go Runners'
+ if: ${{ startsWith(matrix.os, 'windows-') }}
+ run: |
+ $gopath=(get-command go).source | split-path -parent
+ $gccpath=(get-command gcc).source | split-path -parent
+ & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Launch-VsDevShell.ps1"
+ cd $env:GITHUB_WORKSPACE
+ $env:CMAKE_SYSTEM_VERSION="10.0.22621.0"
+ $env:PATH="$gopath;$gccpath;$env:PATH"
+ echo $env:PATH
+ make -C llama -j 4
+ - name: 'Build Unix Go Runners'
+ if: ${{ ! startsWith(matrix.os, 'windows-') }}
+ run: make -C llama -j 4
+ - run: go build .
+
+ lint:
+ strategy:
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-2019]
+ arch: [amd64, arm64]
+ exclude:
+ - os: ubuntu-latest
+ arch: arm64
+ - os: windows-2019
+ arch: arm64
+ - os: macos-latest
+ arch: amd64
+ runs-on: ${{ matrix.os }}
+ env:
+ GOARCH: ${{ matrix.arch }}
+ CGO_ENABLED: '1'
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: false
+ - run: |
+ case ${{ matrix.arch }} in
+ amd64) echo ARCH=x86_64 ;;
+ arm64) echo ARCH=arm64 ;;
+ esac >>$GITHUB_ENV
+ shell: bash
+ - uses: golangci/golangci-lint-action@v6
+ with:
+ args: --timeout 8m0s -v
+ test:
+ strategy:
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-2019]
+ arch: [amd64]
+ exclude:
+ - os: ubuntu-latest
+ arch: arm64
+ - os: windows-2019
+ arch: arm64
+ runs-on: ${{ matrix.os }}
+ env:
+ GOARCH: ${{ matrix.arch }}
+ CGO_ENABLED: '1'
+ OLLAMA_CPU_TARGET: 'static'
+ OLLAMA_SKIP_CPU_GENERATE: '1'
+ OLLAMA_SKIP_METAL_GENERATE: '1'
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache: true
+ - run: |
+ case ${{ matrix.arch }} in
+ amd64) echo ARCH=amd64 ;;
+ arm64) echo ARCH=arm64 ;;
+ esac >>$GITHUB_ENV
+ shell: bash
+ - run: go generate ./...
+ - run: go build
+ - run: go test -v ./...
+
+ patches:
+ needs: [changes]
+ if: ${{ needs.changes.outputs.RUNNERS == 'True' }}
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ - name: Verify patches carry all the changes
+ run: |
+ cd llama && ./sync.sh && git diff --compact-summary --exit-code .
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 515cd442..ce6dce6c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,8 +6,13 @@
.swp
dist
ollama
-ggml-metal.metal
.cache
*.exe
.idea
-test_data
\ No newline at end of file
+test_data
+*.crt
+llm/build
+build/*/*/*
+!build/**/placeholder
+llama/build
+__debug_bin*
\ No newline at end of file
diff --git a/.gitmodules b/.gitmodules
index 5a65e4df..b92f645d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,5 +1,4 @@
-[submodule "llm/llama.cpp/gguf"]
- path = llm/llama.cpp/gguf
- url = https://github.com/ggerganov/llama.cpp.git
- ignore = dirty
- shallow = true
+[submodule "llama.cpp"]
+ path = llm/llama.cpp
+ url = https://github.com/ggerganov/llama.cpp.git
+ shallow = true
\ No newline at end of file
diff --git a/.golangci.yaml b/.golangci.yaml
new file mode 100644
index 00000000..2e0ed3c7
--- /dev/null
+++ b/.golangci.yaml
@@ -0,0 +1,47 @@
+run:
+ timeout: 5m
+linters:
+ enable:
+ - asasalint
+ - bidichk
+ - bodyclose
+ - containedctx
+ - contextcheck
+ - errcheck
+ - exportloopref
+ - gci
+ - gocheckcompilerdirectives
+ - gofmt
+ - gofumpt
+ - gosimple
+ - govet
+ - ineffassign
+ - intrange
+ - makezero
+ - misspell
+ - nilerr
+ - nolintlint
+ - nosprintfhostport
+ - staticcheck
+ - tenv
+ - unconvert
+ - unused
+ - usestdlibvars
+ - wastedassign
+ - whitespace
+linters-settings:
+ gci:
+ sections: [standard, default, localmodule]
+ staticcheck:
+ checks:
+ - all
+ - -SA1019 # omit Deprecated check
+severity:
+ default-severity: error
+ rules:
+ - linters:
+ - gofmt
+ - goimports
+ - intrange
+ - usestdlibvars
+ severity: info
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000..f003a69d
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,37 @@
+# Contributing to Ollama
+
+Thank you for your interest in contributing to Ollama! Here are a few guidelines to help get you started.
+
+## Set up
+
+See the [development documentation](./docs/development.md) for instructions on how to build and run Ollama locally.
+
+## Pull requests
+
+### Ideal issues
+
+* [Bugs](https://github.com/ollama/ollama/issues?q=is%3Aissue+is%3Aopen+label%3Abug): issues where Ollama stops working or where it results in an unexpected error.
+* [Performance](https://github.com/ollama/ollama/issues?q=is%3Aissue+is%3Aopen+label%3Aperformance): issues to make Ollama faster at model inference, downloading or uploading.
+* [Security](https://github.com/ollama/ollama/blob/main/SECURITY.md): issues that could lead to a security vulnerability. As mentioned in [SECURITY.md](https://github.com/ollama/ollama/blob/main/SECURITY.md), please do not disclose security vulnerabilities publicly.
+
+### Issues that are harder to review
+
+* New features: new features (e.g. API fields, environment variables) add surface area to Ollama and make it harder to maintain in the long run as they cannot be removed without potentially breaking users in the future.
+* Refactoring: large code improvements are important, but can be harder or take longer to review and merge.
+* Documentation: small updates to fill in or correct missing documentation is helpful, however large documentation additions can be hard to maintain over time.
+
+### Issues that may not be accepted
+
+* Changes that break backwards compatibility in Ollama's API (including the OpenAI-compatible API)
+* Changes that add significant friction to the user experience
+* Changes that create a large future maintenance burden for maintainers and contributors
+
+### Best practices
+
+* Commit messages: please leave both a title and a description in your commit messages. The title should be a short summary of the changes, with a leading word that explains the section of the code being changed (e.g. `api: fix parsing of prompt field`) . In the description, leave a short 2-3 sentences that explain more about the change and its impact.
+* Tests: please add test coverage to changes where possible.
+* Minimize dependencies: avoid adding new dependencies unless absolutely necessary.
+
+## Need help?
+
+If you need help with anything, feel free to reach out to us on our [Discord server](https://discord.gg/ollama).
diff --git a/Dockerfile b/Dockerfile
index c50665b6..eb07280b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,29 +1,250 @@
-FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
+ARG GOLANG_VERSION=1.22.5
+ARG CMAKE_VERSION=3.22.1
+ARG CUDA_VERSION_11=11.3.1
+ARG CUDA_V11_ARCHITECTURES="50;52;53;60;61;62;70;72;75;80;86"
+ARG CUDA_VERSION_12=12.4.0
+ARG CUDA_V12_ARCHITECTURES="60;61;62;70;72;75;80;86;87;89;90;90a"
+ARG ROCM_VERSION=6.1.2
-ARG TARGETARCH
-ARG GOFLAGS="'-ldflags=-w -s'"
+# Copy the minimal context we need to run the generate scripts
+FROM scratch AS llm-code
+COPY .git .git
+COPY .gitmodules .gitmodules
+COPY llm llm
-WORKDIR /go/src/github.com/jmorganca/ollama
-RUN apt-get update && apt-get install -y git build-essential cmake
-ADD https://dl.google.com/go/go1.21.3.linux-$TARGETARCH.tar.gz /tmp/go1.21.3.tar.gz
-RUN mkdir -p /usr/local && tar xz -C /usr/local </tmp/go1.21.3.tar.gz
+FROM --platform=linux/amd64 nvidia/cuda:$CUDA_VERSION_11-devel-centos7 AS cuda-11-build-amd64
+ARG CMAKE_VERSION
+COPY ./scripts/rh_linux_deps.sh /
+RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
+ENV PATH=/opt/rh/devtoolset-10/root/usr/bin:$PATH
+COPY --from=llm-code / /go/src/github.com/ollama/ollama/
+WORKDIR /go/src/github.com/ollama/ollama/llm/generate
+ARG CGO_CFLAGS
+ARG CUDA_V11_ARCHITECTURES
+ENV GOARCH=amd64
+RUN --mount=type=cache,target=/root/.ccache \
+ OLLAMA_SKIP_STATIC_GENERATE=1 \
+ OLLAMA_SKIP_CPU_GENERATE=1 \
+ CMAKE_CUDA_ARCHITECTURES="${CUDA_V11_ARCHITECTURES}" \
+ CUDA_VARIANT="_v11" \
+ bash gen_linux.sh
+FROM --platform=linux/amd64 nvidia/cuda:$CUDA_VERSION_12-devel-centos7 AS cuda-12-build-amd64
+ARG CMAKE_VERSION
+COPY ./scripts/rh_linux_deps.sh /
+RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
+ENV PATH=/opt/rh/devtoolset-10/root/usr/bin:$PATH
+COPY --from=llm-code / /go/src/github.com/ollama/ollama/
+WORKDIR /go/src/github.com/ollama/ollama/llm/generate
+ARG CGO_CFLAGS
+ARG CUDA_V12_ARCHITECTURES
+ENV GOARCH=amd64
+RUN --mount=type=cache,target=/root/.ccache \
+ OLLAMA_SKIP_STATIC_GENERATE=1 \
+ OLLAMA_SKIP_CPU_GENERATE=1 \
+ CMAKE_CUDA_ARCHITECTURES="${CUDA_V12_ARCHITECTURES}" \
+ CUDA_VARIANT="_v12" \
+ OLLAMA_CUSTOM_CUDA_DEFS="-DGGML_CUDA_USE_GRAPHS=on" \
+ bash gen_linux.sh
+
+FROM --platform=linux/arm64 nvidia/cuda:$CUDA_VERSION_11-devel-rockylinux8 AS cuda-11-build-runner-arm64
+ARG CMAKE_VERSION
+COPY ./scripts/rh_linux_deps.sh /
+RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
+ENV PATH=/opt/rh/gcc-toolset-10/root/usr/bin:$PATH
+COPY --from=llm-code / /go/src/github.com/ollama/ollama/
+WORKDIR /go/src/github.com/ollama/ollama/llm/generate
+ARG CGO_CFLAGS
+ARG CUDA_V11_ARCHITECTURES
+ENV GOARCH=arm64
+RUN OLLAMA_SKIP_STATIC_GENERATE=1 \
+ OLLAMA_SKIP_CPU_GENERATE=1 \
+ CMAKE_CUDA_ARCHITECTURES="${CUDA_V11_ARCHITECTURES}" \
+ CUDA_VARIANT="_v11" \
+ bash gen_linux.sh
+
+FROM --platform=linux/arm64 nvidia/cuda:$CUDA_VERSION_12-devel-rockylinux8 AS cuda-12-build-runner-arm64
+ARG CMAKE_VERSION
+COPY ./scripts/rh_linux_deps.sh /
+RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
+ENV PATH=/opt/rh/gcc-toolset-10/root/usr/bin:$PATH
+COPY --from=llm-code / /go/src/github.com/ollama/ollama/
+WORKDIR /go/src/github.com/ollama/ollama/llm/generate
+ARG CGO_CFLAGS
+ARG CUDA_V12_ARCHITECTURES
+ENV GOARCH=arm64
+RUN --mount=type=cache,target=/root/.ccache \
+ OLLAMA_SKIP_STATIC_GENERATE=1 \
+ OLLAMA_SKIP_CPU_GENERATE=1 \
+ CMAKE_CUDA_ARCHITECTURES="${CUDA_V12_ARCHITECTURES}" \
+ CUDA_VARIANT="_v12" \
+ OLLAMA_CUSTOM_CUDA_DEFS="-DGGML_CUDA_USE_GRAPHS=on" \
+ bash gen_linux.sh
+
+
+FROM --platform=linux/amd64 rocm/dev-centos-7:${ROCM_VERSION}-complete AS rocm-build-amd64
+ARG CMAKE_VERSION
+COPY ./scripts/rh_linux_deps.sh /
+RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
+ENV PATH=/opt/rh/devtoolset-10/root/usr/bin:$PATH
+ENV LIBRARY_PATH=/opt/amdgpu/lib64
+COPY --from=llm-code / /go/src/github.com/ollama/ollama/
+WORKDIR /go/src/github.com/ollama/ollama/llm/generate
+ARG CGO_CFLAGS
+ARG AMDGPU_TARGETS
+ENV GOARCH=amd64
+RUN --mount=type=cache,target=/root/.ccache \
+ OLLAMA_SKIP_STATIC_GENERATE=1 OLLAMA_SKIP_CPU_GENERATE=1 bash gen_linux.sh
+RUN mkdir -p ../../dist/linux-amd64-rocm/lib/ollama && \
+ (cd /opt/rocm/lib && tar cf - rocblas/library) | (cd ../../dist/linux-amd64-rocm/lib/ollama && tar xf - )
+
+FROM --platform=linux/amd64 centos:7 AS cpu-builder-amd64
+ARG CMAKE_VERSION
+ARG GOLANG_VERSION
+COPY ./scripts/rh_linux_deps.sh /
+RUN CMAKE_VERSION=${CMAKE_VERSION} GOLANG_VERSION=${GOLANG_VERSION} sh /rh_linux_deps.sh
+ENV PATH=/opt/rh/devtoolset-10/root/usr/bin:$PATH
+COPY --from=llm-code / /go/src/github.com/ollama/ollama/
+ARG OLLAMA_CUSTOM_CPU_DEFS
+ARG CGO_CFLAGS
+ENV GOARCH=amd64
+WORKDIR /go/src/github.com/ollama/ollama/llm/generate
+
+FROM --platform=linux/amd64 cpu-builder-amd64 AS cpu-build-amd64
+RUN --mount=type=cache,target=/root/.ccache \
+ OLLAMA_SKIP_STATIC_GENERATE=1 OLLAMA_CPU_TARGET="cpu" bash gen_linux.sh
+FROM --platform=linux/amd64 cpu-builder-amd64 AS cpu_avx-build-amd64
+RUN --mount=type=cache,target=/root/.ccache \
+ OLLAMA_SKIP_STATIC_GENERATE=1 OLLAMA_CPU_TARGET="cpu_avx" bash gen_linux.sh
+FROM --platform=linux/amd64 cpu-builder-amd64 AS cpu_avx2-build-amd64
+RUN --mount=type=cache,target=/root/.ccache \
+ OLLAMA_SKIP_STATIC_GENERATE=1 OLLAMA_CPU_TARGET="cpu_avx2" bash gen_linux.sh
+
+FROM --platform=linux/arm64 rockylinux:8 AS cpu-builder-arm64
+ARG CMAKE_VERSION
+ARG GOLANG_VERSION
+COPY ./scripts/rh_linux_deps.sh /
+RUN CMAKE_VERSION=${CMAKE_VERSION} GOLANG_VERSION=${GOLANG_VERSION} sh /rh_linux_deps.sh
+ENV PATH=/opt/rh/gcc-toolset-10/root/usr/bin:$PATH
+COPY --from=llm-code / /go/src/github.com/ollama/ollama/
+ARG OLLAMA_CUSTOM_CPU_DEFS
+ARG CGO_CFLAGS
+ENV GOARCH=arm64
+WORKDIR /go/src/github.com/ollama/ollama/llm/generate
+
+FROM --platform=linux/arm64 cpu-builder-arm64 AS cpu-build-arm64
+RUN --mount=type=cache,target=/root/.ccache \
+ OLLAMA_SKIP_STATIC_GENERATE=1 OLLAMA_CPU_TARGET="cpu" bash gen_linux.sh
+
+
+# Intermediate stages used for ./scripts/build_linux.sh
+FROM --platform=linux/amd64 cpu-build-amd64 AS build-amd64
+ENV CGO_ENABLED=1
+WORKDIR /go/src/github.com/ollama/ollama
COPY . .
-ENV GOARCH=$TARGETARCH
-ENV GOFLAGS=$GOFLAGS
-RUN /usr/local/go/bin/go generate ./... \
- && /usr/local/go/bin/go build .
+COPY --from=cpu_avx-build-amd64 /go/src/github.com/ollama/ollama/build/ build/
+COPY --from=cpu_avx2-build-amd64 /go/src/github.com/ollama/ollama/build/ build/
+COPY --from=cuda-11-build-amd64 /go/src/github.com/ollama/ollama/dist/ dist/
+COPY --from=cuda-11-build-amd64 /go/src/github.com/ollama/ollama/build/ build/
+COPY --from=cuda-12-build-amd64 /go/src/github.com/ollama/ollama/dist/ dist/
+COPY --from=cuda-12-build-amd64 /go/src/github.com/ollama/ollama/build/ build/
+COPY --from=rocm-build-amd64 /go/src/github.com/ollama/ollama/dist/ dist/
+COPY --from=rocm-build-amd64 /go/src/github.com/ollama/ollama/build/ build/
+ARG GOFLAGS
+ARG CGO_CFLAGS
+RUN --mount=type=cache,target=/root/.ccache \
+ go build -trimpath -o dist/linux-amd64/bin/ollama .
+RUN cd dist/linux-$GOARCH && \
+ tar --exclude runners -cf - . | pigz --best > ../ollama-linux-$GOARCH.tgz
+RUN cd dist/linux-$GOARCH-rocm && \
+ tar -cf - . | pigz --best > ../ollama-linux-$GOARCH-rocm.tgz
-FROM ubuntu:22.04
-RUN apt-get update && apt-get install -y ca-certificates
-COPY --from=0 /go/src/github.com/jmorganca/ollama/ollama /bin/ollama
+FROM --platform=linux/arm64 cpu-build-arm64 AS build-arm64
+ENV CGO_ENABLED=1
+ARG GOLANG_VERSION
+WORKDIR /go/src/github.com/ollama/ollama
+COPY . .
+COPY --from=cuda-11-build-runner-arm64 /go/src/github.com/ollama/ollama/dist/ dist/
+COPY --from=cuda-11-build-runner-arm64 /go/src/github.com/ollama/ollama/build/ build/
+COPY --from=cuda-12-build-runner-arm64 /go/src/github.com/ollama/ollama/dist/ dist/
+COPY --from=cuda-12-build-runner-arm64 /go/src/github.com/ollama/ollama/build/ build/
+ARG GOFLAGS
+ARG CGO_CFLAGS
+RUN --mount=type=cache,target=/root/.ccache \
+ go build -trimpath -o dist/linux-arm64/bin/ollama .
+RUN cd dist/linux-$GOARCH && \
+ tar --exclude runners -cf - . | pigz --best > ../ollama-linux-$GOARCH.tgz
+
+FROM --platform=linux/amd64 scratch AS dist-amd64
+COPY --from=build-amd64 /go/src/github.com/ollama/ollama/dist/ollama-linux-*.tgz /
+FROM --platform=linux/arm64 scratch AS dist-arm64
+COPY --from=build-arm64 /go/src/github.com/ollama/ollama/dist/ollama-linux-*.tgz /
+FROM dist-$TARGETARCH as dist
+
+
+# Optimized container images do not cary nested payloads
+FROM --platform=linux/amd64 cpu-builder-amd64 AS container-build-amd64
+WORKDIR /go/src/github.com/ollama/ollama
+COPY . .
+ARG GOFLAGS
+ARG CGO_CFLAGS
+RUN --mount=type=cache,target=/root/.ccache \
+ go build -trimpath -o dist/linux-amd64/bin/ollama .
+
+FROM --platform=linux/arm64 cpu-builder-arm64 AS container-build-arm64
+WORKDIR /go/src/github.com/ollama/ollama
+COPY . .
+ARG GOFLAGS
+ARG CGO_CFLAGS
+RUN --mount=type=cache,target=/root/.ccache \
+ go build -trimpath -o dist/linux-arm64/bin/ollama .
+
+FROM --platform=linux/amd64 ubuntu:22.04 AS runtime-amd64
+RUN apt-get update && \
+ apt-get install -y ca-certificates && \
+ apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY --from=container-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/bin/ /bin/
+COPY --from=cpu-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/lib/ /lib/
+COPY --from=cpu_avx-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/lib/ /lib/
+COPY --from=cpu_avx2-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/lib/ /lib/
+COPY --from=cuda-11-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/lib/ /lib/
+COPY --from=cuda-12-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/lib/ /lib/
+
+FROM --platform=linux/arm64 ubuntu:22.04 AS runtime-arm64
+RUN apt-get update && \
+ apt-get install -y ca-certificates && \
+ apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY --from=container-build-arm64 /go/src/github.com/ollama/ollama/dist/linux-arm64/bin/ /bin/
+COPY --from=cpu-build-arm64 /go/src/github.com/ollama/ollama/dist/linux-arm64/lib/ /lib/
+COPY --from=cuda-11-build-runner-arm64 /go/src/github.com/ollama/ollama/dist/linux-arm64/lib/ /lib/
+COPY --from=cuda-12-build-runner-arm64 /go/src/github.com/ollama/ollama/dist/linux-arm64/lib/ /lib/
+
+# ROCm libraries larger so we keep it distinct from the CPU/CUDA image
+FROM --platform=linux/amd64 ubuntu:22.04 AS runtime-rocm
+# Frontload the rocm libraries which are large, and rarely change to increase chance of a common layer
+# across releases
+COPY --from=rocm-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64-rocm/lib/ /lib/
+RUN apt-get update && \
+ apt-get install -y ca-certificates && \
+ apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY --from=container-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/bin/ /bin/
+COPY --from=cpu-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/lib/ /lib/
+COPY --from=cpu_avx-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/lib/ /lib/
+COPY --from=cpu_avx2-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/lib/ /lib/
+COPY --from=rocm-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/lib/ /lib/
EXPOSE 11434
-ENV OLLAMA_HOST 0.0.0.0
-
-# set some environment variable for better NVIDIA compatibility
-ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64
-ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
+ENV OLLAMA_HOST=0.0.0.0
+
+ENTRYPOINT ["/bin/ollama"]
+CMD ["serve"]
+
+FROM runtime-$TARGETARCH
+EXPOSE 11434
+ENV OLLAMA_HOST=0.0.0.0
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
+ENV NVIDIA_VISIBLE_DEVICES=all
ENTRYPOINT ["/bin/ollama"]
CMD ["serve"]
diff --git a/Dockerfile.build b/Dockerfile.build
deleted file mode 100644
index c8170919..00000000
--- a/Dockerfile.build
+++ /dev/null
@@ -1,74 +0,0 @@
-# Ubuntu 20.04 amd64 dependencies
-FROM --platform=linux/amd64 ubuntu:20.04 AS base-amd64
-ARG CUDA_VERSION=11.3.1-1
-ARG CMAKE_VERSION=3.22.1
-# ROCm only supports amd64
-ARG ROCM_VERSION=6.0
-ARG CLBLAST_VER=1.6.1
-
-# Note: https://rocm.docs.amd.com/en/latest/release/user_kernel_space_compat_matrix.html
-RUN apt-get update && \
- apt-get install -y wget gnupg && \
- wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin && \
- mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
- apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub && \
- echo "deb [by-hash=no] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" > /etc/apt/sources.list.d/cuda.list && \
- wget "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh" -O /tmp/cmake-installer.sh && \
- chmod +x /tmp/cmake-installer.sh && /tmp/cmake-installer.sh --skip-license --prefix=/usr && \
- mkdir --parents --mode=0755 /etc/apt/keyrings && \
- wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor > /etc/apt/keyrings/rocm.gpg && \
- echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${ROCM_VERSION} focal main" > /etc/apt/sources.list.d/rocm.list && \
- echo "Package: *" > /etc/apt/preferences.d/rocm-pin-600 && \
- echo "Pin: release o=repo.radeon.com" >> /etc/apt/preferences.d/rocm-pin-600 && \
- echo "Pin-Priority: 600" >> /etc/apt/preferences.d/rocm-pin-600 && \
- apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get -y install cuda=${CUDA_VERSION} rocm-hip-libraries rocm-device-libs rocm-libs rocm-ocl-icd rocm-hip-sdk rocm-hip-libraries rocm-cmake rocm-clang-ocl rocm-dev
-
-# CLBlast
-RUN wget -qO- https://github.com/CNugteren/CLBlast/archive/refs/tags/${CLBLAST_VER}.tar.gz | tar zxv -C /tmp/ && \
- cd /tmp/CLBlast-${CLBLAST_VER} && mkdir build && cd build && cmake .. && make && make install
-
-ENV ROCM_PATH=/opt/rocm
-
-# Ubuntu 22.04 arm64 dependencies
-FROM --platform=linux/arm64 ubuntu:20.04 AS base-arm64
-ARG CUDA_VERSION=11.3.1-1
-ARG CMAKE_VERSION=3.27.6
-RUN apt-get update && \
- apt-get install -y wget gnupg && \
- wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/sbsa/cuda-ubuntu2004.pin && \
- mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
- apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/sbsa//3bf863cc.pub && \
- echo "deb [by-hash=no] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/sbsa/ /" > /etc/apt/sources.list.d/cuda.list && \
- wget "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-aarch64.sh" -O /tmp/cmake-installer.sh && \
- chmod +x /tmp/cmake-installer.sh && /tmp/cmake-installer.sh --skip-license --prefix=/usr && \
- apt-get update && \
- apt-cache madison cuda && \
- DEBIAN_FRONTEND=noninteractive apt-get -y install cuda=${CUDA_VERSION}
-
-FROM base-${TARGETARCH}
-ARG TARGETARCH
-ARG GOFLAGS="'-ldflags -w -s'"
-ARG CGO_CFLAGS
-ARG GOLANG_VERSION=1.21.3
-
-# Common toolchain
-RUN apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-10 g++-10 cpp-10 git ocl-icd-opencl-dev && \
- update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10
-
-# install go
-ADD https://dl.google.com/go/go${GOLANG_VERSION}.linux-$TARGETARCH.tar.gz /tmp/go${GOLANG_VERSION}.tar.gz
-RUN mkdir -p /usr/local && tar xz -C /usr/local </tmp/go${GOLANG_VERSION}.tar.gz
-
-# build the final binary
-WORKDIR /go/src/github.com/jmorganca/ollama
-COPY . .
-
-ENV GOOS=linux
-ENV GOARCH=$TARGETARCH
-ENV GOFLAGS=$GOFLAGS
-ENV CGO_CFLAGS=${CGO_CFLAGS}
-
-RUN /usr/local/go/bin/go generate ./... && \
- /usr/local/go/bin/go build .
diff --git a/README.md b/README.md
index ba4708b5..6893783a 100644
--- a/README.md
+++ b/README.md
@@ -1,31 +1,28 @@
<div align="center">
- <picture>
- <source media="(prefers-color-scheme: dark)" height="200px" srcset="https://github.com/jmorganca/ollama/assets/3325447/56ea1849-1284-4645-8970-956de6e51c3c">
- <img alt="logo" height="200px" src="https://github.com/jmorganca/ollama/assets/3325447/0d0b44e2-8f4a-4e99-9b52-a5c1c741c8f7">
- </picture>
+ <img alt="ollama" height="200px" src="https://github.com/ollama/ollama/assets/3325447/0d0b44e2-8f4a-4e99-9b52-a5c1c741c8f7">
</div>
# Ollama
[](https://discord.gg/ollama)
-Get up and running with large language models locally.
+Get up and running with large language models.
### macOS
-[Download](https://ollama.ai/download/Ollama-darwin.zip)
+[Download](https://ollama.com/download/Ollama-darwin.zip)
-### Windows
+### Windows preview
-Coming soon! For now, you can install Ollama on Windows via WSL2.
+[Download](https://ollama.com/download/OllamaSetup.exe)
-### Linux & WSL2
+### Linux
```
-curl https://ollama.ai/install.sh | sh
+curl -fsSL https://ollama.com/install.sh | sh
```
-[Manual install instructions](https://github.com/jmorganca/ollama/blob/main/docs/linux.md)
+[Manual install instructions](https://github.com/ollama/ollama/blob/main/docs/linux.md)
### Docker
@@ -43,37 +40,48 @@ echo '[[ $commands[ollama] ]] && source <(ollama completion zsh)' >> ~/.zshrc
ollama completion fish > ~/.config/fish/completions/ollama.fish
```
+### Libraries
+
+- [ollama-python](https://github.com/ollama/ollama-python)
+- [ollama-js](https://github.com/ollama/ollama-js)
+
## Quickstart
-To run and chat with [Llama 2](https://ollama.ai/library/llama2):
+To run and chat with [Llama 3.2](https://ollama.com/library/llama3.2):
```
-ollama run llama2
+ollama run llama3.2
```
## Model library
-Ollama supports a list of open-source models available on [ollama.ai/library](https://ollama.ai/library 'ollama model library')
+Ollama supports a list of models available on [ollama.com/library](https://ollama.com/library 'ollama model library')
-Here are some example open-source models that can be downloaded:
+Here are some example models that can be downloaded:
| Model | Parameters | Size | Download |
| ------------------ | ---------- | ----- | ------------------------------ |
-| Llama 2 | 7B | 3.8GB | `ollama run llama2` |
+| Llama 3.2 | 3B | 2.0GB | `ollama run llama3.2` |
+| Llama 3.2 | 1B | 1.3GB | `ollama run llama3.2:1b` |
+| Llama 3.1 | 8B | 4.7GB | `ollama run llama3.1` |
+| Llama 3.1 | 70B | 40GB | `ollama run llama3.1:70b` |
+| Llama 3.1 | 405B | 231GB | `ollama run llama3.1:405b` |
+| Phi 3 Mini | 3.8B | 2.3GB | `ollama run phi3` |
+| Phi 3 Medium | 14B | 7.9GB | `ollama run phi3:medium` |
+| Gemma 2 | 2B | 1.6GB | `ollama run gemma2:2b` |
+| Gemma 2 | 9B | 5.5GB | `ollama run gemma2` |
+| Gemma 2 | 27B | 16GB | `ollama run gemma2:27b` |
| Mistral | 7B | 4.1GB | `ollama run mistral` |
-| Dolphin Phi | 2.7B | 1.6GB | `ollama run dolphin-phi` |
-| Phi-2 | 2.7B | 1.7GB | `ollama run phi` |
+| Moondream 2 | 1.4B | 829MB | `ollama run moondream` |
| Neural Chat | 7B | 4.1GB | `ollama run neural-chat` |
| Starling | 7B | 4.1GB | `ollama run starling-lm` |
| Code Llama | 7B | 3.8GB | `ollama run codellama` |
| Llama 2 Uncensored | 7B | 3.8GB | `ollama run llama2-uncensored` |
-| Llama 2 13B | 13B | 7.3GB | `ollama run llama2:13b` |
-| Llama 2 70B | 70B | 39GB | `ollama run llama2:70b` |
-| Orca Mini | 3B | 1.9GB | `ollama run orca-mini` |
-| Vicuna | 7B | 3.8GB | `ollama run vicuna` |
| LLaVA | 7B | 4.5GB | `ollama run llava` |
+| Solar | 10.7B | 6.1GB | `ollama run solar` |
-> Note: You should have at least 8 GB of RAM available to run the 7B models, 16 GB to run the 13B models, and 32 GB to run the 33B models.
+> [!NOTE]
+> You should have at least 8 GB of RAM available to run the 7B models, 16 GB to run the 13B models, and 32 GB to run the 33B models.
## Customize a model
@@ -105,16 +113,16 @@ See the [guide](docs/import.md) on importing models for more information.
### Customize a prompt
-Models from the Ollama library can be customized with a prompt. For example, to customize the `llama2` model:
+Models from the Ollama library can be customized with a prompt. For example, to customize the `llama3.2` model:
```
-ollama pull llama2
+ollama pull llama3.2
```
Create a `Modelfile`:
```
-FROM llama2
+FROM llama3.2
# set the temperature to 1 [higher is more creative, lower is more coherent]
PARAMETER temperature 1
@@ -149,7 +157,7 @@ ollama create mymodel -f ./Modelfile
### Pull a model
```
-ollama pull llama2
+ollama pull llama3.2
```
> This command can also be used to update a local model. Only the diff will be pulled.
@@ -157,13 +165,13 @@ ollama pull llama2
### Remove a model
```
-ollama rm llama2
+ollama rm llama3.2
```
### Copy a model
```
-ollama cp llama2 my-llama2
+ollama cp llama3.2 my-model
```
### Multiline input
@@ -180,48 +188,51 @@ I'm a basic program that prints the famous "Hello, world!" message to the consol
### Multimodal models
```
->>> What's in this image? /Users/jmorgan/Desktop/smile.png
+ollama run llava "What's in this image? /Users/jmorgan/Desktop/smile.png"
The image features a yellow smiley face, which is likely the central focus of the picture.
```
-### Pass in prompt as arguments
+### Pass the prompt as an argument
```
-$ ollama run llama2 "Summarize this file: $(cat README.md)"
+$ ollama run llama3.2 "Summarize this file: $(cat README.md)"
Ollama is a lightweight, extensible framework for building and running language models on the local machine. It provides a simple API for creating, running, and managing models, as well as a library of pre-built models that can be easily used in a variety of applications.
```
+### Show model information
+
+```
+ollama show llama3.2
+```
+
### List models on your computer
```
ollama list
```
+### List which models are currently loaded
+
+```
+ollama ps
+```
+
+### Stop a model which is currently running
+
+```
+ollama stop llama3.2
+```
+
### Start Ollama
`ollama serve` is used when you want to start ollama without running the desktop application.
## Building
-Install `cmake` and `go`:
-
-```
-brew install cmake go
-```
-
-Then generate dependencies:
-```
-go generate ./...
-```
-Then build the binary:
-```
-go build .
-```
-
-More detailed instructions can be found in the [developer guide](https://github.com/jmorganca/ollama/blob/main/docs/development.md)
-
+See the [developer guide](https://github.com/ollama/ollama/blob/main/docs/development.md)
### Running local builds
+
Next, start the server:
```
@@ -231,7 +242,7 @@ Next, start the server:
Finally, in a separate shell, run a model:
```
-./ollama run llama2
+./ollama run llama3.2
```
## REST API
@@ -242,7 +253,7 @@ Ollama has a REST API for running and managing models.
```
curl http://localhost:11434/api/generate -d '{
- "model": "llama2",
+ "model": "llama3.2",
"prompt":"Why is the sky blue?"
}'
```
@@ -251,7 +262,7 @@ curl http://localhost:11434/api/generate -d '{
```
curl http://localhost:11434/api/chat -d '{
- "model": "mistral",
+ "model": "llama3.2",
"messages": [
{ "role": "user", "content": "why is the sky blue?" }
]
@@ -263,17 +274,75 @@ See the [API documentation](./docs/api.md) for all endpoints.
## Community Integrations
### Web & Desktop
+
+- [Open WebUI](https://github.com/open-webui/open-webui)
+- [Enchanted (macOS native)](https://github.com/AugustDev/enchanted)
+- [Hollama](https://github.com/fmaclen/hollama)
+- [Lollms-Webui](https://github.com/ParisNeo/lollms-webui)
+- [LibreChat](https://github.com/danny-avila/LibreChat)
- [Bionic GPT](https://github.com/bionic-gpt/bionic-gpt)
- [HTML UI](https://github.com/rtcfirefly/ollama-ui)
+- [Saddle](https://github.com/jikkuatwork/saddle)
- [Chatbot UI](https://github.com/ivanfioravanti/chatbot-ollama)
+- [Chatbot UI v2](https://github.com/mckaywrigley/chatbot-ui)
- [Typescript UI](https://github.com/ollama-interface/Ollama-Gui?tab=readme-ov-file)
- [Minimalistic React UI for Ollama Models](https://github.com/richawo/minimal-llm-ui)
-- [Web UI](https://github.com/ollama-webui/ollama-webui)
- [Ollamac](https://github.com/kevinhermawan/Ollamac)
-- [big-AGI](https://github.com/enricoros/big-agi/blob/main/docs/config-ollama.md)
+- [big-AGI](https://github.com/enricoros/big-AGI/blob/main/docs/config-local-ollama.md)
- [Cheshire Cat assistant framework](https://github.com/cheshire-cat-ai/core)
- [Amica](https://github.com/semperai/amica)
- [chatd](https://github.com/BruceMacD/chatd)
+- [Ollama-SwiftUI](https://github.com/kghandour/Ollama-SwiftUI)
+- [Dify.AI](https://github.com/langgenius/dify)
+- [MindMac](https://mindmac.app)
+- [NextJS Web Interface for Ollama](https://github.com/jakobhoeg/nextjs-ollama-llm-ui)
+- [Msty](https://msty.app)
+- [Chatbox](https://github.com/Bin-Huang/Chatbox)
+- [WinForm Ollama Copilot](https://github.com/tgraupmann/WinForm_Ollama_Copilot)
+- [NextChat](https://github.com/ChatGPTNextWeb/ChatGPT-Next-Web) with [Get Started Doc](https://docs.nextchat.dev/models/ollama)
+- [Alpaca WebUI](https://github.com/mmo80/alpaca-webui)
+- [OllamaGUI](https://github.com/enoch1118/ollamaGUI)
+- [OpenAOE](https://github.com/InternLM/OpenAOE)
+- [Odin Runes](https://github.com/leonid20000/OdinRunes)
+- [LLM-X](https://github.com/mrdjohnson/llm-x) (Progressive Web App)
+- [AnythingLLM (Docker + MacOs/Windows/Linux native app)](https://github.com/Mintplex-Labs/anything-llm)
+- [Ollama Basic Chat: Uses HyperDiv Reactive UI](https://github.com/rapidarchitect/ollama_basic_chat)
+- [Ollama-chats RPG](https://github.com/drazdra/ollama-chats)
+- [QA-Pilot](https://github.com/reid41/QA-Pilot) (Chat with Code Repository)
+- [ChatOllama](https://github.com/sugarforever/chat-ollama) (Open Source Chatbot based on Ollama with Knowledge Bases)
+- [CRAG Ollama Chat](https://github.com/Nagi-ovo/CRAG-Ollama-Chat) (Simple Web Search with Corrective RAG)
+- [RAGFlow](https://github.com/infiniflow/ragflow) (Open-source Retrieval-Augmented Generation engine based on deep document understanding)
+- [StreamDeploy](https://github.com/StreamDeploy-DevRel/streamdeploy-llm-app-scaffold) (LLM Application Scaffold)
+- [chat](https://github.com/swuecho/chat) (chat web app for teams)
+- [Lobe Chat](https://github.com/lobehub/lobe-chat) with [Integrating Doc](https://lobehub.com/docs/self-hosting/examples/ollama)
+- [Ollama RAG Chatbot](https://github.com/datvodinh/rag-chatbot.git) (Local Chat with multiple PDFs using Ollama and RAG)
+- [BrainSoup](https://www.nurgo-software.com/products/brainsoup) (Flexible native client with RAG & multi-agent automation)
+- [macai](https://github.com/Renset/macai) (macOS client for Ollama, ChatGPT, and other compatible API back-ends)
+- [Olpaka](https://github.com/Otacon/olpaka) (User-friendly Flutter Web App for Ollama)
+- [OllamaSpring](https://github.com/CrazyNeil/OllamaSpring) (Ollama Client for macOS)
+- [LLocal.in](https://github.com/kartikm7/llocal) (Easy to use Electron Desktop Client for Ollama)
+- [AiLama](https://github.com/zeyoyt/ailama) (A Discord User App that allows you to interact with Ollama anywhere in discord )
+- [Ollama with Google Mesop](https://github.com/rapidarchitect/ollama_mesop/) (Mesop Chat Client implementation with Ollama)
+- [Painting Droid](https://github.com/mateuszmigas/painting-droid) (Painting app with AI integrations)
+- [Kerlig AI](https://www.kerlig.com/) (AI writing assistant for macOS)
+- [AI Studio](https://github.com/MindWorkAI/AI-Studio)
+- [Sidellama](https://github.com/gyopak/sidellama) (browser-based LLM client)
+- [LLMStack](https://github.com/trypromptly/LLMStack) (No-code multi-agent framework to build LLM agents and workflows)
+- [BoltAI for Mac](https://boltai.com) (AI Chat Client for Mac)
+- [Harbor](https://github.com/av/harbor) (Containerized LLM Toolkit with Ollama as default backend)
+- [Go-CREW](https://www.jonathanhecl.com/go-crew/) (Powerful Offline RAG in Golang)
+- [PartCAD](https://github.com/openvmp/partcad/) (CAD model generation with OpenSCAD and CadQuery)
+- [Ollama4j Web UI](https://github.com/ollama4j/ollama4j-web-ui) - Java-based Web UI for Ollama built with Vaadin, Spring Boot and Ollama4j
+- [PyOllaMx](https://github.com/kspviswa/pyOllaMx) - macOS application capable of chatting with both Ollama and Apple MLX models.
+- [Claude Dev](https://github.com/saoudrizwan/claude-dev) - VSCode extension for multi-file/whole-repo coding
+- [Cherry Studio](https://github.com/kangfenmao/cherry-studio) (Desktop client with Ollama support)
+- [ConfiChat](https://github.com/1runeberg/confichat) (Lightweight, standalone, multi-platform, and privacy focused LLM chat interface with optional encryption)
+- [Archyve](https://github.com/nickthecook/archyve) (RAG-enabling document library)
+- [crewAI with Mesop](https://github.com/rapidarchitect/ollama-crew-mesop) (Mesop Web Interface to run crewAI with Ollama)
+- [LLMChat](https://github.com/trendy-design/llmchat) (Privacy focused, 100% local, intuitive all-in-one chat interface)
+- [ARGO](https://github.com/xark-argo/argo) (Locally download and run Ollama and Huggingface models with RAG on Mac/Windows/Linux)
+- [G1](https://github.com/bklieger-groq/g1) (Prototype of using prompting strategies to improve the LLM's reasoning through o1-like reasoning chains.)
+- [Ollama App](https://github.com/JHubi1/ollama-app) (Modern and easy-to-use multi-platform client for Ollama)
### Terminal
@@ -282,38 +351,86 @@ See the [API documentation](./docs/api.md) for all endpoints.
- [Emacs client](https://github.com/zweifisch/ollama)
- [gen.nvim](https://github.com/David-Kunz/gen.nvim)
- [ollama.nvim](https://github.com/nomnivore/ollama.nvim)
+- [ollero.nvim](https://github.com/marco-souza/ollero.nvim)
+- [ollama-chat.nvim](https://github.com/gerazov/ollama-chat.nvim)
- [ogpt.nvim](https://github.com/huynle/ogpt.nvim)
- [gptel Emacs client](https://github.com/karthink/gptel)
- [Oatmeal](https://github.com/dustinblackman/oatmeal)
- [cmdh](https://github.com/pgibler/cmdh)
+- [ooo](https://github.com/npahlfer/ooo)
+- [shell-pilot](https://github.com/reid41/shell-pilot)
+- [tenere](https://github.com/pythops/tenere)
+- [llm-ollama](https://github.com/taketwo/llm-ollama) for [Datasette's LLM CLI](https://llm.datasette.io/en/stable/).
+- [typechat-cli](https://github.com/anaisbetts/typechat-cli)
+- [ShellOracle](https://github.com/djcopley/ShellOracle)
+- [tlm](https://github.com/yusufcanb/tlm)
+- [podman-ollama](https://github.com/ericcurtin/podman-ollama)
+- [gollama](https://github.com/sammcj/gollama)
+- [Ollama eBook Summary](https://github.com/cognitivetech/ollama-ebook-summary/)
+- [Ollama Mixture of Experts (MOE) in 50 lines of code](https://github.com/rapidarchitect/ollama_moe)
+- [vim-intelligence-bridge](https://github.com/pepo-ec/vim-intelligence-bridge) Simple interaction of "Ollama" with the Vim editor
+
+### Apple Vision Pro
+- [Enchanted](https://github.com/AugustDev/enchanted)
### Database
-- [MindsDB](https://github.com/mindsdb/mindsdb/blob/staging/mindsdb/integrations/handlers/ollama_handler/README.md)
+- [MindsDB](https://github.com/mindsdb/mindsdb/blob/staging/mindsdb/integrations/handlers/ollama_handler/README.md) (Connects Ollama models with nearly 200 data platforms and apps)
+- [chromem-go](https://github.com/philippgille/chromem-go/blob/v0.5.0/embed_ollama.go) with [example](https://github.com/philippgille/chromem-go/tree/v0.5.0/examples/rag-wikipedia-ollama)
### Package managers
- [Pacman](https://archlinux.org/packages/extra/x86_64/ollama/)
+- [Gentoo](https://github.com/gentoo/guru/tree/master/app-misc/ollama)
+- [Helm Chart](https://artifacthub.io/packages/helm/ollama-helm/ollama)
+- [Guix channel](https://codeberg.org/tusharhero/ollama-guix)
+- [Nix package](https://search.nixos.org/packages?channel=24.05&show=ollama&from=0&size=50&sort=relevance&type=packages&query=ollama)
+- [Flox](https://flox.dev/blog/ollama-part-one)
### Libraries
-- [LangChain](https://python.langchain.com/docs/integrations/llms/ollama) and [LangChain.js](https://js.langchain.com/docs/modules/model_io/models/llms/integrations/ollama) with [example](https://js.langchain.com/docs/use_cases/question_answering/local_retrieval_qa)
+- [LangChain](https://python.langchain.com/docs/integrations/llms/ollama) and [LangChain.js](https://js.langchain.com/docs/integrations/chat/ollama/) with [example](https://js.langchain.com/docs/tutorials/local_rag/)
+- [Firebase Genkit](https://firebase.google.com/docs/genkit/plugins/ollama)
+- [crewAI](https://github.com/crewAIInc/crewAI)
- [LangChainGo](https://github.com/tmc/langchaingo/) with [example](https://github.com/tmc/langchaingo/tree/main/examples/ollama-completion-example)
-- [LlamaIndex](https://gpt-index.readthedocs.io/en/stable/examples/llm/ollama.html)
+- [LangChain4j](https://github.com/langchain4j/langchain4j) with [example](https://github.com/langchain4j/langchain4j-examples/tree/main/ollama-examples/src/main/java)
+- [LangChainRust](https://github.com/Abraxas-365/langchain-rust) with [example](https://github.com/Abraxas-365/langchain-rust/blob/main/examples/llm_ollama.rs)
+- [LlamaIndex](https://docs.llamaindex.ai/en/stable/examples/llm/ollama/) and [LlamaIndexTS](https://ts.llamaindex.ai/modules/llms/available_llms/ollama)
- [LiteLLM](https://github.com/BerriAI/litellm)
+- [OllamaFarm for Go](https://github.com/presbrey/ollamafarm)
- [OllamaSharp for .NET](https://github.com/awaescher/OllamaSharp)
+- [Ollama for Ruby](https://github.com/gbaptista/ollama-ai)
- [Ollama-rs for Rust](https://github.com/pepperoni21/ollama-rs)
-- [Ollama4j for Java](https://github.com/amithkoujalgi/ollama4j)
+- [Ollama-hpp for C++](https://github.com/jmont-dev/ollama-hpp)
+- [Ollama4j for Java](https://github.com/ollama4j/ollama4j)
- [ModelFusion Typescript Library](https://modelfusion.dev/integration/model-provider/ollama)
- [OllamaKit for Swift](https://github.com/kevinhermawan/OllamaKit)
- [Ollama for Dart](https://github.com/breitburg/dart-ollama)
- [Ollama for Laravel](https://github.com/cloudstudio/ollama-laravel)
- [LangChainDart](https://github.com/davidmigloz/langchain_dart)
+- [Semantic Kernel - Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/ai/ollama)
+- [Haystack](https://github.com/deepset-ai/haystack-integrations/blob/main/integrations/ollama.md)
+- [Elixir LangChain](https://github.com/brainlid/langchain)
+- [Ollama for R - rollama](https://github.com/JBGruber/rollama)
+- [Ollama for R - ollama-r](https://github.com/hauselin/ollama-r)
+- [Ollama-ex for Elixir](https://github.com/lebrunel/ollama-ex)
+- [Ollama Connector for SAP ABAP](https://github.com/b-tocs/abap_btocs_ollama)
+- [Testcontainers](https://testcontainers.com/modules/ollama/)
+- [Portkey](https://portkey.ai/docs/welcome/integration-guides/ollama)
+- [PromptingTools.jl](https://github.com/svilupp/PromptingTools.jl) with an [example](https://svilupp.github.io/PromptingTools.jl/dev/examples/working_with_ollama)
+- [LlamaScript](https://github.com/Project-Llama/llamascript)
+- [Gollm](https://docs.gollm.co/examples/ollama-example)
+- [Ollamaclient for Golang](https://github.com/xyproto/ollamaclient)
+- [High-level function abstraction in Go](https://gitlab.com/tozd/go/fun)
+- [Ollama PHP](https://github.com/ArdaGnsrn/ollama-php)
+- [Agents-Flex for Java](https://github.com/agents-flex/agents-flex) with [example](https://github.com/agents-flex/agents-flex/tree/main/agents-flex-llm/agents-flex-llm-ollama/src/test/java/com/agentsflex/llm/ollama)
### Mobile
- [Enchanted](https://github.com/AugustDev/enchanted)
-- [Maid](https://github.com/danemadsen/Maid)
+- [Maid](https://github.com/Mobile-Artificial-Intelligence/maid)
+- [Ollama App](https://github.com/JHubi1/ollama-app) (Modern and easy-to-use multi-platform client for Ollama)
+- [ConfiChat](https://github.com/1runeberg/confichat) (Lightweight, standalone, multi-platform, and privacy focused LLM chat interface with optional encryption)
### Extensions & Plugins
@@ -322,10 +439,34 @@ See the [API documentation](./docs/api.md) for all endpoints.
- [Continue](https://github.com/continuedev/continue)
- [Obsidian Ollama plugin](https://github.com/hinterdupfinger/obsidian-ollama)
- [Logseq Ollama plugin](https://github.com/omagdy7/ollama-logseq)
+- [NotesOllama](https://github.com/andersrex/notesollama) (Apple Notes Ollama plugin)
- [Dagger Chatbot](https://github.com/samalba/dagger-chatbot)
- [Discord AI Bot](https://github.com/mekb-turtle/discord-ai-bot)
- [Ollama Telegram Bot](https://github.com/ruecat/ollama-telegram)
- [Hass Ollama Conversation](https://github.com/ej52/hass-ollama-conversation)
- [Rivet plugin](https://github.com/abrenneke/rivet-plugin-ollama)
-- [Llama Coder](https://github.com/ex3ndr/llama-coder) (Copilot alternative using Ollama)
- [Obsidian BMO Chatbot plugin](https://github.com/longy2k/obsidian-bmo-chatbot)
+- [Cliobot](https://github.com/herval/cliobot) (Telegram bot with Ollama support)
+- [Copilot for Obsidian plugin](https://github.com/logancyang/obsidian-copilot)
+- [Obsidian Local GPT plugin](https://github.com/pfrankov/obsidian-local-gpt)
+- [Open Interpreter](https://docs.openinterpreter.com/language-model-setup/local-models/ollama)
+- [Llama Coder](https://github.com/ex3ndr/llama-coder) (Copilot alternative using Ollama)
+- [Ollama Copilot](https://github.com/bernardo-bruning/ollama-copilot) (Proxy that allows you to use ollama as a copilot like Github copilot)
+- [twinny](https://github.com/rjmacarthy/twinny) (Copilot and Copilot chat alternative using Ollama)
+- [Wingman-AI](https://github.com/RussellCanfield/wingman-ai) (Copilot code and chat alternative using Ollama and Hugging Face)
+- [Page Assist](https://github.com/n4ze3m/page-assist) (Chrome Extension)
+- [Plasmoid Ollama Control](https://github.com/imoize/plasmoid-ollamacontrol) (KDE Plasma extension that allows you to quickly manage/control Ollama model)
+- [AI Telegram Bot](https://github.com/tusharhero/aitelegrambot) (Telegram bot using Ollama in backend)
+- [AI ST Completion](https://github.com/yaroslavyaroslav/OpenAI-sublime-text) (Sublime Text 4 AI assistant plugin with Ollama support)
+- [Discord-Ollama Chat Bot](https://github.com/kevinthedang/discord-ollama) (Generalized TypeScript Discord Bot w/ Tuning Documentation)
+- [Discord AI chat/moderation bot](https://github.com/rapmd73/Companion) Chat/moderation bot written in python. Uses Ollama to create personalities.
+- [Headless Ollama](https://github.com/nischalj10/headless-ollama) (Scripts to automatically install ollama client & models on any OS for apps that depends on ollama server)
+- [vnc-lm](https://github.com/jk011ru/vnc-lm) (A containerized Discord bot with support for attachments and web links)
+- [LSP-AI](https://github.com/SilasMarvin/lsp-ai) (Open-source language server for AI-powered functionality)
+- [QodeAssist](https://github.com/Palm1r/QodeAssist) (AI-powered coding assistant plugin for Qt Creator)
+- [Obsidian Quiz Generator plugin](https://github.com/ECuiDev/obsidian-quiz-generator)
+
+### Supported backends
+
+- [llama.cpp](https://github.com/ggerganov/llama.cpp) project founded by Georgi Gerganov.
+
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 00000000..d38bb7c4
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,25 @@
+# Security
+
+The Ollama maintainer team takes security seriously and will actively work to resolve security issues.
+
+## Reporting a vulnerability
+
+If you discover a security vulnerability, please do not open a public issue. Instead, please report it by emailing hello@ollama.com. We ask that you give us sufficient time to investigate and address the vulnerability before disclosing it publicly.
+
+Please include the following details in your report:
+- A description of the vulnerability
+- Steps to reproduce the issue
+- Your assessment of the potential impact
+- Any possible mitigations
+
+## Security best practices
+
+While the maintainer team does their best to secure Ollama, users are encouraged to implement their own security best practices, such as:
+
+- Regularly updating to the latest version of Ollama
+- Securing access to hosted instances of Ollama
+- Monitoring systems for unusual activity
+
+## Contact
+
+For any other questions or concerns related to security, please contact us at hello@ollama.com
diff --git a/api/client.go b/api/client.go
index 72da30b8..2528fb21 100644
--- a/api/client.go
+++ b/api/client.go
@@ -1,3 +1,16 @@
+// Package api implements the client-side API for code wishing to interact
+// with the ollama service. The methods of the [Client] type correspond to
+// the ollama REST API as described in [the API documentation].
+// The ollama command-line client itself uses this package to interact with
+// the backend service.
+//
+// # Examples
+//
+// Several examples of using this package are available [in the GitHub
+// repository].
+//
+// [the API documentation]: https://github.com/ollama/ollama/blob/main/docs/api.md
+// [in the GitHub repository]: https://github.com/ollama/ollama/tree/main/examples
package api
import (
@@ -8,20 +21,20 @@ import (
"errors"
"fmt"
"io"
- "net"
"net/http"
"net/url"
- "os"
"runtime"
- "strings"
- "github.com/jmorganca/ollama/format"
- "github.com/jmorganca/ollama/version"
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/format"
+ "github.com/ollama/ollama/version"
)
+// Client encapsulates client state for interacting with the ollama
+// service. Use [ClientFromEnvironment] to create new Clients.
type Client struct {
base *url.URL
- http http.Client
+ http *http.Client
}
func checkError(resp *http.Response, body []byte) error {
@@ -40,56 +53,27 @@ func checkError(resp *http.Response, body []byte) error {
return apiError
}
+// ClientFromEnvironment creates a new [Client] using configuration from the
+// environment variable OLLAMA_HOST, which points to the network host and
+// port on which the ollama service is listenting. The format of this variable
+// is:
+//
+// <scheme>://<host>:<port>
+//
+// If the variable is not specified, a default ollama host and port will be
+// used.
func ClientFromEnvironment() (*Client, error) {
- defaultPort := "11434"
+ return &Client{
+ base: envconfig.Host(),
+ http: http.DefaultClient,
+ }, nil
+}
- scheme, hostport, ok := strings.Cut(os.Getenv("OLLAMA_HOST"), "://")
- switch {
- case !ok:
- scheme, hostport = "http", os.Getenv("OLLAMA_HOST")
- case scheme == "http":
- defaultPort = "80"
- case scheme == "https":
- defaultPort = "443"
+func NewClient(base *url.URL, http *http.Client) *Client {
+ return &Client{
+ base: base,
+ http: http,
}
-
- // trim trailing slashes
- hostport = strings.TrimRight(hostport, "/")
-
- host, port, err := net.SplitHostPort(hostport)
- if err != nil {
- host, port = "127.0.0.1", defaultPort
- if ip := net.ParseIP(strings.Trim(hostport, "[]")); ip != nil {
- host = ip.String()
- } else if hostport != "" {
- host = hostport
- }
- }
-
- client := Client{
- base: &url.URL{
- Scheme: scheme,
- Host: net.JoinHostPort(host, port),
- },
- }
-
- mockRequest, err := http.NewRequest(http.MethodHead, client.base.String(), nil)
- if err != nil {
- return nil, err
- }
-
- proxyURL, err := http.ProxyFromEnvironment(mockRequest)
- if err != nil {
- return nil, err
- }
-
- client.http = http.Client{
- Transport: &http.Transport{
- Proxy: http.ProxyURL(proxyURL),
- },
- }
-
- return &client, nil
}
func (c *Client) do(ctx context.Context, method, path string, reqData, respData any) error {
@@ -189,7 +173,7 @@ func (c *Client) stream(ctx context.Context, method, path string, data any, fn f
}
if errorResponse.Error != "" {
- return fmt.Errorf(errorResponse.Error)
+ return errors.New(errorResponse.Error)
}
if response.StatusCode >= http.StatusBadRequest {
@@ -208,8 +192,14 @@ func (c *Client) stream(ctx context.Context, method, path string, data any, fn f
return nil
}
+// GenerateResponseFunc is a function that [Client.Generate] invokes every time
+// a response is received from the service. If this function returns an error,
+// [Client.Generate] will stop generating and return this error.
type GenerateResponseFunc func(GenerateResponse) error
+// Generate generates a response for a given prompt. The req parameter should
+// be populated with prompt details. fn is called for each response (there may
+// be multiple responses, e.g. in case streaming is enabled).
func (c *Client) Generate(ctx context.Context, req *GenerateRequest, fn GenerateResponseFunc) error {
return c.stream(ctx, http.MethodPost, "/api/generate", req, func(bts []byte) error {
var resp GenerateResponse
@@ -221,8 +211,15 @@ func (c *Client) Generate(ctx context.Context, req *GenerateRequest, fn Generate
})
}
+// ChatResponseFunc is a function that [Client.Chat] invokes every time
+// a response is received from the service. If this function returns an error,
+// [Client.Chat] will stop generating and return this error.
type ChatResponseFunc func(ChatResponse) error
+// Chat generates the next message in a chat. [ChatRequest] may contain a
+// sequence of messages which can be used to maintain chat history with a model.
+// fn is called for each response (there may be multiple responses, e.g. if case
+// streaming is enabled).
func (c *Client) Chat(ctx context.Context, req *ChatRequest, fn ChatResponseFunc) error {
return c.stream(ctx, http.MethodPost, "/api/chat", req, func(bts []byte) error {
var resp ChatResponse
@@ -234,8 +231,14 @@ func (c *Client) Chat(ctx context.Context, req *ChatRequest, fn ChatResponseFunc
})
}
+// PullProgressFunc is a function that [Client.Pull] invokes every time there
+// is progress with a "pull" request sent to the service. If this function
+// returns an error, [Client.Pull] will stop the process and return this error.
type PullProgressFunc func(ProgressResponse) error
+// Pull downloads a model from the ollama library. fn is called each time
+// progress is made on the request and can be used to display a progress bar,
+// etc.
func (c *Client) Pull(ctx context.Context, req *PullRequest, fn PullProgressFunc) error {
return c.stream(ctx, http.MethodPost, "/api/pull", req, func(bts []byte) error {
var resp ProgressResponse
@@ -247,8 +250,14 @@ func (c *Client) Pull(ctx context.Context, req *PullRequest, fn PullProgressFunc
})
}
+// PushProgressFunc is a function that [Client.Push] invokes when progress is
+// made.
+// It's similar to other progress function types like [PullProgressFunc].
type PushProgressFunc func(ProgressResponse) error
+// Push uploads a model to the model library; requires registering for ollama.ai
+// and adding a public key first. fn is called each time progress is made on
+// the request and can be used to display a progress bar, etc.
func (c *Client) Push(ctx context.Context, req *PushRequest, fn PushProgressFunc) error {
return c.stream(ctx, http.MethodPost, "/api/push", req, func(bts []byte) error {
var resp ProgressResponse
@@ -260,8 +269,15 @@ func (c *Client) Push(ctx context.Context, req *PushRequest, fn PushProgressFunc
})
}
+// CreateProgressFunc is a function that [Client.Create] invokes when progress
+// is made.
+// It's similar to other progress function types like [PullProgressFunc].
type CreateProgressFunc func(ProgressResponse) error
+// Create creates a model from a [Modelfile]. fn is a progress function that
+// behaves similarly to other methods (see [Client.Pull]).
+//
+// [Modelfile]: https://github.com/ollama/ollama/blob/main/docs/modelfile.md
func (c *Client) Create(ctx context.Context, req *CreateRequest, fn CreateProgressFunc) error {
return c.stream(ctx, http.MethodPost, "/api/create", req, func(bts []byte) error {
var resp ProgressResponse
@@ -273,6 +289,7 @@ func (c *Client) Create(ctx context.Context, req *CreateRequest, fn CreateProgre
})
}
+// List lists models that are available locally.
func (c *Client) List(ctx context.Context) (*ListResponse, error) {
var lr ListResponse
if err := c.do(ctx, http.MethodGet, "/api/tags", nil, &lr); err != nil {
@@ -281,6 +298,17 @@ func (c *Client) List(ctx context.Context) (*ListResponse, error) {
return &lr, nil
}
+// ListRunning lists running models.
+func (c *Client) ListRunning(ctx context.Context) (*ProcessResponse, error) {
+ var lr ProcessResponse
+ if err := c.do(ctx, http.MethodGet, "/api/ps", nil, &lr); err != nil {
+ return nil, err
+ }
+ return &lr, nil
+}
+
+// Copy copies a model - creating a model with another name from an existing
+// model.
func (c *Client) Copy(ctx context.Context, req *CopyRequest) error {
if err := c.do(ctx, http.MethodPost, "/api/copy", req, nil); err != nil {
return err
@@ -288,6 +316,7 @@ func (c *Client) Copy(ctx context.Context, req *CopyRequest) error {
return nil
}
+// Delete deletes a model and its data.
func (c *Client) Delete(ctx context.Context, req *DeleteRequest) error {
if err := c.do(ctx, http.MethodDelete, "/api/delete", req, nil); err != nil {
return err
@@ -295,6 +324,7 @@ func (c *Client) Delete(ctx context.Context, req *DeleteRequest) error {
return nil
}
+// Show obtains model information, including details, modelfile, license etc.
func (c *Client) Show(ctx context.Context, req *ShowRequest) (*ShowResponse, error) {
var resp ShowResponse
if err := c.do(ctx, http.MethodPost, "/api/show", req, &resp); err != nil {
@@ -303,6 +333,8 @@ func (c *Client) Show(ctx context.Context, req *ShowRequest) (*ShowResponse, err
return &resp, nil
}
+// Heartbeat checks if the server has started and is responsive; if yes, it
+// returns nil, otherwise an error.
func (c *Client) Heartbeat(ctx context.Context) error {
if err := c.do(ctx, http.MethodHead, "/", nil, nil); err != nil {
return err
@@ -310,21 +342,31 @@ func (c *Client) Heartbeat(ctx context.Context) error {
return nil
}
-func (c *Client) CreateBlob(ctx context.Context, digest string, r io.Reader) error {
- if err := c.do(ctx, http.MethodHead, fmt.Sprintf("/api/blobs/%s", digest), nil, nil); err != nil {
- var statusError StatusError
- if !errors.As(err, &statusError) || statusError.StatusCode != http.StatusNotFound {
- return err
- }
-
- if err := c.do(ctx, http.MethodPost, fmt.Sprintf("/api/blobs/%s", digest), r, nil); err != nil {
- return err
- }
+// Embed generates embeddings from a model.
+func (c *Client) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) {
+ var resp EmbedResponse
+ if err := c.do(ctx, http.MethodPost, "/api/embed", req, &resp); err != nil {
+ return nil, err
}
-
- return nil
+ return &resp, nil
}
+// Embeddings generates an embedding from a model.
+func (c *Client) Embeddings(ctx context.Context, req *EmbeddingRequest) (*EmbeddingResponse, error) {
+ var resp EmbeddingResponse
+ if err := c.do(ctx, http.MethodPost, "/api/embeddings", req, &resp); err != nil {
+ return nil, err
+ }
+ return &resp, nil
+}
+
+// CreateBlob creates a blob from a file on the server. digest is the
+// expected SHA256 digest of the file, and r represents the file.
+func (c *Client) CreateBlob(ctx context.Context, digest string, r io.Reader) error {
+ return c.do(ctx, http.MethodPost, fmt.Sprintf("/api/blobs/%s", digest), r, nil)
+}
+
+// Version returns the Ollama server version as a string.
func (c *Client) Version(ctx context.Context) (string, error) {
var version struct {
Version string `json:"version"`
diff --git a/api/client.py b/api/client.py
deleted file mode 100644
index a2eb50e8..00000000
--- a/api/client.py
+++ /dev/null
@@ -1,284 +0,0 @@
-import os
-import json
-import requests
-import os
-import hashlib
-import json
-from pathlib import Path
-
-BASE_URL = os.environ.get('OLLAMA_HOST', 'http://localhost:11434')
-
-# Generate a response for a given prompt with a provided model. This is a streaming endpoint, so will be a series of responses.
-# The final response object will include statistics and additional data from the request. Use the callback function to override
-# the default handler.
-def generate(model_name, prompt, system=None, template=None, format="", context=None, options=None, callback=None):
- try:
- url = f"{BASE_URL}/api/generate"
- payload = {
- "model": model_name,
- "prompt": prompt,
- "system": system,
- "template": template,
- "context": context,
- "options": options,
- "format": format,
- }
-
- # Remove keys with None values
- payload = {k: v for k, v in payload.items() if v is not None}
-
- with requests.post(url, json=payload, stream=True) as response:
- response.raise_for_status()
-
- # Creating a variable to hold the context history of the final chunk
- final_context = None
-
- # Variable to hold concatenated response strings if no callback is provided
- full_response = ""
-
- # Iterating over the response line by line and displaying the details
- for line in response.iter_lines():
- if line:
- # Parsing each line (JSON chunk) and extracting the details
- chunk = json.loads(line)
-
- # If a callback function is provided, call it with the chunk
- if callback:
- callback(chunk)
- else:
- # If this is not the last chunk, add the "response" field value to full_response and print it
- if not chunk.get("done"):
- response_piece = chunk.get("response", "")
- full_response += response_piece
- print(response_piece, end="", flush=True)
-
- # Check if it's the last chunk (done is true)
- if chunk.get("done"):
- final_context = chunk.get("context")
-
- # Return the full response and the final context
- return full_response, final_context
- except requests.exceptions.RequestException as e:
- print(f"An error occurred: {e}")
- return None, None
-
-
-# Create a blob file on the server if it doesn't exist.
-def create_blob(digest, file_path):
- url = f"{BASE_URL}/api/blobs/{digest}"
-
- # Check if the blob exists
- response = requests.head(url)
- if response.status_code != 404:
- return # Blob already exists, no need to upload
- response.raise_for_status()
-
- # Upload the blob
- with open(file_path, 'rb') as file_data:
- requests.post(url, data=file_data)
-
-
-# Create a model from a Modelfile. Use the callback function to override the default handler.
-def create(model_name, filename, callback=None):
- try:
- file_path = Path(filename).expanduser().resolve()
- processed_lines = []
-
- # Read and process the modelfile
- with open(file_path, 'r') as f:
- for line in f:
- # Skip empty or whitespace-only lines
- if not line.strip():
- continue
-
- command, args = line.split(maxsplit=1)
-
- if command.upper() in ["FROM", "ADAPTER"]:
- path = Path(args.strip()).expanduser()
-
- # Check if path is relative and resolve it
- if not path.is_absolute():
- path = (file_path.parent / path)
-
- # Skip if file does not exist for "model", this is handled by the server
- if not path.exists():
- processed_lines.append(line)
- continue
-
- # Calculate SHA-256 hash
- with open(path, 'rb') as bin_file:
- hash = hashlib.sha256()
- hash.update(bin_file.read())
- blob = f"sha256:{hash.hexdigest()}"
-
- # Add the file to the remote server
- create_blob(blob, path)
-
- # Replace path with digest in the line
- line = f"{command} @{blob}\n"
-
- processed_lines.append(line)
-
- # Combine processed lines back into a single string
- modelfile_content = '\n'.join(processed_lines)
-
- url = f"{BASE_URL}/api/create"
- payload = {"name": model_name, "modelfile": modelfile_content}
-
- # Making a POST request with the stream parameter set to True to handle streaming responses
- with requests.post(url, json=payload, stream=True) as response:
- response.raise_for_status()
- # Iterating over the response line by line and displaying the status
- for line in response.iter_lines():
- if line:
- chunk = json.loads(line)
- if callback:
- callback(chunk)
- else:
- print(f"Status: {chunk.get('status')}")
-
- except Exception as e:
- print(f"An error occurred: {e}")
-
-
-# Pull a model from a the model registry. Cancelled pulls are resumed from where they left off, and multiple
-# calls to will share the same download progress. Use the callback function to override the default handler.
-def pull(model_name, insecure=False, callback=None):
- try:
- url = f"{BASE_URL}/api/pull"
- payload = {
- "name": model_name,
- "insecure": insecure
- }
-
- # Making a POST request with the stream parameter set to True to handle streaming responses
- with requests.post(url, json=payload, stream=True) as response:
- response.raise_for_status()
-
- # Iterating over the response line by line and displaying the details
- for line in response.iter_lines():
- if line:
- # Parsing each line (JSON chunk) and extracting the details
- chunk = json.loads(line)
-
- # If a callback function is provided, call it with the chunk
- if callback:
- callback(chunk)
- else:
- # Print the status message directly to the console
- print(chunk.get('status', ''), end='', flush=True)
-
- # If there's layer data, you might also want to print that (adjust as necessary)
- if 'digest' in chunk:
- print(f" - Digest: {chunk['digest']}", end='', flush=True)
- print(f" - Total: {chunk['total']}", end='', flush=True)
- print(f" - Completed: {chunk['completed']}", end='\n', flush=True)
- else:
- print()
- except requests.exceptions.RequestException as e:
- print(f"An error occurred: {e}")
-
-# Push a model to the model registry. Use the callback function to override the default handler.
-def push(model_name, insecure=False, callback=None):
- try:
- url = f"{BASE_URL}/api/push"
- payload = {
- "name": model_name,
- "insecure": insecure
- }
-
- # Making a POST request with the stream parameter set to True to handle streaming responses
- with requests.post(url, json=payload, stream=True) as response:
- response.raise_for_status()
-
- # Iterating over the response line by line and displaying the details
- for line in response.iter_lines():
- if line:
- # Parsing each line (JSON chunk) and extracting the details
- chunk = json.loads(line)
-
- # If a callback function is provided, call it with the chunk
- if callback:
- callback(chunk)
- else:
- # Print the status message directly to the console
- print(chunk.get('status', ''), end='', flush=True)
-
- # If there's layer data, you might also want to print that (adjust as necessary)
- if 'digest' in chunk:
- print(f" - Digest: {chunk['digest']}", end='', flush=True)
- print(f" - Total: {chunk['total']}", end='', flush=True)
- print(f" - Completed: {chunk['completed']}", end='\n', flush=True)
- else:
- print()
- except requests.exceptions.RequestException as e:
- print(f"An error occurred: {e}")
-
-# List models that are available locally.
-def list():
- try:
- response = requests.get(f"{BASE_URL}/api/tags")
- response.raise_for_status()
- data = response.json()
- models = data.get('models', [])
- return models
-
- except requests.exceptions.RequestException as e:
- print(f"An error occurred: {e}")
- return None
-
-# Copy a model. Creates a model with another name from an existing model.
-def copy(source, destination):
- try:
- # Create the JSON payload
- payload = {
- "source": source,
- "destination": destination
- }
-
- response = requests.post(f"{BASE_URL}/api/copy", json=payload)
- response.raise_for_status()
-
- # If the request was successful, return a message indicating that the copy was successful
- return "Copy successful"
-
- except requests.exceptions.RequestException as e:
- print(f"An error occurred: {e}")
- return None
-
-# Delete a model and its data.
-def delete(model_name):
- try:
- url = f"{BASE_URL}/api/delete"
- payload = {"name": model_name}
- response = requests.delete(url, json=payload)
- response.raise_for_status()
- return "Delete successful"
- except requests.exceptions.RequestException as e:
- print(f"An error occurred: {e}")
- return None
-
-# Show info about a model.
-def show(model_name):
- try:
- url = f"{BASE_URL}/api/show"
- payload = {"name": model_name}
- response = requests.post(url, json=payload)
- response.raise_for_status()
-
- # Parse the JSON response and return it
- data = response.json()
- return data
- except requests.exceptions.RequestException as e:
- print(f"An error occurred: {e}")
- return None
-
-def heartbeat():
- try:
- url = f"{BASE_URL}/"
- response = requests.head(url)
- response.raise_for_status()
- return "Ollama is running"
- except requests.exceptions.RequestException as e:
- print(f"An error occurred: {e}")
- return "Ollama is not running"
diff --git a/api/client_test.go b/api/client_test.go
index 0eafedca..23fe9334 100644
--- a/api/client_test.go
+++ b/api/client_test.go
@@ -1,6 +1,8 @@
package api
-import "testing"
+import (
+ "testing"
+)
func TestClientFromEnvironment(t *testing.T) {
type testCase struct {
diff --git a/api/types.go b/api/types.go
index 43f642c3..df7bab21 100644
--- a/api/types.go
+++ b/api/types.go
@@ -3,6 +3,7 @@ package api
import (
"encoding/json"
"fmt"
+ "log/slog"
"math"
"os"
"reflect"
@@ -11,6 +12,7 @@ import (
"time"
)
+// StatusError is an error with and HTTP status code.
type StatusError struct {
StatusCode int
Status string
@@ -31,41 +33,161 @@ func (e StatusError) Error() string {
}
}
+// ImageData represents the raw binary data of an image file.
type ImageData []byte
+// GenerateRequest describes a request sent by [Client.Generate]. While you
+// have to specify the Model and Prompt fields, all the other fields have
+// reasonable defaults for basic uses.
type GenerateRequest struct {
- Model string `json:"model"`
- Prompt string `json:"prompt"`
- System string `json:"system"`
- Template string `json:"template"`
- Context []int `json:"context,omitempty"`
- Stream *bool `json:"stream,omitempty"`
- Raw bool `json:"raw,omitempty"`
- Format string `json:"format"`
- Images []ImageData `json:"images,omitempty"`
+ // Model is the model name; it should be a name familiar to Ollama from
+ // the library at https://ollama.com/library
+ Model string `json:"model"`
+ // Prompt is the textual prompt to send to the model.
+ Prompt string `json:"prompt"`
+
+ // Suffix is the text that comes after the inserted text.
+ Suffix string `json:"suffix"`
+
+ // System overrides the model's default system message/prompt.
+ System string `json:"system"`
+
+ // Template overrides the model's default prompt template.
+ Template string `json:"template"`
+
+ // Context is the context parameter returned from a previous call to
+ // Generate call. It can be used to keep a short conversational memory.
+ Context []int `json:"context,omitempty"`
+
+ // Stream specifies whether the response is streaming; it is true by default.
+ Stream *bool `json:"stream,omitempty"`
+
+ // Raw set to true means that no formatting will be applied to the prompt.
+ Raw bool `json:"raw,omitempty"`
+
+ // Format specifies the format to return a response in.
+ Format string `json:"format"`
+
+ // KeepAlive controls how long the model will stay loaded in memory following
+ // this request.
+ KeepAlive *Duration `json:"keep_alive,omitempty"`
+
+ // Images is an optional list of base64-encoded images accompanying this
+ // request, for multimodal models.
+ Images []ImageData `json:"images,omitempty"`
+
+ // Options lists model-specific options. For example, temperature can be
+ // set through this field, if the model supports it.
Options map[string]interface{} `json:"options"`
}
+// ChatRequest describes a request sent by [Client.Chat].
type ChatRequest struct {
- Model string `json:"model"`
- Messages []Message `json:"messages"`
- Stream *bool `json:"stream,omitempty"`
- Format string `json:"format"`
+ // Model is the model name, as in [GenerateRequest].
+ Model string `json:"model"`
+ // Messages is the messages of the chat - can be used to keep a chat memory.
+ Messages []Message `json:"messages"`
+
+ // Stream enable streaming of returned response; true by default.
+ Stream *bool `json:"stream,omitempty"`
+
+ // Format is the format to return the response in (e.g. "json").
+ Format string `json:"format"`
+
+ // KeepAlive controls how long the model will stay loaded into memory
+ // followin the request.
+ KeepAlive *Duration `json:"keep_alive,omitempty"`
+
+ // Tools is an optional list of tools the model has access to.
+ Tools `json:"tools,omitempty"`
+
+ // Options lists model-specific options.
Options map[string]interface{} `json:"options"`
}
-type Message struct {
- Role string `json:"role"` // one of ["system", "user", "assistant"]
- Content string `json:"content"`
- Images []ImageData `json:"images,omitempty"`
+type Tools []Tool
+
+func (t Tools) String() string {
+ bts, _ := json.Marshal(t)
+ return string(bts)
}
+func (t Tool) String() string {
+ bts, _ := json.Marshal(t)
+ return string(bts)
+}
+
+// Message is a single message in a chat sequence. The message contains the
+// role ("system", "user", or "assistant"), the content and an optional list
+// of images.
+type Message struct {
+ Role string `json:"role"`
+ Content string `json:"content"`
+ Images []ImageData `json:"images,omitempty"`
+ ToolCalls []ToolCall `json:"tool_calls,omitempty"`
+}
+
+func (m *Message) UnmarshalJSON(b []byte) error {
+ type Alias Message
+ var a Alias
+ if err := json.Unmarshal(b, &a); err != nil {
+ return err
+ }
+
+ *m = Message(a)
+ m.Role = strings.ToLower(m.Role)
+ return nil
+}
+
+type ToolCall struct {
+ Function ToolCallFunction `json:"function"`
+}
+
+type ToolCallFunction struct {
+ Name string `json:"name"`
+ Arguments ToolCallFunctionArguments `json:"arguments"`
+}
+
+type ToolCallFunctionArguments map[string]any
+
+func (t *ToolCallFunctionArguments) String() string {
+ bts, _ := json.Marshal(t)
+ return string(bts)
+}
+
+type Tool struct {
+ Type string `json:"type"`
+ Function ToolFunction `json:"function"`
+}
+
+type ToolFunction struct {
+ Name string `json:"name"`
+ Description string `json:"description"`
+ Parameters struct {
+ Type string `json:"type"`
+ Required []string `json:"required"`
+ Properties map[string]struct {
+ Type string `json:"type"`
+ Description string `json:"description"`
+ Enum []string `json:"enum,omitempty"`
+ } `json:"properties"`
+ } `json:"parameters"`
+}
+
+func (t *ToolFunction) String() string {
+ bts, _ := json.Marshal(t)
+ return string(bts)
+}
+
+// ChatResponse is the response returned by [Client.Chat]. Its fields are
+// similar to [GenerateResponse].
type ChatResponse struct {
- Model string `json:"model"`
- CreatedAt time.Time `json:"created_at"`
- Message Message `json:"message"`
+ Model string `json:"model"`
+ CreatedAt time.Time `json:"created_at"`
+ Message Message `json:"message"`
+ DoneReason string `json:"done_reason,omitempty"`
Done bool `json:"done"`
@@ -81,7 +203,8 @@ type Metrics struct {
EvalDuration time.Duration `json:"eval_duration,omitempty"`
}
-// Options specfied in GenerateRequest, if you add a new option here add it to the API docs also
+// Options specified in [GenerateRequest], if you add a new option here add it
+// to the API docs also.
type Options struct {
Runner
@@ -91,6 +214,7 @@ type Options struct {
NumPredict int `json:"num_predict,omitempty"`
TopK int `json:"top_k,omitempty"`
TopP float32 `json:"top_p,omitempty"`
+ MinP float32 `json:"min_p,omitempty"`
TFSZ float32 `json:"tfs_z,omitempty"`
TypicalP float32 `json:"typical_p,omitempty"`
RepeatLastN int `json:"repeat_last_n,omitempty"`
@@ -107,72 +231,142 @@ type Options struct {
// Runner options which must be set when the model is loaded into memory
type Runner struct {
- UseNUMA bool `json:"numa,omitempty"`
- NumCtx int `json:"num_ctx,omitempty"`
- NumBatch int `json:"num_batch,omitempty"`
- NumGQA int `json:"num_gqa,omitempty"`
- NumGPU int `json:"num_gpu,omitempty"`
- MainGPU int `json:"main_gpu,omitempty"`
- LowVRAM bool `json:"low_vram,omitempty"`
- F16KV bool `json:"f16_kv,omitempty"`
- LogitsAll bool `json:"logits_all,omitempty"`
- VocabOnly bool `json:"vocab_only,omitempty"`
- UseMMap bool `json:"use_mmap,omitempty"`
- UseMLock bool `json:"use_mlock,omitempty"`
- EmbeddingOnly bool `json:"embedding_only,omitempty"`
- RopeFrequencyBase float32 `json:"rope_frequency_base,omitempty"`
- RopeFrequencyScale float32 `json:"rope_frequency_scale,omitempty"`
- NumThread int `json:"num_thread,omitempty"`
+ NumCtx int `json:"num_ctx,omitempty"`
+ NumBatch int `json:"num_batch,omitempty"`
+ NumGPU int `json:"num_gpu,omitempty"`
+ MainGPU int `json:"main_gpu,omitempty"`
+ LowVRAM bool `json:"low_vram,omitempty"`
+ F16KV bool `json:"f16_kv,omitempty"`
+ LogitsAll bool `json:"logits_all,omitempty"`
+ VocabOnly bool `json:"vocab_only,omitempty"`
+ UseMMap *bool `json:"use_mmap,omitempty"`
+ UseMLock bool `json:"use_mlock,omitempty"`
+ NumThread int `json:"num_thread,omitempty"`
}
-type EmbeddingRequest struct {
- Model string `json:"model"`
- Prompt string `json:"prompt"`
+// EmbedRequest is the request passed to [Client.Embed].
+type EmbedRequest struct {
+ // Model is the model name.
+ Model string `json:"model"`
+ // Input is the input to embed.
+ Input any `json:"input"`
+
+ // KeepAlive controls how long the model will stay loaded in memory following
+ // this request.
+ KeepAlive *Duration `json:"keep_alive,omitempty"`
+
+ Truncate *bool `json:"truncate,omitempty"`
+
+ // Options lists model-specific options.
Options map[string]interface{} `json:"options"`
}
+// EmbedResponse is the response from [Client.Embed].
+type EmbedResponse struct {
+ Model string `json:"model"`
+ Embeddings [][]float32 `json:"embeddings"`
+
+ TotalDuration time.Duration `json:"total_duration,omitempty"`
+ LoadDuration time.Duration `json:"load_duration,omitempty"`
+ PromptEvalCount int `json:"prompt_eval_count,omitempty"`
+}
+
+// EmbeddingRequest is the request passed to [Client.Embeddings].
+type EmbeddingRequest struct {
+ // Model is the model name.
+ Model string `json:"model"`
+
+ // Prompt is the textual prompt to embed.
+ Prompt string `json:"prompt"`
+
+ // KeepAlive controls how long the model will stay loaded in memory following
+ // this request.
+ KeepAlive *Duration `json:"keep_alive,omitempty"`
+
+ // Options lists model-specific options.
+ Options map[string]interface{} `json:"options"`
+}
+
+// EmbeddingResponse is the response from [Client.Embeddings].
type EmbeddingResponse struct {
Embedding []float64 `json:"embedding"`
}
+// CreateRequest is the request passed to [Client.Create].
type CreateRequest struct {
- Name string `json:"name"`
- Path string `json:"path"`
+ Model string `json:"model"`
Modelfile string `json:"modelfile"`
Stream *bool `json:"stream,omitempty"`
+ Quantize string `json:"quantize,omitempty"`
+
+ // Deprecated: set the model name with Model instead
+ Name string `json:"name"`
+
+ // Deprecated: set the file content with Modelfile instead
+ Path string `json:"path"`
+
+ // Deprecated: use Quantize instead
+ Quantization string `json:"quantization,omitempty"`
}
+// DeleteRequest is the request passed to [Client.Delete].
type DeleteRequest struct {
+ Model string `json:"model"`
+
+ // Deprecated: set the model name with Model instead
Name string `json:"name"`
}
+// ShowRequest is the request passed to [Client.Show].
type ShowRequest struct {
+ Model string `json:"model"`
+ System string `json:"system"`
+
+ // Template is deprecated
+ Template string `json:"template"`
+ Verbose bool `json:"verbose"`
+
+ Options map[string]interface{} `json:"options"`
+
+ // Deprecated: set the model name with Model instead
Name string `json:"name"`
}
+// ShowResponse is the response returned from [Client.Show].
type ShowResponse struct {
- License string `json:"license,omitempty"`
- Modelfile string `json:"modelfile,omitempty"`
- Parameters string `json:"parameters,omitempty"`
- Template string `json:"template,omitempty"`
- System string `json:"system,omitempty"`
- Details ModelDetails `json:"details,omitempty"`
+ License string `json:"license,omitempty"`
+ Modelfile string `json:"modelfile,omitempty"`
+ Parameters string `json:"parameters,omitempty"`
+ Template string `json:"template,omitempty"`
+ System string `json:"system,omitempty"`
+ Details ModelDetails `json:"details,omitempty"`
+ Messages []Message `json:"messages,omitempty"`
+ ModelInfo map[string]any `json:"model_info,omitempty"`
+ ProjectorInfo map[string]any `json:"projector_info,omitempty"`
+ ModifiedAt time.Time `json:"modified_at,omitempty"`
}
+// CopyRequest is the request passed to [Client.Copy].
type CopyRequest struct {
Source string `json:"source"`
Destination string `json:"destination"`
}
+// PullRequest is the request passed to [Client.Pull].
type PullRequest struct {
- Name string `json:"name"`
+ Model string `json:"model"`
Insecure bool `json:"insecure,omitempty"`
Username string `json:"username"`
Password string `json:"password"`
Stream *bool `json:"stream,omitempty"`
+
+ // Deprecated: set the model name with Model instead
+ Name string `json:"name"`
}
+// ProgressResponse is the response passed to progress functions like
+// [PullProgressFunc] and [PushProgressFunc].
type ProgressResponse struct {
Status string `json:"status"`
Digest string `json:"digest,omitempty"`
@@ -180,42 +374,87 @@ type ProgressResponse struct {
Completed int64 `json:"completed,omitempty"`
}
+// PushRequest is the request passed to [Client.Push].
type PushRequest struct {
- Name string `json:"name"`
+ Model string `json:"model"`
Insecure bool `json:"insecure,omitempty"`
Username string `json:"username"`
Password string `json:"password"`
Stream *bool `json:"stream,omitempty"`
+
+ // Deprecated: set the model name with Model instead
+ Name string `json:"name"`
}
+// ListResponse is the response from [Client.List].
type ListResponse struct {
- Models []ModelResponse `json:"models"`
+ Models []ListModelResponse `json:"models"`
}
-type ModelResponse struct {
+// ProcessResponse is the response from [Client.Process].
+type ProcessResponse struct {
+ Models []ProcessModelResponse `json:"models"`
+}
+
+// ListModelResponse is a single model description in [ListResponse].
+type ListModelResponse struct {
Name string `json:"name"`
+ Model string `json:"model"`
ModifiedAt time.Time `json:"modified_at"`
Size int64 `json:"size"`
Digest string `json:"digest"`
Details ModelDetails `json:"details,omitempty"`
}
+// ProcessModelResponse is a single model description in [ProcessResponse].
+type ProcessModelResponse struct {
+ Name string `json:"name"`
+ Model string `json:"model"`
+ Size int64 `json:"size"`
+ Digest string `json:"digest"`
+ Details ModelDetails `json:"details,omitempty"`
+ ExpiresAt time.Time `json:"expires_at"`
+ SizeVRAM int64 `json:"size_vram"`
+}
+
+type RetrieveModelResponse struct {
+ Id string `json:"id"`
+ Object string `json:"object"`
+ Created int64 `json:"created"`
+ OwnedBy string `json:"owned_by"`
+}
+
type TokenResponse struct {
Token string `json:"token"`
}
+// GenerateResponse is the response passed into [GenerateResponseFunc].
type GenerateResponse struct {
- Model string `json:"model"`
- CreatedAt time.Time `json:"created_at"`
- Response string `json:"response"`
+ // Model is the model name that generated the response.
+ Model string `json:"model"`
- Done bool `json:"done"`
+ // CreatedAt is the timestamp of the response.
+ CreatedAt time.Time `json:"created_at"`
+
+ // Response is the textual response itself.
+ Response string `json:"response"`
+
+ // Done specifies if the response is complete.
+ Done bool `json:"done"`
+
+ // DoneReason is the reason the model stopped generating text.
+ DoneReason string `json:"done_reason,omitempty"`
+
+ // Context is an encoding of the conversation used in this response; this
+ // can be sent in the next request to keep a conversational memory.
Context []int `json:"context,omitempty"`
Metrics
}
+// ModelDetails provides details about a model.
type ModelDetails struct {
+ ParentModel string `json:"parent_model"`
Format string `json:"format"`
Family string `json:"family"`
Families []string `json:"families"`
@@ -251,8 +490,6 @@ func (m *Metrics) Summary() {
}
}
-var ErrInvalidOpts = fmt.Errorf("invalid options")
-
func (opts *Options) FromMap(m map[string]interface{}) error {
valueOpts := reflect.ValueOf(opts).Elem() // names of the fields in the options struct
typeOpts := reflect.TypeOf(opts).Elem() // types of the fields in the options struct
@@ -266,81 +503,94 @@ func (opts *Options) FromMap(m map[string]interface{}) error {
}
}
- invalidOpts := []string{}
for key, val := range m {
- if opt, ok := jsonOpts[key]; ok {
- field := valueOpts.FieldByName(opt.Name)
- if field.IsValid() && field.CanSet() {
- if val == nil {
- continue
- }
+ opt, ok := jsonOpts[key]
+ if !ok {
+ slog.Warn("invalid option provided", "option", key)
+ continue
+ }
- switch field.Kind() {
- case reflect.Int:
- switch t := val.(type) {
- case int64:
- field.SetInt(t)
- case float64:
- // when JSON unmarshals numbers, it uses float64, not int
- field.SetInt(int64(t))
- default:
- return fmt.Errorf("option %q must be of type integer", key)
+ field := valueOpts.FieldByName(opt.Name)
+ if field.IsValid() && field.CanSet() {
+ if val == nil {
+ continue
+ }
+
+ switch field.Kind() {
+ case reflect.Int:
+ switch t := val.(type) {
+ case int64:
+ field.SetInt(t)
+ case float64:
+ // when JSON unmarshals numbers, it uses float64, not int
+ field.SetInt(int64(t))
+ default:
+ return fmt.Errorf("option %q must be of type integer", key)
+ }
+ case reflect.Bool:
+ val, ok := val.(bool)
+ if !ok {
+ return fmt.Errorf("option %q must be of type boolean", key)
+ }
+ field.SetBool(val)
+ case reflect.Float32:
+ // JSON unmarshals to float64
+ val, ok := val.(float64)
+ if !ok {
+ return fmt.Errorf("option %q must be of type float32", key)
+ }
+ field.SetFloat(val)
+ case reflect.String:
+ val, ok := val.(string)
+ if !ok {
+ return fmt.Errorf("option %q must be of type string", key)
+ }
+ field.SetString(val)
+ case reflect.Slice:
+ // JSON unmarshals to []interface{}, not []string
+ val, ok := val.([]interface{})
+ if !ok {
+ return fmt.Errorf("option %q must be of type array", key)
+ }
+ // convert []interface{} to []string
+ slice := make([]string, len(val))
+ for i, item := range val {
+ str, ok := item.(string)
+ if !ok {
+ return fmt.Errorf("option %q must be of an array of strings", key)
}
- case reflect.Bool:
+ slice[i] = str
+ }
+ field.Set(reflect.ValueOf(slice))
+ case reflect.Pointer:
+ var b bool
+ if field.Type() == reflect.TypeOf(&b) {
val, ok := val.(bool)
if !ok {
return fmt.Errorf("option %q must be of type boolean", key)
}
- field.SetBool(val)
- case reflect.Float32:
- // JSON unmarshals to float64
- val, ok := val.(float64)
- if !ok {
- return fmt.Errorf("option %q must be of type float32", key)
- }
- field.SetFloat(val)
- case reflect.String:
- val, ok := val.(string)
- if !ok {
- return fmt.Errorf("option %q must be of type string", key)
- }
- field.SetString(val)
- case reflect.Slice:
- // JSON unmarshals to []interface{}, not []string
- val, ok := val.([]interface{})
- if !ok {
- return fmt.Errorf("option %q must be of type array", key)
- }
- // convert []interface{} to []string
- slice := make([]string, len(val))
- for i, item := range val {
- str, ok := item.(string)
- if !ok {
- return fmt.Errorf("option %q must be of an array of strings", key)
- }
- slice[i] = str
- }
- field.Set(reflect.ValueOf(slice))
- default:
- return fmt.Errorf("unknown type loading config params: %v", field.Kind())
+ field.Set(reflect.ValueOf(&val))
+ } else {
+ return fmt.Errorf("unknown type loading config params: %v %v", field.Kind(), field.Type())
}
+ default:
+ return fmt.Errorf("unknown type loading config params: %v", field.Kind())
}
- } else {
- invalidOpts = append(invalidOpts, key)
}
}
- if len(invalidOpts) > 0 {
- return fmt.Errorf("%w: %v", ErrInvalidOpts, strings.Join(invalidOpts, ", "))
- }
return nil
}
+// DefaultOptions is the default set of options for [GenerateRequest]; these
+// values are used unless the user specifies other values explicitly.
func DefaultOptions() Options {
return Options{
// options set on request to runner
- NumPredict: -1,
- NumKeep: 0,
+ NumPredict: -1,
+
+ // set a minimal num_keep to avoid issues on context shifts
+ NumKeep: 4,
Temperature: 0.8,
TopK: 40,
TopP: 0.9,
@@ -358,19 +608,14 @@ func DefaultOptions() Options {
Runner: Runner{
// options set when the model is loaded
- NumCtx: 2048,
- RopeFrequencyBase: 10000.0,
- RopeFrequencyScale: 1.0,
- NumBatch: 512,
- NumGPU: -1, // -1 here indicates that NumGPU should be set dynamically
- NumGQA: 1,
- NumThread: 0, // let the runtime decide
- LowVRAM: false,
- F16KV: true,
- UseMLock: false,
- UseMMap: true,
- UseNUMA: false,
- EmbeddingOnly: true,
+ NumCtx: 2048,
+ NumBatch: 512,
+ NumGPU: -1, // -1 here indicates that NumGPU should be set dynamically
+ NumThread: 0, // let the runtime decide
+ LowVRAM: false,
+ F16KV: true,
+ UseMLock: false,
+ UseMMap: nil,
},
}
}
@@ -379,6 +624,13 @@ type Duration struct {
time.Duration
}
+func (d Duration) MarshalJSON() ([]byte, error) {
+ if d.Duration < 0 {
+ return []byte("-1"), nil
+ }
+ return []byte("\"" + d.Duration.String() + "\""), nil
+}
+
func (d *Duration) UnmarshalJSON(b []byte) (err error) {
var v any
if err := json.Unmarshal(b, &v); err != nil {
@@ -390,15 +642,20 @@ func (d *Duration) UnmarshalJSON(b []byte) (err error) {
switch t := v.(type) {
case float64:
if t < 0 {
- t = math.MaxFloat64
+ d.Duration = time.Duration(math.MaxInt64)
+ } else {
+ d.Duration = time.Duration(int(t) * int(time.Second))
}
-
- d.Duration = time.Duration(t)
case string:
d.Duration, err = time.ParseDuration(t)
if err != nil {
return err
}
+ if d.Duration < 0 {
+ d.Duration = time.Duration(math.MaxInt64)
+ }
+ default:
+ return fmt.Errorf("Unsupported type: '%s'", reflect.TypeOf(v))
}
return nil
@@ -454,6 +711,17 @@ func FormatParams(params map[string][]string) (map[string]interface{}, error) {
case reflect.Slice:
// TODO: only string slices are supported right now
out[key] = vals
+ case reflect.Pointer:
+ var b bool
+ if field.Type() == reflect.TypeOf(&b) {
+ boolVal, err := strconv.ParseBool(vals[0])
+ if err != nil {
+ return nil, fmt.Errorf("invalid bool value %s", vals)
+ }
+ out[key] = &boolVal
+ } else {
+ return nil, fmt.Errorf("unknown type %s for %s", field.Kind(), key)
+ }
default:
return nil, fmt.Errorf("unknown type %s for %s", field.Kind(), key)
}
diff --git a/api/types_test.go b/api/types_test.go
new file mode 100644
index 00000000..a9de5a9a
--- /dev/null
+++ b/api/types_test.go
@@ -0,0 +1,233 @@
+package api
+
+import (
+ "encoding/json"
+ "errors"
+ "math"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestKeepAliveParsingFromJSON(t *testing.T) {
+ tests := []struct {
+ name string
+ req string
+ exp *Duration
+ }{
+ {
+ name: "Positive Integer",
+ req: `{ "keep_alive": 42 }`,
+ exp: &Duration{42 * time.Second},
+ },
+ {
+ name: "Positive Float",
+ req: `{ "keep_alive": 42.5 }`,
+ exp: &Duration{42 * time.Second},
+ },
+ {
+ name: "Positive Integer String",
+ req: `{ "keep_alive": "42m" }`,
+ exp: &Duration{42 * time.Minute},
+ },
+ {
+ name: "Negative Integer",
+ req: `{ "keep_alive": -1 }`,
+ exp: &Duration{math.MaxInt64},
+ },
+ {
+ name: "Negative Float",
+ req: `{ "keep_alive": -3.14 }`,
+ exp: &Duration{math.MaxInt64},
+ },
+ {
+ name: "Negative Integer String",
+ req: `{ "keep_alive": "-1m" }`,
+ exp: &Duration{math.MaxInt64},
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ var dec ChatRequest
+ err := json.Unmarshal([]byte(test.req), &dec)
+ require.NoError(t, err)
+
+ assert.Equal(t, test.exp, dec.KeepAlive)
+ })
+ }
+}
+
+func TestDurationMarshalUnmarshal(t *testing.T) {
+ tests := []struct {
+ name string
+ input time.Duration
+ expected time.Duration
+ }{
+ {
+ "negative duration",
+ time.Duration(-1),
+ time.Duration(math.MaxInt64),
+ },
+ {
+ "positive duration",
+ 42 * time.Second,
+ 42 * time.Second,
+ },
+ {
+ "another positive duration",
+ 42 * time.Minute,
+ 42 * time.Minute,
+ },
+ {
+ "zero duration",
+ time.Duration(0),
+ time.Duration(0),
+ },
+ {
+ "max duration",
+ time.Duration(math.MaxInt64),
+ time.Duration(math.MaxInt64),
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ b, err := json.Marshal(Duration{test.input})
+ require.NoError(t, err)
+
+ var d Duration
+ err = json.Unmarshal(b, &d)
+ require.NoError(t, err)
+
+ assert.Equal(t, test.expected, d.Duration, "input %v, marshalled %v, got %v", test.input, string(b), d.Duration)
+ })
+ }
+}
+
+func TestUseMmapParsingFromJSON(t *testing.T) {
+ tr := true
+ fa := false
+ tests := []struct {
+ name string
+ req string
+ exp *bool
+ }{
+ {
+ name: "Undefined",
+ req: `{ }`,
+ exp: nil,
+ },
+ {
+ name: "True",
+ req: `{ "use_mmap": true }`,
+ exp: &tr,
+ },
+ {
+ name: "False",
+ req: `{ "use_mmap": false }`,
+ exp: &fa,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ var oMap map[string]interface{}
+ err := json.Unmarshal([]byte(test.req), &oMap)
+ require.NoError(t, err)
+ opts := DefaultOptions()
+ err = opts.FromMap(oMap)
+ require.NoError(t, err)
+ assert.Equal(t, test.exp, opts.UseMMap)
+ })
+ }
+}
+
+func TestUseMmapFormatParams(t *testing.T) {
+ tr := true
+ fa := false
+ tests := []struct {
+ name string
+ req map[string][]string
+ exp *bool
+ err error
+ }{
+ {
+ name: "True",
+ req: map[string][]string{
+ "use_mmap": {"true"},
+ },
+ exp: &tr,
+ err: nil,
+ },
+ {
+ name: "False",
+ req: map[string][]string{
+ "use_mmap": {"false"},
+ },
+ exp: &fa,
+ err: nil,
+ },
+ {
+ name: "Numeric True",
+ req: map[string][]string{
+ "use_mmap": {"1"},
+ },
+ exp: &tr,
+ err: nil,
+ },
+ {
+ name: "Numeric False",
+ req: map[string][]string{
+ "use_mmap": {"0"},
+ },
+ exp: &fa,
+ err: nil,
+ },
+ {
+ name: "invalid string",
+ req: map[string][]string{
+ "use_mmap": {"foo"},
+ },
+ exp: nil,
+ err: errors.New("invalid bool value [foo]"),
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ resp, err := FormatParams(test.req)
+ require.Equal(t, test.err, err)
+ respVal, ok := resp["use_mmap"]
+ if test.exp != nil {
+ assert.True(t, ok, "resp: %v", resp)
+ assert.Equal(t, *test.exp, *respVal.(*bool))
+ }
+ })
+ }
+}
+
+func TestMessage_UnmarshalJSON(t *testing.T) {
+ tests := []struct {
+ input string
+ expected string
+ }{
+ {`{"role": "USER", "content": "Hello!"}`, "user"},
+ {`{"role": "System", "content": "Initialization complete."}`, "system"},
+ {`{"role": "assistant", "content": "How can I help you?"}`, "assistant"},
+ {`{"role": "TOOl", "content": "Access granted."}`, "tool"},
+ }
+
+ for _, test := range tests {
+ var msg Message
+ if err := json.Unmarshal([]byte(test.input), &msg); err != nil {
+ t.Errorf("Unexpected error: %v", err)
+ }
+
+ if msg.Role != test.expected {
+ t.Errorf("role not lowercased: got %v, expected %v", msg.Role, test.expected)
+ }
+ }
+}
diff --git a/app/.gitignore b/app/.gitignore
index 8296128d..0aa24794 100644
--- a/app/.gitignore
+++ b/app/.gitignore
@@ -1,92 +1 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-lerna-debug.log*
-
-# Diagnostic reports (https://nodejs.org/api/report.html)
-report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
-
-# Runtime data
-pids
-*.pid
-*.seed
-*.pid.lock
-.DS_Store
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-*.lcov
-
-# nyc test coverage
-.nyc_output
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
-
-# Dependency directories
-node_modules/
-jspm_packages/
-
-# TypeScript v1 declaration files
-typings/
-
-# TypeScript cache
-*.tsbuildinfo
-
-# Optional npm cache directory
-.npm
-
-# Optional eslint cache
-.eslintcache
-
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
-
-# dotenv environment variables file
-.env
-.env.test
-
-# parcel-bundler cache (https://parceljs.org/)
-.cache
-
-# next.js build output
-.next
-
-# nuxt.js build output
-.nuxt
-
-# vuepress build output
-.vuepress/dist
-
-# Serverless directories
-.serverless/
-
-# FuseBox cache
-.fusebox/
-
-# DynamoDB Local files
-.dynamodb/
-
-# Webpack
-.webpack/
-
-# Vite
-.vite/
-
-# Electron-Forge
-out/
+ollama.syso
diff --git a/app/README.md b/app/README.md
index cc34d745..883d7ab7 100644
--- a/app/README.md
+++ b/app/README.md
@@ -1,21 +1,22 @@
-# Desktop
+# Ollama App
-This app builds upon Ollama to provide a desktop experience for running models.
+## Linux
-## Developing
+TODO
-First, build the `ollama` binary:
+## MacOS
+
+TODO
+
+## Windows
+
+If you want to build the installer, youll need to install
+- https://jrsoftware.org/isinfo.php
+
+
+In the top directory of this repo, run the following powershell script
+to build the ollama CLI, ollama app, and ollama installer.
```
-cd ..
-go build .
+powershell -ExecutionPolicy Bypass -File .\scripts\build_windows.ps1
```
-
-Then run the desktop app with `npm start`:
-
-```
-cd app
-npm install
-npm start
-```
-
diff --git a/app/assets/app.ico b/app/assets/app.ico
new file mode 100644
index 00000000..875924f2
Binary files /dev/null and b/app/assets/app.ico differ
diff --git a/app/assets/assets.go b/app/assets/assets.go
new file mode 100644
index 00000000..6fed2d0e
--- /dev/null
+++ b/app/assets/assets.go
@@ -0,0 +1,17 @@
+package assets
+
+import (
+ "embed"
+ "io/fs"
+)
+
+//go:embed *.ico
+var icons embed.FS
+
+func ListIcons() ([]string, error) {
+ return fs.Glob(icons, "*")
+}
+
+func GetIcon(filename string) ([]byte, error) {
+ return icons.ReadFile(filename)
+}
diff --git a/app/assets/setup.bmp b/app/assets/setup.bmp
new file mode 100644
index 00000000..ff58b909
Binary files /dev/null and b/app/assets/setup.bmp differ
diff --git a/app/assets/tray.ico b/app/assets/tray.ico
new file mode 100644
index 00000000..e63616c5
Binary files /dev/null and b/app/assets/tray.ico differ
diff --git a/app/assets/tray_upgrade.ico b/app/assets/tray_upgrade.ico
new file mode 100644
index 00000000..d2083051
Binary files /dev/null and b/app/assets/tray_upgrade.ico differ
diff --git a/app/lifecycle/getstarted_nonwindows.go b/app/lifecycle/getstarted_nonwindows.go
new file mode 100644
index 00000000..2af87ab9
--- /dev/null
+++ b/app/lifecycle/getstarted_nonwindows.go
@@ -0,0 +1,9 @@
+//go:build !windows
+
+package lifecycle
+
+import "errors"
+
+func GetStarted() error {
+ return errors.New("not implemented")
+}
diff --git a/app/lifecycle/getstarted_windows.go b/app/lifecycle/getstarted_windows.go
new file mode 100644
index 00000000..f39dc31c
--- /dev/null
+++ b/app/lifecycle/getstarted_windows.go
@@ -0,0 +1,43 @@
+package lifecycle
+
+import (
+ "fmt"
+ "log/slog"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "syscall"
+)
+
+func GetStarted() error {
+ const CREATE_NEW_CONSOLE = 0x00000010
+ var err error
+ bannerScript := filepath.Join(AppDir, "ollama_welcome.ps1")
+ args := []string{
+ // TODO once we're signed, the execution policy bypass should be removed
+ "powershell", "-noexit", "-ExecutionPolicy", "Bypass", "-nologo", "-file", bannerScript,
+ }
+ args[0], err = exec.LookPath(args[0])
+ if err != nil {
+ return err
+ }
+
+ // Make sure the script actually exists
+ _, err = os.Stat(bannerScript)
+ if err != nil {
+ return fmt.Errorf("getting started banner script error %s", err)
+ }
+
+ slog.Info(fmt.Sprintf("opening getting started terminal with %v", args))
+ attrs := &os.ProcAttr{
+ Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
+ Sys: &syscall.SysProcAttr{CreationFlags: CREATE_NEW_CONSOLE, HideWindow: false},
+ }
+ proc, err := os.StartProcess(args[0], args, attrs)
+ if err != nil {
+ return fmt.Errorf("unable to start getting started shell %w", err)
+ }
+
+ slog.Debug(fmt.Sprintf("getting started terminal PID: %d", proc.Pid))
+ return proc.Release()
+}
diff --git a/app/lifecycle/lifecycle.go b/app/lifecycle/lifecycle.go
new file mode 100644
index 00000000..ab624e81
--- /dev/null
+++ b/app/lifecycle/lifecycle.go
@@ -0,0 +1,92 @@
+package lifecycle
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "log/slog"
+ "os"
+ "os/signal"
+ "syscall"
+
+ "github.com/ollama/ollama/app/store"
+ "github.com/ollama/ollama/app/tray"
+)
+
+func Run() {
+ InitLogging()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ var done chan int
+
+ t, err := tray.NewTray()
+ if err != nil {
+ log.Fatalf("Failed to start: %s", err)
+ }
+ callbacks := t.GetCallbacks()
+
+ signals := make(chan os.Signal, 1)
+ signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
+
+ go func() {
+ slog.Debug("starting callback loop")
+ for {
+ select {
+ case <-callbacks.Quit:
+ slog.Debug("quit called")
+ t.Quit()
+ case <-signals:
+ slog.Debug("shutting down due to signal")
+ t.Quit()
+ case <-callbacks.Update:
+ err := DoUpgrade(cancel, done)
+ if err != nil {
+ slog.Warn(fmt.Sprintf("upgrade attempt failed: %s", err))
+ }
+ case <-callbacks.ShowLogs:
+ ShowLogs()
+ case <-callbacks.DoFirstUse:
+ err := GetStarted()
+ if err != nil {
+ slog.Warn(fmt.Sprintf("Failed to launch getting started shell: %s", err))
+ }
+ }
+ }
+ }()
+
+ // Are we first use?
+ if !store.GetFirstTimeRun() {
+ slog.Debug("First time run")
+ err = t.DisplayFirstUseNotification()
+ if err != nil {
+ slog.Debug(fmt.Sprintf("XXX failed to display first use notification %v", err))
+ }
+ store.SetFirstTimeRun(true)
+ } else {
+ slog.Debug("Not first time, skipping first run notification")
+ }
+
+ if IsServerRunning(ctx) {
+ slog.Info("Detected another instance of ollama running, exiting")
+ os.Exit(1)
+ } else {
+ done, err = SpawnServer(ctx, CLIName)
+ if err != nil {
+ // TODO - should we retry in a backoff loop?
+ // TODO - should we pop up a warning and maybe add a menu item to view application logs?
+ slog.Error(fmt.Sprintf("Failed to spawn ollama server %s", err))
+ done = make(chan int, 1)
+ done <- 1
+ }
+ }
+
+ StartBackgroundUpdaterChecker(ctx, t.UpdateAvailable)
+
+ t.Run()
+ cancel()
+ slog.Info("Waiting for ollama server to shutdown...")
+ if done != nil {
+ <-done
+ }
+ slog.Info("Ollama app exiting")
+}
diff --git a/app/lifecycle/logging.go b/app/lifecycle/logging.go
new file mode 100644
index 00000000..9985fc3f
--- /dev/null
+++ b/app/lifecycle/logging.go
@@ -0,0 +1,80 @@
+package lifecycle
+
+import (
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "github.com/ollama/ollama/envconfig"
+)
+
+func InitLogging() {
+ level := slog.LevelInfo
+
+ if envconfig.Debug() {
+ level = slog.LevelDebug
+ }
+
+ var logFile *os.File
+ var err error
+ // Detect if we're a GUI app on windows, and if not, send logs to console
+ if os.Stderr.Fd() != 0 {
+ // Console app detected
+ logFile = os.Stderr
+ // TODO - write one-line to the app.log file saying we're running in console mode to help avoid confusion
+ } else {
+ rotateLogs(AppLogFile)
+ logFile, err = os.OpenFile(AppLogFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o755)
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to create server log %v", err))
+ return
+ }
+ }
+ handler := slog.NewTextHandler(logFile, &slog.HandlerOptions{
+ Level: level,
+ AddSource: true,
+ ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr {
+ if attr.Key == slog.SourceKey {
+ source := attr.Value.Any().(*slog.Source)
+ source.File = filepath.Base(source.File)
+ }
+ return attr
+ },
+ })
+
+ slog.SetDefault(slog.New(handler))
+
+ slog.Info("ollama app started")
+}
+
+func rotateLogs(logFile string) {
+ if _, err := os.Stat(logFile); os.IsNotExist(err) {
+ return
+ }
+ index := strings.LastIndex(logFile, ".")
+ pre := logFile[:index]
+ post := "." + logFile[index+1:]
+ for i := LogRotationCount; i > 0; i-- {
+ older := pre + "-" + strconv.Itoa(i) + post
+ newer := pre + "-" + strconv.Itoa(i-1) + post
+ if i == 1 {
+ newer = pre + post
+ }
+ if _, err := os.Stat(newer); err == nil {
+ if _, err := os.Stat(older); err == nil {
+ err := os.Remove(older)
+ if err != nil {
+ slog.Warn("Failed to remove older log", "older", older, "error", err)
+ continue
+ }
+ }
+ err := os.Rename(newer, older)
+ if err != nil {
+ slog.Warn("Failed to rotate log", "older", older, "newer", newer, "error", err)
+ }
+ }
+ }
+}
diff --git a/app/lifecycle/logging_nonwindows.go b/app/lifecycle/logging_nonwindows.go
new file mode 100644
index 00000000..205e47d7
--- /dev/null
+++ b/app/lifecycle/logging_nonwindows.go
@@ -0,0 +1,9 @@
+//go:build !windows
+
+package lifecycle
+
+import "log/slog"
+
+func ShowLogs() {
+ slog.Warn("not implemented")
+}
diff --git a/app/lifecycle/logging_test.go b/app/lifecycle/logging_test.go
new file mode 100644
index 00000000..8d5cdf6e
--- /dev/null
+++ b/app/lifecycle/logging_test.go
@@ -0,0 +1,44 @@
+package lifecycle
+
+import (
+ "os"
+ "path/filepath"
+ "strconv"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestRotateLogs(t *testing.T) {
+ logDir := t.TempDir()
+ logFile := filepath.Join(logDir, "testlog.log")
+
+ // No log exists
+ rotateLogs(logFile)
+
+ require.NoError(t, os.WriteFile(logFile, []byte("1"), 0o644))
+ assert.FileExists(t, logFile)
+ // First rotation
+ rotateLogs(logFile)
+ assert.FileExists(t, filepath.Join(logDir, "testlog-1.log"))
+ assert.NoFileExists(t, filepath.Join(logDir, "testlog-2.log"))
+ assert.NoFileExists(t, logFile)
+
+ // Should be a no-op without a new log
+ rotateLogs(logFile)
+ assert.FileExists(t, filepath.Join(logDir, "testlog-1.log"))
+ assert.NoFileExists(t, filepath.Join(logDir, "testlog-2.log"))
+ assert.NoFileExists(t, logFile)
+
+ for i := 2; i <= LogRotationCount+1; i++ {
+ require.NoError(t, os.WriteFile(logFile, []byte(strconv.Itoa(i)), 0o644))
+ assert.FileExists(t, logFile)
+ rotateLogs(logFile)
+ assert.NoFileExists(t, logFile)
+ for j := 1; j < i; j++ {
+ assert.FileExists(t, filepath.Join(logDir, "testlog-"+strconv.Itoa(j)+".log"))
+ }
+ assert.NoFileExists(t, filepath.Join(logDir, "testlog-"+strconv.Itoa(i+1)+".log"))
+ }
+}
diff --git a/app/lifecycle/logging_windows.go b/app/lifecycle/logging_windows.go
new file mode 100644
index 00000000..8f20337f
--- /dev/null
+++ b/app/lifecycle/logging_windows.go
@@ -0,0 +1,19 @@
+package lifecycle
+
+import (
+ "fmt"
+ "log/slog"
+ "os/exec"
+ "syscall"
+)
+
+func ShowLogs() {
+ cmd_path := "c:\\Windows\\system32\\cmd.exe"
+ slog.Debug(fmt.Sprintf("viewing logs with start %s", AppDataDir))
+ cmd := exec.Command(cmd_path, "/c", "start", AppDataDir)
+ cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: false, CreationFlags: 0x08000000}
+ err := cmd.Start()
+ if err != nil {
+ slog.Error(fmt.Sprintf("Failed to open log dir: %s", err))
+ }
+}
diff --git a/app/lifecycle/paths.go b/app/lifecycle/paths.go
new file mode 100644
index 00000000..4d9f4c5a
--- /dev/null
+++ b/app/lifecycle/paths.go
@@ -0,0 +1,79 @@
+package lifecycle
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+)
+
+var (
+ AppName = "ollama app"
+ CLIName = "ollama"
+ AppDir = "/opt/Ollama"
+ AppDataDir = "/opt/Ollama"
+ // TODO - should there be a distinct log dir?
+ UpdateStageDir = "/tmp"
+ AppLogFile = "/tmp/ollama_app.log"
+ ServerLogFile = "/tmp/ollama.log"
+ UpgradeLogFile = "/tmp/ollama_update.log"
+ Installer = "OllamaSetup.exe"
+ LogRotationCount = 5
+)
+
+func init() {
+ if runtime.GOOS == "windows" {
+ AppName += ".exe"
+ CLIName += ".exe"
+ // Logs, configs, downloads go to LOCALAPPDATA
+ localAppData := os.Getenv("LOCALAPPDATA")
+ AppDataDir = filepath.Join(localAppData, "Ollama")
+ UpdateStageDir = filepath.Join(AppDataDir, "updates")
+ AppLogFile = filepath.Join(AppDataDir, "app.log")
+ ServerLogFile = filepath.Join(AppDataDir, "server.log")
+ UpgradeLogFile = filepath.Join(AppDataDir, "upgrade.log")
+
+ // Executables are stored in APPDATA
+ AppDir = filepath.Join(localAppData, "Programs", "Ollama")
+
+ // Make sure we have PATH set correctly for any spawned children
+ paths := strings.Split(os.Getenv("PATH"), ";")
+ // Start with whatever we find in the PATH/LD_LIBRARY_PATH
+ found := false
+ for _, path := range paths {
+ d, err := filepath.Abs(path)
+ if err != nil {
+ continue
+ }
+ if strings.EqualFold(AppDir, d) {
+ found = true
+ }
+ }
+ if !found {
+ paths = append(paths, AppDir)
+
+ pathVal := strings.Join(paths, ";")
+ slog.Debug("setting PATH=" + pathVal)
+ err := os.Setenv("PATH", pathVal)
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to update PATH: %s", err))
+ }
+ }
+
+ // Make sure our logging dir exists
+ _, err := os.Stat(AppDataDir)
+ if errors.Is(err, os.ErrNotExist) {
+ if err := os.MkdirAll(AppDataDir, 0o755); err != nil {
+ slog.Error(fmt.Sprintf("create ollama dir %s: %v", AppDataDir, err))
+ }
+ }
+ } else if runtime.GOOS == "darwin" {
+ // TODO
+ AppName += ".app"
+ // } else if runtime.GOOS == "linux" {
+ // TODO
+ }
+}
diff --git a/app/lifecycle/server.go b/app/lifecycle/server.go
new file mode 100644
index 00000000..37957399
--- /dev/null
+++ b/app/lifecycle/server.go
@@ -0,0 +1,180 @@
+package lifecycle
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "time"
+
+ "github.com/ollama/ollama/api"
+)
+
+func getCLIFullPath(command string) string {
+ var cmdPath string
+ appExe, err := os.Executable()
+ if err == nil {
+ cmdPath = filepath.Join(filepath.Dir(appExe), command)
+ _, err := os.Stat(cmdPath)
+ if err == nil {
+ return cmdPath
+ }
+ }
+ cmdPath, err = exec.LookPath(command)
+ if err == nil {
+ _, err := os.Stat(cmdPath)
+ if err == nil {
+ return cmdPath
+ }
+ }
+ pwd, err := os.Getwd()
+ if err == nil {
+ cmdPath = filepath.Join(pwd, command)
+ _, err = os.Stat(cmdPath)
+ if err == nil {
+ return cmdPath
+ }
+ }
+
+ return command
+}
+
+func start(ctx context.Context, command string) (*exec.Cmd, error) {
+ cmd := getCmd(ctx, getCLIFullPath(command))
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ return nil, fmt.Errorf("failed to spawn server stdout pipe: %w", err)
+ }
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ return nil, fmt.Errorf("failed to spawn server stderr pipe: %w", err)
+ }
+
+ rotateLogs(ServerLogFile)
+ logFile, err := os.OpenFile(ServerLogFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o755)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create server log: %w", err)
+ }
+
+ logDir := filepath.Dir(ServerLogFile)
+ _, err = os.Stat(logDir)
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ return nil, fmt.Errorf("stat ollama server log dir %s: %v", logDir, err)
+ }
+
+ if err := os.MkdirAll(logDir, 0o755); err != nil {
+ return nil, fmt.Errorf("create ollama server log dir %s: %v", logDir, err)
+ }
+ }
+
+ go func() {
+ defer logFile.Close()
+ io.Copy(logFile, stdout) //nolint:errcheck
+ }()
+ go func() {
+ defer logFile.Close()
+ io.Copy(logFile, stderr) //nolint:errcheck
+ }()
+
+ // Re-wire context done behavior to attempt a graceful shutdown of the server
+ cmd.Cancel = func() error {
+ if cmd.Process != nil {
+ err := terminate(cmd)
+ if err != nil {
+ slog.Warn("error trying to gracefully terminate server", "err", err)
+ return cmd.Process.Kill()
+ }
+
+ tick := time.NewTicker(10 * time.Millisecond)
+ defer tick.Stop()
+
+ for {
+ select {
+ case <-tick.C:
+ exited, err := isProcessExited(cmd.Process.Pid)
+ if err != nil {
+ return err
+ }
+
+ if exited {
+ return nil
+ }
+ case <-time.After(5 * time.Second):
+ slog.Warn("graceful server shutdown timeout, killing", "pid", cmd.Process.Pid)
+ return cmd.Process.Kill()
+ }
+ }
+ }
+ return nil
+ }
+
+ // run the command and wait for it to finish
+ if err := cmd.Start(); err != nil {
+ return nil, fmt.Errorf("failed to start server %w", err)
+ }
+ if cmd.Process != nil {
+ slog.Info(fmt.Sprintf("started ollama server with pid %d", cmd.Process.Pid))
+ }
+ slog.Info(fmt.Sprintf("ollama server logs %s", ServerLogFile))
+
+ return cmd, nil
+}
+
+func SpawnServer(ctx context.Context, command string) (chan int, error) {
+ done := make(chan int)
+
+ go func() {
+ // Keep the server running unless we're shuttind down the app
+ crashCount := 0
+ for {
+ slog.Info("starting server...")
+ cmd, err := start(ctx, command)
+ if err != nil {
+ crashCount++
+ slog.Error(fmt.Sprintf("failed to start server %s", err))
+ time.Sleep(500 * time.Millisecond * time.Duration(crashCount))
+ continue
+ }
+
+ cmd.Wait() //nolint:errcheck
+ var code int
+ if cmd.ProcessState != nil {
+ code = cmd.ProcessState.ExitCode()
+ }
+
+ select {
+ case <-ctx.Done():
+ slog.Info(fmt.Sprintf("server shutdown with exit code %d", code))
+ done <- code
+ return
+ default:
+ crashCount++
+ slog.Warn(fmt.Sprintf("server crash %d - exit code %d - respawning", crashCount, code))
+ time.Sleep(500 * time.Millisecond * time.Duration(crashCount))
+ break
+ }
+ }
+ }()
+
+ return done, nil
+}
+
+func IsServerRunning(ctx context.Context) bool {
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ slog.Info("unable to connect to server")
+ return false
+ }
+ err = client.Heartbeat(ctx)
+ if err != nil {
+ slog.Debug(fmt.Sprintf("heartbeat from server: %s", err))
+ slog.Info("unable to connect to server")
+ return false
+ }
+ return true
+}
diff --git a/app/lifecycle/server_unix.go b/app/lifecycle/server_unix.go
new file mode 100644
index 00000000..70573913
--- /dev/null
+++ b/app/lifecycle/server_unix.go
@@ -0,0 +1,38 @@
+//go:build !windows
+
+package lifecycle
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "os/exec"
+ "syscall"
+)
+
+func getCmd(ctx context.Context, cmd string) *exec.Cmd {
+ return exec.CommandContext(ctx, cmd, "serve")
+}
+
+func terminate(cmd *exec.Cmd) error {
+ return cmd.Process.Signal(os.Interrupt)
+}
+
+func isProcessExited(pid int) (bool, error) {
+ proc, err := os.FindProcess(pid)
+ if err != nil {
+ return false, fmt.Errorf("failed to find process: %v", err)
+ }
+
+ err = proc.Signal(syscall.Signal(0))
+ if err != nil {
+ if errors.Is(err, os.ErrProcessDone) || errors.Is(err, syscall.ESRCH) {
+ return true, nil
+ }
+
+ return false, fmt.Errorf("error signaling process: %v", err)
+ }
+
+ return false, nil
+}
diff --git a/app/lifecycle/server_windows.go b/app/lifecycle/server_windows.go
new file mode 100644
index 00000000..5f9fe124
--- /dev/null
+++ b/app/lifecycle/server_windows.go
@@ -0,0 +1,91 @@
+package lifecycle
+
+import (
+ "context"
+ "fmt"
+ "os/exec"
+ "syscall"
+
+ "golang.org/x/sys/windows"
+)
+
+func getCmd(ctx context.Context, exePath string) *exec.Cmd {
+ cmd := exec.CommandContext(ctx, exePath, "serve")
+ cmd.SysProcAttr = &syscall.SysProcAttr{
+ HideWindow: true,
+ CreationFlags: windows.CREATE_NEW_PROCESS_GROUP,
+ }
+
+ return cmd
+}
+
+func terminate(cmd *exec.Cmd) error {
+ dll, err := windows.LoadDLL("kernel32.dll")
+ if err != nil {
+ return err
+ }
+ //nolint:errcheck
+ defer dll.Release()
+
+ pid := cmd.Process.Pid
+
+ f, err := dll.FindProc("AttachConsole")
+ if err != nil {
+ return err
+ }
+
+ r1, _, err := f.Call(uintptr(pid))
+ if r1 == 0 && err != syscall.ERROR_ACCESS_DENIED {
+ return err
+ }
+
+ f, err = dll.FindProc("SetConsoleCtrlHandler")
+ if err != nil {
+ return err
+ }
+
+ r1, _, err = f.Call(0, 1)
+ if r1 == 0 {
+ return err
+ }
+
+ f, err = dll.FindProc("GenerateConsoleCtrlEvent")
+ if err != nil {
+ return err
+ }
+
+ r1, _, err = f.Call(windows.CTRL_BREAK_EVENT, uintptr(pid))
+ if r1 == 0 {
+ return err
+ }
+
+ r1, _, err = f.Call(windows.CTRL_C_EVENT, uintptr(pid))
+ if r1 == 0 {
+ return err
+ }
+
+ return nil
+}
+
+const STILL_ACTIVE = 259
+
+func isProcessExited(pid int) (bool, error) {
+ hProcess, err := windows.OpenProcess(windows.PROCESS_QUERY_INFORMATION, false, uint32(pid))
+ if err != nil {
+ return false, fmt.Errorf("failed to open process: %v", err)
+ }
+ //nolint:errcheck
+ defer windows.CloseHandle(hProcess)
+
+ var exitCode uint32
+ err = windows.GetExitCodeProcess(hProcess, &exitCode)
+ if err != nil {
+ return false, fmt.Errorf("failed to get exit code: %v", err)
+ }
+
+ if exitCode == STILL_ACTIVE {
+ return false, nil
+ }
+
+ return true, nil
+}
diff --git a/app/lifecycle/updater.go b/app/lifecycle/updater.go
new file mode 100644
index 00000000..4d3c7d8d
--- /dev/null
+++ b/app/lifecycle/updater.go
@@ -0,0 +1,229 @@
+package lifecycle
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "mime"
+ "net/http"
+ "net/url"
+ "os"
+ "path"
+ "path/filepath"
+ "runtime"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/ollama/ollama/auth"
+ "github.com/ollama/ollama/version"
+)
+
+var (
+ UpdateCheckURLBase = "https://ollama.com/api/update"
+ UpdateDownloaded = false
+ UpdateCheckInterval = 60 * 60 * time.Second
+)
+
+// TODO - maybe move up to the API package?
+type UpdateResponse struct {
+ UpdateURL string `json:"url"`
+ UpdateVersion string `json:"version"`
+}
+
+func IsNewReleaseAvailable(ctx context.Context) (bool, UpdateResponse) {
+ var updateResp UpdateResponse
+
+ requestURL, err := url.Parse(UpdateCheckURLBase)
+ if err != nil {
+ return false, updateResp
+ }
+
+ query := requestURL.Query()
+ query.Add("os", runtime.GOOS)
+ query.Add("arch", runtime.GOARCH)
+ query.Add("version", version.Version)
+ query.Add("ts", strconv.FormatInt(time.Now().Unix(), 10))
+
+ nonce, err := auth.NewNonce(rand.Reader, 16)
+ if err != nil {
+ return false, updateResp
+ }
+
+ query.Add("nonce", nonce)
+ requestURL.RawQuery = query.Encode()
+
+ data := []byte(fmt.Sprintf("%s,%s", http.MethodGet, requestURL.RequestURI()))
+ signature, err := auth.Sign(ctx, data)
+ if err != nil {
+ return false, updateResp
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL.String(), nil)
+ if err != nil {
+ slog.Warn(fmt.Sprintf("failed to check for update: %s", err))
+ return false, updateResp
+ }
+ req.Header.Set("Authorization", signature)
+ req.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version()))
+
+ slog.Debug("checking for available update", "requestURL", requestURL)
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ slog.Warn(fmt.Sprintf("failed to check for update: %s", err))
+ return false, updateResp
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode == http.StatusNoContent {
+ slog.Debug("check update response 204 (current version is up to date)")
+ return false, updateResp
+ }
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ slog.Warn(fmt.Sprintf("failed to read body response: %s", err))
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ slog.Info(fmt.Sprintf("check update error %d - %.96s", resp.StatusCode, string(body)))
+ return false, updateResp
+ }
+ err = json.Unmarshal(body, &updateResp)
+ if err != nil {
+ slog.Warn(fmt.Sprintf("malformed response checking for update: %s", err))
+ return false, updateResp
+ }
+ // Extract the version string from the URL in the github release artifact path
+ updateResp.UpdateVersion = path.Base(path.Dir(updateResp.UpdateURL))
+
+ slog.Info("New update available at " + updateResp.UpdateURL)
+ return true, updateResp
+}
+
+func DownloadNewRelease(ctx context.Context, updateResp UpdateResponse) error {
+ // Do a head first to check etag info
+ req, err := http.NewRequestWithContext(ctx, http.MethodHead, updateResp.UpdateURL, nil)
+ if err != nil {
+ return err
+ }
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("error checking update: %w", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("unexpected status attempting to download update %d", resp.StatusCode)
+ }
+ resp.Body.Close()
+ etag := strings.Trim(resp.Header.Get("etag"), "\"")
+ if etag == "" {
+ slog.Debug("no etag detected, falling back to filename based dedup")
+ etag = "_"
+ }
+ filename := Installer
+ _, params, err := mime.ParseMediaType(resp.Header.Get("content-disposition"))
+ if err == nil {
+ filename = params["filename"]
+ }
+
+ stageFilename := filepath.Join(UpdateStageDir, etag, filename)
+
+ // Check to see if we already have it downloaded
+ _, err = os.Stat(stageFilename)
+ if err == nil {
+ slog.Info("update already downloaded")
+ return nil
+ }
+
+ cleanupOldDownloads()
+
+ req.Method = http.MethodGet
+ resp, err = http.DefaultClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("error checking update: %w", err)
+ }
+ defer resp.Body.Close()
+ etag = strings.Trim(resp.Header.Get("etag"), "\"")
+ if etag == "" {
+ slog.Debug("no etag detected, falling back to filename based dedup") // TODO probably can get rid of this redundant log
+ etag = "_"
+ }
+
+ stageFilename = filepath.Join(UpdateStageDir, etag, filename)
+
+ _, err = os.Stat(filepath.Dir(stageFilename))
+ if errors.Is(err, os.ErrNotExist) {
+ if err := os.MkdirAll(filepath.Dir(stageFilename), 0o755); err != nil {
+ return fmt.Errorf("create ollama dir %s: %v", filepath.Dir(stageFilename), err)
+ }
+ }
+
+ payload, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return fmt.Errorf("failed to read body response: %w", err)
+ }
+ fp, err := os.OpenFile(stageFilename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
+ if err != nil {
+ return fmt.Errorf("write payload %s: %w", stageFilename, err)
+ }
+ defer fp.Close()
+ if n, err := fp.Write(payload); err != nil || n != len(payload) {
+ return fmt.Errorf("write payload %s: %d vs %d -- %w", stageFilename, n, len(payload), err)
+ }
+ slog.Info("new update downloaded " + stageFilename)
+
+ UpdateDownloaded = true
+ return nil
+}
+
+func cleanupOldDownloads() {
+ files, err := os.ReadDir(UpdateStageDir)
+ if err != nil && errors.Is(err, os.ErrNotExist) {
+ // Expected behavior on first run
+ return
+ } else if err != nil {
+ slog.Warn(fmt.Sprintf("failed to list stage dir: %s", err))
+ return
+ }
+ for _, file := range files {
+ fullname := filepath.Join(UpdateStageDir, file.Name())
+ slog.Debug("cleaning up old download: " + fullname)
+ err = os.RemoveAll(fullname)
+ if err != nil {
+ slog.Warn(fmt.Sprintf("failed to cleanup stale update download %s", err))
+ }
+ }
+}
+
+func StartBackgroundUpdaterChecker(ctx context.Context, cb func(string) error) {
+ go func() {
+ // Don't blast an update message immediately after startup
+ // time.Sleep(30 * time.Second)
+ time.Sleep(3 * time.Second)
+
+ for {
+ available, resp := IsNewReleaseAvailable(ctx)
+ if available {
+ err := DownloadNewRelease(ctx, resp)
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to download new release: %s", err))
+ }
+ err = cb(resp.UpdateVersion)
+ if err != nil {
+ slog.Warn(fmt.Sprintf("failed to register update available with tray: %s", err))
+ }
+ }
+ select {
+ case <-ctx.Done():
+ slog.Debug("stopping background update checker")
+ return
+ default:
+ time.Sleep(UpdateCheckInterval)
+ }
+ }
+ }()
+}
diff --git a/app/lifecycle/updater_nonwindows.go b/app/lifecycle/updater_nonwindows.go
new file mode 100644
index 00000000..1d2dda80
--- /dev/null
+++ b/app/lifecycle/updater_nonwindows.go
@@ -0,0 +1,12 @@
+//go:build !windows
+
+package lifecycle
+
+import (
+ "context"
+ "errors"
+)
+
+func DoUpgrade(cancel context.CancelFunc, done chan int) error {
+ return errors.New("not implemented")
+}
diff --git a/app/lifecycle/updater_windows.go b/app/lifecycle/updater_windows.go
new file mode 100644
index 00000000..1d3830d4
--- /dev/null
+++ b/app/lifecycle/updater_windows.go
@@ -0,0 +1,78 @@
+package lifecycle
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "os"
+ "os/exec"
+ "path/filepath"
+)
+
+func DoUpgrade(cancel context.CancelFunc, done chan int) error {
+ files, err := filepath.Glob(filepath.Join(UpdateStageDir, "*", "*.exe")) // TODO generalize for multiplatform
+ if err != nil {
+ return fmt.Errorf("failed to lookup downloads: %s", err)
+ }
+ if len(files) == 0 {
+ return errors.New("no update downloads found")
+ } else if len(files) > 1 {
+ // Shouldn't happen
+ slog.Warn(fmt.Sprintf("multiple downloads found, using first one %v", files))
+ }
+ installerExe := files[0]
+
+ slog.Info("starting upgrade with " + installerExe)
+ slog.Info("upgrade log file " + UpgradeLogFile)
+
+ // When running in debug mode, we'll be "verbose" and let the installer pop up and prompt
+ installArgs := []string{
+ "/CLOSEAPPLICATIONS", // Quit the tray app if it's still running
+ "/LOG=" + filepath.Base(UpgradeLogFile), // Only relative seems reliable, so set pwd
+ "/FORCECLOSEAPPLICATIONS", // Force close the tray app - might be needed
+ }
+ // make the upgrade as quiet as possible (no GUI, no prompts)
+ installArgs = append(installArgs,
+ "/SP", // Skip the "This will install... Do you wish to continue" prompt
+ "/SUPPRESSMSGBOXES",
+ "/SILENT",
+ "/VERYSILENT",
+ )
+
+ // Safeguard in case we have requests in flight that need to drain...
+ slog.Info("Waiting for server to shutdown")
+ cancel()
+ if done != nil {
+ <-done
+ } else {
+ // Shouldn't happen
+ slog.Warn("done chan was nil, not actually waiting")
+ }
+
+ slog.Debug(fmt.Sprintf("starting installer: %s %v", installerExe, installArgs))
+ os.Chdir(filepath.Dir(UpgradeLogFile)) //nolint:errcheck
+ cmd := exec.Command(installerExe, installArgs...)
+
+ if err := cmd.Start(); err != nil {
+ return fmt.Errorf("unable to start ollama app %w", err)
+ }
+
+ if cmd.Process != nil {
+ err = cmd.Process.Release()
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to release server process: %s", err))
+ }
+ } else {
+ // TODO - some details about why it didn't start, or is this a pedantic error case?
+ return errors.New("installer process did not start")
+ }
+
+ // TODO should we linger for a moment and check to make sure it's actually running by checking the pid?
+
+ slog.Info("Installer started in background, exiting")
+
+ os.Exit(0)
+ // Not reached
+ return nil
+}
diff --git a/app/main.go b/app/main.go
new file mode 100644
index 00000000..db829795
--- /dev/null
+++ b/app/main.go
@@ -0,0 +1,12 @@
+package main
+
+// Compile with the following to get rid of the cmd pop up on windows
+// go build -ldflags="-H windowsgui" .
+
+import (
+ "github.com/ollama/ollama/app/lifecycle"
+)
+
+func main() {
+ lifecycle.Run()
+}
diff --git a/app/ollama.iss b/app/ollama.iss
new file mode 100644
index 00000000..4038815a
--- /dev/null
+++ b/app/ollama.iss
@@ -0,0 +1,205 @@
+; Inno Setup Installer for Ollama
+;
+; To build the installer use the build script invoked from the top of the source tree
+;
+; powershell -ExecutionPolicy Bypass -File .\scripts\build_windows.ps
+
+
+#define MyAppName "Ollama"
+#if GetEnv("PKG_VERSION") != ""
+ #define MyAppVersion GetEnv("PKG_VERSION")
+#else
+ #define MyAppVersion "0.0.0"
+#endif
+#define MyAppPublisher "Ollama"
+#define MyAppURL "https://ollama.com/"
+#define MyAppExeName "ollama app.exe"
+#define MyIcon ".\assets\app.ico"
+
+[Setup]
+; NOTE: The value of AppId uniquely identifies this application. Do not use the same AppId value in installers for other applications.
+; (To generate a new GUID, click Tools | Generate GUID inside the IDE.)
+AppId={{44E83376-CE68-45EB-8FC1-393500EB558C}
+AppName={#MyAppName}
+AppVersion={#MyAppVersion}
+VersionInfoVersion={#MyAppVersion}
+;AppVerName={#MyAppName} {#MyAppVersion}
+AppPublisher={#MyAppPublisher}
+AppPublisherURL={#MyAppURL}
+AppSupportURL={#MyAppURL}
+AppUpdatesURL={#MyAppURL}
+ArchitecturesAllowed=x64compatible arm64
+ArchitecturesInstallIn64BitMode=x64compatible arm64
+DefaultDirName={localappdata}\Programs\{#MyAppName}
+DefaultGroupName={#MyAppName}
+DisableProgramGroupPage=yes
+PrivilegesRequired=lowest
+OutputBaseFilename="OllamaSetup"
+SetupIconFile={#MyIcon}
+UninstallDisplayIcon={uninstallexe}
+Compression=lzma2
+SolidCompression=no
+WizardStyle=modern
+ChangesEnvironment=yes
+OutputDir=..\dist\
+
+; Disable logging once everything's battle tested
+; Filename will be %TEMP%\Setup Log*.txt
+SetupLogging=yes
+CloseApplications=yes
+RestartApplications=no
+RestartIfNeededByRun=no
+
+; https://jrsoftware.org/ishelp/index.php?topic=setup_wizardimagefile
+WizardSmallImageFile=.\assets\setup.bmp
+
+; TODO verifty actual min windows version...
+; OG Win 10
+MinVersion=10.0.10240
+
+; First release that supports WinRT UI Composition for win32 apps
+; MinVersion=10.0.17134
+; First release with XAML Islands - possible UI path forward
+; MinVersion=10.0.18362
+
+; quiet...
+DisableDirPage=yes
+DisableFinishedPage=yes
+DisableReadyMemo=yes
+DisableReadyPage=yes
+DisableStartupPrompt=yes
+DisableWelcomePage=yes
+
+; TODO - percentage can't be set less than 100, so how to make it shorter?
+; WizardSizePercent=100,80
+
+#if GetEnv("KEY_CONTAINER")
+SignTool=MySignTool
+SignedUninstaller=yes
+#endif
+
+SetupMutex=OllamaSetupMutex
+
+[Languages]
+Name: "english"; MessagesFile: "compiler:Default.isl"
+
+[LangOptions]
+DialogFontSize=12
+
+[Files]
+#if DirExists("..\dist\windows-amd64")
+Source: "..\dist\windows-amd64-app.exe"; DestDir: "{app}"; DestName: "{#MyAppExeName}" ;Check: not IsArm64(); Flags: ignoreversion 64bit
+Source: "..\dist\windows-amd64\ollama.exe"; DestDir: "{app}"; Check: not IsArm64(); Flags: ignoreversion 64bit
+Source: "..\dist\windows-amd64\lib\ollama\*"; DestDir: "{app}\lib\ollama\"; Check: not IsArm64(); Flags: ignoreversion 64bit recursesubdirs
+#endif
+
+#if DirExists("..\dist\windows-arm64")
+Source: "..\dist\windows-arm64\vc_redist.arm64.exe"; DestDir: "{tmp}"; Check: IsArm64() and vc_redist_needed(); Flags: deleteafterinstall
+Source: "..\dist\windows-arm64-app.exe"; DestDir: "{app}"; DestName: "{#MyAppExeName}" ;Check: IsArm64(); Flags: ignoreversion 64bit
+Source: "..\dist\windows-arm64\ollama.exe"; DestDir: "{app}"; Check: IsArm64(); Flags: ignoreversion 64bit
+Source: "..\dist\windows-arm64\lib\ollama\*"; DestDir: "{app}\lib\ollama\"; Check: IsArm64(); Flags: ignoreversion 64bit recursesubdirs
+#endif
+
+Source: "..\dist\ollama_welcome.ps1"; DestDir: "{app}"; Flags: ignoreversion
+Source: ".\assets\app.ico"; DestDir: "{app}"; Flags: ignoreversion
+
+[Icons]
+Name: "{group}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"; IconFilename: "{app}\app.ico"
+Name: "{userstartup}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"; IconFilename: "{app}\app.ico"
+Name: "{userprograms}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"; IconFilename: "{app}\app.ico"
+
+[Run]
+#if DirExists("..\dist\windows-arm64")
+Filename: "{tmp}\vc_redist.arm64.exe"; Parameters: "/install /passive /norestart"; Check: IsArm64() and vc_redist_needed(); StatusMsg: "Installing VC++ Redistributables..."; Flags: waituntilterminated
+#endif
+Filename: "{cmd}"; Parameters: "/C set PATH={app};%PATH% & ""{app}\{#MyAppExeName}"""; Flags: postinstall nowait runhidden
+
+[UninstallRun]
+; Filename: "{cmd}"; Parameters: "/C ""taskkill /im ''{#MyAppExeName}'' /f /t"; Flags: runhidden
+; Filename: "{cmd}"; Parameters: "/C ""taskkill /im ollama.exe /f /t"; Flags: runhidden
+Filename: "taskkill"; Parameters: "/im ""{#MyAppExeName}"" /f /t"; Flags: runhidden
+Filename: "taskkill"; Parameters: "/im ""ollama.exe"" /f /t"; Flags: runhidden
+; HACK! need to give the server and app enough time to exit
+; TODO - convert this to a Pascal code script so it waits until they're no longer running, then completes
+Filename: "{cmd}"; Parameters: "/c timeout 5"; Flags: runhidden
+
+[UninstallDelete]
+Type: filesandordirs; Name: "{%TEMP}\ollama*"
+Type: filesandordirs; Name: "{%LOCALAPPDATA}\Ollama"
+Type: filesandordirs; Name: "{%LOCALAPPDATA}\Programs\Ollama"
+Type: filesandordirs; Name: "{%USERPROFILE}\.ollama\models"
+Type: filesandordirs; Name: "{%USERPROFILE}\.ollama\history"
+; NOTE: if the user has a custom OLLAMA_MODELS it will be preserved
+
+[InstallDelete]
+Type: filesandordirs; Name: "{%TEMP}\ollama*"
+Type: filesandordirs; Name: "{%LOCALAPPDATA}\Programs\Ollama"
+
+[Messages]
+WizardReady=Ollama Windows Preview
+ReadyLabel1=%nLet's get you up and running with your own large language models.
+SetupAppRunningError=Another Ollama installer is running.%n%nPlease cancel or finish the other installer, then click OK to continue with this install, or Cancel to exit.
+
+
+;FinishedHeadingLabel=Run your first model
+;FinishedLabel=%nRun this command in a PowerShell or cmd terminal.%n%n%n ollama run llama3.2
+;ClickFinish=%n
+
+[Registry]
+Root: HKCU; Subkey: "Environment"; \
+ ValueType: expandsz; ValueName: "Path"; ValueData: "{olddata};{app}"; \
+ Check: NeedsAddPath('{app}')
+
+[Code]
+
+function NeedsAddPath(Param: string): boolean;
+var
+ OrigPath: string;
+begin
+ if not RegQueryStringValue(HKEY_CURRENT_USER,
+ 'Environment',
+ 'Path', OrigPath)
+ then begin
+ Result := True;
+ exit;
+ end;
+ { look for the path with leading and trailing semicolon }
+ { Pos() returns 0 if not found }
+ Result := Pos(';' + ExpandConstant(Param) + ';', ';' + OrigPath + ';') = 0;
+end;
+
+{ --- VC Runtime libraries discovery code - Only install vc_redist if it isn't already installed ----- }
+const VCRTL_MIN_V1 = 14;
+const VCRTL_MIN_V2 = 40;
+const VCRTL_MIN_V3 = 33807;
+const VCRTL_MIN_V4 = 0;
+
+ // check if the minimum required vc redist is installed (by looking the registry)
+function vc_redist_needed (): Boolean;
+var
+ sRegKey: string;
+ v1: Cardinal;
+ v2: Cardinal;
+ v3: Cardinal;
+ v4: Cardinal;
+begin
+ sRegKey := 'SOFTWARE\WOW6432Node\Microsoft\VisualStudio\14.0\VC\Runtimes\arm64';
+ if (RegQueryDWordValue (HKEY_LOCAL_MACHINE, sRegKey, 'Major', v1) and
+ RegQueryDWordValue (HKEY_LOCAL_MACHINE, sRegKey, 'Minor', v2) and
+ RegQueryDWordValue (HKEY_LOCAL_MACHINE, sRegKey, 'Bld', v3) and
+ RegQueryDWordValue (HKEY_LOCAL_MACHINE, sRegKey, 'RBld', v4)) then
+ begin
+ Log ('VC Redist version: ' + IntToStr (v1) +
+ '.' + IntToStr (v2) + '.' + IntToStr (v3) +
+ '.' + IntToStr (v4));
+ { Version info was found. Return true if later or equal to our
+ minimal required version RTL_MIN_Vx }
+ Result := not (
+ (v1 > VCRTL_MIN_V1) or ((v1 = VCRTL_MIN_V1) and
+ ((v2 > VCRTL_MIN_V2) or ((v2 = VCRTL_MIN_V2) and
+ ((v3 > VCRTL_MIN_V3) or ((v3 = VCRTL_MIN_V3) and
+ (v4 >= VCRTL_MIN_V4)))))));
+ end
+ else
+ Result := TRUE;
+end;
diff --git a/app/ollama.rc b/app/ollama.rc
new file mode 100644
index 00000000..acd84493
--- /dev/null
+++ b/app/ollama.rc
@@ -0,0 +1,29 @@
+#include <winver.h>
+
+VS_VERSION_INFO VERSIONINFO
+ FILEFLAGSMASK 0x3fL
+#ifdef _DEBUG
+ FILEFLAGS 0x1L
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS 0x40004L
+ FILETYPE 0x1L
+ FILESUBTYPE 0x0L
+BEGIN
+ BLOCK "StringFileInfo"
+ BEGIN
+ BLOCK "040904b0"
+ BEGIN
+ VALUE "FileDescription", "Ollama"
+ VALUE "InternalName", "Ollama"
+ VALUE "OriginalFilename", "ollama app.exe"
+ VALUE "ProductName", "Ollama"
+ END
+ END
+
+ BLOCK "VarFileInfo"
+ BEGIN
+ VALUE "Translation", 0x409, 1200
+ END
+END
diff --git a/app/ollama_welcome.ps1 b/app/ollama_welcome.ps1
new file mode 100644
index 00000000..e9695748
--- /dev/null
+++ b/app/ollama_welcome.ps1
@@ -0,0 +1,8 @@
+# TODO - consider ANSI colors and maybe ASCII art...
+write-host ""
+write-host "Welcome to Ollama!"
+write-host ""
+write-host "Run your first model:"
+write-host ""
+write-host "`tollama run llama3.2"
+write-host ""
\ No newline at end of file
diff --git a/app/store/store.go b/app/store/store.go
new file mode 100644
index 00000000..b743e8a8
--- /dev/null
+++ b/app/store/store.go
@@ -0,0 +1,97 @@
+package store
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "sync"
+
+ "github.com/google/uuid"
+)
+
+type Store struct {
+ ID string `json:"id"`
+ FirstTimeRun bool `json:"first-time-run"`
+}
+
+var (
+ lock sync.Mutex
+ store Store
+)
+
+func GetID() string {
+ lock.Lock()
+ defer lock.Unlock()
+ if store.ID == "" {
+ initStore()
+ }
+ return store.ID
+}
+
+func GetFirstTimeRun() bool {
+ lock.Lock()
+ defer lock.Unlock()
+ if store.ID == "" {
+ initStore()
+ }
+ return store.FirstTimeRun
+}
+
+func SetFirstTimeRun(val bool) {
+ lock.Lock()
+ defer lock.Unlock()
+ if store.FirstTimeRun == val {
+ return
+ }
+ store.FirstTimeRun = val
+ writeStore(getStorePath())
+}
+
+// lock must be held
+func initStore() {
+ storeFile, err := os.Open(getStorePath())
+ if err == nil {
+ defer storeFile.Close()
+ err = json.NewDecoder(storeFile).Decode(&store)
+ if err == nil {
+ slog.Debug(fmt.Sprintf("loaded existing store %s - ID: %s", getStorePath(), store.ID))
+ return
+ }
+ } else if !errors.Is(err, os.ErrNotExist) {
+ slog.Debug(fmt.Sprintf("unexpected error searching for store: %s", err))
+ }
+ slog.Debug("initializing new store")
+ store.ID = uuid.New().String()
+ writeStore(getStorePath())
+}
+
+func writeStore(storeFilename string) {
+ ollamaDir := filepath.Dir(storeFilename)
+ _, err := os.Stat(ollamaDir)
+ if errors.Is(err, os.ErrNotExist) {
+ if err := os.MkdirAll(ollamaDir, 0o755); err != nil {
+ slog.Error(fmt.Sprintf("create ollama dir %s: %v", ollamaDir, err))
+ return
+ }
+ }
+ payload, err := json.Marshal(store)
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to marshal store: %s", err))
+ return
+ }
+ fp, err := os.OpenFile(storeFilename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
+ if err != nil {
+ slog.Error(fmt.Sprintf("write store payload %s: %v", storeFilename, err))
+ return
+ }
+ defer fp.Close()
+ if n, err := fp.Write(payload); err != nil || n != len(payload) {
+ slog.Error(fmt.Sprintf("write store payload %s: %d vs %d -- %v", storeFilename, n, len(payload), err))
+ return
+ }
+ slog.Debug("Store contents: " + string(payload))
+ slog.Info(fmt.Sprintf("wrote store: %s", storeFilename))
+}
diff --git a/app/store/store_darwin.go b/app/store/store_darwin.go
new file mode 100644
index 00000000..e53d8525
--- /dev/null
+++ b/app/store/store_darwin.go
@@ -0,0 +1,13 @@
+package store
+
+import (
+ "os"
+ "path/filepath"
+)
+
+func getStorePath() string {
+ // TODO - system wide location?
+
+ home := os.Getenv("HOME")
+ return filepath.Join(home, "Library", "Application Support", "Ollama", "config.json")
+}
diff --git a/app/store/store_linux.go b/app/store/store_linux.go
new file mode 100644
index 00000000..3aac9b01
--- /dev/null
+++ b/app/store/store_linux.go
@@ -0,0 +1,16 @@
+package store
+
+import (
+ "os"
+ "path/filepath"
+)
+
+func getStorePath() string {
+ if os.Geteuid() == 0 {
+ // TODO where should we store this on linux for system-wide operation?
+ return "/etc/ollama/config.json"
+ }
+
+ home := os.Getenv("HOME")
+ return filepath.Join(home, ".ollama", "config.json")
+}
diff --git a/app/store/store_windows.go b/app/store/store_windows.go
new file mode 100644
index 00000000..ba06b82c
--- /dev/null
+++ b/app/store/store_windows.go
@@ -0,0 +1,11 @@
+package store
+
+import (
+ "os"
+ "path/filepath"
+)
+
+func getStorePath() string {
+ localAppData := os.Getenv("LOCALAPPDATA")
+ return filepath.Join(localAppData, "Ollama", "config.json")
+}
diff --git a/app/tray/commontray/types.go b/app/tray/commontray/types.go
new file mode 100644
index 00000000..ed633dc9
--- /dev/null
+++ b/app/tray/commontray/types.go
@@ -0,0 +1,24 @@
+package commontray
+
+var (
+ Title = "Ollama"
+ ToolTip = "Ollama"
+
+ UpdateIconName = "tray_upgrade"
+ IconName = "tray"
+)
+
+type Callbacks struct {
+ Quit chan struct{}
+ Update chan struct{}
+ DoFirstUse chan struct{}
+ ShowLogs chan struct{}
+}
+
+type OllamaTray interface {
+ GetCallbacks() Callbacks
+ Run()
+ UpdateAvailable(ver string) error
+ DisplayFirstUseNotification() error
+ Quit()
+}
diff --git a/app/tray/tray.go b/app/tray/tray.go
new file mode 100644
index 00000000..dfa63435
--- /dev/null
+++ b/app/tray/tray.go
@@ -0,0 +1,28 @@
+package tray
+
+import (
+ "fmt"
+ "runtime"
+
+ "github.com/ollama/ollama/app/assets"
+ "github.com/ollama/ollama/app/tray/commontray"
+)
+
+func NewTray() (commontray.OllamaTray, error) {
+ extension := ".png"
+ if runtime.GOOS == "windows" {
+ extension = ".ico"
+ }
+ iconName := commontray.UpdateIconName + extension
+ updateIcon, err := assets.GetIcon(iconName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to load icon %s: %w", iconName, err)
+ }
+ iconName = commontray.IconName + extension
+ icon, err := assets.GetIcon(iconName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to load icon %s: %w", iconName, err)
+ }
+
+ return InitPlatformTray(icon, updateIcon)
+}
diff --git a/app/tray/tray_nonwindows.go b/app/tray/tray_nonwindows.go
new file mode 100644
index 00000000..a03d233e
--- /dev/null
+++ b/app/tray/tray_nonwindows.go
@@ -0,0 +1,13 @@
+//go:build !windows
+
+package tray
+
+import (
+ "errors"
+
+ "github.com/ollama/ollama/app/tray/commontray"
+)
+
+func InitPlatformTray(icon, updateIcon []byte) (commontray.OllamaTray, error) {
+ return nil, errors.New("not implemented")
+}
diff --git a/app/tray/tray_windows.go b/app/tray/tray_windows.go
new file mode 100644
index 00000000..086fc794
--- /dev/null
+++ b/app/tray/tray_windows.go
@@ -0,0 +1,10 @@
+package tray
+
+import (
+ "github.com/ollama/ollama/app/tray/commontray"
+ "github.com/ollama/ollama/app/tray/wintray"
+)
+
+func InitPlatformTray(icon, updateIcon []byte) (commontray.OllamaTray, error) {
+ return wintray.InitTray(icon, updateIcon)
+}
diff --git a/app/tray/wintray/eventloop.go b/app/tray/wintray/eventloop.go
new file mode 100644
index 00000000..157828a3
--- /dev/null
+++ b/app/tray/wintray/eventloop.go
@@ -0,0 +1,181 @@
+//go:build windows
+
+package wintray
+
+import (
+ "fmt"
+ "log/slog"
+ "sync"
+ "unsafe"
+
+ "golang.org/x/sys/windows"
+)
+
+var quitOnce sync.Once
+
+func (t *winTray) Run() {
+ nativeLoop()
+}
+
+func nativeLoop() {
+ // Main message pump.
+ slog.Debug("starting event handling loop")
+ m := &struct {
+ WindowHandle windows.Handle
+ Message uint32
+ Wparam uintptr
+ Lparam uintptr
+ Time uint32
+ Pt point
+ LPrivate uint32
+ }{}
+ for {
+ ret, _, err := pGetMessage.Call(uintptr(unsafe.Pointer(m)), 0, 0, 0)
+
+ // If the function retrieves a message other than WM_QUIT, the return value is nonzero.
+ // If the function retrieves the WM_QUIT message, the return value is zero.
+ // If there is an error, the return value is -1
+ // https://msdn.microsoft.com/en-us/library/windows/desktop/ms644936(v=vs.85).aspx
+ switch int32(ret) {
+ case -1:
+ slog.Error(fmt.Sprintf("get message failure: %v", err))
+ return
+ case 0:
+ return
+ default:
+ pTranslateMessage.Call(uintptr(unsafe.Pointer(m))) //nolint:errcheck
+ pDispatchMessage.Call(uintptr(unsafe.Pointer(m))) //nolint:errcheck
+ }
+ }
+}
+
+// WindowProc callback function that processes messages sent to a window.
+// https://msdn.microsoft.com/en-us/library/windows/desktop/ms633573(v=vs.85).aspx
+func (t *winTray) wndProc(hWnd windows.Handle, message uint32, wParam, lParam uintptr) (lResult uintptr) {
+ const (
+ WM_RBUTTONUP = 0x0205
+ WM_LBUTTONUP = 0x0202
+ WM_COMMAND = 0x0111
+ WM_ENDSESSION = 0x0016
+ WM_CLOSE = 0x0010
+ WM_DESTROY = 0x0002
+ WM_MOUSEMOVE = 0x0200
+ WM_LBUTTONDOWN = 0x0201
+ )
+ switch message {
+ case WM_COMMAND:
+ menuItemId := int32(wParam)
+ // https://docs.microsoft.com/en-us/windows/win32/menurc/wm-command#menus
+ switch menuItemId {
+ case quitMenuID:
+ select {
+ case t.callbacks.Quit <- struct{}{}:
+ // should not happen but in case not listening
+ default:
+ slog.Error("no listener on Quit")
+ }
+ case updateMenuID:
+ select {
+ case t.callbacks.Update <- struct{}{}:
+ // should not happen but in case not listening
+ default:
+ slog.Error("no listener on Update")
+ }
+ case diagLogsMenuID:
+ select {
+ case t.callbacks.ShowLogs <- struct{}{}:
+ // should not happen but in case not listening
+ default:
+ slog.Error("no listener on ShowLogs")
+ }
+ default:
+ slog.Debug(fmt.Sprintf("Unexpected menu item id: %d", menuItemId))
+ }
+ case WM_CLOSE:
+ boolRet, _, err := pDestroyWindow.Call(uintptr(t.window))
+ if boolRet == 0 {
+ slog.Error(fmt.Sprintf("failed to destroy window: %s", err))
+ }
+ err = t.wcex.unregister()
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to uregister windo %s", err))
+ }
+ case WM_DESTROY:
+ // same as WM_ENDSESSION, but throws 0 exit code after all
+ defer pPostQuitMessage.Call(uintptr(int32(0))) //nolint:errcheck
+ fallthrough
+ case WM_ENDSESSION:
+ t.muNID.Lock()
+ if t.nid != nil {
+ err := t.nid.delete()
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to delete nid: %s", err))
+ }
+ }
+ t.muNID.Unlock()
+ case t.wmSystrayMessage:
+ switch lParam {
+ case WM_MOUSEMOVE, WM_LBUTTONDOWN:
+ // Ignore these...
+ case WM_RBUTTONUP, WM_LBUTTONUP:
+ err := t.showMenu()
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to show menu: %s", err))
+ }
+ case 0x405: // TODO - how is this magic value derived for the notification left click
+ if t.pendingUpdate {
+ select {
+ case t.callbacks.Update <- struct{}{}:
+ // should not happen but in case not listening
+ default:
+ slog.Error("no listener on Update")
+ }
+ } else {
+ select {
+ case t.callbacks.DoFirstUse <- struct{}{}:
+ // should not happen but in case not listening
+ default:
+ slog.Error("no listener on DoFirstUse")
+ }
+ }
+ case 0x404: // Middle click or close notification
+ // slog.Debug("doing nothing on close of first time notification")
+ default:
+ // 0x402 also seems common - what is it?
+ slog.Debug(fmt.Sprintf("unmanaged app message, lParm: 0x%x", lParam))
+ }
+ case t.wmTaskbarCreated: // on explorer.exe restarts
+ t.muNID.Lock()
+ err := t.nid.add()
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to refresh the taskbar on explorer restart: %s", err))
+ }
+ t.muNID.Unlock()
+ default:
+ // Calls the default window procedure to provide default processing for any window messages that an application does not process.
+ // https://msdn.microsoft.com/en-us/library/windows/desktop/ms633572(v=vs.85).aspx
+ lResult, _, _ = pDefWindowProc.Call(
+ uintptr(hWnd),
+ uintptr(message),
+ wParam,
+ lParam,
+ )
+ }
+ return
+}
+
+func (t *winTray) Quit() {
+ quitOnce.Do(quit)
+}
+
+func quit() {
+ boolRet, _, err := pPostMessage.Call(
+ uintptr(wt.window),
+ WM_CLOSE,
+ 0,
+ 0,
+ )
+ if boolRet == 0 {
+ slog.Error(fmt.Sprintf("failed to post close message on shutdown %s", err))
+ }
+}
diff --git a/app/tray/wintray/menus.go b/app/tray/wintray/menus.go
new file mode 100644
index 00000000..59624444
--- /dev/null
+++ b/app/tray/wintray/menus.go
@@ -0,0 +1,71 @@
+//go:build windows
+
+package wintray
+
+import (
+ "fmt"
+ "log/slog"
+ "unsafe"
+
+ "golang.org/x/sys/windows"
+)
+
+const (
+ updateAvailableMenuID = 1
+ updateMenuID = updateAvailableMenuID + 1
+ separatorMenuID = updateMenuID + 1
+ diagLogsMenuID = separatorMenuID + 1
+ diagSeparatorMenuID = diagLogsMenuID + 1
+ quitMenuID = diagSeparatorMenuID + 1
+)
+
+func (t *winTray) initMenus() error {
+ if err := t.addOrUpdateMenuItem(diagLogsMenuID, 0, diagLogsMenuTitle, false); err != nil {
+ return fmt.Errorf("unable to create menu entries %w\n", err)
+ }
+ if err := t.addSeparatorMenuItem(diagSeparatorMenuID, 0); err != nil {
+ return fmt.Errorf("unable to create menu entries %w", err)
+ }
+ if err := t.addOrUpdateMenuItem(quitMenuID, 0, quitMenuTitle, false); err != nil {
+ return fmt.Errorf("unable to create menu entries %w\n", err)
+ }
+ return nil
+}
+
+func (t *winTray) UpdateAvailable(ver string) error {
+ if !t.updateNotified {
+ slog.Debug("updating menu and sending notification for new update")
+ if err := t.addOrUpdateMenuItem(updateAvailableMenuID, 0, updateAvailableMenuTitle, true); err != nil {
+ return fmt.Errorf("unable to create menu entries %w", err)
+ }
+ if err := t.addOrUpdateMenuItem(updateMenuID, 0, updateMenutTitle, false); err != nil {
+ return fmt.Errorf("unable to create menu entries %w", err)
+ }
+ if err := t.addSeparatorMenuItem(separatorMenuID, 0); err != nil {
+ return fmt.Errorf("unable to create menu entries %w", err)
+ }
+ iconFilePath, err := iconBytesToFilePath(wt.updateIcon)
+ if err != nil {
+ return fmt.Errorf("unable to write icon data to temp file: %w", err)
+ }
+ if err := wt.setIcon(iconFilePath); err != nil {
+ return fmt.Errorf("unable to set icon: %w", err)
+ }
+ t.updateNotified = true
+
+ t.pendingUpdate = true
+ // Now pop up the notification
+ t.muNID.Lock()
+ defer t.muNID.Unlock()
+ copy(t.nid.InfoTitle[:], windows.StringToUTF16(updateTitle))
+ copy(t.nid.Info[:], windows.StringToUTF16(fmt.Sprintf(updateMessage, ver)))
+ t.nid.Flags |= NIF_INFO
+ t.nid.Timeout = 10
+ t.nid.Size = uint32(unsafe.Sizeof(*wt.nid))
+ err = t.nid.modify()
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/app/tray/wintray/messages.go b/app/tray/wintray/messages.go
new file mode 100644
index 00000000..d364c716
--- /dev/null
+++ b/app/tray/wintray/messages.go
@@ -0,0 +1,15 @@
+//go:build windows
+
+package wintray
+
+const (
+ firstTimeTitle = "Ollama is running"
+ firstTimeMessage = "Click here to get started"
+ updateTitle = "Update available"
+ updateMessage = "Ollama version %s is ready to install"
+
+ quitMenuTitle = "Quit Ollama"
+ updateAvailableMenuTitle = "An update is available"
+ updateMenutTitle = "Restart to update"
+ diagLogsMenuTitle = "View logs"
+)
diff --git a/app/tray/wintray/notifyicon.go b/app/tray/wintray/notifyicon.go
new file mode 100644
index 00000000..47071669
--- /dev/null
+++ b/app/tray/wintray/notifyicon.go
@@ -0,0 +1,66 @@
+//go:build windows
+
+package wintray
+
+import (
+ "unsafe"
+
+ "golang.org/x/sys/windows"
+)
+
+// Contains information that the system needs to display notifications in the notification area.
+// Used by Shell_NotifyIcon.
+// https://msdn.microsoft.com/en-us/library/windows/desktop/bb773352(v=vs.85).aspx
+// https://msdn.microsoft.com/en-us/library/windows/desktop/bb762159
+type notifyIconData struct {
+ Size uint32
+ Wnd windows.Handle
+ ID, Flags, CallbackMessage uint32
+ Icon windows.Handle
+ Tip [128]uint16
+ State, StateMask uint32
+ Info [256]uint16
+ // Timeout, Version uint32
+ Timeout uint32
+
+ InfoTitle [64]uint16
+ InfoFlags uint32
+ GuidItem windows.GUID
+ BalloonIcon windows.Handle
+}
+
+func (nid *notifyIconData) add() error {
+ const NIM_ADD = 0x00000000
+ res, _, err := pShellNotifyIcon.Call(
+ uintptr(NIM_ADD),
+ uintptr(unsafe.Pointer(nid)),
+ )
+ if res == 0 {
+ return err
+ }
+ return nil
+}
+
+func (nid *notifyIconData) modify() error {
+ const NIM_MODIFY = 0x00000001
+ res, _, err := pShellNotifyIcon.Call(
+ uintptr(NIM_MODIFY),
+ uintptr(unsafe.Pointer(nid)),
+ )
+ if res == 0 {
+ return err
+ }
+ return nil
+}
+
+func (nid *notifyIconData) delete() error {
+ const NIM_DELETE = 0x00000002
+ res, _, err := pShellNotifyIcon.Call(
+ uintptr(NIM_DELETE),
+ uintptr(unsafe.Pointer(nid)),
+ )
+ if res == 0 {
+ return err
+ }
+ return nil
+}
diff --git a/app/tray/wintray/tray.go b/app/tray/wintray/tray.go
new file mode 100644
index 00000000..6f827893
--- /dev/null
+++ b/app/tray/wintray/tray.go
@@ -0,0 +1,488 @@
+//go:build windows
+
+package wintray
+
+import (
+ "crypto/md5"
+ "encoding/hex"
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "sort"
+ "sync"
+ "syscall"
+ "unsafe"
+
+ "golang.org/x/sys/windows"
+
+ "github.com/ollama/ollama/app/tray/commontray"
+)
+
+// Helpful sources: https://github.com/golang/exp/blob/master/shiny/driver/internal/win32
+
+// Contains information about loaded resources
+type winTray struct {
+ instance,
+ icon,
+ cursor,
+ window windows.Handle
+
+ loadedImages map[string]windows.Handle
+ muLoadedImages sync.RWMutex
+
+ // menus keeps track of the submenus keyed by the menu item ID, plus 0
+ // which corresponds to the main popup menu.
+ menus map[uint32]windows.Handle
+ muMenus sync.RWMutex
+ menuOf map[uint32]windows.Handle
+ muMenuOf sync.RWMutex
+ // menuItemIcons maintains the bitmap of each menu item (if applies). It's
+ // needed to show the icon correctly when showing a previously hidden menu
+ // item again.
+ // menuItemIcons map[uint32]windows.Handle
+ // muMenuItemIcons sync.RWMutex
+ visibleItems map[uint32][]uint32
+ muVisibleItems sync.RWMutex
+
+ nid *notifyIconData
+ muNID sync.RWMutex
+ wcex *wndClassEx
+
+ wmSystrayMessage,
+ wmTaskbarCreated uint32
+
+ pendingUpdate bool
+ updateNotified bool // Only pop up the notification once - TODO consider daily nag?
+ // Callbacks
+ callbacks commontray.Callbacks
+ normalIcon []byte
+ updateIcon []byte
+}
+
+var wt winTray
+
+func (t *winTray) GetCallbacks() commontray.Callbacks {
+ return t.callbacks
+}
+
+func InitTray(icon, updateIcon []byte) (*winTray, error) {
+ wt.callbacks.Quit = make(chan struct{})
+ wt.callbacks.Update = make(chan struct{})
+ wt.callbacks.ShowLogs = make(chan struct{})
+ wt.callbacks.DoFirstUse = make(chan struct{})
+ wt.normalIcon = icon
+ wt.updateIcon = updateIcon
+ if err := wt.initInstance(); err != nil {
+ return nil, fmt.Errorf("Unable to init instance: %w\n", err)
+ }
+
+ if err := wt.createMenu(); err != nil {
+ return nil, fmt.Errorf("Unable to create menu: %w\n", err)
+ }
+
+ iconFilePath, err := iconBytesToFilePath(wt.normalIcon)
+ if err != nil {
+ return nil, fmt.Errorf("Unable to write icon data to temp file: %w", err)
+ }
+ if err := wt.setIcon(iconFilePath); err != nil {
+ return nil, fmt.Errorf("Unable to set icon: %w", err)
+ }
+
+ return &wt, wt.initMenus()
+}
+
+func (t *winTray) initInstance() error {
+ const (
+ className = "OllamaClass"
+ windowName = ""
+ )
+
+ t.wmSystrayMessage = WM_USER + 1
+ t.visibleItems = make(map[uint32][]uint32)
+ t.menus = make(map[uint32]windows.Handle)
+ t.menuOf = make(map[uint32]windows.Handle)
+
+ t.loadedImages = make(map[string]windows.Handle)
+
+ taskbarEventNamePtr, _ := windows.UTF16PtrFromString("TaskbarCreated")
+ // https://msdn.microsoft.com/en-us/library/windows/desktop/ms644947
+ res, _, err := pRegisterWindowMessage.Call(
+ uintptr(unsafe.Pointer(taskbarEventNamePtr)),
+ )
+ if res == 0 { // success 0xc000-0xfff
+ return fmt.Errorf("failed to register window: %w", err)
+ }
+ t.wmTaskbarCreated = uint32(res)
+
+ instanceHandle, _, err := pGetModuleHandle.Call(0)
+ if instanceHandle == 0 {
+ return err
+ }
+ t.instance = windows.Handle(instanceHandle)
+
+ // https://msdn.microsoft.com/en-us/library/windows/desktop/ms648072(v=vs.85).aspx
+ iconHandle, _, err := pLoadIcon.Call(0, uintptr(IDI_APPLICATION))
+ if iconHandle == 0 {
+ return err
+ }
+ t.icon = windows.Handle(iconHandle)
+
+ // https://msdn.microsoft.com/en-us/library/windows/desktop/ms648391(v=vs.85).aspx
+ cursorHandle, _, err := pLoadCursor.Call(0, uintptr(IDC_ARROW))
+ if cursorHandle == 0 {
+ return err
+ }
+ t.cursor = windows.Handle(cursorHandle)
+
+ classNamePtr, err := windows.UTF16PtrFromString(className)
+ if err != nil {
+ return err
+ }
+
+ windowNamePtr, err := windows.UTF16PtrFromString(windowName)
+ if err != nil {
+ return err
+ }
+
+ t.wcex = &wndClassEx{
+ Style: CS_HREDRAW | CS_VREDRAW,
+ WndProc: windows.NewCallback(t.wndProc),
+ Instance: t.instance,
+ Icon: t.icon,
+ Cursor: t.cursor,
+ Background: windows.Handle(6), // (COLOR_WINDOW + 1)
+ ClassName: classNamePtr,
+ IconSm: t.icon,
+ }
+ if err := t.wcex.register(); err != nil {
+ return err
+ }
+
+ windowHandle, _, err := pCreateWindowEx.Call(
+ uintptr(0),
+ uintptr(unsafe.Pointer(classNamePtr)),
+ uintptr(unsafe.Pointer(windowNamePtr)),
+ uintptr(WS_OVERLAPPEDWINDOW),
+ uintptr(CW_USEDEFAULT),
+ uintptr(CW_USEDEFAULT),
+ uintptr(CW_USEDEFAULT),
+ uintptr(CW_USEDEFAULT),
+ uintptr(0),
+ uintptr(0),
+ uintptr(t.instance),
+ uintptr(0),
+ )
+ if windowHandle == 0 {
+ return err
+ }
+ t.window = windows.Handle(windowHandle)
+
+ pShowWindow.Call(uintptr(t.window), uintptr(SW_HIDE)) //nolint:errcheck
+
+ boolRet, _, err := pUpdateWindow.Call(uintptr(t.window))
+ if boolRet == 0 {
+ slog.Error(fmt.Sprintf("failed to update window: %s", err))
+ }
+
+ t.muNID.Lock()
+ defer t.muNID.Unlock()
+ t.nid = ¬ifyIconData{
+ Wnd: t.window,
+ ID: 100,
+ Flags: NIF_MESSAGE,
+ CallbackMessage: t.wmSystrayMessage,
+ }
+ t.nid.Size = uint32(unsafe.Sizeof(*t.nid))
+
+ return t.nid.add()
+}
+
+func (t *winTray) createMenu() error {
+ menuHandle, _, err := pCreatePopupMenu.Call()
+ if menuHandle == 0 {
+ return err
+ }
+ t.menus[0] = windows.Handle(menuHandle)
+
+ // https://msdn.microsoft.com/en-us/library/windows/desktop/ms647575(v=vs.85).aspx
+ mi := struct {
+ Size, Mask, Style, Max uint32
+ Background windows.Handle
+ ContextHelpID uint32
+ MenuData uintptr
+ }{
+ Mask: MIM_APPLYTOSUBMENUS,
+ }
+ mi.Size = uint32(unsafe.Sizeof(mi))
+
+ res, _, err := pSetMenuInfo.Call(
+ uintptr(t.menus[0]),
+ uintptr(unsafe.Pointer(&mi)),
+ )
+ if res == 0 {
+ return err
+ }
+ return nil
+}
+
+// Contains information about a menu item.
+// https://msdn.microsoft.com/en-us/library/windows/desktop/ms647578(v=vs.85).aspx
+type menuItemInfo struct {
+ Size, Mask, Type, State uint32
+ ID uint32
+ SubMenu, Checked, Unchecked windows.Handle
+ ItemData uintptr
+ TypeData *uint16
+ Cch uint32
+ BMPItem windows.Handle
+}
+
+func (t *winTray) addOrUpdateMenuItem(menuItemId uint32, parentId uint32, title string, disabled bool) error {
+ titlePtr, err := windows.UTF16PtrFromString(title)
+ if err != nil {
+ return err
+ }
+
+ mi := menuItemInfo{
+ Mask: MIIM_FTYPE | MIIM_STRING | MIIM_ID | MIIM_STATE,
+ Type: MFT_STRING,
+ ID: menuItemId,
+ TypeData: titlePtr,
+ Cch: uint32(len(title)),
+ }
+ mi.Size = uint32(unsafe.Sizeof(mi))
+ if disabled {
+ mi.State |= MFS_DISABLED
+ }
+
+ var res uintptr
+ t.muMenus.RLock()
+ menu := t.menus[parentId]
+ t.muMenus.RUnlock()
+ if t.getVisibleItemIndex(parentId, menuItemId) != -1 {
+ // We set the menu item info based on the menuID
+ boolRet, _, err := pSetMenuItemInfo.Call(
+ uintptr(menu),
+ uintptr(menuItemId),
+ 0,
+ uintptr(unsafe.Pointer(&mi)),
+ )
+ if boolRet == 0 {
+ return fmt.Errorf("failed to set menu item: %w", err)
+ }
+ }
+
+ if res == 0 {
+ // Menu item does not already exist, create it
+ t.muMenus.RLock()
+ submenu, exists := t.menus[menuItemId]
+ t.muMenus.RUnlock()
+ if exists {
+ mi.Mask |= MIIM_SUBMENU
+ mi.SubMenu = submenu
+ }
+ t.addToVisibleItems(parentId, menuItemId)
+ position := t.getVisibleItemIndex(parentId, menuItemId)
+ res, _, err = pInsertMenuItem.Call(
+ uintptr(menu),
+ uintptr(position),
+ 1,
+ uintptr(unsafe.Pointer(&mi)),
+ )
+ if res == 0 {
+ t.delFromVisibleItems(parentId, menuItemId)
+ return err
+ }
+ t.muMenuOf.Lock()
+ t.menuOf[menuItemId] = menu
+ t.muMenuOf.Unlock()
+ }
+
+ return nil
+}
+
+func (t *winTray) addSeparatorMenuItem(menuItemId, parentId uint32) error {
+ mi := menuItemInfo{
+ Mask: MIIM_FTYPE | MIIM_ID | MIIM_STATE,
+ Type: MFT_SEPARATOR,
+ ID: menuItemId,
+ }
+
+ mi.Size = uint32(unsafe.Sizeof(mi))
+
+ t.addToVisibleItems(parentId, menuItemId)
+ position := t.getVisibleItemIndex(parentId, menuItemId)
+ t.muMenus.RLock()
+ menu := uintptr(t.menus[parentId])
+ t.muMenus.RUnlock()
+ res, _, err := pInsertMenuItem.Call(
+ menu,
+ uintptr(position),
+ 1,
+ uintptr(unsafe.Pointer(&mi)),
+ )
+ if res == 0 {
+ return err
+ }
+
+ return nil
+}
+
+// func (t *winTray) hideMenuItem(menuItemId, parentId uint32) error {
+// const ERROR_SUCCESS syscall.Errno = 0
+
+// t.muMenus.RLock()
+// menu := uintptr(t.menus[parentId])
+// t.muMenus.RUnlock()
+// res, _, err := pRemoveMenu.Call(
+// menu,
+// uintptr(menuItemId),
+// MF_BYCOMMAND,
+// )
+// if res == 0 && err.(syscall.Errno) != ERROR_SUCCESS {
+// return err
+// }
+// t.delFromVisibleItems(parentId, menuItemId)
+
+// return nil
+// }
+
+func (t *winTray) showMenu() error {
+ p := point{}
+ boolRet, _, err := pGetCursorPos.Call(uintptr(unsafe.Pointer(&p)))
+ if boolRet == 0 {
+ return err
+ }
+ boolRet, _, err = pSetForegroundWindow.Call(uintptr(t.window))
+ if boolRet == 0 {
+ slog.Warn(fmt.Sprintf("failed to bring menu to foreground: %s", err))
+ }
+
+ boolRet, _, err = pTrackPopupMenu.Call(
+ uintptr(t.menus[0]),
+ TPM_BOTTOMALIGN|TPM_LEFTALIGN,
+ uintptr(p.X),
+ uintptr(p.Y),
+ 0,
+ uintptr(t.window),
+ 0,
+ )
+ if boolRet == 0 {
+ return err
+ }
+
+ return nil
+}
+
+func (t *winTray) delFromVisibleItems(parent, val uint32) {
+ t.muVisibleItems.Lock()
+ defer t.muVisibleItems.Unlock()
+ visibleItems := t.visibleItems[parent]
+ for i, itemval := range visibleItems {
+ if val == itemval {
+ t.visibleItems[parent] = append(visibleItems[:i], visibleItems[i+1:]...)
+ break
+ }
+ }
+}
+
+func (t *winTray) addToVisibleItems(parent, val uint32) {
+ t.muVisibleItems.Lock()
+ defer t.muVisibleItems.Unlock()
+ if visibleItems, exists := t.visibleItems[parent]; !exists {
+ t.visibleItems[parent] = []uint32{val}
+ } else {
+ newvisible := append(visibleItems, val)
+ sort.Slice(newvisible, func(i, j int) bool { return newvisible[i] < newvisible[j] })
+ t.visibleItems[parent] = newvisible
+ }
+}
+
+func (t *winTray) getVisibleItemIndex(parent, val uint32) int {
+ t.muVisibleItems.RLock()
+ defer t.muVisibleItems.RUnlock()
+ for i, itemval := range t.visibleItems[parent] {
+ if val == itemval {
+ return i
+ }
+ }
+ return -1
+}
+
+func iconBytesToFilePath(iconBytes []byte) (string, error) {
+ bh := md5.Sum(iconBytes)
+ dataHash := hex.EncodeToString(bh[:])
+ iconFilePath := filepath.Join(os.TempDir(), "ollama_temp_icon_"+dataHash)
+
+ if _, err := os.Stat(iconFilePath); os.IsNotExist(err) {
+ if err := os.WriteFile(iconFilePath, iconBytes, 0o644); err != nil {
+ return "", err
+ }
+ }
+ return iconFilePath, nil
+}
+
+// Loads an image from file and shows it in tray.
+// Shell_NotifyIcon: https://msdn.microsoft.com/en-us/library/windows/desktop/bb762159(v=vs.85).aspx
+func (t *winTray) setIcon(src string) error {
+ h, err := t.loadIconFrom(src)
+ if err != nil {
+ return err
+ }
+
+ t.muNID.Lock()
+ defer t.muNID.Unlock()
+ t.nid.Icon = h
+ t.nid.Flags |= NIF_ICON | NIF_TIP
+ if toolTipUTF16, err := syscall.UTF16FromString(commontray.ToolTip); err == nil {
+ copy(t.nid.Tip[:], toolTipUTF16)
+ } else {
+ return err
+ }
+ t.nid.Size = uint32(unsafe.Sizeof(*t.nid))
+
+ return t.nid.modify()
+}
+
+// Loads an image from file to be shown in tray or menu item.
+// LoadImage: https://msdn.microsoft.com/en-us/library/windows/desktop/ms648045(v=vs.85).aspx
+func (t *winTray) loadIconFrom(src string) (windows.Handle, error) {
+ // Save and reuse handles of loaded images
+ t.muLoadedImages.RLock()
+ h, ok := t.loadedImages[src]
+ t.muLoadedImages.RUnlock()
+ if !ok {
+ srcPtr, err := windows.UTF16PtrFromString(src)
+ if err != nil {
+ return 0, err
+ }
+ res, _, err := pLoadImage.Call(
+ 0,
+ uintptr(unsafe.Pointer(srcPtr)),
+ IMAGE_ICON,
+ 0,
+ 0,
+ LR_LOADFROMFILE|LR_DEFAULTSIZE,
+ )
+ if res == 0 {
+ return 0, err
+ }
+ h = windows.Handle(res)
+ t.muLoadedImages.Lock()
+ t.loadedImages[src] = h
+ t.muLoadedImages.Unlock()
+ }
+ return h, nil
+}
+
+func (t *winTray) DisplayFirstUseNotification() error {
+ t.muNID.Lock()
+ defer t.muNID.Unlock()
+ copy(t.nid.InfoTitle[:], windows.StringToUTF16(firstTimeTitle))
+ copy(t.nid.Info[:], windows.StringToUTF16(firstTimeMessage))
+ t.nid.Flags |= NIF_INFO
+ t.nid.Size = uint32(unsafe.Sizeof(*wt.nid))
+
+ return t.nid.modify()
+}
diff --git a/app/tray/wintray/w32api.go b/app/tray/wintray/w32api.go
new file mode 100644
index 00000000..7c7c0ac8
--- /dev/null
+++ b/app/tray/wintray/w32api.go
@@ -0,0 +1,90 @@
+//go:build windows
+
+package wintray
+
+import (
+ "runtime"
+
+ "golang.org/x/sys/windows"
+)
+
+var (
+ k32 = windows.NewLazySystemDLL("Kernel32.dll")
+ u32 = windows.NewLazySystemDLL("User32.dll")
+ s32 = windows.NewLazySystemDLL("Shell32.dll")
+
+ pCreatePopupMenu = u32.NewProc("CreatePopupMenu")
+ pCreateWindowEx = u32.NewProc("CreateWindowExW")
+ pDefWindowProc = u32.NewProc("DefWindowProcW")
+ pDestroyWindow = u32.NewProc("DestroyWindow")
+ pDispatchMessage = u32.NewProc("DispatchMessageW")
+ pGetCursorPos = u32.NewProc("GetCursorPos")
+ pGetMessage = u32.NewProc("GetMessageW")
+ pGetModuleHandle = k32.NewProc("GetModuleHandleW")
+ pInsertMenuItem = u32.NewProc("InsertMenuItemW")
+ pLoadCursor = u32.NewProc("LoadCursorW")
+ pLoadIcon = u32.NewProc("LoadIconW")
+ pLoadImage = u32.NewProc("LoadImageW")
+ pPostMessage = u32.NewProc("PostMessageW")
+ pPostQuitMessage = u32.NewProc("PostQuitMessage")
+ pRegisterClass = u32.NewProc("RegisterClassExW")
+ pRegisterWindowMessage = u32.NewProc("RegisterWindowMessageW")
+ pSetForegroundWindow = u32.NewProc("SetForegroundWindow")
+ pSetMenuInfo = u32.NewProc("SetMenuInfo")
+ pSetMenuItemInfo = u32.NewProc("SetMenuItemInfoW")
+ pShellNotifyIcon = s32.NewProc("Shell_NotifyIconW")
+ pShowWindow = u32.NewProc("ShowWindow")
+ pTrackPopupMenu = u32.NewProc("TrackPopupMenu")
+ pTranslateMessage = u32.NewProc("TranslateMessage")
+ pUnregisterClass = u32.NewProc("UnregisterClassW")
+ pUpdateWindow = u32.NewProc("UpdateWindow")
+)
+
+const (
+ CS_HREDRAW = 0x0002
+ CS_VREDRAW = 0x0001
+ CW_USEDEFAULT = 0x80000000
+ IDC_ARROW = 32512 // Standard arrow
+ IDI_APPLICATION = 32512
+ IMAGE_ICON = 1 // Loads an icon
+ LR_DEFAULTSIZE = 0x00000040 // Loads default-size icon for windows(SM_CXICON x SM_CYICON) if cx, cy are set to zero
+ LR_LOADFROMFILE = 0x00000010 // Loads the stand-alone image from the file
+ MF_BYCOMMAND = 0x00000000
+ MFS_DISABLED = 0x00000003
+ MFT_SEPARATOR = 0x00000800
+ MFT_STRING = 0x00000000
+ MIIM_BITMAP = 0x00000080
+ MIIM_FTYPE = 0x00000100
+ MIIM_ID = 0x00000002
+ MIIM_STATE = 0x00000001
+ MIIM_STRING = 0x00000040
+ MIIM_SUBMENU = 0x00000004
+ MIM_APPLYTOSUBMENUS = 0x80000000
+ NIF_ICON = 0x00000002
+ NIF_TIP = 0x00000004
+ NIF_INFO = 0x00000010
+ NIF_MESSAGE = 0x00000001
+ SW_HIDE = 0
+ TPM_BOTTOMALIGN = 0x0020
+ TPM_LEFTALIGN = 0x0000
+ WM_CLOSE = 0x0010
+ WM_USER = 0x0400
+ WS_CAPTION = 0x00C00000
+ WS_MAXIMIZEBOX = 0x00010000
+ WS_MINIMIZEBOX = 0x00020000
+ WS_OVERLAPPED = 0x00000000
+ WS_OVERLAPPEDWINDOW = WS_OVERLAPPED | WS_CAPTION | WS_SYSMENU | WS_THICKFRAME | WS_MINIMIZEBOX | WS_MAXIMIZEBOX
+ WS_SYSMENU = 0x00080000
+ WS_THICKFRAME = 0x00040000
+)
+
+// Not sure if this is actually needed on windows
+func init() {
+ runtime.LockOSThread()
+}
+
+// The POINT structure defines the x- and y- coordinates of a point.
+// https://msdn.microsoft.com/en-us/library/windows/desktop/dd162805(v=vs.85).aspx
+type point struct {
+ X, Y int32
+}
diff --git a/app/tray/wintray/winclass.go b/app/tray/wintray/winclass.go
new file mode 100644
index 00000000..9ce71d00
--- /dev/null
+++ b/app/tray/wintray/winclass.go
@@ -0,0 +1,45 @@
+//go:build windows
+
+package wintray
+
+import (
+ "unsafe"
+
+ "golang.org/x/sys/windows"
+)
+
+// Contains window class information.
+// It is used with the RegisterClassEx and GetClassInfoEx functions.
+// https://msdn.microsoft.com/en-us/library/ms633577.aspx
+type wndClassEx struct {
+ Size, Style uint32
+ WndProc uintptr
+ ClsExtra, WndExtra int32
+ Instance, Icon, Cursor, Background windows.Handle
+ MenuName, ClassName *uint16
+ IconSm windows.Handle
+}
+
+// Registers a window class for subsequent use in calls to the CreateWindow or CreateWindowEx function.
+// https://msdn.microsoft.com/en-us/library/ms633587.aspx
+func (w *wndClassEx) register() error {
+ w.Size = uint32(unsafe.Sizeof(*w))
+ res, _, err := pRegisterClass.Call(uintptr(unsafe.Pointer(w)))
+ if res == 0 {
+ return err
+ }
+ return nil
+}
+
+// Unregisters a window class, freeing the memory required for the class.
+// https://msdn.microsoft.com/en-us/library/ms644899.aspx
+func (w *wndClassEx) unregister() error {
+ res, _, err := pUnregisterClass.Call(
+ uintptr(unsafe.Pointer(w.ClassName)),
+ uintptr(w.Instance),
+ )
+ if res == 0 {
+ return err
+ }
+ return nil
+}
diff --git a/auth/auth.go b/auth/auth.go
new file mode 100644
index 00000000..e1d85412
--- /dev/null
+++ b/auth/auth.go
@@ -0,0 +1,92 @@
+package auth
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "golang.org/x/crypto/ssh"
+)
+
+const defaultPrivateKey = "id_ed25519"
+
+func keyPath() (string, error) {
+ home, err := os.UserHomeDir()
+ if err != nil {
+ return "", err
+ }
+
+ return filepath.Join(home, ".ollama", defaultPrivateKey), nil
+}
+
+func GetPublicKey() (string, error) {
+ keyPath, err := keyPath()
+ if err != nil {
+ return "", err
+ }
+
+ privateKeyFile, err := os.ReadFile(keyPath)
+ if err != nil {
+ slog.Info(fmt.Sprintf("Failed to load private key: %v", err))
+ return "", err
+ }
+
+ privateKey, err := ssh.ParsePrivateKey(privateKeyFile)
+ if err != nil {
+ return "", err
+ }
+
+ publicKey := ssh.MarshalAuthorizedKey(privateKey.PublicKey())
+
+ return strings.TrimSpace(string(publicKey)), nil
+}
+
+func NewNonce(r io.Reader, length int) (string, error) {
+ nonce := make([]byte, length)
+ if _, err := io.ReadFull(r, nonce); err != nil {
+ return "", err
+ }
+
+ return base64.RawURLEncoding.EncodeToString(nonce), nil
+}
+
+func Sign(ctx context.Context, bts []byte) (string, error) {
+ keyPath, err := keyPath()
+ if err != nil {
+ return "", err
+ }
+
+ privateKeyFile, err := os.ReadFile(keyPath)
+ if err != nil {
+ slog.Info(fmt.Sprintf("Failed to load private key: %v", err))
+ return "", err
+ }
+
+ privateKey, err := ssh.ParsePrivateKey(privateKeyFile)
+ if err != nil {
+ return "", err
+ }
+
+ // get the pubkey, but remove the type
+ publicKey := ssh.MarshalAuthorizedKey(privateKey.PublicKey())
+ parts := bytes.Split(publicKey, []byte(" "))
+ if len(parts) < 2 {
+ return "", errors.New("malformed public key")
+ }
+
+ signedData, err := privateKey.Sign(rand.Reader, bts)
+ if err != nil {
+ return "", err
+ }
+
+ // signature is <pubkey>:<signature>
+ return fmt.Sprintf("%s:%s", bytes.TrimSpace(parts[1]), base64.StdEncoding.EncodeToString(signedData.Blob)), nil
+}
diff --git a/build/darwin/amd64/placeholder b/build/darwin/amd64/placeholder
new file mode 100644
index 00000000..87dc2738
--- /dev/null
+++ b/build/darwin/amd64/placeholder
@@ -0,0 +1 @@
+This is here to make sure the build/ directory exists for the go:embed command
diff --git a/build/darwin/arm64/placeholder b/build/darwin/arm64/placeholder
new file mode 100644
index 00000000..87dc2738
--- /dev/null
+++ b/build/darwin/arm64/placeholder
@@ -0,0 +1 @@
+This is here to make sure the build/ directory exists for the go:embed command
diff --git a/build/embed_darwin_amd64.go b/build/embed_darwin_amd64.go
new file mode 100644
index 00000000..af1458ea
--- /dev/null
+++ b/build/embed_darwin_amd64.go
@@ -0,0 +1,8 @@
+package build
+
+import "embed"
+
+// Darwin payloads separated by architecture to avoid duplicate payloads when cross compiling
+
+//go:embed darwin/amd64/*
+var EmbedFS embed.FS
diff --git a/build/embed_darwin_arm64.go b/build/embed_darwin_arm64.go
new file mode 100644
index 00000000..d885365d
--- /dev/null
+++ b/build/embed_darwin_arm64.go
@@ -0,0 +1,8 @@
+package build
+
+import "embed"
+
+// Darwin payloads separated by architecture to avoid duplicate payloads when cross compiling
+
+//go:embed darwin/arm64/*
+var EmbedFS embed.FS
diff --git a/build/embed_linux.go b/build/embed_linux.go
new file mode 100644
index 00000000..4cf7be4c
--- /dev/null
+++ b/build/embed_linux.go
@@ -0,0 +1,6 @@
+package build
+
+import "embed"
+
+//go:embed linux/*
+var EmbedFS embed.FS
diff --git a/build/embed_unused.go b/build/embed_unused.go
new file mode 100644
index 00000000..00fbe02e
--- /dev/null
+++ b/build/embed_unused.go
@@ -0,0 +1,8 @@
+//go:build !linux && !darwin
+
+package build
+
+import "embed"
+
+// unused on windows
+var EmbedFS embed.FS
diff --git a/build/linux/amd64/placeholder b/build/linux/amd64/placeholder
new file mode 100644
index 00000000..87dc2738
--- /dev/null
+++ b/build/linux/amd64/placeholder
@@ -0,0 +1 @@
+This is here to make sure the build/ directory exists for the go:embed command
diff --git a/build/linux/arm64/placeholder b/build/linux/arm64/placeholder
new file mode 100644
index 00000000..87dc2738
--- /dev/null
+++ b/build/linux/arm64/placeholder
@@ -0,0 +1 @@
+This is here to make sure the build/ directory exists for the go:embed command
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 46d00a3d..04099fd5 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -1,6 +1,8 @@
package cmd
import (
+ "archive/zip"
+ "bufio"
"bytes"
"context"
"crypto/ed25519"
@@ -11,35 +13,40 @@ import (
"fmt"
"io"
"log"
+ "math"
"net"
"net/http"
"os"
- "os/exec"
"os/signal"
"path/filepath"
"regexp"
"runtime"
+ "slices"
+ "strconv"
"strings"
+ "sync/atomic"
"syscall"
"time"
+ "github.com/containerd/console"
+ "github.com/mattn/go-runewidth"
"github.com/olekukonko/tablewriter"
"github.com/spf13/cobra"
"golang.org/x/crypto/ssh"
- "golang.org/x/exp/slices"
"golang.org/x/term"
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/format"
- "github.com/jmorganca/ollama/parser"
- "github.com/jmorganca/ollama/progress"
- "github.com/jmorganca/ollama/readline"
- "github.com/jmorganca/ollama/server"
- "github.com/jmorganca/ollama/version"
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/auth"
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/format"
+ "github.com/ollama/ollama/parser"
+ "github.com/ollama/ollama/progress"
+ "github.com/ollama/ollama/server"
+ "github.com/ollama/ollama/types/errtypes"
+ "github.com/ollama/ollama/types/model"
+ "github.com/ollama/ollama/version"
)
-type ImageData []byte
-
func CreateHandler(cmd *cobra.Command, args []string) error {
filename, _ := cmd.Flags().GetString("file")
filename, err := filepath.Abs(filename)
@@ -55,14 +62,13 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
p := progress.NewProgress(os.Stderr)
defer p.Stop()
- bars := make(map[string]*progress.Bar)
-
- modelfile, err := os.ReadFile(filename)
+ f, err := os.Open(filename)
if err != nil {
return err
}
+ defer f.Close()
- commands, err := parser.Parse(bytes.NewReader(modelfile))
+ modelfile, err := parser.ParseFile(f)
if err != nil {
return err
}
@@ -75,11 +81,12 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
status := "transferring model data"
spinner := progress.NewSpinner(status)
p.Add(status, spinner)
+ defer p.Stop()
- for _, c := range commands {
- switch c.Name {
+ for i := range modelfile.Commands {
+ switch modelfile.Commands[i].Name {
case "model", "adapter":
- path := c.Args
+ path := modelfile.Commands[i].Args
if path == "~" {
path = home
} else if strings.HasPrefix(path, "~/") {
@@ -90,29 +97,35 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
path = filepath.Join(filepath.Dir(filename), path)
}
- bin, err := os.Open(path)
- if errors.Is(err, os.ErrNotExist) && c.Name == "model" {
+ fi, err := os.Stat(path)
+ if errors.Is(err, os.ErrNotExist) && modelfile.Commands[i].Name == "model" {
continue
} else if err != nil {
return err
}
- defer bin.Close()
- hash := sha256.New()
- if _, err := io.Copy(hash, bin); err != nil {
- return err
+ if fi.IsDir() {
+ // this is likely a safetensors or pytorch directory
+ // TODO make this work w/ adapters
+ tempfile, err := tempZipFiles(path)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tempfile)
+
+ path = tempfile
}
- bin.Seek(0, io.SeekStart)
- digest := fmt.Sprintf("sha256:%x", hash.Sum(nil))
- if err = client.CreateBlob(cmd.Context(), digest, bin); err != nil {
+ digest, err := createBlob(cmd, client, path, spinner)
+ if err != nil {
return err
}
- modelfile = bytes.ReplaceAll(modelfile, []byte(c.Args), []byte("@"+digest))
+ modelfile.Commands[i].Args = "@" + digest
}
}
+ bars := make(map[string]*progress.Bar)
fn := func(resp api.ProgressResponse) error {
if resp.Digest != "" {
spinner.Stop()
@@ -136,7 +149,9 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
return nil
}
- request := api.CreateRequest{Name: args[0], Modelfile: string(modelfile)}
+ quantize, _ := cmd.Flags().GetString("quantize")
+
+ request := api.CreateRequest{Name: args[0], Modelfile: modelfile.String(), Quantize: quantize}
if err := client.Create(cmd.Context(), &request, fn); err != nil {
return err
}
@@ -144,26 +159,364 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
return nil
}
+func tempZipFiles(path string) (string, error) {
+ tempfile, err := os.CreateTemp("", "ollama-tf")
+ if err != nil {
+ return "", err
+ }
+ defer tempfile.Close()
+
+ detectContentType := func(path string) (string, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return "", err
+ }
+ defer f.Close()
+
+ var b bytes.Buffer
+ b.Grow(512)
+
+ if _, err := io.CopyN(&b, f, 512); err != nil && !errors.Is(err, io.EOF) {
+ return "", err
+ }
+
+ contentType, _, _ := strings.Cut(http.DetectContentType(b.Bytes()), ";")
+ return contentType, nil
+ }
+
+ glob := func(pattern, contentType string) ([]string, error) {
+ matches, err := filepath.Glob(pattern)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, safetensor := range matches {
+ if ct, err := detectContentType(safetensor); err != nil {
+ return nil, err
+ } else if ct != contentType {
+ return nil, fmt.Errorf("invalid content type: expected %s for %s", ct, safetensor)
+ }
+ }
+
+ return matches, nil
+ }
+
+ var files []string
+ if st, _ := glob(filepath.Join(path, "model*.safetensors"), "application/octet-stream"); len(st) > 0 {
+ // safetensors files might be unresolved git lfs references; skip if they are
+ // covers model-x-of-y.safetensors, model.fp32-x-of-y.safetensors, model.safetensors
+ files = append(files, st...)
+ } else if st, _ := glob(filepath.Join(path, "adapters.safetensors"), "application/octet-stream"); len(st) > 0 {
+ // covers adapters.safetensors
+ files = append(files, st...)
+ } else if st, _ := glob(filepath.Join(path, "adapter_model.safetensors"), "application/octet-stream"); len(st) > 0 {
+ // covers adapter_model.safetensors
+ files = append(files, st...)
+ } else if pt, _ := glob(filepath.Join(path, "pytorch_model*.bin"), "application/zip"); len(pt) > 0 {
+ // pytorch files might also be unresolved git lfs references; skip if they are
+ // covers pytorch_model-x-of-y.bin, pytorch_model.fp32-x-of-y.bin, pytorch_model.bin
+ files = append(files, pt...)
+ } else if pt, _ := glob(filepath.Join(path, "consolidated*.pth"), "application/zip"); len(pt) > 0 {
+ // pytorch files might also be unresolved git lfs references; skip if they are
+ // covers consolidated.x.pth, consolidated.pth
+ files = append(files, pt...)
+ } else {
+ return "", errors.New("no safetensors or torch files found")
+ }
+
+ // add configuration files, json files are detected as text/plain
+ js, err := glob(filepath.Join(path, "*.json"), "text/plain")
+ if err != nil {
+ return "", err
+ }
+ files = append(files, js...)
+
+ // bert models require a nested config.json
+ // TODO(mxyng): merge this with the glob above
+ js, err = glob(filepath.Join(path, "**/*.json"), "text/plain")
+ if err != nil {
+ return "", err
+ }
+ files = append(files, js...)
+
+ if tks, _ := glob(filepath.Join(path, "tokenizer.model"), "application/octet-stream"); len(tks) > 0 {
+ // add tokenizer.model if it exists, tokenizer.json is automatically picked up by the previous glob
+ // tokenizer.model might be a unresolved git lfs reference; error if it is
+ files = append(files, tks...)
+ } else if tks, _ := glob(filepath.Join(path, "**/tokenizer.model"), "text/plain"); len(tks) > 0 {
+ // some times tokenizer.model is in a subdirectory (e.g. meta-llama/Meta-Llama-3-8B)
+ files = append(files, tks...)
+ }
+
+ zipfile := zip.NewWriter(tempfile)
+ defer zipfile.Close()
+
+ for _, file := range files {
+ f, err := os.Open(file)
+ if err != nil {
+ return "", err
+ }
+ defer f.Close()
+
+ fi, err := f.Stat()
+ if err != nil {
+ return "", err
+ }
+
+ zfi, err := zip.FileInfoHeader(fi)
+ if err != nil {
+ return "", err
+ }
+
+ zfi.Name, err = filepath.Rel(path, file)
+ if err != nil {
+ return "", err
+ }
+
+ zf, err := zipfile.CreateHeader(zfi)
+ if err != nil {
+ return "", err
+ }
+
+ if _, err := io.Copy(zf, f); err != nil {
+ return "", err
+ }
+ }
+
+ return tempfile.Name(), nil
+}
+
+func createBlob(cmd *cobra.Command, client *api.Client, path string, spinner *progress.Spinner) (string, error) {
+ bin, err := os.Open(path)
+ if err != nil {
+ return "", err
+ }
+ defer bin.Close()
+
+ // Get file info to retrieve the size
+ fileInfo, err := bin.Stat()
+ if err != nil {
+ return "", err
+ }
+ fileSize := fileInfo.Size()
+
+ hash := sha256.New()
+ if _, err := io.Copy(hash, bin); err != nil {
+ return "", err
+ }
+
+ if _, err := bin.Seek(0, io.SeekStart); err != nil {
+ return "", err
+ }
+
+ var pw progressWriter
+ status := "transferring model data 0%"
+ spinner.SetMessage(status)
+
+ done := make(chan struct{})
+ defer close(done)
+
+ go func() {
+ ticker := time.NewTicker(60 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ spinner.SetMessage(fmt.Sprintf("transferring model data %d%%", int(100*pw.n.Load()/fileSize)))
+ case <-done:
+ spinner.SetMessage("transferring model data 100%")
+ return
+ }
+ }
+ }()
+
+ digest := fmt.Sprintf("sha256:%x", hash.Sum(nil))
+ if err = client.CreateBlob(cmd.Context(), digest, io.TeeReader(bin, &pw)); err != nil {
+ return "", err
+ }
+ return digest, nil
+}
+
+type progressWriter struct {
+ n atomic.Int64
+}
+
+func (w *progressWriter) Write(p []byte) (n int, err error) {
+ w.n.Add(int64(len(p)))
+ return len(p), nil
+}
+
+func loadOrUnloadModel(cmd *cobra.Command, opts *runOptions) error {
+ p := progress.NewProgress(os.Stderr)
+ defer p.StopAndClear()
+
+ spinner := progress.NewSpinner("")
+ p.Add("", spinner)
+
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ return err
+ }
+
+ req := &api.GenerateRequest{
+ Model: opts.Model,
+ KeepAlive: opts.KeepAlive,
+ }
+
+ return client.Generate(cmd.Context(), req, func(api.GenerateResponse) error { return nil })
+}
+
+func StopHandler(cmd *cobra.Command, args []string) error {
+ opts := &runOptions{
+ Model: args[0],
+ KeepAlive: &api.Duration{Duration: 0},
+ }
+ if err := loadOrUnloadModel(cmd, opts); err != nil {
+ if strings.Contains(err.Error(), "not found") {
+ return fmt.Errorf("couldn't find model \"%s\" to stop", args[0])
+ }
+ }
+ return nil
+}
+
func RunHandler(cmd *cobra.Command, args []string) error {
+ interactive := true
+
+ opts := runOptions{
+ Model: args[0],
+ WordWrap: os.Getenv("TERM") == "xterm-256color",
+ Options: map[string]interface{}{},
+ }
+
+ format, err := cmd.Flags().GetString("format")
+ if err != nil {
+ return err
+ }
+ opts.Format = format
+
+ keepAlive, err := cmd.Flags().GetString("keepalive")
+ if err != nil {
+ return err
+ }
+ if keepAlive != "" {
+ d, err := time.ParseDuration(keepAlive)
+ if err != nil {
+ return err
+ }
+ opts.KeepAlive = &api.Duration{Duration: d}
+ }
+
+ prompts := args[1:]
+ // prepend stdin to the prompt if provided
+ if !term.IsTerminal(int(os.Stdin.Fd())) {
+ in, err := io.ReadAll(os.Stdin)
+ if err != nil {
+ return err
+ }
+
+ prompts = append([]string{string(in)}, prompts...)
+ opts.WordWrap = false
+ interactive = false
+ }
+ opts.Prompt = strings.Join(prompts, " ")
+ if len(prompts) > 0 {
+ interactive = false
+ }
+
+ nowrap, err := cmd.Flags().GetBool("nowordwrap")
+ if err != nil {
+ return err
+ }
+ opts.WordWrap = !nowrap
+
+ // Fill out the rest of the options based on information about the
+ // model.
client, err := api.ClientFromEnvironment()
if err != nil {
return err
}
name := args[0]
- // check if the model exists on the server
- _, err = client.Show(cmd.Context(), &api.ShowRequest{Name: name})
- var statusError api.StatusError
- switch {
- case errors.As(err, &statusError) && statusError.StatusCode == http.StatusNotFound:
- if err := PullHandler(cmd, args); err != nil {
- return err
+ info, err := func() (*api.ShowResponse, error) {
+ showReq := &api.ShowRequest{Name: name}
+ info, err := client.Show(cmd.Context(), showReq)
+ var se api.StatusError
+ if errors.As(err, &se) && se.StatusCode == http.StatusNotFound {
+ if err := PullHandler(cmd, []string{name}); err != nil {
+ return nil, err
+ }
+ return client.Show(cmd.Context(), &api.ShowRequest{Name: name})
}
- case err != nil:
+ return info, err
+ }()
+ if err != nil {
return err
}
- return RunGenerate(cmd, args)
+ opts.MultiModal = slices.Contains(info.Details.Families, "clip")
+ opts.ParentModel = info.Details.ParentModel
+
+ if interactive {
+ if err := loadOrUnloadModel(cmd, &opts); err != nil {
+ return err
+ }
+
+ for _, msg := range info.Messages {
+ switch msg.Role {
+ case "user":
+ fmt.Printf(">>> %s\n", msg.Content)
+ case "assistant":
+ state := &displayResponseState{}
+ displayResponse(msg.Content, opts.WordWrap, state)
+ fmt.Println()
+ fmt.Println()
+ }
+ }
+
+ return generateInteractive(cmd, opts)
+ }
+ return generate(cmd, opts)
+}
+
+func errFromUnknownKey(unknownKeyErr error) error {
+ // find SSH public key in the error message
+ sshKeyPattern := `ssh-\w+ [^\s"]+`
+ re := regexp.MustCompile(sshKeyPattern)
+ matches := re.FindStringSubmatch(unknownKeyErr.Error())
+
+ if len(matches) > 0 {
+ serverPubKey := matches[0]
+
+ localPubKey, err := auth.GetPublicKey()
+ if err != nil {
+ return unknownKeyErr
+ }
+
+ if runtime.GOOS == "linux" && serverPubKey != localPubKey {
+ // try the ollama service public key
+ svcPubKey, err := os.ReadFile("/usr/share/ollama/.ollama/id_ed25519.pub")
+ if err != nil {
+ return unknownKeyErr
+ }
+ localPubKey = strings.TrimSpace(string(svcPubKey))
+ }
+
+ // check if the returned public key matches the local public key, this prevents adding a remote key to the user's account
+ if serverPubKey != localPubKey {
+ return unknownKeyErr
+ }
+
+ var msg strings.Builder
+ msg.WriteString(unknownKeyErr.Error())
+ msg.WriteString("\n\nYour ollama key is:\n")
+ msg.WriteString(localPubKey)
+ msg.WriteString("\nAdd your key at:\n")
+ msg.WriteString("https://ollama.com/settings/keys")
+
+ return errors.New(msg.String())
+ }
+
+ return unknownKeyErr
}
func PushHandler(cmd *cobra.Command, args []string) error {
@@ -213,6 +566,20 @@ func PushHandler(cmd *cobra.Command, args []string) error {
request := api.PushRequest{Name: args[0], Insecure: insecure}
if err := client.Push(cmd.Context(), &request, fn); err != nil {
+ if spinner != nil {
+ spinner.Stop()
+ }
+ if strings.Contains(err.Error(), "access denied") {
+ return errors.New("you are not authorized to push to this namespace, create the model under a namespace you own")
+ }
+ host := model.ParseName(args[0]).Host
+ isOllamaHost := strings.HasSuffix(host, ".ollama.ai") || strings.HasSuffix(host, ".ollama.com")
+ if strings.Contains(err.Error(), errtypes.UnknownOllamaKeyErrMsg) && isOllamaHost {
+ // the user has not added their ollama key to ollama.com
+ // re-throw an error with a more user-friendly message
+ return errFromUnknownKey(err)
+ }
+
return err
}
@@ -246,7 +613,61 @@ func ListHandler(cmd *cobra.Command, args []string) error {
table.SetHeaderLine(false)
table.SetBorder(false)
table.SetNoWhiteSpace(true)
- table.SetTablePadding("\t")
+ table.SetTablePadding(" ")
+ table.AppendBulk(data)
+ table.Render()
+
+ return nil
+}
+
+func ListRunningHandler(cmd *cobra.Command, args []string) error {
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ return err
+ }
+
+ models, err := client.ListRunning(cmd.Context())
+ if err != nil {
+ return err
+ }
+
+ var data [][]string
+
+ for _, m := range models.Models {
+ if len(args) == 0 || strings.HasPrefix(m.Name, args[0]) {
+ var procStr string
+ switch {
+ case m.SizeVRAM == 0:
+ procStr = "100% CPU"
+ case m.SizeVRAM == m.Size:
+ procStr = "100% GPU"
+ case m.SizeVRAM > m.Size || m.Size == 0:
+ procStr = "Unknown"
+ default:
+ sizeCPU := m.Size - m.SizeVRAM
+ cpuPercent := math.Round(float64(sizeCPU) / float64(m.Size) * 100)
+ procStr = fmt.Sprintf("%d%%/%d%% CPU/GPU", int(cpuPercent), int(100-cpuPercent))
+ }
+
+ var until string
+ delta := time.Since(m.ExpiresAt)
+ if delta > 0 {
+ until = "Stopping..."
+ } else {
+ until = format.HumanTime(m.ExpiresAt, "Never")
+ }
+ data = append(data, []string{m.Name, m.Digest[:12], format.HumanBytes(m.Size), procStr, until})
+ }
+ }
+
+ table := tablewriter.NewWriter(os.Stdout)
+ table.SetHeader([]string{"NAME", "ID", "SIZE", "PROCESSOR", "UNTIL"})
+ table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+ table.SetAlignment(tablewriter.ALIGN_LEFT)
+ table.SetHeaderLine(false)
+ table.SetBorder(false)
+ table.SetNoWhiteSpace(true)
+ table.SetTablePadding(" ")
table.AppendBulk(data)
table.Render()
@@ -259,6 +680,17 @@ func DeleteHandler(cmd *cobra.Command, args []string) error {
return err
}
+ // Unload the model if it's running before deletion
+ opts := &runOptions{
+ Model: args[0],
+ KeepAlive: &api.Duration{Duration: 0},
+ }
+ if err := loadOrUnloadModel(cmd, opts); err != nil {
+ if !strings.Contains(err.Error(), "not found") {
+ return fmt.Errorf("unable to stop existing running model \"%s\": %s", args[0], err)
+ }
+ }
+
for _, name := range args {
req := api.DeleteRequest{Name: name}
if err := client.Delete(cmd.Context(), &req); err != nil {
@@ -275,10 +707,6 @@ func ShowHandler(cmd *cobra.Command, args []string) error {
return err
}
- if len(args) != 1 {
- return errors.New("missing model name")
- }
-
license, errLicense := cmd.Flags().GetBool("license")
modelfile, errModelfile := cmd.Flags().GetBool("modelfile")
parameters, errParams := cmd.Flags().GetBool("parameters")
@@ -321,8 +749,6 @@ func ShowHandler(cmd *cobra.Command, args []string) error {
if flagsSet > 1 {
return errors.New("only one of '--license', '--modelfile', '--parameters', '--system', or '--template' can be specified")
- } else if flagsSet == 0 {
- return errors.New("one of '--license', '--modelfile', '--parameters', '--system', or '--template' must be specified")
}
req := api.ShowRequest{Name: args[0]}
@@ -331,17 +757,103 @@ func ShowHandler(cmd *cobra.Command, args []string) error {
return err
}
- switch showType {
- case "license":
- fmt.Println(resp.License)
- case "modelfile":
- fmt.Println(resp.Modelfile)
- case "parameters":
- fmt.Println(resp.Parameters)
- case "system":
- fmt.Println(resp.System)
- case "template":
- fmt.Println(resp.Template)
+ if flagsSet == 1 {
+ switch showType {
+ case "license":
+ fmt.Println(resp.License)
+ case "modelfile":
+ fmt.Println(resp.Modelfile)
+ case "parameters":
+ fmt.Println(resp.Parameters)
+ case "system":
+ fmt.Println(resp.System)
+ case "template":
+ fmt.Println(resp.Template)
+ }
+
+ return nil
+ }
+
+ return showInfo(resp, os.Stdout)
+}
+
+func showInfo(resp *api.ShowResponse, w io.Writer) error {
+ tableRender := func(header string, rows func() [][]string) {
+ fmt.Fprintln(w, " ", header)
+ table := tablewriter.NewWriter(w)
+ table.SetAlignment(tablewriter.ALIGN_LEFT)
+ table.SetBorder(false)
+ table.SetNoWhiteSpace(true)
+ table.SetTablePadding(" ")
+
+ switch header {
+ case "Template", "System", "License":
+ table.SetColWidth(100)
+ }
+
+ table.AppendBulk(rows())
+ table.Render()
+ fmt.Fprintln(w)
+ }
+
+ tableRender("Model", func() (rows [][]string) {
+ if resp.ModelInfo != nil {
+ arch := resp.ModelInfo["general.architecture"].(string)
+ rows = append(rows, []string{"", "architecture", arch})
+ rows = append(rows, []string{"", "parameters", format.HumanNumber(uint64(resp.ModelInfo["general.parameter_count"].(float64)))})
+ rows = append(rows, []string{"", "context length", strconv.FormatFloat(resp.ModelInfo[fmt.Sprintf("%s.context_length", arch)].(float64), 'f', -1, 64)})
+ rows = append(rows, []string{"", "embedding length", strconv.FormatFloat(resp.ModelInfo[fmt.Sprintf("%s.embedding_length", arch)].(float64), 'f', -1, 64)})
+ } else {
+ rows = append(rows, []string{"", "architecture", resp.Details.Family})
+ rows = append(rows, []string{"", "parameters", resp.Details.ParameterSize})
+ }
+ rows = append(rows, []string{"", "quantization", resp.Details.QuantizationLevel})
+ return
+ })
+
+ if resp.ProjectorInfo != nil {
+ tableRender("Projector", func() (rows [][]string) {
+ arch := resp.ProjectorInfo["general.architecture"].(string)
+ rows = append(rows, []string{"", "architecture", arch})
+ rows = append(rows, []string{"", "parameters", format.HumanNumber(uint64(resp.ProjectorInfo["general.parameter_count"].(float64)))})
+ rows = append(rows, []string{"", "embedding length", strconv.FormatFloat(resp.ProjectorInfo[fmt.Sprintf("%s.vision.embedding_length", arch)].(float64), 'f', -1, 64)})
+ rows = append(rows, []string{"", "dimensions", strconv.FormatFloat(resp.ProjectorInfo[fmt.Sprintf("%s.vision.projection_dim", arch)].(float64), 'f', -1, 64)})
+ return
+ })
+ }
+
+ if resp.Parameters != "" {
+ tableRender("Parameters", func() (rows [][]string) {
+ scanner := bufio.NewScanner(strings.NewReader(resp.Parameters))
+ for scanner.Scan() {
+ if text := scanner.Text(); text != "" {
+ rows = append(rows, append([]string{""}, strings.Fields(text)...))
+ }
+ }
+ return
+ })
+ }
+
+ head := func(s string, n int) (rows [][]string) {
+ scanner := bufio.NewScanner(strings.NewReader(s))
+ for scanner.Scan() && (len(rows) < n || n < 0) {
+ if text := scanner.Text(); text != "" {
+ rows = append(rows, []string{"", strings.TrimSpace(text)})
+ }
+ }
+ return
+ }
+
+ if resp.System != "" {
+ tableRender("System", func() [][]string {
+ return head(resp.System, 2)
+ })
+ }
+
+ if resp.License != "" {
+ tableRender("License", func() [][]string {
+ return head(resp.License, 2)
+ })
}
return nil
@@ -415,66 +927,153 @@ func PullHandler(cmd *cobra.Command, args []string) error {
return nil
}
-func RunGenerate(cmd *cobra.Command, args []string) error {
- interactive := true
-
- opts := generateOptions{
- Model: args[0],
- WordWrap: os.Getenv("TERM") == "xterm-256color",
- Options: map[string]interface{}{},
- Images: []ImageData{},
- }
-
- format, err := cmd.Flags().GetString("format")
- if err != nil {
- return err
- }
- opts.Format = format
-
- prompts := args[1:]
- // prepend stdin to the prompt if provided
- if !term.IsTerminal(int(os.Stdin.Fd())) {
- in, err := io.ReadAll(os.Stdin)
- if err != nil {
- return err
- }
-
- prompts = append([]string{string(in)}, prompts...)
- opts.WordWrap = false
- interactive = false
- }
- opts.Prompt = strings.Join(prompts, " ")
- if len(prompts) > 0 {
- interactive = false
- }
-
- nowrap, err := cmd.Flags().GetBool("nowordwrap")
- if err != nil {
- return err
- }
- opts.WordWrap = !nowrap
-
- if !interactive {
- return generate(cmd, opts)
- }
-
- return generateInteractive(cmd, opts)
-}
-
type generateContextKey string
-type generateOptions struct {
- Model string
- Prompt string
- WordWrap bool
- Format string
- System string
- Template string
- Images []ImageData
- Options map[string]interface{}
+type runOptions struct {
+ Model string
+ ParentModel string
+ Prompt string
+ Messages []api.Message
+ WordWrap bool
+ Format string
+ System string
+ Images []api.ImageData
+ Options map[string]interface{}
+ MultiModal bool
+ KeepAlive *api.Duration
}
-func generate(cmd *cobra.Command, opts generateOptions) error {
+type displayResponseState struct {
+ lineLength int
+ wordBuffer string
+}
+
+func displayResponse(content string, wordWrap bool, state *displayResponseState) {
+ termWidth, _, _ := term.GetSize(int(os.Stdout.Fd()))
+ if wordWrap && termWidth >= 10 {
+ for _, ch := range content {
+ if state.lineLength+1 > termWidth-5 {
+ if runewidth.StringWidth(state.wordBuffer) > termWidth-10 {
+ fmt.Printf("%s%c", state.wordBuffer, ch)
+ state.wordBuffer = ""
+ state.lineLength = 0
+ continue
+ }
+
+ // backtrack the length of the last word and clear to the end of the line
+ a := runewidth.StringWidth(state.wordBuffer)
+ if a > 0 {
+ fmt.Printf("\x1b[%dD", a)
+ }
+ fmt.Printf("\x1b[K\n")
+ fmt.Printf("%s%c", state.wordBuffer, ch)
+ chWidth := runewidth.RuneWidth(ch)
+
+ state.lineLength = runewidth.StringWidth(state.wordBuffer) + chWidth
+ } else {
+ fmt.Print(string(ch))
+ state.lineLength += runewidth.RuneWidth(ch)
+ if runewidth.RuneWidth(ch) >= 2 {
+ state.wordBuffer = ""
+ continue
+ }
+
+ switch ch {
+ case ' ':
+ state.wordBuffer = ""
+ case '\n':
+ state.lineLength = 0
+ default:
+ state.wordBuffer += string(ch)
+ }
+ }
+ }
+ } else {
+ fmt.Printf("%s%s", state.wordBuffer, content)
+ if len(state.wordBuffer) > 0 {
+ state.wordBuffer = ""
+ }
+ }
+}
+
+func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) {
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ return nil, err
+ }
+
+ p := progress.NewProgress(os.Stderr)
+ defer p.StopAndClear()
+
+ spinner := progress.NewSpinner("")
+ p.Add("", spinner)
+
+ cancelCtx, cancel := context.WithCancel(cmd.Context())
+ defer cancel()
+
+ sigChan := make(chan os.Signal, 1)
+ signal.Notify(sigChan, syscall.SIGINT)
+
+ go func() {
+ <-sigChan
+ cancel()
+ }()
+
+ var state *displayResponseState = &displayResponseState{}
+ var latest api.ChatResponse
+ var fullResponse strings.Builder
+ var role string
+
+ fn := func(response api.ChatResponse) error {
+ p.StopAndClear()
+
+ latest = response
+
+ role = response.Message.Role
+ content := response.Message.Content
+ fullResponse.WriteString(content)
+
+ displayResponse(content, opts.WordWrap, state)
+
+ return nil
+ }
+
+ req := &api.ChatRequest{
+ Model: opts.Model,
+ Messages: opts.Messages,
+ Format: opts.Format,
+ Options: opts.Options,
+ }
+
+ if opts.KeepAlive != nil {
+ req.KeepAlive = opts.KeepAlive
+ }
+
+ if err := client.Chat(cancelCtx, req, fn); err != nil {
+ if errors.Is(err, context.Canceled) {
+ return nil, nil
+ }
+ return nil, err
+ }
+
+ if len(opts.Messages) > 0 {
+ fmt.Println()
+ fmt.Println()
+ }
+
+ verbose, err := cmd.Flags().GetBool("verbose")
+ if err != nil {
+ return nil, err
+ }
+
+ if verbose {
+ latest.Summary()
+ }
+
+ return &api.Message{Role: role, Content: fullResponse.String()}, nil
+}
+
+func generate(cmd *cobra.Command, opts runOptions) error {
client, err := api.ClientFromEnvironment()
if err != nil {
return err
@@ -493,11 +1092,6 @@ func generate(cmd *cobra.Command, opts generateOptions) error {
generateContext = []int{}
}
- termWidth, _, err := term.GetSize(int(os.Stdout.Fd()))
- if err != nil {
- opts.WordWrap = false
- }
-
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
@@ -509,94 +1103,44 @@ func generate(cmd *cobra.Command, opts generateOptions) error {
cancel()
}()
- var currentLineLength int
- var wordBuffer string
+ var state *displayResponseState = &displayResponseState{}
fn := func(response api.GenerateResponse) error {
p.StopAndClear()
latest = response
+ content := response.Response
- termWidth, _, _ = term.GetSize(int(os.Stdout.Fd()))
- if opts.WordWrap && termWidth >= 10 {
- for _, ch := range response.Response {
- if currentLineLength+1 > termWidth-5 {
- if len(wordBuffer) > termWidth-10 {
- fmt.Printf("%s%c", wordBuffer, ch)
- wordBuffer = ""
- currentLineLength = 0
- continue
- }
-
- // backtrack the length of the last word and clear to the end of the line
- fmt.Printf("\x1b[%dD\x1b[K\n", len(wordBuffer))
- fmt.Printf("%s%c", wordBuffer, ch)
- currentLineLength = len(wordBuffer) + 1
- } else {
- fmt.Print(string(ch))
- currentLineLength += 1
-
- switch ch {
- case ' ':
- wordBuffer = ""
- case '\n':
- currentLineLength = 0
- default:
- wordBuffer += string(ch)
- }
- }
- }
- } else {
- fmt.Printf("%s%s", wordBuffer, response.Response)
- if len(wordBuffer) > 0 {
- wordBuffer = ""
- }
- }
+ displayResponse(content, opts.WordWrap, state)
return nil
}
- images := make([]api.ImageData, 0)
- for _, i := range opts.Images {
- images = append(images, api.ImageData(i))
- }
- request := api.GenerateRequest{
- Model: opts.Model,
- Prompt: opts.Prompt,
- Context: generateContext,
- Format: opts.Format,
- System: opts.System,
- Template: opts.Template,
- Options: opts.Options,
- Images: images,
- }
-
- if err := client.Generate(ctx, &request, fn); err != nil {
- switch {
- case errors.Is(err, context.Canceled):
- return nil
- case strings.Contains(err.Error(), "unsupported model format"):
- // pull and retry to see if the model has been updated
- parts := strings.Split(opts.Model, string(os.PathSeparator))
- if len(parts) == 1 {
- // this is a library model, log some info
- fmt.Fprintln(os.Stderr, "This model is no longer compatible with Ollama. Pulling a new version...")
- }
- if err := PullHandler(cmd, []string{opts.Model}); err != nil {
- fmt.Printf("Error: %s\n", err)
- return fmt.Errorf("unsupported model, please update this model to gguf format") // relay the original error
- }
- // retry
- if err := client.Generate(ctx, &request, fn); err != nil {
- if errors.Is(err, context.Canceled) {
- return nil
- }
- return err
- }
- default:
+ if opts.MultiModal {
+ opts.Prompt, opts.Images, err = extractFileData(opts.Prompt)
+ if err != nil {
return err
}
}
+
+ request := api.GenerateRequest{
+ Model: opts.Model,
+ Prompt: opts.Prompt,
+ Context: generateContext,
+ Images: opts.Images,
+ Format: opts.Format,
+ System: opts.System,
+ Options: opts.Options,
+ KeepAlive: opts.KeepAlive,
+ }
+
+ if err := client.Generate(ctx, &request, fn); err != nil {
+ if errors.Is(err, context.Canceled) {
+ return nil
+ }
+ return err
+ }
+
if opts.Prompt != "" {
fmt.Println()
fmt.Println()
@@ -621,504 +1165,22 @@ func generate(cmd *cobra.Command, opts generateOptions) error {
return nil
}
-type MultilineState int
-
-const (
- MultilineNone MultilineState = iota
- MultilinePrompt
- MultilineSystem
- MultilineTemplate
-)
-
-func modelIsMultiModal(cmd *cobra.Command, name string) bool {
- // get model details
- client, err := api.ClientFromEnvironment()
- if err != nil {
- fmt.Println("error: couldn't connect to ollama server")
- return false
- }
-
- req := api.ShowRequest{Name: name}
- resp, err := client.Show(cmd.Context(), &req)
- if err != nil {
- return false
- }
-
- return slices.Contains(resp.Details.Families, "clip")
-}
-
-func generateInteractive(cmd *cobra.Command, opts generateOptions) error {
- multiModal := modelIsMultiModal(cmd, opts.Model)
-
- // load the model
- loadOpts := generateOptions{
- Model: opts.Model,
- Prompt: "",
- Images: []ImageData{},
- }
- if err := generate(cmd, loadOpts); err != nil {
- return err
- }
-
- usage := func() {
- fmt.Fprintln(os.Stderr, "Available Commands:")
- fmt.Fprintln(os.Stderr, " /set Set session variables")
- fmt.Fprintln(os.Stderr, " /show Show model information")
- fmt.Fprintln(os.Stderr, " /bye Exit")
- fmt.Fprintln(os.Stderr, " /?, /help Help for a command")
- fmt.Fprintln(os.Stderr, "")
- fmt.Fprintln(os.Stderr, "Use \"\"\" to begin a multi-line message.")
- fmt.Fprintln(os.Stderr, "")
- }
-
- usageSet := func() {
- fmt.Fprintln(os.Stderr, "Available Commands:")
- fmt.Fprintln(os.Stderr, " /set parameter ... Set a parameter")
- fmt.Fprintln(os.Stderr, " /set system <string> Set system message")
- fmt.Fprintln(os.Stderr, " /set template <string> Set prompt template")
- fmt.Fprintln(os.Stderr, " /set history Enable history")
- fmt.Fprintln(os.Stderr, " /set nohistory Disable history")
- fmt.Fprintln(os.Stderr, " /set wordwrap Enable wordwrap")
- fmt.Fprintln(os.Stderr, " /set nowordwrap Disable wordwrap")
- fmt.Fprintln(os.Stderr, " /set format json Enable JSON mode")
- fmt.Fprintln(os.Stderr, " /set noformat Disable formatting")
- fmt.Fprintln(os.Stderr, " /set verbose Show LLM stats")
- fmt.Fprintln(os.Stderr, " /set quiet Disable LLM stats")
- fmt.Fprintln(os.Stderr, "")
- }
-
- usageShow := func() {
- fmt.Fprintln(os.Stderr, "Available Commands:")
- fmt.Fprintln(os.Stderr, " /show license Show model license")
- fmt.Fprintln(os.Stderr, " /show modelfile Show Modelfile for this model")
- fmt.Fprintln(os.Stderr, " /show parameters Show parameters for this model")
- fmt.Fprintln(os.Stderr, " /show system Show system message")
- fmt.Fprintln(os.Stderr, " /show template Show prompt template")
- fmt.Fprintln(os.Stderr, "")
- }
-
- // only list out the most common parameters
- usageParameters := func() {
- fmt.Fprintln(os.Stderr, "Available Parameters:")
- fmt.Fprintln(os.Stderr, " /set parameter seed <int> Random number seed")
- fmt.Fprintln(os.Stderr, " /set parameter num_predict <int> Max number of tokens to predict")
- fmt.Fprintln(os.Stderr, " /set parameter top_k <int> Pick from top k num of tokens")
- fmt.Fprintln(os.Stderr, " /set parameter top_p <float> Pick token based on sum of probabilities")
- fmt.Fprintln(os.Stderr, " /set parameter num_ctx <int> Set the context size")
- fmt.Fprintln(os.Stderr, " /set parameter temperature <float> Set creativity level")
- fmt.Fprintln(os.Stderr, " /set parameter repeat_penalty <float> How strongly to penalize repetitions")
- fmt.Fprintln(os.Stderr, " /set parameter repeat_last_n <int> Set how far back to look for repetitions")
- fmt.Fprintln(os.Stderr, " /set parameter num_gpu <int> The number of layers to send to the GPU")
- fmt.Fprintln(os.Stderr, " /set parameter stop \"<string>\", ... Set the stop parameters")
- fmt.Fprintln(os.Stderr, "")
- }
-
- scanner, err := readline.New(readline.Prompt{
- Prompt: ">>> ",
- AltPrompt: "... ",
- Placeholder: "Send a message (/? for help)",
- AltPlaceholder: `Use """ to end multi-line input`,
- })
- if err != nil {
- return err
- }
-
- fmt.Print(readline.StartBracketedPaste)
- defer fmt.Printf(readline.EndBracketedPaste)
-
- var multiline MultilineState
- var prompt string
-
- for {
- line, err := scanner.Readline()
- switch {
- case errors.Is(err, io.EOF):
- fmt.Println()
- return nil
- case errors.Is(err, readline.ErrInterrupt):
- if line == "" {
- fmt.Println("\nUse Ctrl-D or /bye to exit.")
- }
-
- scanner.Prompt.UseAlt = false
- prompt = ""
-
- continue
- case err != nil:
- return err
- }
-
- switch {
- case strings.HasPrefix(prompt, `"""`):
- // if the prompt so far starts with """ then we're in multiline mode
- // and we need to keep reading until we find a line that ends with """
- cut, found := strings.CutSuffix(line, `"""`)
- prompt += cut
-
- if !found {
- prompt += "\n"
- continue
- }
-
- prompt = strings.TrimPrefix(prompt, `"""`)
- scanner.Prompt.UseAlt = false
-
- switch multiline {
- case MultilineSystem:
- opts.System = prompt
- prompt = ""
- fmt.Println("Set system message.")
- case MultilineTemplate:
- opts.Template = prompt
- prompt = ""
- fmt.Println("Set prompt template.")
- }
- multiline = MultilineNone
- case strings.HasPrefix(line, `"""`) && len(prompt) == 0:
- scanner.Prompt.UseAlt = true
- multiline = MultilinePrompt
- prompt += line + "\n"
- continue
- case scanner.Pasting:
- prompt += line + "\n"
- continue
- case strings.HasPrefix(line, "/list"):
- args := strings.Fields(line)
- if err := ListHandler(cmd, args[1:]); err != nil {
- return err
- }
- case strings.HasPrefix(line, "/set"):
- args := strings.Fields(line)
- if len(args) > 1 {
- switch args[1] {
- case "history":
- scanner.HistoryEnable()
- case "nohistory":
- scanner.HistoryDisable()
- case "wordwrap":
- opts.WordWrap = true
- fmt.Println("Set 'wordwrap' mode.")
- case "nowordwrap":
- opts.WordWrap = false
- fmt.Println("Set 'nowordwrap' mode.")
- case "verbose":
- cmd.Flags().Set("verbose", "true")
- fmt.Println("Set 'verbose' mode.")
- case "quiet":
- cmd.Flags().Set("verbose", "false")
- fmt.Println("Set 'quiet' mode.")
- case "format":
- if len(args) < 3 || args[2] != "json" {
- fmt.Println("Invalid or missing format. For 'json' mode use '/set format json'")
- } else {
- opts.Format = args[2]
- fmt.Printf("Set format to '%s' mode.\n", args[2])
- }
- case "noformat":
- opts.Format = ""
- fmt.Println("Disabled format.")
- case "parameter":
- if len(args) < 4 {
- usageParameters()
- continue
- }
- var params []string
- for _, p := range args[3:] {
- params = append(params, p)
- }
- fp, err := api.FormatParams(map[string][]string{args[2]: params})
- if err != nil {
- fmt.Printf("Couldn't set parameter: %q\n\n", err)
- continue
- }
- fmt.Printf("Set parameter '%s' to '%s'\n\n", args[2], strings.Join(params, ", "))
- opts.Options[args[2]] = fp[args[2]]
- case "system", "template":
- if len(args) < 3 {
- usageSet()
- continue
- }
- line := strings.Join(args[2:], " ")
- line = strings.TrimPrefix(line, `"""`)
- if strings.HasPrefix(args[2], `"""`) {
- cut, found := strings.CutSuffix(line, `"""`)
- prompt += cut
- if found {
- if args[1] == "system" {
- opts.System = prompt
- fmt.Println("Set system message.")
- } else {
- opts.Template = prompt
- fmt.Println("Set prompt template.")
- }
- prompt = ""
- } else {
- prompt = `"""` + prompt + "\n"
- if args[1] == "system" {
- multiline = MultilineSystem
- } else {
- multiline = MultilineTemplate
- }
- scanner.Prompt.UseAlt = true
- }
- } else {
- opts.System = line
- fmt.Println("Set system message.")
- }
- default:
- fmt.Printf("Unknown command '/set %s'. Type /? for help\n", args[1])
- }
- } else {
- usageSet()
- }
- case strings.HasPrefix(line, "/show"):
- args := strings.Fields(line)
- if len(args) > 1 {
- client, err := api.ClientFromEnvironment()
- if err != nil {
- fmt.Println("error: couldn't connect to ollama server")
- return err
- }
- resp, err := client.Show(cmd.Context(), &api.ShowRequest{Name: opts.Model})
- if err != nil {
- fmt.Println("error: couldn't get model")
- return err
- }
-
- switch args[1] {
- case "license":
- if resp.License == "" {
- fmt.Print("No license was specified for this model.\n\n")
- } else {
- fmt.Println(resp.License)
- }
- case "modelfile":
- fmt.Println(resp.Modelfile)
- case "parameters":
- if resp.Parameters == "" {
- fmt.Print("No parameters were specified for this model.\n\n")
- } else {
- if len(opts.Options) > 0 {
- fmt.Println("User defined parameters:")
- for k, v := range opts.Options {
- fmt.Printf("%-*s %v\n", 30, k, v)
- }
- fmt.Println()
- }
- fmt.Println("Model defined parameters:")
- fmt.Println(resp.Parameters)
- }
- case "system":
- switch {
- case opts.System != "":
- fmt.Println(opts.System + "\n")
- case resp.System != "":
- fmt.Println(resp.System + "\n")
- default:
- fmt.Print("No system message was specified for this model.\n\n")
- }
- case "template":
- switch {
- case opts.Template != "":
- fmt.Println(opts.Template + "\n")
- case resp.Template != "":
- fmt.Println(resp.Template)
- default:
- fmt.Print("No prompt template was specified for this model.\n\n")
- }
- default:
- fmt.Printf("Unknown command '/show %s'. Type /? for help\n", args[1])
- }
- } else {
- usageShow()
- }
- case strings.HasPrefix(line, "/help"), strings.HasPrefix(line, "/?"):
- args := strings.Fields(line)
- if len(args) > 1 {
- switch args[1] {
- case "set", "/set":
- usageSet()
- case "show", "/show":
- usageShow()
- }
- } else {
- usage()
- }
- case line == "/exit", line == "/bye":
- return nil
- case strings.HasPrefix(line, "/"):
- args := strings.Fields(line)
- isFile := false
-
- if multiModal {
- for _, f := range extractFileNames(line) {
- if strings.HasPrefix(f, args[0]) {
- isFile = true
- break
- }
- }
- }
-
- if isFile {
- prompt += line
- } else {
- fmt.Printf("Unknown command '%s'. Type /? for help\n", args[0])
- continue
- }
- default:
- prompt += line
- }
-
- if len(prompt) > 0 && multiline == MultilineNone {
- opts.Prompt = prompt
- if multiModal {
- newPrompt, images, err := extractFileData(prompt)
- if err != nil {
- return err
- }
- opts.Prompt = newPrompt
-
- // reset the context if we find another image
- if len(images) > 0 {
- opts.Images = images
- ctx := cmd.Context()
- ctx = context.WithValue(ctx, generateContextKey("context"), []int{})
- cmd.SetContext(ctx)
- }
- if len(opts.Images) == 0 {
- fmt.Println("This model requires you to add a jpeg, png, or svg image.")
- fmt.Println()
- prompt = ""
- continue
- }
- }
- if err := generate(cmd, opts); err != nil {
- return err
- }
-
- prompt = ""
- }
- }
-}
-
-func normalizeFilePath(fp string) string {
- // Define a map of escaped characters and their replacements
- replacements := map[string]string{
- "\\ ": " ", // Escaped space
- "\\(": "(", // Escaped left parenthesis
- "\\)": ")", // Escaped right parenthesis
- "\\[": "[", // Escaped left square bracket
- "\\]": "]", // Escaped right square bracket
- "\\{": "{", // Escaped left curly brace
- "\\}": "}", // Escaped right curly brace
- "\\$": "$", // Escaped dollar sign
- "\\&": "&", // Escaped ampersand
- "\\;": ";", // Escaped semicolon
- "\\'": "'", // Escaped single quote
- "\\\\": "\\", // Escaped backslash
- "\\*": "*", // Escaped asterisk
- "\\?": "?", // Escaped question mark
- }
-
- for escaped, actual := range replacements {
- fp = strings.ReplaceAll(fp, escaped, actual)
- }
- return fp
-}
-
-func extractFileNames(input string) []string {
- // Regex to match file paths starting with / or ./ and include escaped spaces (\ or %20)
- // and followed by more characters and a file extension
- regexPattern := `(?:\./|/)[\S\\ ]+?\.(?i:jpg|jpeg|png|svg)\b`
- re := regexp.MustCompile(regexPattern)
-
- return re.FindAllString(input, -1)
-}
-
-func extractFileData(input string) (string, []ImageData, error) {
- filePaths := extractFileNames(input)
- var imgs []ImageData
-
- for _, fp := range filePaths {
- nfp := normalizeFilePath(fp)
- data, err := getImageData(nfp)
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
- fmt.Printf("Couldn't process image: %q\n", err)
- return "", imgs, err
- }
- fmt.Printf("Added image '%s'\n", nfp)
- input = strings.ReplaceAll(input, fp, "")
- imgs = append(imgs, data)
- }
- return input, imgs, nil
-}
-
-func RunServer(cmd *cobra.Command, _ []string) error {
- host, port, err := net.SplitHostPort(os.Getenv("OLLAMA_HOST"))
- if err != nil {
- host, port = "127.0.0.1", "11434"
- if ip := net.ParseIP(strings.Trim(os.Getenv("OLLAMA_HOST"), "[]")); ip != nil {
- host = ip.String()
- }
- }
-
+func RunServer(_ *cobra.Command, _ []string) error {
if err := initializeKeypair(); err != nil {
return err
}
- ln, err := net.Listen("tcp", net.JoinHostPort(host, port))
+ ln, err := net.Listen("tcp", envconfig.Host().Host)
if err != nil {
return err
}
- return server.Serve(ln)
-}
-
-func getImageData(filePath string) ([]byte, error) {
- file, err := os.Open(filePath)
- if err != nil {
- return nil, err
- }
- defer file.Close()
-
- buf := make([]byte, 512)
- _, err = file.Read(buf)
- if err != nil {
- return nil, err
+ err = server.Serve(ln)
+ if errors.Is(err, http.ErrServerClosed) {
+ return nil
}
- contentType := http.DetectContentType(buf)
- allowedTypes := []string{"image/jpeg", "image/jpg", "image/svg+xml", "image/png"}
- if !slices.Contains(allowedTypes, contentType) {
- return nil, fmt.Errorf("invalid image type: %s", contentType)
- }
-
- info, err := file.Stat()
- if err != nil {
- return nil, err
- }
-
- // Check if the file size exceeds 100MB
- var maxSize int64 = 100 * 1024 * 1024 // 100MB in bytes
- if info.Size() > maxSize {
- return nil, fmt.Errorf("file size exceeds maximum limit (100MB)")
- }
-
- buf = make([]byte, info.Size())
- _, err = file.Seek(0, 0)
- if err != nil {
- return nil, err
- }
-
- _, err = io.ReadFull(file, buf)
- if err != nil {
- return nil, err
- }
-
- return buf, nil
+ return err
}
func initializeKeypair() error {
@@ -1133,89 +1195,51 @@ func initializeKeypair() error {
_, err = os.Stat(privKeyPath)
if os.IsNotExist(err) {
fmt.Printf("Couldn't find '%s'. Generating new private key.\n", privKeyPath)
- _, privKey, err := ed25519.GenerateKey(rand.Reader)
+ cryptoPublicKey, cryptoPrivateKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
return err
}
- privKeyBytes, err := format.OpenSSHPrivateKey(privKey, "")
+ privateKeyBytes, err := ssh.MarshalPrivateKey(cryptoPrivateKey, "")
if err != nil {
return err
}
- err = os.MkdirAll(filepath.Dir(privKeyPath), 0o755)
- if err != nil {
+ if err := os.MkdirAll(filepath.Dir(privKeyPath), 0o755); err != nil {
return fmt.Errorf("could not create directory %w", err)
}
- err = os.WriteFile(privKeyPath, pem.EncodeToMemory(privKeyBytes), 0o600)
+ if err := os.WriteFile(privKeyPath, pem.EncodeToMemory(privateKeyBytes), 0o600); err != nil {
+ return err
+ }
+
+ sshPublicKey, err := ssh.NewPublicKey(cryptoPublicKey)
if err != nil {
return err
}
- sshPrivateKey, err := ssh.NewSignerFromKey(privKey)
- if err != nil {
+ publicKeyBytes := ssh.MarshalAuthorizedKey(sshPublicKey)
+
+ if err := os.WriteFile(pubKeyPath, publicKeyBytes, 0o644); err != nil {
return err
}
- pubKeyData := ssh.MarshalAuthorizedKey(sshPrivateKey.PublicKey())
-
- err = os.WriteFile(pubKeyPath, pubKeyData, 0o644)
- if err != nil {
- return err
- }
-
- fmt.Printf("Your new public key is: \n\n%s\n", string(pubKeyData))
+ fmt.Printf("Your new public key is: \n\n%s\n", publicKeyBytes)
}
return nil
}
-func startMacApp(ctx context.Context, client *api.Client) error {
- exe, err := os.Executable()
- if err != nil {
- return err
- }
- link, err := os.Readlink(exe)
- if err != nil {
- return err
- }
- if !strings.Contains(link, "Ollama.app") {
- return fmt.Errorf("could not find ollama app")
- }
- path := strings.Split(link, "Ollama.app")
- if err := exec.Command("/usr/bin/open", "-a", path[0]+"Ollama.app").Run(); err != nil {
- return err
- }
- // wait for the server to start
- timeout := time.After(5 * time.Second)
- tick := time.Tick(500 * time.Millisecond)
- for {
- select {
- case <-timeout:
- return errors.New("timed out waiting for server to start")
- case <-tick:
- if err := client.Heartbeat(ctx); err == nil {
- return nil // server has started
- }
- }
- }
-}
-
func checkServerHeartbeat(cmd *cobra.Command, _ []string) error {
client, err := api.ClientFromEnvironment()
if err != nil {
return err
}
if err := client.Heartbeat(cmd.Context()); err != nil {
- if !strings.Contains(err.Error(), "connection refused") {
+ if !strings.Contains(err.Error(), " refused") {
return err
}
- if runtime.GOOS == "darwin" {
- if err := startMacApp(cmd.Context(), client); err != nil {
- return fmt.Errorf("could not connect to ollama app, is it running?")
- }
- } else {
- return fmt.Errorf("could not connect to ollama server, run 'ollama serve' to start it")
+ if err := startApp(cmd.Context(), client); err != nil {
+ return errors.New("could not connect to ollama app, is it running?")
}
}
return nil
@@ -1287,10 +1311,29 @@ func versionHandler(cmd *cobra.Command, _ []string) {
}
}
+func appendEnvDocs(cmd *cobra.Command, envs []envconfig.EnvVar) {
+ if len(envs) == 0 {
+ return
+ }
+
+ envUsage := `
+Environment Variables:
+`
+ for _, e := range envs {
+ envUsage += fmt.Sprintf(" %-24s %s\n", e.Name, e.Description)
+ }
+
+ cmd.SetUsageTemplate(cmd.UsageTemplate() + envUsage)
+}
+
func NewCLI() *cobra.Command {
log.SetFlags(log.LstdFlags | log.Lshortfile)
cobra.EnableCommandSorting = false
+ if runtime.GOOS == "windows" {
+ console.ConsoleFromFile(os.Stdin) //nolint:errcheck
+ }
+
rootCmd := &cobra.Command{
Use: "ollama",
Short: "Large language model runner",
@@ -1320,7 +1363,8 @@ func NewCLI() *cobra.Command {
ValidArgsFunction: doNotAutocomplete,
}
- createCmd.Flags().StringP("file", "f", "Modelfile", "Name of the Modelfile (default \"Modelfile\")")
+ createCmd.Flags().StringP("file", "f", "Modelfile", "Name of the Modelfile")
+ createCmd.Flags().StringP("quantize", "q", "", "Quantize model to this level (e.g. q4_0)")
showCmd := &cobra.Command{
Use: "show MODEL",
@@ -1346,11 +1390,20 @@ func NewCLI() *cobra.Command {
ValidArgsFunction: autocompleteModelName,
}
+ runCmd.Flags().String("keepalive", "", "Duration to keep a model loaded (e.g. 5m)")
runCmd.Flags().Bool("verbose", false, "Show timings for response")
runCmd.Flags().Bool("insecure", false, "Use an insecure registry")
runCmd.Flags().Bool("nowordwrap", false, "Don't wrap words to the next line automatically")
runCmd.Flags().String("format", "", "Response format (e.g. json)")
+ stopCmd := &cobra.Command{
+ Use: "stop MODEL",
+ Short: "Stop a running model",
+ Args: cobra.ExactArgs(1),
+ PreRunE: checkServerHeartbeat,
+ RunE: StopHandler,
+ }
+
serveCmd := &cobra.Command{
Use: "serve",
Aliases: []string{"start"},
@@ -1391,8 +1444,15 @@ func NewCLI() *cobra.Command {
ValidArgsFunction: doNotAutocomplete,
}
+ psCmd := &cobra.Command{
+ Use: "ps",
+ Short: "List running models",
+ PreRunE: checkServerHeartbeat,
+ RunE: ListRunningHandler,
+ }
+
copyCmd := &cobra.Command{
- Use: "cp SOURCE TARGET",
+ Use: "cp SOURCE DESTINATION",
Short: "Copy a model",
Args: cobra.ExactArgs(2),
PreRunE: checkServerHeartbeat,
@@ -1421,14 +1481,59 @@ func NewCLI() *cobra.Command {
},
}
+ envVars := envconfig.AsMap()
+
+ envs := []envconfig.EnvVar{envVars["OLLAMA_HOST"]}
+
+ for _, cmd := range []*cobra.Command{
+ createCmd,
+ showCmd,
+ runCmd,
+ stopCmd,
+ pullCmd,
+ pushCmd,
+ listCmd,
+ psCmd,
+ copyCmd,
+ deleteCmd,
+ serveCmd,
+ } {
+ switch cmd {
+ case runCmd:
+ appendEnvDocs(cmd, []envconfig.EnvVar{envVars["OLLAMA_HOST"], envVars["OLLAMA_NOHISTORY"]})
+ case serveCmd:
+ appendEnvDocs(cmd, []envconfig.EnvVar{
+ envVars["OLLAMA_DEBUG"],
+ envVars["OLLAMA_HOST"],
+ envVars["OLLAMA_KEEP_ALIVE"],
+ envVars["OLLAMA_MAX_LOADED_MODELS"],
+ envVars["OLLAMA_MAX_QUEUE"],
+ envVars["OLLAMA_MODELS"],
+ envVars["OLLAMA_NUM_PARALLEL"],
+ envVars["OLLAMA_NOPRUNE"],
+ envVars["OLLAMA_ORIGINS"],
+ envVars["OLLAMA_SCHED_SPREAD"],
+ envVars["OLLAMA_TMPDIR"],
+ envVars["OLLAMA_FLASH_ATTENTION"],
+ envVars["OLLAMA_LLM_LIBRARY"],
+ envVars["OLLAMA_GPU_OVERHEAD"],
+ envVars["OLLAMA_LOAD_TIMEOUT"],
+ })
+ default:
+ appendEnvDocs(cmd, envs)
+ }
+ }
+
rootCmd.AddCommand(
serveCmd,
createCmd,
showCmd,
runCmd,
+ stopCmd,
pullCmd,
pushCmd,
listCmd,
+ psCmd,
copyCmd,
deleteCmd,
completionCmd,
diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go
new file mode 100644
index 00000000..9d23f3e9
--- /dev/null
+++ b/cmd/cmd_test.go
@@ -0,0 +1,272 @@
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/spf13/cobra"
+
+ "github.com/ollama/ollama/api"
+)
+
+func TestShowInfo(t *testing.T) {
+ t.Run("bare details", func(t *testing.T) {
+ var b bytes.Buffer
+ if err := showInfo(&api.ShowResponse{
+ Details: api.ModelDetails{
+ Family: "test",
+ ParameterSize: "7B",
+ QuantizationLevel: "FP16",
+ },
+ }, &b); err != nil {
+ t.Fatal(err)
+ }
+
+ expect := ` Model
+ architecture test
+ parameters 7B
+ quantization FP16
+
+`
+
+ if diff := cmp.Diff(expect, b.String()); diff != "" {
+ t.Errorf("unexpected output (-want +got):\n%s", diff)
+ }
+ })
+
+ t.Run("bare model info", func(t *testing.T) {
+ var b bytes.Buffer
+ if err := showInfo(&api.ShowResponse{
+ ModelInfo: map[string]any{
+ "general.architecture": "test",
+ "general.parameter_count": float64(7_000_000_000),
+ "test.context_length": float64(0),
+ "test.embedding_length": float64(0),
+ },
+ Details: api.ModelDetails{
+ Family: "test",
+ ParameterSize: "7B",
+ QuantizationLevel: "FP16",
+ },
+ }, &b); err != nil {
+ t.Fatal(err)
+ }
+
+ expect := ` Model
+ architecture test
+ parameters 7B
+ context length 0
+ embedding length 0
+ quantization FP16
+
+`
+ if diff := cmp.Diff(expect, b.String()); diff != "" {
+ t.Errorf("unexpected output (-want +got):\n%s", diff)
+ }
+ })
+
+ t.Run("parameters", func(t *testing.T) {
+ var b bytes.Buffer
+ if err := showInfo(&api.ShowResponse{
+ Details: api.ModelDetails{
+ Family: "test",
+ ParameterSize: "7B",
+ QuantizationLevel: "FP16",
+ },
+ Parameters: `
+ stop never
+ stop gonna
+ stop give
+ stop you
+ stop up
+ temperature 99`,
+ }, &b); err != nil {
+ t.Fatal(err)
+ }
+
+ expect := ` Model
+ architecture test
+ parameters 7B
+ quantization FP16
+
+ Parameters
+ stop never
+ stop gonna
+ stop give
+ stop you
+ stop up
+ temperature 99
+
+`
+ if diff := cmp.Diff(expect, b.String()); diff != "" {
+ t.Errorf("unexpected output (-want +got):\n%s", diff)
+ }
+ })
+
+ t.Run("project info", func(t *testing.T) {
+ var b bytes.Buffer
+ if err := showInfo(&api.ShowResponse{
+ Details: api.ModelDetails{
+ Family: "test",
+ ParameterSize: "7B",
+ QuantizationLevel: "FP16",
+ },
+ ProjectorInfo: map[string]any{
+ "general.architecture": "clip",
+ "general.parameter_count": float64(133_700_000),
+ "clip.vision.embedding_length": float64(0),
+ "clip.vision.projection_dim": float64(0),
+ },
+ }, &b); err != nil {
+ t.Fatal(err)
+ }
+
+ expect := ` Model
+ architecture test
+ parameters 7B
+ quantization FP16
+
+ Projector
+ architecture clip
+ parameters 133.70M
+ embedding length 0
+ dimensions 0
+
+`
+ if diff := cmp.Diff(expect, b.String()); diff != "" {
+ t.Errorf("unexpected output (-want +got):\n%s", diff)
+ }
+ })
+
+ t.Run("system", func(t *testing.T) {
+ var b bytes.Buffer
+ if err := showInfo(&api.ShowResponse{
+ Details: api.ModelDetails{
+ Family: "test",
+ ParameterSize: "7B",
+ QuantizationLevel: "FP16",
+ },
+ System: `You are a pirate!
+Ahoy, matey!
+Weigh anchor!
+ `,
+ }, &b); err != nil {
+ t.Fatal(err)
+ }
+
+ expect := ` Model
+ architecture test
+ parameters 7B
+ quantization FP16
+
+ System
+ You are a pirate!
+ Ahoy, matey!
+
+`
+ if diff := cmp.Diff(expect, b.String()); diff != "" {
+ t.Errorf("unexpected output (-want +got):\n%s", diff)
+ }
+ })
+
+ t.Run("license", func(t *testing.T) {
+ var b bytes.Buffer
+ license, err := os.ReadFile(filepath.Join("..", "LICENSE"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := showInfo(&api.ShowResponse{
+ Details: api.ModelDetails{
+ Family: "test",
+ ParameterSize: "7B",
+ QuantizationLevel: "FP16",
+ },
+ License: string(license),
+ }, &b); err != nil {
+ t.Fatal(err)
+ }
+
+ expect := ` Model
+ architecture test
+ parameters 7B
+ quantization FP16
+
+ License
+ MIT License
+ Copyright (c) Ollama
+
+`
+ if diff := cmp.Diff(expect, b.String()); diff != "" {
+ t.Errorf("unexpected output (-want +got):\n%s", diff)
+ }
+ })
+}
+
+func TestDeleteHandler(t *testing.T) {
+ stopped := false
+ mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/api/delete" && r.Method == http.MethodDelete {
+ var req api.DeleteRequest
+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ if req.Name == "test-model" {
+ w.WriteHeader(http.StatusOK)
+ } else {
+ w.WriteHeader(http.StatusNotFound)
+ }
+ return
+ }
+ if r.URL.Path == "/api/generate" && r.Method == http.MethodPost {
+ var req api.GenerateRequest
+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ if req.Model == "test-model" {
+ w.WriteHeader(http.StatusOK)
+ if err := json.NewEncoder(w).Encode(api.GenerateResponse{
+ Done: true,
+ }); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ }
+ stopped = true
+ return
+ } else {
+ w.WriteHeader(http.StatusNotFound)
+ if err := json.NewEncoder(w).Encode(api.GenerateResponse{
+ Done: false,
+ }); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ }
+ }
+ }
+ }))
+
+ t.Setenv("OLLAMA_HOST", mockServer.URL)
+ t.Cleanup(mockServer.Close)
+
+ cmd := &cobra.Command{}
+ cmd.SetContext(context.TODO())
+ if err := DeleteHandler(cmd, []string{"test-model"}); err != nil {
+ t.Fatalf("DeleteHandler failed: %v", err)
+ }
+ if !stopped {
+ t.Fatal("Model was not stopped before deletion")
+ }
+
+ err := DeleteHandler(cmd, []string{"test-model-not-found"})
+ if err == nil || !strings.Contains(err.Error(), "unable to stop existing running model \"test-model-not-found\"") {
+ t.Fatalf("DeleteHandler failed: expected error about stopping non-existent model, got %v", err)
+ }
+}
diff --git a/cmd/interactive.go b/cmd/interactive.go
new file mode 100644
index 00000000..1b4b187b
--- /dev/null
+++ b/cmd/interactive.go
@@ -0,0 +1,594 @@
+package cmd
+
+import (
+ "cmp"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "path/filepath"
+ "regexp"
+ "slices"
+ "strings"
+
+ "github.com/spf13/cobra"
+ "golang.org/x/exp/maps"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/parser"
+ "github.com/ollama/ollama/readline"
+ "github.com/ollama/ollama/types/errtypes"
+)
+
+type MultilineState int
+
+const (
+ MultilineNone MultilineState = iota
+ MultilinePrompt
+ MultilineSystem
+)
+
+func generateInteractive(cmd *cobra.Command, opts runOptions) error {
+ usage := func() {
+ fmt.Fprintln(os.Stderr, "Available Commands:")
+ fmt.Fprintln(os.Stderr, " /set Set session variables")
+ fmt.Fprintln(os.Stderr, " /show Show model information")
+ fmt.Fprintln(os.Stderr, " /load <model> Load a session or model")
+ fmt.Fprintln(os.Stderr, " /save <model> Save your current session")
+ fmt.Fprintln(os.Stderr, " /clear Clear session context")
+ fmt.Fprintln(os.Stderr, " /bye Exit")
+ fmt.Fprintln(os.Stderr, " /?, /help Help for a command")
+ fmt.Fprintln(os.Stderr, " /? shortcuts Help for keyboard shortcuts")
+ fmt.Fprintln(os.Stderr, "")
+ fmt.Fprintln(os.Stderr, "Use \"\"\" to begin a multi-line message.")
+
+ if opts.MultiModal {
+ fmt.Fprintf(os.Stderr, "Use %s to include .jpg or .png images.\n", filepath.FromSlash("/path/to/file"))
+ }
+
+ fmt.Fprintln(os.Stderr, "")
+ }
+
+ usageSet := func() {
+ fmt.Fprintln(os.Stderr, "Available Commands:")
+ fmt.Fprintln(os.Stderr, " /set parameter ... Set a parameter")
+ fmt.Fprintln(os.Stderr, " /set system <string> Set system message")
+ fmt.Fprintln(os.Stderr, " /set history Enable history")
+ fmt.Fprintln(os.Stderr, " /set nohistory Disable history")
+ fmt.Fprintln(os.Stderr, " /set wordwrap Enable wordwrap")
+ fmt.Fprintln(os.Stderr, " /set nowordwrap Disable wordwrap")
+ fmt.Fprintln(os.Stderr, " /set format json Enable JSON mode")
+ fmt.Fprintln(os.Stderr, " /set noformat Disable formatting")
+ fmt.Fprintln(os.Stderr, " /set verbose Show LLM stats")
+ fmt.Fprintln(os.Stderr, " /set quiet Disable LLM stats")
+ fmt.Fprintln(os.Stderr, "")
+ }
+
+ usageShortcuts := func() {
+ fmt.Fprintln(os.Stderr, "Available keyboard shortcuts:")
+ fmt.Fprintln(os.Stderr, " Ctrl + a Move to the beginning of the line (Home)")
+ fmt.Fprintln(os.Stderr, " Ctrl + e Move to the end of the line (End)")
+ fmt.Fprintln(os.Stderr, " Alt + b Move back (left) one word")
+ fmt.Fprintln(os.Stderr, " Alt + f Move forward (right) one word")
+ fmt.Fprintln(os.Stderr, " Ctrl + k Delete the sentence after the cursor")
+ fmt.Fprintln(os.Stderr, " Ctrl + u Delete the sentence before the cursor")
+ fmt.Fprintln(os.Stderr, " Ctrl + w Delete the word before the cursor")
+ fmt.Fprintln(os.Stderr, "")
+ fmt.Fprintln(os.Stderr, " Ctrl + l Clear the screen")
+ fmt.Fprintln(os.Stderr, " Ctrl + c Stop the model from responding")
+ fmt.Fprintln(os.Stderr, " Ctrl + d Exit ollama (/bye)")
+ fmt.Fprintln(os.Stderr, "")
+ }
+
+ usageShow := func() {
+ fmt.Fprintln(os.Stderr, "Available Commands:")
+ fmt.Fprintln(os.Stderr, " /show info Show details for this model")
+ fmt.Fprintln(os.Stderr, " /show license Show model license")
+ fmt.Fprintln(os.Stderr, " /show modelfile Show Modelfile for this model")
+ fmt.Fprintln(os.Stderr, " /show parameters Show parameters for this model")
+ fmt.Fprintln(os.Stderr, " /show system Show system message")
+ fmt.Fprintln(os.Stderr, " /show template Show prompt template")
+ fmt.Fprintln(os.Stderr, "")
+ }
+
+ // only list out the most common parameters
+ usageParameters := func() {
+ fmt.Fprintln(os.Stderr, "Available Parameters:")
+ fmt.Fprintln(os.Stderr, " /set parameter seed <int> Random number seed")
+ fmt.Fprintln(os.Stderr, " /set parameter num_predict <int> Max number of tokens to predict")
+ fmt.Fprintln(os.Stderr, " /set parameter top_k <int> Pick from top k num of tokens")
+ fmt.Fprintln(os.Stderr, " /set parameter top_p <float> Pick token based on sum of probabilities")
+ fmt.Fprintln(os.Stderr, " /set parameter min_p <float> Pick token based on top token probability * min_p")
+ fmt.Fprintln(os.Stderr, " /set parameter num_ctx <int> Set the context size")
+ fmt.Fprintln(os.Stderr, " /set parameter temperature <float> Set creativity level")
+ fmt.Fprintln(os.Stderr, " /set parameter repeat_penalty <float> How strongly to penalize repetitions")
+ fmt.Fprintln(os.Stderr, " /set parameter repeat_last_n <int> Set how far back to look for repetitions")
+ fmt.Fprintln(os.Stderr, " /set parameter num_gpu <int> The number of layers to send to the GPU")
+ fmt.Fprintln(os.Stderr, " /set parameter stop <string> <string> ... Set the stop parameters")
+ fmt.Fprintln(os.Stderr, "")
+ }
+
+ scanner, err := readline.New(readline.Prompt{
+ Prompt: ">>> ",
+ AltPrompt: "... ",
+ Placeholder: "Send a message (/? for help)",
+ AltPlaceholder: `Use """ to end multi-line input`,
+ })
+ if err != nil {
+ return err
+ }
+
+ if envconfig.NoHistory() {
+ scanner.HistoryDisable()
+ }
+
+ fmt.Print(readline.StartBracketedPaste)
+ defer fmt.Printf(readline.EndBracketedPaste)
+
+ var sb strings.Builder
+ var multiline MultilineState
+
+ for {
+ line, err := scanner.Readline()
+ switch {
+ case errors.Is(err, io.EOF):
+ fmt.Println()
+ return nil
+ case errors.Is(err, readline.ErrInterrupt):
+ if line == "" {
+ fmt.Println("\nUse Ctrl + d or /bye to exit.")
+ }
+
+ scanner.Prompt.UseAlt = false
+ sb.Reset()
+
+ continue
+ case err != nil:
+ return err
+ }
+
+ switch {
+ case multiline != MultilineNone:
+ // check if there's a multiline terminating string
+ before, ok := strings.CutSuffix(line, `"""`)
+ sb.WriteString(before)
+ if !ok {
+ fmt.Fprintln(&sb)
+ continue
+ }
+
+ switch multiline {
+ case MultilineSystem:
+ opts.System = sb.String()
+ opts.Messages = append(opts.Messages, api.Message{Role: "system", Content: opts.System})
+ fmt.Println("Set system message.")
+ sb.Reset()
+ }
+
+ multiline = MultilineNone
+ scanner.Prompt.UseAlt = false
+ case strings.HasPrefix(line, `"""`):
+ line := strings.TrimPrefix(line, `"""`)
+ line, ok := strings.CutSuffix(line, `"""`)
+ sb.WriteString(line)
+ if !ok {
+ // no multiline terminating string; need more input
+ fmt.Fprintln(&sb)
+ multiline = MultilinePrompt
+ scanner.Prompt.UseAlt = true
+ }
+ case scanner.Pasting:
+ fmt.Fprintln(&sb, line)
+ continue
+ case strings.HasPrefix(line, "/list"):
+ args := strings.Fields(line)
+ if err := ListHandler(cmd, args[1:]); err != nil {
+ return err
+ }
+ case strings.HasPrefix(line, "/load"):
+ args := strings.Fields(line)
+ if len(args) != 2 {
+ fmt.Println("Usage:\n /load <modelname>")
+ continue
+ }
+ opts.Model = args[1]
+ opts.Messages = []api.Message{}
+ fmt.Printf("Loading model '%s'\n", opts.Model)
+ if err := loadOrUnloadModel(cmd, &opts); err != nil {
+ return err
+ }
+ continue
+ case strings.HasPrefix(line, "/save"):
+ args := strings.Fields(line)
+ if len(args) != 2 {
+ fmt.Println("Usage:\n /save <modelname>")
+ continue
+ }
+
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ fmt.Println("error: couldn't connect to ollama server")
+ return err
+ }
+
+ req := &api.CreateRequest{
+ Name: args[1],
+ Modelfile: buildModelfile(opts),
+ }
+ fn := func(resp api.ProgressResponse) error { return nil }
+ err = client.Create(cmd.Context(), req, fn)
+ if err != nil {
+ if strings.Contains(err.Error(), errtypes.InvalidModelNameErrMsg) {
+ fmt.Printf("error: The model name '%s' is invalid\n", args[1])
+ continue
+ }
+ return err
+ }
+ fmt.Printf("Created new model '%s'\n", args[1])
+ continue
+ case strings.HasPrefix(line, "/clear"):
+ opts.Messages = []api.Message{}
+ if opts.System != "" {
+ newMessage := api.Message{Role: "system", Content: opts.System}
+ opts.Messages = append(opts.Messages, newMessage)
+ }
+ fmt.Println("Cleared session context")
+ continue
+ case strings.HasPrefix(line, "/set"):
+ args := strings.Fields(line)
+ if len(args) > 1 {
+ switch args[1] {
+ case "history":
+ scanner.HistoryEnable()
+ case "nohistory":
+ scanner.HistoryDisable()
+ case "wordwrap":
+ opts.WordWrap = true
+ fmt.Println("Set 'wordwrap' mode.")
+ case "nowordwrap":
+ opts.WordWrap = false
+ fmt.Println("Set 'nowordwrap' mode.")
+ case "verbose":
+ if err := cmd.Flags().Set("verbose", "true"); err != nil {
+ return err
+ }
+ fmt.Println("Set 'verbose' mode.")
+ case "quiet":
+ if err := cmd.Flags().Set("verbose", "false"); err != nil {
+ return err
+ }
+ fmt.Println("Set 'quiet' mode.")
+ case "format":
+ if len(args) < 3 || args[2] != "json" {
+ fmt.Println("Invalid or missing format. For 'json' mode use '/set format json'")
+ } else {
+ opts.Format = args[2]
+ fmt.Printf("Set format to '%s' mode.\n", args[2])
+ }
+ case "noformat":
+ opts.Format = ""
+ fmt.Println("Disabled format.")
+ case "parameter":
+ if len(args) < 4 {
+ usageParameters()
+ continue
+ }
+ params := args[3:]
+ fp, err := api.FormatParams(map[string][]string{args[2]: params})
+ if err != nil {
+ fmt.Printf("Couldn't set parameter: %q\n", err)
+ continue
+ }
+ fmt.Printf("Set parameter '%s' to '%s'\n", args[2], strings.Join(params, ", "))
+ opts.Options[args[2]] = fp[args[2]]
+ case "system":
+ if len(args) < 3 {
+ usageSet()
+ continue
+ }
+
+ multiline = MultilineSystem
+
+ line := strings.Join(args[2:], " ")
+ line, ok := strings.CutPrefix(line, `"""`)
+ if !ok {
+ multiline = MultilineNone
+ } else {
+ // only cut suffix if the line is multiline
+ line, ok = strings.CutSuffix(line, `"""`)
+ if ok {
+ multiline = MultilineNone
+ }
+ }
+
+ sb.WriteString(line)
+ if multiline != MultilineNone {
+ scanner.Prompt.UseAlt = true
+ continue
+ }
+
+ opts.System = sb.String() // for display in modelfile
+ newMessage := api.Message{Role: "system", Content: sb.String()}
+ // Check if the slice is not empty and the last message is from 'system'
+ if len(opts.Messages) > 0 && opts.Messages[len(opts.Messages)-1].Role == "system" {
+ // Replace the last message
+ opts.Messages[len(opts.Messages)-1] = newMessage
+ } else {
+ opts.Messages = append(opts.Messages, newMessage)
+ }
+ fmt.Println("Set system message.")
+ sb.Reset()
+
+ sb.Reset()
+ continue
+ default:
+ fmt.Printf("Unknown command '/set %s'. Type /? for help\n", args[1])
+ }
+ } else {
+ usageSet()
+ }
+ case strings.HasPrefix(line, "/show"):
+ args := strings.Fields(line)
+ if len(args) > 1 {
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ fmt.Println("error: couldn't connect to ollama server")
+ return err
+ }
+ req := &api.ShowRequest{
+ Name: opts.Model,
+ System: opts.System,
+ Options: opts.Options,
+ }
+ resp, err := client.Show(cmd.Context(), req)
+ if err != nil {
+ fmt.Println("error: couldn't get model")
+ return err
+ }
+
+ switch args[1] {
+ case "info":
+ _ = showInfo(resp, os.Stderr)
+ case "license":
+ if resp.License == "" {
+ fmt.Println("No license was specified for this model.")
+ } else {
+ fmt.Println(resp.License)
+ }
+ case "modelfile":
+ fmt.Println(resp.Modelfile)
+ case "parameters":
+ if resp.Parameters == "" {
+ fmt.Println("No parameters were specified for this model.")
+ } else {
+ if len(opts.Options) > 0 {
+ fmt.Println("User defined parameters:")
+ for k, v := range opts.Options {
+ fmt.Printf("%-*s %v\n", 30, k, v)
+ }
+ fmt.Println()
+ }
+ fmt.Println("Model defined parameters:")
+ fmt.Println(resp.Parameters)
+ }
+ case "system":
+ switch {
+ case opts.System != "":
+ fmt.Println(opts.System + "\n")
+ case resp.System != "":
+ fmt.Println(resp.System + "\n")
+ default:
+ fmt.Println("No system message was specified for this model.")
+ }
+ case "template":
+ if resp.Template != "" {
+ fmt.Println(resp.Template)
+ } else {
+ fmt.Println("No prompt template was specified for this model.")
+ }
+ default:
+ fmt.Printf("Unknown command '/show %s'. Type /? for help\n", args[1])
+ }
+ } else {
+ usageShow()
+ }
+ case strings.HasPrefix(line, "/help"), strings.HasPrefix(line, "/?"):
+ args := strings.Fields(line)
+ if len(args) > 1 {
+ switch args[1] {
+ case "set", "/set":
+ usageSet()
+ case "show", "/show":
+ usageShow()
+ case "shortcut", "shortcuts":
+ usageShortcuts()
+ }
+ } else {
+ usage()
+ }
+ case strings.HasPrefix(line, "/exit"), strings.HasPrefix(line, "/bye"):
+ return nil
+ case strings.HasPrefix(line, "/"):
+ args := strings.Fields(line)
+ isFile := false
+
+ if opts.MultiModal {
+ for _, f := range extractFileNames(line) {
+ if strings.HasPrefix(f, args[0]) {
+ isFile = true
+ break
+ }
+ }
+ }
+
+ if !isFile {
+ fmt.Printf("Unknown command '%s'. Type /? for help\n", args[0])
+ continue
+ }
+
+ sb.WriteString(line)
+ default:
+ sb.WriteString(line)
+ }
+
+ if sb.Len() > 0 && multiline == MultilineNone {
+ newMessage := api.Message{Role: "user", Content: sb.String()}
+
+ if opts.MultiModal {
+ msg, images, err := extractFileData(sb.String())
+ if err != nil {
+ return err
+ }
+
+ newMessage.Content = msg
+ newMessage.Images = images
+ }
+
+ opts.Messages = append(opts.Messages, newMessage)
+
+ assistant, err := chat(cmd, opts)
+ if err != nil {
+ return err
+ }
+ if assistant != nil {
+ opts.Messages = append(opts.Messages, *assistant)
+ }
+
+ sb.Reset()
+ }
+ }
+}
+
+func buildModelfile(opts runOptions) string {
+ var f parser.File
+ f.Commands = append(f.Commands, parser.Command{Name: "model", Args: cmp.Or(opts.ParentModel, opts.Model)})
+
+ if opts.System != "" {
+ f.Commands = append(f.Commands, parser.Command{Name: "system", Args: opts.System})
+ }
+
+ keys := maps.Keys(opts.Options)
+ slices.Sort(keys)
+ for _, k := range keys {
+ v := opts.Options[k]
+ var cmds []parser.Command
+ switch t := v.(type) {
+ case []string:
+ for _, s := range t {
+ cmds = append(cmds, parser.Command{Name: k, Args: s})
+ }
+ default:
+ cmds = append(cmds, parser.Command{Name: k, Args: fmt.Sprintf("%v", t)})
+ }
+
+ f.Commands = append(f.Commands, cmds...)
+ }
+
+ for _, msg := range opts.Messages {
+ f.Commands = append(f.Commands, parser.Command{Name: "message", Args: fmt.Sprintf("%s: %s", msg.Role, msg.Content)})
+ }
+
+ return f.String()
+}
+
+func normalizeFilePath(fp string) string {
+ // Define a map of escaped characters and their replacements
+ replacements := map[string]string{
+ "\\ ": " ", // Escaped space
+ "\\(": "(", // Escaped left parenthesis
+ "\\)": ")", // Escaped right parenthesis
+ "\\[": "[", // Escaped left square bracket
+ "\\]": "]", // Escaped right square bracket
+ "\\{": "{", // Escaped left curly brace
+ "\\}": "}", // Escaped right curly brace
+ "\\$": "$", // Escaped dollar sign
+ "\\&": "&", // Escaped ampersand
+ "\\;": ";", // Escaped semicolon
+ "\\'": "'", // Escaped single quote
+ "\\\\": "\\", // Escaped backslash
+ "\\*": "*", // Escaped asterisk
+ "\\?": "?", // Escaped question mark
+ }
+
+ for escaped, actual := range replacements {
+ fp = strings.ReplaceAll(fp, escaped, actual)
+ }
+ return fp
+}
+
+func extractFileNames(input string) []string {
+ // Regex to match file paths starting with optional drive letter, / ./ \ or .\ and include escaped or unescaped spaces (\ or %20)
+ // and followed by more characters and a file extension
+ // This will capture non filename strings, but we'll check for file existence to remove mismatches
+ regexPattern := `(?:[a-zA-Z]:)?(?:\./|/|\\)[\S\\ ]+?\.(?i:jpg|jpeg|png|svg)\b`
+ re := regexp.MustCompile(regexPattern)
+
+ return re.FindAllString(input, -1)
+}
+
+func extractFileData(input string) (string, []api.ImageData, error) {
+ filePaths := extractFileNames(input)
+ var imgs []api.ImageData
+
+ for _, fp := range filePaths {
+ nfp := normalizeFilePath(fp)
+ data, err := getImageData(nfp)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+ fmt.Fprintf(os.Stderr, "Couldn't process image: %q\n", err)
+ return "", imgs, err
+ }
+ fmt.Fprintf(os.Stderr, "Added image '%s'\n", nfp)
+ input = strings.ReplaceAll(input, fp, "")
+ imgs = append(imgs, data)
+ }
+ return input, imgs, nil
+}
+
+func getImageData(filePath string) ([]byte, error) {
+ file, err := os.Open(filePath)
+ if err != nil {
+ return nil, err
+ }
+ defer file.Close()
+
+ buf := make([]byte, 512)
+ _, err = file.Read(buf)
+ if err != nil {
+ return nil, err
+ }
+
+ contentType := http.DetectContentType(buf)
+ allowedTypes := []string{"image/jpeg", "image/jpg", "image/png"}
+ if !slices.Contains(allowedTypes, contentType) {
+ return nil, fmt.Errorf("invalid image type: %s", contentType)
+ }
+
+ info, err := file.Stat()
+ if err != nil {
+ return nil, err
+ }
+
+ // Check if the file size exceeds 100MB
+ var maxSize int64 = 100 * 1024 * 1024 // 100MB in bytes
+ if info.Size() > maxSize {
+ return nil, errors.New("file size exceeds maximum limit (100MB)")
+ }
+
+ buf = make([]byte, info.Size())
+ _, err = file.Seek(0, 0)
+ if err != nil {
+ return nil, err
+ }
+
+ _, err = io.ReadFull(file, buf)
+ if err != nil {
+ return nil, err
+ }
+
+ return buf, nil
+}
diff --git a/cmd/interactive_test.go b/cmd/interactive_test.go
new file mode 100644
index 00000000..bb7e0aba
--- /dev/null
+++ b/cmd/interactive_test.go
@@ -0,0 +1,107 @@
+package cmd
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/ollama/ollama/api"
+)
+
+func TestExtractFilenames(t *testing.T) {
+ // Unix style paths
+ input := ` some preamble
+ ./relative\ path/one.png inbetween1 ./not a valid two.jpg inbetween2
+/unescaped space /three.jpeg inbetween3 /valid\ path/dir/four.png "./quoted with spaces/five.svg`
+ res := extractFileNames(input)
+ assert.Len(t, res, 5)
+ assert.Contains(t, res[0], "one.png")
+ assert.Contains(t, res[1], "two.jpg")
+ assert.Contains(t, res[2], "three.jpeg")
+ assert.Contains(t, res[3], "four.png")
+ assert.Contains(t, res[4], "five.svg")
+ assert.NotContains(t, res[4], '"')
+ assert.NotContains(t, res, "inbtween")
+
+ // Windows style paths
+ input = ` some preamble
+ c:/users/jdoe/one.png inbetween1 c:/program files/someplace/two.jpg inbetween2
+ /absolute/nospace/three.jpeg inbetween3 /absolute/with space/four.png inbetween4
+./relative\ path/five.svg inbetween5 "./relative with/spaces/six.png inbetween6
+d:\path with\spaces\seven.svg inbetween7 c:\users\jdoe\eight.png inbetween8
+ d:\program files\someplace\nine.png inbetween9 "E:\program files\someplace\ten.svg some ending
+`
+ res = extractFileNames(input)
+ assert.Len(t, res, 10)
+ assert.NotContains(t, res, "inbtween")
+ assert.Contains(t, res[0], "one.png")
+ assert.Contains(t, res[0], "c:")
+ assert.Contains(t, res[1], "two.jpg")
+ assert.Contains(t, res[1], "c:")
+ assert.Contains(t, res[2], "three.jpeg")
+ assert.Contains(t, res[3], "four.png")
+ assert.Contains(t, res[4], "five.svg")
+ assert.Contains(t, res[5], "six.png")
+ assert.Contains(t, res[6], "seven.svg")
+ assert.Contains(t, res[6], "d:")
+ assert.Contains(t, res[7], "eight.png")
+ assert.Contains(t, res[7], "c:")
+ assert.Contains(t, res[8], "nine.png")
+ assert.Contains(t, res[8], "d:")
+ assert.Contains(t, res[9], "ten.svg")
+ assert.Contains(t, res[9], "E:")
+}
+
+func TestModelfileBuilder(t *testing.T) {
+ opts := runOptions{
+ Model: "hork",
+ System: "You are part horse and part shark, but all hork. Do horklike things",
+ Messages: []api.Message{
+ {Role: "user", Content: "Hey there hork!"},
+ {Role: "assistant", Content: "Yes it is true, I am half horse, half shark."},
+ },
+ Options: map[string]any{
+ "temperature": 0.9,
+ "seed": 42,
+ "penalize_newline": false,
+ "stop": []string{"hi", "there"},
+ },
+ }
+
+ t.Run("model", func(t *testing.T) {
+ expect := `FROM hork
+SYSTEM You are part horse and part shark, but all hork. Do horklike things
+PARAMETER penalize_newline false
+PARAMETER seed 42
+PARAMETER stop hi
+PARAMETER stop there
+PARAMETER temperature 0.9
+MESSAGE user Hey there hork!
+MESSAGE assistant Yes it is true, I am half horse, half shark.
+`
+
+ actual := buildModelfile(opts)
+ if diff := cmp.Diff(expect, actual); diff != "" {
+ t.Errorf("mismatch (-want +got):\n%s", diff)
+ }
+ })
+
+ t.Run("parent model", func(t *testing.T) {
+ opts.ParentModel = "horseshark"
+ expect := `FROM horseshark
+SYSTEM You are part horse and part shark, but all hork. Do horklike things
+PARAMETER penalize_newline false
+PARAMETER seed 42
+PARAMETER stop hi
+PARAMETER stop there
+PARAMETER temperature 0.9
+MESSAGE user Hey there hork!
+MESSAGE assistant Yes it is true, I am half horse, half shark.
+`
+ actual := buildModelfile(opts)
+ if diff := cmp.Diff(expect, actual); diff != "" {
+ t.Errorf("mismatch (-want +got):\n%s", diff)
+ }
+ })
+}
diff --git a/cmd/start.go b/cmd/start.go
new file mode 100644
index 00000000..0c4eed08
--- /dev/null
+++ b/cmd/start.go
@@ -0,0 +1,27 @@
+//go:build darwin || windows
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/ollama/ollama/api"
+)
+
+func waitForServer(ctx context.Context, client *api.Client) error {
+ // wait for the server to start
+ timeout := time.After(5 * time.Second)
+ tick := time.Tick(500 * time.Millisecond)
+ for {
+ select {
+ case <-timeout:
+ return errors.New("timed out waiting for server to start")
+ case <-tick:
+ if err := client.Heartbeat(ctx); err == nil {
+ return nil // server has started
+ }
+ }
+ }
+}
diff --git a/cmd/start_darwin.go b/cmd/start_darwin.go
new file mode 100644
index 00000000..1a9a1ae8
--- /dev/null
+++ b/cmd/start_darwin.go
@@ -0,0 +1,30 @@
+package cmd
+
+import (
+ "context"
+ "errors"
+ "os"
+ "os/exec"
+ "strings"
+
+ "github.com/ollama/ollama/api"
+)
+
+func startApp(ctx context.Context, client *api.Client) error {
+ exe, err := os.Executable()
+ if err != nil {
+ return err
+ }
+ link, err := os.Readlink(exe)
+ if err != nil {
+ return err
+ }
+ if !strings.Contains(link, "Ollama.app") {
+ return errors.New("could not find ollama app")
+ }
+ path := strings.Split(link, "Ollama.app")
+ if err := exec.Command("/usr/bin/open", "-a", path[0]+"Ollama.app").Run(); err != nil {
+ return err
+ }
+ return waitForServer(ctx, client)
+}
diff --git a/cmd/start_default.go b/cmd/start_default.go
new file mode 100644
index 00000000..5eabb286
--- /dev/null
+++ b/cmd/start_default.go
@@ -0,0 +1,14 @@
+//go:build !windows && !darwin
+
+package cmd
+
+import (
+ "context"
+ "errors"
+
+ "github.com/ollama/ollama/api"
+)
+
+func startApp(ctx context.Context, client *api.Client) error {
+ return errors.New("could not connect to ollama server, run 'ollama serve' to start it")
+}
diff --git a/cmd/start_windows.go b/cmd/start_windows.go
new file mode 100644
index 00000000..5bca2433
--- /dev/null
+++ b/cmd/start_windows.go
@@ -0,0 +1,58 @@
+package cmd
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "syscall"
+
+ "github.com/ollama/ollama/api"
+)
+
+func startApp(ctx context.Context, client *api.Client) error {
+ // log.Printf("XXX Attempting to find and start ollama app")
+ AppName := "ollama app.exe"
+ exe, err := os.Executable()
+ if err != nil {
+ return err
+ }
+ appExe := filepath.Join(filepath.Dir(exe), AppName)
+ _, err = os.Stat(appExe)
+ if errors.Is(err, os.ErrNotExist) {
+ // Try the standard install location
+ localAppData := os.Getenv("LOCALAPPDATA")
+ appExe = filepath.Join(localAppData, "Ollama", AppName)
+ _, err := os.Stat(appExe)
+ if errors.Is(err, os.ErrNotExist) {
+ // Finally look in the path
+ appExe, err = exec.LookPath(AppName)
+ if err != nil {
+ return errors.New("could not locate ollama app")
+ }
+ }
+ }
+ // log.Printf("XXX attempting to start app %s", appExe)
+
+ cmd_path := "c:\\Windows\\system32\\cmd.exe"
+ cmd := exec.Command(cmd_path, "/c", appExe)
+ // TODO - these hide flags aren't working - still pops up a command window for some reason
+ cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: 0x08000000, HideWindow: true}
+
+ // TODO this didn't help either...
+ cmd.Stdin = strings.NewReader("")
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+
+ if err := cmd.Start(); err != nil {
+ return fmt.Errorf("unable to start ollama app %w", err)
+ }
+
+ if cmd.Process != nil {
+ defer cmd.Process.Release() //nolint:errcheck
+ }
+ return waitForServer(ctx, client)
+}
diff --git a/convert/convert.go b/convert/convert.go
new file mode 100644
index 00000000..44783b6e
--- /dev/null
+++ b/convert/convert.go
@@ -0,0 +1,232 @@
+package convert
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "log/slog"
+ "strings"
+
+ "github.com/ollama/ollama/llm"
+)
+
+type ModelParameters struct {
+ Architectures []string `json:"architectures"`
+ VocabSize uint32 `json:"vocab_size"`
+}
+
+type AdapterParameters struct {
+ Alpha uint32 `json:"lora_alpha"`
+ LoraLayers uint32 `json:"lora_layers"`
+ LoraParameters struct {
+ Rank uint32 `json:"rank"`
+ Alpha float32 `json:"alpha"`
+ Scale float32 `json:"scale"`
+ } `json:"lora_parameters"`
+}
+
+func (ModelParameters) KV(t *Tokenizer) llm.KV {
+ kv := llm.KV{
+ "general.file_type": uint32(1),
+ "general.quantization_version": uint32(2),
+ "tokenizer.ggml.pre": t.Pre,
+ "tokenizer.ggml.model": t.Vocabulary.Model,
+ "tokenizer.ggml.tokens": t.Vocabulary.Tokens,
+ "tokenizer.ggml.scores": t.Vocabulary.Scores,
+ "tokenizer.ggml.token_type": t.Vocabulary.Types,
+ }
+
+ if len(t.Merges) > 0 {
+ kv["tokenizer.ggml.merges"] = t.Merges
+ }
+
+ if t.Template != "" {
+ kv["tokenizer.chat_template"] = t.Template
+ }
+
+ for _, sv := range t.SpecialVocabulary {
+ kv[fmt.Sprintf("tokenizer.ggml.%s_token_id", sv.Key())] = uint32(sv.ID)
+ kv[fmt.Sprintf("tokenizer.ggml.add_%s_token", sv.Key())] = sv.AddToken
+ }
+
+ return kv
+}
+
+func (p AdapterParameters) KV() llm.KV {
+ var alpha float32
+ if p.LoraParameters.Alpha == 0 {
+ alpha = float32(p.Alpha)
+ } else {
+ alpha = p.LoraParameters.Alpha
+ }
+
+ kv := llm.KV{
+ "adapter.lora.alpha": alpha,
+ "adapter.type": "lora",
+ "general.file_type": uint32(1),
+ "general.type": "adapter",
+ "general.version": "v0.2",
+ }
+
+ return kv
+}
+
+func (ModelParameters) specialTokenTypes() []string {
+ return []string{
+ "bos", "eos", "unk", "sep", "pad", "cls", "mask",
+ }
+}
+
+func (ModelParameters) writeFile(ws io.WriteSeeker, kv llm.KV, ts []llm.Tensor) error {
+ return llm.WriteGGUF(ws, kv, ts)
+}
+
+func (AdapterParameters) writeFile(ws io.WriteSeeker, kv llm.KV, ts []llm.Tensor) error {
+ return llm.WriteGGUF(ws, kv, ts)
+}
+
+type ModelConverter interface {
+ // KV maps parameters to LLM key-values
+ KV(*Tokenizer) llm.KV
+ // Tensors maps input tensors to LLM tensors. Model specific modifications can be done here.
+ Tensors([]Tensor) []llm.Tensor
+ // Replacements returns a list of string pairs to replace in tensor names.
+ // See [strings.Replacer](https://pkg.go.dev/strings#Replacer) for details
+ Replacements() []string
+
+ // specialTokenTypes returns any special token types the model uses
+ specialTokenTypes() []string
+ // writeFile writes the model to the provided io.WriteSeeker
+ writeFile(io.WriteSeeker, llm.KV, []llm.Tensor) error
+}
+
+type moreParser interface {
+ parseMore(fs.FS) error
+}
+
+type AdapterConverter interface {
+ // KV maps parameters to LLM key-values
+ KV(llm.KV) llm.KV
+ // Tensors maps input tensors to LLM tensors. Adapter specific modifications can be done here.
+ Tensors([]Tensor) []llm.Tensor
+ // Replacements returns a list of string pairs to replace in tensor names.
+ // See [strings.Replacer](https://pkg.go.dev/strings#Replacer) for details
+ Replacements() []string
+
+ writeFile(io.WriteSeeker, llm.KV, []llm.Tensor) error
+}
+
+func ConvertAdapter(fsys fs.FS, ws io.WriteSeeker, baseKV llm.KV) error {
+ bts, err := fs.ReadFile(fsys, "adapter_config.json")
+ if err != nil {
+ return err
+ }
+
+ var p AdapterParameters
+ if err := json.Unmarshal(bts, &p); err != nil {
+ return err
+ }
+
+ arch, ok := baseKV["general.architecture"]
+ if !ok {
+ return errors.New("architecture not set for the base model")
+ }
+
+ var conv AdapterConverter
+ switch arch {
+ case "llama":
+ conv = &llamaAdapter{}
+ case "gemma2":
+ conv = &gemma2Adapter{}
+ default:
+ return errors.New("unsupported architecture")
+ }
+
+ ts, err := parseTensors(fsys, strings.NewReplacer(conv.Replacements()...))
+ if err != nil {
+ return err
+ }
+
+ if err := json.Unmarshal(bts, conv); err != nil {
+ return err
+ }
+
+ return conv.writeFile(ws, conv.KV(baseKV), conv.Tensors(ts))
+}
+
+// Convert writes an Ollama compatible model to the provided io.WriteSeeker based on configurations
+// and files it finds in the input path.
+// Supported input model formats include safetensors.
+// Supported input tokenizers files include tokenizer.json (preferred) and tokenizer.model.
+func ConvertModel(fsys fs.FS, ws io.WriteSeeker) error {
+ bts, err := fs.ReadFile(fsys, "config.json")
+ if err != nil {
+ return err
+ }
+
+ var p ModelParameters
+ if err := json.Unmarshal(bts, &p); err != nil {
+ return err
+ }
+
+ if len(p.Architectures) < 1 {
+ return errors.New("unknown architecture")
+ }
+
+ var conv ModelConverter
+ switch p.Architectures[0] {
+ case "LlamaForCausalLM", "MistralForCausalLM":
+ conv = &llamaModel{}
+ case "MixtralForCausalLM":
+ conv = &mixtralModel{}
+ case "GemmaForCausalLM":
+ conv = &gemmaModel{}
+ case "Gemma2ForCausalLM":
+ conv = &gemma2Model{}
+ case "Phi3ForCausalLM":
+ conv = &phi3Model{}
+ case "BertModel":
+ conv = &bertModel{}
+ default:
+ return errors.New("unsupported architecture")
+ }
+
+ if err := json.Unmarshal(bts, conv); err != nil {
+ return err
+ }
+
+ if t, ok := conv.(moreParser); ok {
+ if err := t.parseMore(fsys); err != nil {
+ return err
+ }
+ }
+
+ t, err := parseTokenizer(fsys, conv.specialTokenTypes())
+ if err != nil {
+ return err
+ }
+
+ vocabSize := int(p.VocabSize)
+ switch {
+ case vocabSize > len(t.Vocabulary.Tokens):
+ slog.Warn("vocabulary is smaller than expected, padding with dummy tokens", "expect", vocabSize, "actual", len(t.Vocabulary.Tokens))
+ for i := range vocabSize - len(t.Vocabulary.Tokens) {
+ t.Vocabulary.Tokens = append(t.Vocabulary.Tokens, fmt.Sprintf("[PAD%d]", i))
+ t.Vocabulary.Scores = append(t.Vocabulary.Scores, -1)
+ t.Vocabulary.Types = append(t.Vocabulary.Types, tokenTypeUserDefined)
+ }
+ case vocabSize < len(t.Vocabulary.Tokens):
+ return fmt.Errorf("vocabulary is larger than expected '%d' instead of '%d'", len(t.Vocabulary.Tokens), vocabSize)
+ default:
+ slog.Debug("vocabulary", "size", len(t.Vocabulary.Tokens))
+ }
+
+ ts, err := parseTensors(fsys, strings.NewReplacer(conv.Replacements()...))
+ if err != nil {
+ return err
+ }
+
+ return conv.writeFile(ws, conv.KV(t), conv.Tensors(ts))
+}
diff --git a/convert/convert_bert.go b/convert/convert_bert.go
new file mode 100644
index 00000000..ea5facaa
--- /dev/null
+++ b/convert/convert_bert.go
@@ -0,0 +1,174 @@
+package convert
+
+import (
+ "cmp"
+ "encoding/json"
+ "io/fs"
+ "path/filepath"
+ "slices"
+ "strings"
+
+ "github.com/ollama/ollama/llm"
+)
+
+type bertModel struct {
+ ModelParameters
+ NLayers uint32 `json:"n_layers"`
+ NumHiddenLayers uint32 `json:"num_hidden_layers"`
+ NLayer uint32 `json:"n_layer"`
+ MaxPositionEmbeddings uint32 `json:"max_position_embeddings"`
+ NCtx uint32 `json:"n_ctx"`
+ HiddenSize uint32 `json:"hidden_size"`
+ NEmbd uint32 `json:"n_embd"`
+ IntermediateSize uint32 `json:"intermediate_size"`
+ NInner uint32 `json:"n_inner"`
+ NumAttentionHeads uint32 `json:"num_attention_heads"`
+ NHead uint32 `json:"n_head"`
+ NumKeyValueHeads uint32 `json:"num_key_value_heads"`
+ LayerNormEPS float32 `json:"layer_norm_eps"`
+ LayerNormEpsilon float32 `json:"layer_norm_epsilon"`
+ NormEpsilon float32 `json:"norm_epsilon"`
+
+ PoolingType uint32
+}
+
+var (
+ _ ModelConverter = (*bertModel)(nil)
+ _ moreParser = (*bertModel)(nil)
+)
+
+func (p *bertModel) parseMore(fsys fs.FS) error {
+ bts, err := fs.ReadFile(fsys, "modules.json")
+ if err != nil {
+ return err
+ }
+
+ var modules []struct {
+ Type string `json:"type"`
+ Path string `json:"path"`
+ }
+
+ if err := json.Unmarshal(bts, &modules); err != nil {
+ return err
+ }
+
+ var pooling string
+ for _, m := range modules {
+ if m.Type == "sentence_transformers.models.Pooling" {
+ pooling = m.Path
+ break
+ }
+ }
+
+ if pooling != "" {
+ bts, err := fs.ReadFile(fsys, filepath.Join(pooling, "config.json"))
+ if err != nil {
+ return err
+ }
+
+ var pc struct {
+ PoolingModeCLSToken bool `json:"pooling_mode_cls_token"`
+ PoolingModeMeanTokens bool `json:"pooling_mode_mean_tokens"`
+ }
+
+ if err := json.Unmarshal(bts, &pc); err != nil {
+ return err
+ }
+
+ if pc.PoolingModeMeanTokens {
+ p.PoolingType = 1
+ } else if pc.PoolingModeCLSToken {
+ p.PoolingType = 2
+ }
+ }
+
+ return nil
+}
+
+func (p *bertModel) KV(t *Tokenizer) llm.KV {
+ kv := p.ModelParameters.KV(t)
+ kv["general.architecture"] = "bert"
+ kv["bert.attention.causal"] = false
+ kv["bert.pooling_type"] = p.PoolingType
+
+ kv["bert.block_count"] = cmp.Or(p.NLayers, p.NumHiddenLayers, p.NLayer)
+
+ if contextLength := cmp.Or(p.MaxPositionEmbeddings, p.NCtx); contextLength > 0 {
+ kv["bert.context_length"] = contextLength
+ }
+
+ if embeddingLength := cmp.Or(p.HiddenSize, p.NEmbd); embeddingLength > 0 {
+ kv["bert.embedding_length"] = cmp.Or(p.HiddenSize, p.NEmbd)
+ }
+
+ if feedForwardLength := cmp.Or(p.IntermediateSize, p.NInner); feedForwardLength > 0 {
+ kv["bert.feed_forward_length"] = cmp.Or(p.IntermediateSize, p.NInner)
+ }
+
+ if headCount := cmp.Or(p.NumAttentionHeads, p.NHead); headCount > 0 {
+ kv["bert.attention.head_count"] = cmp.Or(p.NumAttentionHeads, p.NHead)
+ }
+
+ if layerNormEpsilon := cmp.Or(p.LayerNormEPS, p.LayerNormEpsilon, p.NormEpsilon); layerNormEpsilon > 0 {
+ kv["bert.attention.layer_norm_epsilon"] = layerNormEpsilon
+ }
+
+ kv["tokenizer.ggml.model"] = "bert"
+ kv["tokenizer.ggml.token_type_count"] = uint32(2)
+
+ // convert to phantom space tokens
+ for i, e := range t.Tokens {
+ if strings.HasPrefix(e, "[") && strings.HasSuffix(e, "]") {
+ // noop
+ } else if strings.HasPrefix(e, "##") {
+ t.Tokens[i] = e[2:]
+ } else {
+ t.Tokens[i] = "\u2581" + e
+ }
+ }
+
+ kv["tokenizer.ggml.tokens"] = t.Tokens
+
+ return kv
+}
+
+func (p *bertModel) Tensors(ts []Tensor) []llm.Tensor {
+ var out []llm.Tensor
+ for _, t := range ts {
+ if slices.Contains([]string{
+ "embeddings.position_ids",
+ "pooler.dense.weight",
+ "pooler.dense.bias",
+ }, t.Name()) {
+ continue
+ }
+
+ out = append(out, llm.Tensor{
+ Name: t.Name(),
+ Kind: t.Kind(),
+ Shape: t.Shape(),
+ WriterTo: t,
+ })
+ }
+
+ return out
+}
+
+func (bertModel) Replacements() []string {
+ return []string{
+ "encoder.layer", "blk",
+ "encoder.layers", "blk",
+ "embeddings.word_embeddings", "token_embd",
+ "embeddings.token_type_embeddings", "token_types",
+ "embeddings.LayerNorm", "token_embd_norm",
+ "embeddings.position_embeddings", "position_embd",
+ "attention.self.query", "attn_q",
+ "attention.self.key", "attn_k",
+ "attention.self.value", "attn_v",
+ "attention.output.dense", "attn_output",
+ "attention.output.LayerNorm", "attn_output_norm",
+ "intermediate.dense", "ffn_up",
+ "output.dense", "ffn_down",
+ "output.LayerNorm", "layer_output_norm",
+ }
+}
diff --git a/convert/convert_gemma.go b/convert/convert_gemma.go
new file mode 100644
index 00000000..b8865294
--- /dev/null
+++ b/convert/convert_gemma.go
@@ -0,0 +1,100 @@
+package convert
+
+import (
+ "strings"
+
+ "github.com/pdevine/tensor"
+ "github.com/pdevine/tensor/native"
+
+ "github.com/ollama/ollama/llm"
+)
+
+type gemmaModel struct {
+ ModelParameters
+ MaxPositionEmbeddings uint32 `json:"max_position_embeddings"`
+ HiddenSize uint32 `json:"hidden_size"`
+ HiddenLayers uint32 `json:"num_hidden_layers"`
+ IntermediateSize uint32 `json:"intermediate_size"`
+ NumAttentionHeads uint32 `json:"num_attention_heads"`
+ NumKeyValueHeads uint32 `json:"num_key_value_heads"`
+ RMSNormEPS float32 `json:"rms_norm_eps"`
+ HeadDim uint32 `json:"head_dim"`
+}
+
+var _ ModelConverter = (*gemmaModel)(nil)
+
+func (p *gemmaModel) KV(t *Tokenizer) llm.KV {
+ kv := p.ModelParameters.KV(t)
+ kv["general.architecture"] = "gemma"
+ kv["gemma.context_length"] = p.MaxPositionEmbeddings
+ kv["gemma.embedding_length"] = p.HiddenSize
+ kv["gemma.block_count"] = p.HiddenLayers
+ kv["gemma.feed_forward_length"] = p.IntermediateSize
+ kv["gemma.attention.head_count"] = p.NumAttentionHeads
+ kv["gemma.attention.head_count_kv"] = p.NumKeyValueHeads
+ kv["gemma.attention.layer_norm_rms_epsilon"] = p.RMSNormEPS
+ kv["gemma.attention.key_length"] = p.HeadDim
+ kv["gemma.attention.value_length"] = p.HeadDim
+ kv["tokenizer.ggml.eot_token_id"] = uint32(107)
+ kv["tokenizer.ggml.middle_token_id"] = uint32(68)
+ kv["tokenizer.ggml.prefix_token_id"] = uint32(67)
+ kv["tokenizer.ggml.suffix_token_id"] = uint32(69)
+ return kv
+}
+
+func (p *gemmaModel) Tensors(ts []Tensor) []llm.Tensor {
+ var out []llm.Tensor
+ for _, t := range ts {
+ if strings.HasSuffix(t.Name(), "_norm.weight") {
+ t.SetRepacker(p.addOne)
+ }
+
+ out = append(out, llm.Tensor{
+ Name: t.Name(),
+ Kind: t.Kind(),
+ Shape: t.Shape(),
+ WriterTo: t,
+ })
+ }
+
+ return out
+}
+
+func (p *gemmaModel) Replacements() []string {
+ return []string{
+ "model.embed_tokens", "token_embd",
+ "model.norm", "output_norm",
+ "model.layers", "blk",
+ "input_layernorm", "attn_norm",
+ "self_attn.q_proj", "attn_q",
+ "self_attn.k_proj", "attn_k",
+ "self_attn.v_proj", "attn_v",
+ "self_attn.o_proj", "attn_output",
+ "mlp.gate_proj", "ffn_gate",
+ "mlp.down_proj", "ffn_down",
+ "mlp.up_proj", "ffn_up",
+ "post_attention_layernorm", "ffn_norm",
+ }
+}
+
+func (*gemmaModel) addOne(_ string, data []float32, shape []uint64) ([]float32, error) {
+ n := tensor.New(tensor.WithShape(int(shape[0])), tensor.WithBacking(data))
+ ones := tensor.Ones(tensor.Float32, int(shape[0]))
+
+ n, err := n.Add(ones)
+ if err != nil {
+ return nil, err
+ }
+
+ ts, err := native.SelectF32(n, 0)
+ if err != nil {
+ return nil, err
+ }
+
+ var f32s []float32
+ for _, t := range ts {
+ f32s = append(f32s, t...)
+ }
+
+ return f32s, nil
+}
diff --git a/convert/convert_gemma2.go b/convert/convert_gemma2.go
new file mode 100644
index 00000000..0f98c1e3
--- /dev/null
+++ b/convert/convert_gemma2.go
@@ -0,0 +1,53 @@
+package convert
+
+import (
+ "github.com/ollama/ollama/llm"
+)
+
+type gemma2Model struct {
+ gemmaModel
+ SlidingWindow uint32 `json:"sliding_window"`
+ AttentionLogitSoftcap float32 `json:"attn_logit_softcapping"`
+ FinalLogitSoftcap float32 `json:"final_logit_softcapping"`
+}
+
+func (p *gemma2Model) KV(t *Tokenizer) llm.KV {
+ kv := p.ModelParameters.KV(t)
+ kv["general.architecture"] = "gemma2"
+ kv["gemma2.context_length"] = p.MaxPositionEmbeddings
+ kv["gemma2.embedding_length"] = p.HiddenSize
+ kv["gemma2.block_count"] = p.HiddenLayers
+ kv["gemma2.feed_forward_length"] = p.IntermediateSize
+ kv["gemma2.attention.head_count"] = p.NumAttentionHeads
+ kv["gemma2.attention.head_count_kv"] = p.NumKeyValueHeads
+ kv["gemma2.attention.layer_norm_rms_epsilon"] = p.RMSNormEPS
+ kv["gemma2.attention.key_length"] = p.HeadDim
+ kv["gemma2.attention.value_length"] = p.HeadDim
+ kv["gemma2.attention.sliding_window"] = p.SlidingWindow
+ kv["gemma2.attn_logit_softcapping"] = p.AttentionLogitSoftcap
+ kv["gemma2.final_logit_softcapping"] = p.FinalLogitSoftcap
+ kv["tokenizer.ggml.eot_token_id"] = uint32(107)
+ kv["tokenizer.ggml.middle_token_id"] = uint32(68)
+ kv["tokenizer.ggml.prefix_token_id"] = uint32(67)
+ kv["tokenizer.ggml.suffix_token_id"] = uint32(69)
+ return kv
+}
+
+func (p *gemma2Model) Replacements() []string {
+ return []string{
+ "model.embed_tokens", "token_embd",
+ "model.norm", "output_norm",
+ "model.layers", "blk",
+ "input_layernorm", "attn_norm",
+ "self_attn.q_proj", "attn_q",
+ "self_attn.k_proj", "attn_k",
+ "self_attn.v_proj", "attn_v",
+ "self_attn.o_proj", "attn_output",
+ "mlp.gate_proj", "ffn_gate",
+ "mlp.down_proj", "ffn_down",
+ "mlp.up_proj", "ffn_up",
+ "post_attention_layernorm", "post_attention_norm",
+ "pre_feedforward_layernorm", "ffn_norm",
+ "post_feedforward_layernorm", "post_ffw_norm",
+ }
+}
diff --git a/convert/convert_gemma2_adapter.go b/convert/convert_gemma2_adapter.go
new file mode 100644
index 00000000..a89a25f4
--- /dev/null
+++ b/convert/convert_gemma2_adapter.go
@@ -0,0 +1,91 @@
+package convert
+
+import (
+ "strings"
+
+ "github.com/pdevine/tensor"
+ "github.com/pdevine/tensor/native"
+
+ "github.com/ollama/ollama/llm"
+)
+
+type gemma2Adapter struct {
+ AdapterParameters
+}
+
+var _ AdapterConverter = (*gemma2Adapter)(nil)
+
+func (p *gemma2Adapter) KV(baseKV llm.KV) llm.KV {
+ kv := p.AdapterParameters.KV()
+ kv["general.architecture"] = "gemma2"
+ return kv
+}
+
+func (p *gemma2Adapter) Tensors(ts []Tensor) []llm.Tensor {
+ var out []llm.Tensor
+ for _, t := range ts {
+ shape := t.Shape()
+ if (strings.HasSuffix(t.Name(), "weight.lora_a") && shape[0] > shape[1]) ||
+ (strings.HasSuffix(t.Name(), "weight.lora_b") && shape[0] < shape[1]) {
+ shape[0], shape[1] = shape[1], shape[0]
+ t.SetRepacker(p.repack)
+ }
+
+ out = append(out, llm.Tensor{
+ Name: t.Name(),
+ Kind: t.Kind(),
+ Shape: t.Shape(),
+ WriterTo: t,
+ })
+ }
+
+ return out
+}
+
+func (p *gemma2Adapter) Replacements() []string {
+ return []string{
+ "base_model.model.", "",
+ "model.layers", "blk",
+ "self_attn.q_proj", "attn_q",
+ "self_attn.k_proj", "attn_k",
+ "self_attn.v_proj", "attn_v",
+ "self_attn.o_proj", "attn_output",
+ "mlp.gate_proj", "ffn_gate",
+ "mlp.down_proj", "ffn_down",
+ "mlp.up_proj", "ffn_up",
+ "lora_A.weight", "weight.lora_a",
+ "lora_B.weight", "weight.lora_b",
+ "lora_a", "weight.lora_a",
+ "lora_b", "weight.lora_b",
+ }
+}
+
+func (p *gemma2Adapter) repack(name string, data []float32, shape []uint64) ([]float32, error) {
+ dims := []int{int(shape[1]), int(shape[0])}
+
+ n := tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data))
+
+ if err := n.T(1, 0); err != nil {
+ return nil, err
+ }
+
+ if err := n.Reshape(dims...); err != nil {
+ return nil, err
+ }
+
+ if err := n.Transpose(); err != nil {
+ return nil, err
+ }
+
+ ts, err := native.SelectF32(n, 1)
+ if err != nil {
+ return nil, err
+ }
+
+ var f32s []float32
+ for _, t := range ts {
+ f32s = append(f32s, t...)
+ }
+
+ return f32s, nil
+}
diff --git a/convert/convert_llama.go b/convert/convert_llama.go
new file mode 100644
index 00000000..5dedb829
--- /dev/null
+++ b/convert/convert_llama.go
@@ -0,0 +1,213 @@
+package convert
+
+import (
+ "cmp"
+ "fmt"
+ "math"
+ "strings"
+
+ "github.com/pdevine/tensor"
+ "github.com/pdevine/tensor/native"
+
+ "github.com/ollama/ollama/llm"
+)
+
+type llamaModel struct {
+ ModelParameters
+ NLayers uint32 `json:"n_layers"`
+ NumHiddenLayers uint32 `json:"num_hidden_layers"`
+ NLayer uint32 `json:"n_layer"`
+ MaxPositionEmbeddings uint32 `json:"max_position_embeddings"`
+ NCtx uint32 `json:"n_ctx"`
+ HiddenSize uint32 `json:"hidden_size"`
+ NEmbd uint32 `json:"n_embd"`
+ IntermediateSize uint32 `json:"intermediate_size"`
+ NInner uint32 `json:"n_inner"`
+ NumAttentionHeads uint32 `json:"num_attention_heads"`
+ NHead uint32 `json:"n_head"`
+ NumKeyValueHeads uint32 `json:"num_key_value_heads"`
+ RopeTheta float32 `json:"rope_theta"`
+ RopeScaling struct {
+ Type string `json:"type"`
+ RopeType string `json:"rope_type"`
+ Factor float32 `json:"factor"`
+ LowFrequencyFactor float32 `json:"low_freq_factor"`
+ HighFrequencyFactor float32 `json:"high_freq_factor"`
+ OriginalMaxPositionalEmbeddings uint32 `json:"original_max_positional_embeddings"`
+
+ factors ropeFactor
+ } `json:"rope_scaling"`
+ RMSNormEPS float32 `json:"rms_norm_eps"`
+ LayerNormEPS float32 `json:"layer_norm_eps"`
+ LayerNormEpsilon float32 `json:"layer_norm_epsilon"`
+ NormEpsilon float32 `json:"norm_epsilon"`
+ HeadDim uint32 `json:"head_dim"`
+}
+
+var _ ModelConverter = (*llamaModel)(nil)
+
+func (p *llamaModel) KV(t *Tokenizer) llm.KV {
+ kv := p.ModelParameters.KV(t)
+ kv["general.architecture"] = "llama"
+ kv["llama.vocab_size"] = p.VocabSize
+
+ kv["llama.block_count"] = cmp.Or(p.NLayers, p.NumHiddenLayers, p.NLayer)
+
+ if contextLength := cmp.Or(p.MaxPositionEmbeddings, p.NCtx); contextLength > 0 {
+ kv["llama.context_length"] = contextLength
+ }
+
+ if embeddingLength := cmp.Or(p.HiddenSize, p.NEmbd); embeddingLength > 0 {
+ kv["llama.embedding_length"] = cmp.Or(p.HiddenSize, p.NEmbd)
+ }
+
+ if feedForwardLength := cmp.Or(p.IntermediateSize, p.NInner); feedForwardLength > 0 {
+ kv["llama.feed_forward_length"] = cmp.Or(p.IntermediateSize, p.NInner)
+ }
+
+ if headCount := cmp.Or(p.NumAttentionHeads, p.NHead); headCount > 0 {
+ kv["llama.attention.head_count"] = cmp.Or(p.NumAttentionHeads, p.NHead)
+ kv["llama.rope.dimension_count"] = p.HiddenSize / headCount
+ }
+
+ if p.RopeTheta > 0 {
+ kv["llama.rope.freq_base"] = p.RopeTheta
+ }
+
+ if p.RopeScaling.Type == "linear" {
+ kv["llama.rope.scaling.type"] = p.RopeScaling.Type
+ kv["llama.rope.scaling.factor"] = p.RopeScaling.Factor
+ } else if p.RopeScaling.RopeType == "llama3" {
+ dim := p.HiddenSize / p.NumAttentionHeads
+ for i := uint32(0); i < dim; i += 2 {
+ factor := cmp.Or(p.RopeScaling.Factor, 8.0)
+ factorLow := cmp.Or(p.RopeScaling.LowFrequencyFactor, 1.0)
+ factorHigh := cmp.Or(p.RopeScaling.HighFrequencyFactor, 4.0)
+
+ original := cmp.Or(p.RopeScaling.OriginalMaxPositionalEmbeddings, 8192)
+ lambdaLow := float32(original) / factorLow
+ lambdaHigh := float32(original) / factorHigh
+
+ lambda := 2 * math.Pi * math.Pow(float64(p.RopeTheta), float64(i)/float64(dim))
+ if lambda < float64(lambdaHigh) {
+ p.RopeScaling.factors = append(p.RopeScaling.factors, 1.0)
+ } else if lambda > float64(lambdaLow) {
+ p.RopeScaling.factors = append(p.RopeScaling.factors, factor)
+ } else {
+ smooth := (float32(original)/float32(lambda) - factorLow) / (factorHigh - factorLow)
+ p.RopeScaling.factors = append(p.RopeScaling.factors, 1.0/((1-smooth)/factor+smooth))
+ }
+ }
+ }
+
+ if p.NumKeyValueHeads > 0 {
+ kv["llama.attention.head_count_kv"] = p.NumKeyValueHeads
+ }
+
+ if p.RMSNormEPS > 0 {
+ kv["llama.attention.layer_norm_rms_epsilon"] = p.RMSNormEPS
+ }
+
+ if layerNormEpsilon := cmp.Or(p.LayerNormEPS, p.LayerNormEpsilon, p.NormEpsilon); layerNormEpsilon > 0 {
+ kv["llama.attention.layer_norm_epsilon"] = layerNormEpsilon
+ }
+
+ if p.HeadDim > 0 {
+ kv["llama.attention.key_length"] = p.HeadDim
+ kv["llama.attention.value_length"] = p.HeadDim
+ }
+
+ return kv
+}
+
+func (p *llamaModel) Tensors(ts []Tensor) []llm.Tensor {
+ var out []llm.Tensor
+
+ if p.RopeScaling.factors != nil {
+ out = append(out, llm.Tensor{
+ Name: "rope_freqs.weight",
+ Kind: 0,
+ Shape: []uint64{uint64(len(p.RopeScaling.factors))},
+ WriterTo: p.RopeScaling.factors,
+ })
+ }
+
+ for _, t := range ts {
+ if strings.HasSuffix(t.Name(), "attn_q.weight") ||
+ strings.HasSuffix(t.Name(), "attn_k.weight") {
+ t.SetRepacker(p.repack)
+ }
+
+ out = append(out, llm.Tensor{
+ Name: t.Name(),
+ Kind: t.Kind(),
+ Shape: t.Shape(),
+ WriterTo: t,
+ })
+ }
+
+ return out
+}
+
+func (p *llamaModel) Replacements() []string {
+ return []string{
+ "lm_head", "output",
+ "model.embed_tokens", "token_embd",
+ "model.norm", "output_norm",
+ "model.layers", "blk",
+ "input_layernorm", "attn_norm",
+ "self_attn.q_proj", "attn_q",
+ "self_attn.k_proj", "attn_k",
+ "self_attn.v_proj", "attn_v",
+ "self_attn.o_proj", "attn_output",
+ "mlp.gate_proj", "ffn_gate",
+ "mlp.down_proj", "ffn_down",
+ "mlp.up_proj", "ffn_up",
+ "post_attention_layernorm", "ffn_norm",
+ }
+}
+
+func (p *llamaModel) repack(name string, data []float32, shape []uint64) ([]float32, error) {
+ var dims []int
+ for _, dim := range shape {
+ dims = append(dims, int(dim))
+ }
+
+ var heads uint32
+ if strings.HasSuffix(name, "attn_q.weight") {
+ heads = p.NumAttentionHeads
+ } else if strings.HasSuffix(name, "attn_k.weight") {
+ heads = cmp.Or(p.NumKeyValueHeads, p.NumAttentionHeads)
+ } else {
+ return nil, fmt.Errorf("unknown tensor for repack: %s", name)
+ }
+
+ n := tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data))
+ if err := n.Reshape(append([]int{int(heads), 2, dims[0] / int(heads) / 2}, dims[1:]...)...); err != nil {
+ return nil, err
+ }
+
+ if err := n.T(0, 2, 1, 3); err != nil {
+ return nil, err
+ }
+
+ if err := n.Reshape(dims...); err != nil {
+ return nil, err
+ }
+
+ if err := n.Transpose(); err != nil {
+ return nil, err
+ }
+
+ ts, err := native.SelectF32(n, 1)
+ if err != nil {
+ return nil, err
+ }
+
+ var f32s []float32
+ for _, t := range ts {
+ f32s = append(f32s, t...)
+ }
+
+ return f32s, nil
+}
diff --git a/convert/convert_llama_adapter.go b/convert/convert_llama_adapter.go
new file mode 100644
index 00000000..08ddee10
--- /dev/null
+++ b/convert/convert_llama_adapter.go
@@ -0,0 +1,169 @@
+package convert
+
+import (
+ "cmp"
+ "strings"
+
+ "github.com/pdevine/tensor"
+ "github.com/pdevine/tensor/native"
+
+ "github.com/ollama/ollama/llm"
+)
+
+type llamaAdapter struct {
+ AdapterParameters
+ NumAttentionHeads uint32 `json:"num_attention_heads"`
+ NumKeyValueHeads uint32 `json:"num_key_value_heads"`
+}
+
+var _ AdapterConverter = (*llamaAdapter)(nil)
+
+func (p *llamaAdapter) KV(baseKV llm.KV) llm.KV {
+ kv := p.AdapterParameters.KV()
+ kv["general.architecture"] = "llama"
+ kv["llama.attention.head_count"] = baseKV["llama.attention.head_count"]
+ kv["llama.attention.head_count_kv"] = baseKV["llama.attention.head_count_kv"]
+
+ p.NumAttentionHeads = baseKV["llama.attention.head_count"].(uint32)
+
+ return kv
+}
+
+func (p *llamaAdapter) Tensors(ts []Tensor) []llm.Tensor {
+ var out []llm.Tensor
+ for _, t := range ts {
+ shape := t.Shape()
+ if (strings.HasSuffix(t.Name(), "weight.lora_a") && shape[0] > shape[1]) ||
+ (strings.HasSuffix(t.Name(), "weight.lora_b") && shape[0] < shape[1]) {
+ shape[0], shape[1] = shape[1], shape[0]
+ t.SetRepacker(p.repackAndTranspose)
+ } else {
+ t.SetRepacker(p.repack)
+ }
+
+ out = append(out, llm.Tensor{
+ Name: t.Name(),
+ Kind: t.Kind(),
+ Shape: shape,
+ WriterTo: t,
+ })
+ }
+
+ return out
+}
+
+func (p *llamaAdapter) Replacements() []string {
+ return []string{
+ "base_model.model.", "",
+ "model.layers", "blk",
+ "self_attn.q_proj", "attn_q",
+ "self_attn.k_proj", "attn_k",
+ "self_attn.v_proj", "attn_v",
+ "self_attn.o_proj", "attn_output",
+ "mlp.gate_proj", "ffn_gate",
+ "mlp.down_proj", "ffn_down",
+ "mlp.up_proj", "ffn_up",
+ "lora_A.weight", "weight.lora_a",
+ "lora_B.weight", "weight.lora_b",
+ "lora_a", "weight.lora_a",
+ "lora_b", "weight.lora_b",
+ }
+}
+
+func (p *llamaAdapter) repack(name string, data []float32, shape []uint64) ([]float32, error) {
+ dims := []int{int(shape[1]), int(shape[0])}
+
+ var heads uint32
+ if strings.HasSuffix(name, "attn_q.weight.lora_a") {
+ heads = p.NumAttentionHeads
+ } else if strings.HasSuffix(name, "attn_k.weight.lora_a") {
+ heads = cmp.Or(p.NumKeyValueHeads, p.NumAttentionHeads)
+ } else {
+ return data, nil
+ }
+
+ n := tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data))
+
+ if err := n.Reshape(append([]int{int(heads), 2, dims[0] / int(heads) / 2}, dims[1:]...)...); err != nil {
+ return nil, err
+ }
+
+ if err := n.T(0, 2, 1, 3); err != nil {
+ return nil, err
+ }
+
+ if err := n.Reshape(dims...); err != nil {
+ return nil, err
+ }
+
+ if err := n.Transpose(); err != nil {
+ return nil, err
+ }
+
+ ts, err := native.SelectF32(n, 1)
+ if err != nil {
+ return nil, err
+ }
+
+ var f32s []float32
+ for _, t := range ts {
+ f32s = append(f32s, t...)
+ }
+
+ return f32s, nil
+}
+
+func (p *llamaAdapter) repackAndTranspose(name string, data []float32, shape []uint64) ([]float32, error) {
+ dims := []int{int(shape[1]), int(shape[0])}
+
+ n := tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data))
+
+ var heads uint32
+ if strings.HasSuffix(name, "attn_q.weight.lora_a") {
+ heads = p.NumAttentionHeads
+ } else if strings.HasSuffix(name, "attn_k.weight.lora_a") {
+ heads = cmp.Or(p.NumKeyValueHeads, p.NumAttentionHeads)
+ }
+
+ if heads > 0 {
+ if err := n.Reshape(append([]int{int(heads), 2, dims[0] / int(heads) / 2}, dims[1:]...)...); err != nil {
+ return nil, err
+ }
+
+ if err := n.T(0, 2, 1, 3); err != nil {
+ return nil, err
+ }
+
+ if err := n.Reshape(dims...); err != nil {
+ return nil, err
+ }
+
+ if err := n.Transpose(); err != nil {
+ return nil, err
+ }
+ }
+
+ if err := n.T(1, 0); err != nil {
+ return nil, err
+ }
+
+ if err := n.Reshape(dims...); err != nil {
+ return nil, err
+ }
+
+ if err := n.Transpose(); err != nil {
+ return nil, err
+ }
+
+ ts, err := native.SelectF32(n, 1)
+ if err != nil {
+ return nil, err
+ }
+
+ var f32s []float32
+ for _, t := range ts {
+ f32s = append(f32s, t...)
+ }
+
+ return f32s, nil
+}
diff --git a/convert/convert_mixtral.go b/convert/convert_mixtral.go
new file mode 100644
index 00000000..43b7c8b1
--- /dev/null
+++ b/convert/convert_mixtral.go
@@ -0,0 +1,94 @@
+package convert
+
+import (
+ "fmt"
+ "io"
+ "slices"
+ "strings"
+
+ "github.com/ollama/ollama/llm"
+)
+
+type mixtralModel struct {
+ llamaModel
+ NumLocalExperts uint32 `json:"num_local_experts"`
+ NumExpertsPerToken uint32 `json:"num_experts_per_tok"`
+}
+
+func (p *mixtralModel) KV(t *Tokenizer) llm.KV {
+ kv := p.llamaModel.KV(t)
+
+ if p.NumLocalExperts > 0 {
+ kv["llama.expert_count"] = p.NumLocalExperts
+ }
+
+ if p.NumExpertsPerToken > 0 {
+ kv["llama.expert_used_count"] = p.NumExpertsPerToken
+ }
+
+ return kv
+}
+
+func (p *mixtralModel) Tensors(ts []Tensor) []llm.Tensor {
+ oldnew := []string{
+ "model.layers", "blk",
+ "w1", "ffn_gate_exps",
+ "w2", "ffn_down_exps",
+ "w3", "ffn_up_exps",
+ }
+
+ for i := range p.NumLocalExperts {
+ oldnew = append(oldnew, fmt.Sprintf(".block_sparse_moe.experts.%d.", i), ".")
+ }
+
+ // group experts of the same layer (model.layers.%d) and type (w[123]) into a single tensor
+ namer := strings.NewReplacer(oldnew...)
+ experts := make(map[string]experts)
+
+ // merge experts into a single tensor while removing them from ts
+ ts = slices.DeleteFunc(ts, func(t Tensor) bool {
+ if !strings.Contains(t.Name(), ".block_sparse_moe.experts.") {
+ return false
+ }
+
+ name := namer.Replace(t.Name())
+ experts[name] = append(experts[name], t)
+ return true
+ })
+
+ var out []llm.Tensor
+ for n, e := range experts {
+ // TODO(mxyng): sanity check experts
+ out = append(out, llm.Tensor{
+ Name: n,
+ Kind: e[0].Kind(),
+ Shape: append([]uint64{uint64(len(e))}, e[0].Shape()...),
+ WriterTo: e,
+ })
+ }
+
+ return append(out, p.llamaModel.Tensors(ts)...)
+}
+
+func (p *mixtralModel) Replacements() []string {
+ return append(
+ p.llamaModel.Replacements(),
+ "block_sparse_moe.gate", "ffn_gate_inp",
+ )
+}
+
+type experts []Tensor
+
+func (e experts) WriteTo(w io.Writer) (int64, error) {
+ // TODO(mxyng): experts _should_ be numerically sorted by expert but this should check
+ for _, t := range e {
+ // the canonical merged experts tensor stacks all experts along a new, 0 axis,
+ // e.g. `tensor.Stack(0, e[0], e[1:]...)`, which requires allocating temporary buffers
+ // this accomplishes the same thing by writing each expert tensor in sequence
+ if _, err := t.WriteTo(w); err != nil {
+ return 0, err
+ }
+ }
+
+ return 0, nil
+}
diff --git a/convert/convert_phi3.go b/convert/convert_phi3.go
new file mode 100644
index 00000000..3de0d404
--- /dev/null
+++ b/convert/convert_phi3.go
@@ -0,0 +1,123 @@
+package convert
+
+import (
+ "cmp"
+ "encoding/binary"
+ "io"
+ "math"
+ "strings"
+ "sync"
+
+ "github.com/ollama/ollama/llm"
+)
+
+type phi3Model struct {
+ ModelParameters
+ NumHiddenLayers uint32 `json:"num_hidden_layers"`
+ NLayers uint32 `json:"n_layers"`
+ HiddenSize uint32 `json:"hidden_size"`
+ NEmbd uint32 `json:"n_embd"`
+ IntermediateSize uint32 `json:"intermediate_size"`
+ NumAttentionHeads uint32 `json:"num_attention_heads"`
+ NHead uint32 `json:"n_head"`
+ NumKeyValueHeads uint32 `json:"num_key_value_heads"`
+ NHeadKV uint32 `json:"n_head_kv"`
+ RopeTheta float32 `json:"rope_theta"`
+ RopeScaling struct {
+ Type string `json:"type"`
+ LongFactor ropeFactor `json:"long_factor"`
+ ShortFactor ropeFactor `json:"short_factor"`
+ } `json:"rope_scaling"`
+ RMSNormEPS float32 `json:"rms_norm_eps"`
+ NPositions uint32 `json:"n_positions"`
+ MaxPositionEmbeddings uint32 `json:"max_position_embeddings"`
+ OriginalMaxPositionEmbeddings uint32 `json:"original_max_position_embeddings"`
+ SlidingWindow uint32 `json:"sliding_window"`
+}
+
+var _ ModelConverter = (*phi3Model)(nil)
+
+func (p *phi3Model) KV(t *Tokenizer) llm.KV {
+ kv := p.ModelParameters.KV(t)
+ kv["general.architecture"] = "phi3"
+ kv["phi3.context_length"] = p.MaxPositionEmbeddings
+ kv["phi3.embedding_length"] = cmp.Or(p.HiddenSize, p.NEmbd)
+ kv["phi3.feed_forward_length"] = p.IntermediateSize
+ kv["phi3.block_count"] = cmp.Or(p.NumHiddenLayers, p.NLayers)
+ kv["phi3.attention.head_count"] = cmp.Or(p.NumAttentionHeads, p.NHead)
+ kv["phi3.attention.head_count_kv"] = cmp.Or(p.NumKeyValueHeads, p.NHeadKV)
+ kv["phi3.attention.layer_norm_rms_epsilon"] = p.RMSNormEPS
+ kv["phi3.rope.dimension_count"] = p.HiddenSize / cmp.Or(p.NumAttentionHeads, p.NHead)
+ kv["phi3.rope.freq_base"] = p.RopeTheta
+ kv["phi3.rope.scaling.original_context_length"] = p.OriginalMaxPositionEmbeddings
+ kv["phi3.attention.sliding_window"] = p.SlidingWindow
+
+ scale := float64(p.MaxPositionEmbeddings) / float64(p.OriginalMaxPositionEmbeddings)
+
+ switch p.RopeScaling.Type {
+ case "":
+ // no scaling
+ case "su", "longrope":
+ kv["phi3.rope.scaling.attn_factor"] = float32(max(math.Sqrt(1+math.Log(scale)/math.Log(float64(p.OriginalMaxPositionEmbeddings))), 1.0))
+ case "yarn":
+ kv["phi3.rope.scaling.attn_factor"] = float32(max(0.1*math.Log(scale)+1.0, 1.0))
+ default:
+ panic("unknown rope scaling type")
+ }
+
+ return kv
+}
+
+func (p *phi3Model) Tensors(ts []Tensor) []llm.Tensor {
+ var addRopeFactors sync.Once
+
+ out := make([]llm.Tensor, 0, len(ts)+2)
+ for _, t := range ts {
+ if strings.HasPrefix(t.Name(), "blk.0.") {
+ addRopeFactors.Do(func() {
+ out = append(out, llm.Tensor{
+ Name: "rope_factors_long.weight",
+ Kind: 0,
+ Shape: []uint64{uint64(len(p.RopeScaling.LongFactor))},
+ WriterTo: p.RopeScaling.LongFactor,
+ }, llm.Tensor{
+ Name: "rope_factors_short.weight",
+ Kind: 0,
+ Shape: []uint64{uint64(len(p.RopeScaling.ShortFactor))},
+ WriterTo: p.RopeScaling.ShortFactor,
+ })
+ })
+ }
+
+ out = append(out, llm.Tensor{
+ Name: t.Name(),
+ Kind: t.Kind(),
+ Shape: t.Shape(),
+ WriterTo: t,
+ })
+ }
+
+ return out
+}
+
+func (p *phi3Model) Replacements() []string {
+ return []string{
+ "lm_head", "output",
+ "model.embed_tokens", "token_embd",
+ "model.norm", "output_norm",
+ "model.layers", "blk",
+ "input_layernorm", "attn_norm",
+ "self_attn.qkv_proj", "attn_qkv",
+ "self_attn.o_proj", "attn_output",
+ "mlp.down_proj", "ffn_down",
+ "mlp.gate_up_proj", "ffn_up",
+ "post_attention_layernorm", "ffn_norm",
+ }
+}
+
+type ropeFactor []float32
+
+func (r ropeFactor) WriteTo(w io.Writer) (int64, error) {
+ err := binary.Write(w, binary.LittleEndian, r)
+ return 0, err
+}
diff --git a/convert/convert_test.go b/convert/convert_test.go
new file mode 100644
index 00000000..2969673d
--- /dev/null
+++ b/convert/convert_test.go
@@ -0,0 +1,476 @@
+package convert
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "encoding/binary"
+ "encoding/hex"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io"
+ "io/fs"
+ "log/slog"
+ "math"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+
+ "golang.org/x/exp/maps"
+
+ "github.com/ollama/ollama/llm"
+)
+
+type tensorData struct {
+ Offsets []int `json:"data_offsets"`
+ Type string `json:"dtype"`
+ Shape []int `json:"shape"`
+}
+
+func convertFull(t *testing.T, fsys fs.FS) (*os.File, llm.KV, llm.Tensors) {
+ t.Helper()
+
+ f, err := os.CreateTemp(t.TempDir(), "f16")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+
+ if err := ConvertModel(fsys, f); err != nil {
+ t.Fatal(err)
+ }
+
+ r, err := os.Open(f.Name())
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { r.Close() })
+
+ m, _, err := llm.DecodeGGML(r, math.MaxInt)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := r.Seek(0, io.SeekStart); err != nil {
+ t.Fatal(err)
+ }
+
+ return r, m.KV(), m.Tensors()
+}
+
+func generateResultsJSON(t *testing.T, f *os.File, kv llm.KV, tensors llm.Tensors) map[string]string {
+ actual := make(map[string]string)
+ for k, v := range kv {
+ if s, ok := v.(json.Marshaler); !ok {
+ actual[k] = fmt.Sprintf("%v", v)
+ } else {
+ bts, err := json.Marshal(s)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ actual[k] = fmt.Sprintf("%x", sha256.Sum256(bts))
+ }
+ }
+
+ for _, tensor := range tensors.Items {
+ sha256sum := sha256.New()
+ sr := io.NewSectionReader(f, int64(tensors.Offset+tensor.Offset), int64(tensor.Size()))
+ if _, err := io.Copy(sha256sum, sr); err != nil {
+ t.Fatal(err)
+ }
+
+ actual[tensor.Name] = hex.EncodeToString(sha256sum.Sum(nil))
+ }
+
+ return actual
+}
+
+func TestMain(m *testing.M) {
+ var level slog.Level
+ flag.TextVar(&level, "level", slog.LevelInfo, "log level")
+ flag.Parse()
+ slog.SetLogLoggerLevel(level)
+ os.Exit(m.Run())
+}
+
+func TestConvertModel(t *testing.T) {
+ cases := []string{
+ "Meta-Llama-3-8B-Instruct",
+ "Meta-Llama-3.1-8B-Instruct",
+ "Mistral-7B-Instruct-v0.2",
+ "Mixtral-8x7B-Instruct-v0.1",
+ "gemma-2b-it",
+ "gemma-2-2b-it",
+ // microsoft/Phi-3-mini-128-instruct@d548c233192db00165d842bf8edff054bb3212f8
+ "Phi-3-mini-128k-instruct",
+ "all-MiniLM-L6-v2",
+ "gemma-2-9b-it",
+ }
+
+ for i := range cases {
+ tt := cases[i]
+ t.Run(tt, func(t *testing.T) {
+ t.Parallel()
+
+ p := filepath.Join("testdata", tt)
+ if testing.Short() {
+ t.Skip("skipping in short mode")
+ } else if _, err := os.Stat(p); err != nil {
+ t.Skipf("%s not found", p)
+ }
+
+ f, kv, tensors := convertFull(t, os.DirFS(p))
+ actual := generateResultsJSON(t, f, kv, tensors)
+
+ expectFile, err := os.Open(filepath.Join("testdata", fmt.Sprintf("%s.json", tt)))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var expect map[string]string
+ if err := json.NewDecoder(expectFile).Decode(&expect); err != nil {
+ t.Fatal(err)
+ }
+
+ keys := maps.Keys(expect)
+ slices.Sort(keys)
+ for _, k := range keys {
+ if v, ok := actual[k]; !ok {
+ t.Errorf("missing %s", k)
+ } else if v != expect[k] {
+ t.Errorf("unexpected %s: want %s, got %s", k, expect[k], v)
+ }
+ }
+ })
+ }
+}
+
+func TestConvertInvalidTensorNames(t *testing.T) {
+ f, err := os.CreateTemp(t.TempDir(), "testmodel")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+
+ tempDir := t.TempDir()
+
+ td := map[string]*tensorData{}
+ offset := 4096
+
+ td["model.layers.0.self_attn.q_proj.weight"] = &tensorData{
+ Offsets: []int{0, offset},
+ Type: "F32",
+ Shape: []int{4096, 4096},
+ }
+ td["blk.0.attn_q.weight"] = &tensorData{
+ Offsets: []int{offset, offset * 2},
+ Type: "F32",
+ Shape: []int{4096, 4096},
+ }
+ generateSafetensorTestData(t, tempDir, td)
+
+ err = ConvertModel(os.DirFS(tempDir), f)
+ if err == nil || !strings.HasPrefix(err.Error(), "duplicate tensor name") {
+ t.Errorf("expected error but didn't get one")
+ }
+}
+
+func TestConvertInvalidDatatype(t *testing.T) {
+ f, err := os.CreateTemp(t.TempDir(), "testmodel")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+
+ tempDir := t.TempDir()
+
+ td := map[string]*tensorData{}
+ offset := 4096 * 14336
+
+ td["model.layers.0.mlp.down_proj.weight"] = &tensorData{
+ Offsets: []int{0, offset},
+ Type: "I8",
+ Shape: []int{4096, 14336},
+ }
+ td["model.layers.0.mlp.down_proj.weight_format"] = &tensorData{
+ Offsets: []int{offset, offset},
+ Type: "U8",
+ Shape: []int{},
+ }
+ generateSafetensorTestData(t, tempDir, td)
+
+ err = ConvertModel(os.DirFS(tempDir), f)
+ if err == nil || err.Error() != "unsupported safetensors model" {
+ t.Errorf("expected error but didn't get one")
+ }
+}
+
+func generateSafetensorTestData(t *testing.T, tempDir string, tensorData map[string]*tensorData) {
+ data, err := json.Marshal(tensorData)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var buf bytes.Buffer
+
+ l := int64(len(data))
+ err = binary.Write(&buf, binary.LittleEndian, l)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = buf.Write(data)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fdata, err := os.Create(filepath.Join(tempDir, "model-00001-of-00001.safetensors"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer fdata.Close()
+
+ _, err = fdata.Write(buf.Bytes())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ configData := `
+{
+ "architectures": [
+ "LlamaForCausalLM"
+ ]
+}
+`
+
+ f, err := os.Create(filepath.Join(tempDir, "config.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+
+ _, err = f.WriteString(configData)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ tokenizerData := `
+{
+}
+`
+
+ f, err = os.Create(filepath.Join(tempDir, "tokenizer.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+
+ _, err = f.WriteString(tokenizerData)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestConvertAdapter(t *testing.T) {
+ type AdapterCase struct {
+ Name string
+ BaseKV map[string]any
+ Expected map[string]string
+ }
+
+ cases := []AdapterCase{
+ {
+ Name: "discollama",
+ BaseKV: map[string]any{
+ "general.architecture": "llama",
+ "llama.attention.head_count": uint32(32),
+ "llama.attention.head_count_kv": uint32(8),
+ },
+ Expected: map[string]string{
+ "general.architecture": "llama",
+ "general.file_type": "1",
+ "general.parameter_count": "106496",
+ "general.type": "adapter",
+ "general.version": "v0.2",
+ "adapter.lora.alpha": "16",
+ "adapter.type": "lora",
+ "llama.attention.head_count": "32",
+ "llama.attention.head_count_kv": "8",
+ "blk.31.attn_q.weight.lora_a": "0eb3318b02cd313429bcc7621b539fdbb10240fea190c56c9e5f93fcd37a4e50",
+ "blk.31.attn_q.weight.lora_b": "0eb3318b02cd313429bcc7621b539fdbb10240fea190c56c9e5f93fcd37a4e50",
+ "blk.31.attn_v.weight.lora_a": "0eb3318b02cd313429bcc7621b539fdbb10240fea190c56c9e5f93fcd37a4e50",
+ "blk.31.attn_v.weight.lora_b": "071dcafe89df065d6e1c935ecb8fdf6479b3c202eb912e7da938597673ff5857",
+ },
+ },
+ }
+
+ for _, c := range cases {
+ t.Run(c.Name, func(t *testing.T) {
+ t.Parallel()
+
+ f, err := os.CreateTemp(t.TempDir(), "f16")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+
+ tempDir := t.TempDir()
+ generateLoraTestData(t, tempDir)
+
+ if err = ConvertAdapter(os.DirFS(tempDir), f, c.BaseKV); err != nil {
+ t.Fatal(err)
+ }
+
+ r, err := os.Open(f.Name())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer r.Close()
+
+ m, _, err := llm.DecodeGGML(r, math.MaxInt)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := r.Seek(0, io.SeekStart); err != nil {
+ t.Fatal(err)
+ }
+
+ actual := generateResultsJSON(t, r, m.KV(), m.Tensors())
+
+ keys := maps.Keys(c.Expected)
+ slices.Sort(keys)
+ for _, k := range keys {
+ if v, ok := actual[k]; !ok {
+ t.Errorf("missing %s", k)
+ } else if v != c.Expected[k] {
+ t.Errorf("unexpected %s: want %s, got %s", k, c.Expected[k], v)
+ }
+ }
+ })
+ }
+}
+
+func generateLoraTestData(t *testing.T, tempDir string) {
+ offset := 4096 * 8 * 4
+
+ td := map[string]*tensorData{"__metadata__": nil}
+ td["model.layers.31.self_attn.q_proj.lora_a"] = &tensorData{
+ Offsets: []int{0, offset},
+ Type: "F32",
+ Shape: []int{4096, 8},
+ }
+ td["model.layers.31.self_attn.q_proj.lora_b"] = &tensorData{
+ Offsets: []int{offset, offset * 2},
+ Type: "F32",
+ Shape: []int{8, 4096},
+ }
+ td["model.layers.31.self_attn.v_proj.lora_a"] = &tensorData{
+ Offsets: []int{offset * 2, offset * 3},
+ Type: "F32",
+ Shape: []int{4096, 8},
+ }
+ td["model.layers.31.self_attn.v_proj.lora_b"] = &tensorData{
+ Offsets: []int{offset * 3, offset*3 + 8*1024*4},
+ Type: "F32",
+ Shape: []int{8, 1024},
+ }
+
+ data, err := json.Marshal(td)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var buf bytes.Buffer
+
+ l := int64(len(data))
+ err = binary.Write(&buf, binary.LittleEndian, l)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = buf.Write(data)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // write some data for the tensors
+
+ ones := make([]float32, 4096*8)
+ for i := range ones {
+ ones[i] = float32(1)
+ }
+
+ for range 3 {
+ err = binary.Write(&buf, binary.LittleEndian, ones)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ ones = make([]float32, 1024*8)
+ for i := range ones {
+ ones[i] = float32(1)
+ }
+
+ err = binary.Write(&buf, binary.LittleEndian, ones)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fdata, err := os.Create(filepath.Join(tempDir, "adapters.safetensors"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer fdata.Close()
+
+ _, err = fdata.Write(buf.Bytes())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ configData := `
+{
+ "adapter_path": "adapters-test",
+ "batch_size": 8,
+ "config": "config-tiny.json",
+ "data": "../discollama-completion",
+ "grad_checkpoint": null,
+ "iters": 1000,
+ "learning_rate": 1e-05,
+ "lora_layers": 1,
+ "lora_parameters": {
+ "rank": 8,
+ "alpha": 16,
+ "dropout": 0.0,
+ "scale": 2.0
+ },
+ "lr_schedule": null,
+ "max_seq_length": 2048,
+ "model": "/Users/pdevine/git/Meta-Llama-3-8B-Instruct",
+ "resume_adapter_file": null,
+ "save_every": 100,
+ "seed": 0,
+ "steps_per_eval": 200,
+ "steps_per_report": 10,
+ "test": false,
+ "test_batches": 500,
+ "train": true,
+ "use_dora": false,
+ "val_batches": 25
+}
+`
+ f, err := os.Create(filepath.Join(tempDir, "adapter_config.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+
+ _, err = f.WriteString(configData)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/convert/fs.go b/convert/fs.go
new file mode 100644
index 00000000..31132dbe
--- /dev/null
+++ b/convert/fs.go
@@ -0,0 +1,58 @@
+package convert
+
+import (
+ "archive/zip"
+ "errors"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+)
+
+type ZipReader struct {
+ r *zip.Reader
+ p string
+
+ // limit is the maximum size of a file that can be read directly
+ // from the zip archive. Files larger than this size will be extracted
+ limit int64
+}
+
+func NewZipReader(r *zip.Reader, p string, limit int64) fs.FS {
+ return &ZipReader{r, p, limit}
+}
+
+func (z *ZipReader) Open(name string) (fs.File, error) {
+ r, err := z.r.Open(name)
+ if err != nil {
+ return nil, err
+ }
+ defer r.Close()
+
+ if fi, err := r.Stat(); err != nil {
+ return nil, err
+ } else if fi.Size() < z.limit {
+ return r, nil
+ }
+
+ if !filepath.IsLocal(name) {
+ return nil, zip.ErrInsecurePath
+ }
+
+ n := filepath.Join(z.p, name)
+ if _, err := os.Stat(n); errors.Is(err, os.ErrNotExist) {
+ w, err := os.Create(n)
+ if err != nil {
+ return nil, err
+ }
+ defer w.Close()
+
+ if _, err := io.Copy(w, r); err != nil {
+ return nil, err
+ }
+ } else if err != nil {
+ return nil, err
+ }
+
+ return os.Open(n)
+}
diff --git a/convert/reader.go b/convert/reader.go
new file mode 100644
index 00000000..c1218e66
--- /dev/null
+++ b/convert/reader.go
@@ -0,0 +1,86 @@
+package convert
+
+import (
+ "errors"
+ "io"
+ "io/fs"
+ "strings"
+)
+
+type Tensor interface {
+ Name() string
+ Shape() []uint64
+ Kind() uint32
+ SetRepacker(repacker)
+ WriteTo(io.Writer) (int64, error)
+}
+
+type tensorBase struct {
+ name string
+ shape []uint64
+ repacker
+}
+
+func (t tensorBase) Name() string {
+ return t.name
+}
+
+func (t tensorBase) Shape() []uint64 {
+ return t.shape
+}
+
+const (
+ tensorKindF32 uint32 = iota
+ tensorKindF16
+)
+
+func (t tensorBase) Kind() uint32 {
+ if strings.HasSuffix(t.name, ".ffn_gate_inp.weight") ||
+ t.name == "token_types.weight" {
+ // these tensors are always F32
+ return 0
+ }
+
+ switch len(t.shape) {
+ case 0:
+ panic("invalid tensor shape")
+ case 1:
+ return tensorKindF32
+ default:
+ return tensorKindF16
+ }
+}
+
+func (t *tensorBase) SetRepacker(fn repacker) {
+ t.repacker = fn
+}
+
+type repacker func(string, []float32, []uint64) ([]float32, error)
+
+func parseTensors(fsys fs.FS, replacer *strings.Replacer) ([]Tensor, error) {
+ patterns := []struct {
+ Pattern string
+ Func func(fs.FS, *strings.Replacer, ...string) ([]Tensor, error)
+ }{
+ {"model-*-of-*.safetensors", parseSafetensors},
+ {"model.safetensors", parseSafetensors},
+ {"adapters.safetensors", parseSafetensors},
+ {"adapter_model.safetensors", parseSafetensors},
+ {"pytorch_model-*-of-*.bin", parseTorch},
+ {"pytorch_model.bin", parseTorch},
+ {"consolidated.*.pth", parseTorch},
+ }
+
+ for _, pattern := range patterns {
+ matches, err := fs.Glob(fsys, pattern.Pattern)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(matches) > 0 {
+ return pattern.Func(fsys, replacer, matches...)
+ }
+ }
+
+ return nil, errors.New("unknown tensor format")
+}
diff --git a/convert/reader_safetensors.go b/convert/reader_safetensors.go
new file mode 100644
index 00000000..b21d219c
--- /dev/null
+++ b/convert/reader_safetensors.go
@@ -0,0 +1,163 @@
+package convert
+
+import (
+ "bytes"
+ "encoding/binary"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "slices"
+ "strings"
+
+ "github.com/d4l3k/go-bfloat16"
+ "github.com/x448/float16"
+ "golang.org/x/exp/maps"
+)
+
+type safetensorMetadata struct {
+ Type string `json:"dtype"`
+ Shape []uint64 `json:"shape"`
+ Offsets []int64 `json:"data_offsets"`
+}
+
+func parseSafetensors(fsys fs.FS, replacer *strings.Replacer, ps ...string) ([]Tensor, error) {
+ var ts []Tensor
+ for _, p := range ps {
+ f, err := fsys.Open(p)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ var n int64
+ if err := binary.Read(f, binary.LittleEndian, &n); err != nil {
+ return nil, err
+ }
+
+ b := bytes.NewBuffer(make([]byte, 0, n))
+ if _, err = io.CopyN(b, f, n); err != nil {
+ return nil, err
+ }
+
+ var headers map[string]safetensorMetadata
+ if err := json.NewDecoder(b).Decode(&headers); err != nil {
+ return nil, err
+ }
+
+ keys := maps.Keys(headers)
+ slices.Sort(keys)
+
+ names := make(map[string]struct{}, len(keys))
+
+ for _, key := range keys {
+ if value := headers[key]; value.Type != "" {
+ // bitsandbytes quantized models are unsupported
+ if len(value.Shape) == 0 {
+ return nil, errors.New("unsupported safetensors model")
+ }
+ ggufName := replacer.Replace(key)
+ if _, ok := names[ggufName]; ok {
+ return nil, fmt.Errorf("duplicate tensor name '%s' was found for this model", ggufName)
+ }
+ names[ggufName] = struct{}{}
+ ts = append(ts, safetensor{
+ fs: fsys,
+ path: p,
+ dtype: value.Type,
+ offset: safetensorsPad(n, value.Offsets[0]),
+ size: safetensorsPad(n, value.Offsets[1]) - safetensorsPad(n, value.Offsets[0]),
+ tensorBase: &tensorBase{
+ name: ggufName,
+ shape: value.Shape,
+ },
+ })
+ }
+ }
+ }
+
+ return ts, nil
+}
+
+// safetensorsPad returns the padded size of the safetensors file given a length n and offset s
+func safetensorsPad(n, offset int64) int64 {
+ return 8 + n + offset
+}
+
+type safetensor struct {
+ fs fs.FS
+ path string
+ dtype string
+ offset int64
+ size int64
+ *tensorBase
+}
+
+func (st safetensor) WriteTo(w io.Writer) (int64, error) {
+ f, err := st.fs.Open(st.path)
+ if err != nil {
+ return 0, err
+ }
+ defer f.Close()
+
+ if seeker, ok := f.(io.Seeker); ok {
+ if _, err := seeker.Seek(st.offset, io.SeekStart); err != nil {
+ return 0, err
+ }
+ } else {
+ if _, err := io.CopyN(io.Discard, f, st.offset); err != nil {
+ return 0, err
+ }
+ }
+
+ var f32s []float32
+ switch st.dtype {
+ case "F32":
+ f32s = make([]float32, st.size/4)
+ if err = binary.Read(f, binary.LittleEndian, f32s); err != nil {
+ return 0, err
+ }
+ case "F16":
+ u16s := make([]uint16, st.size/2)
+ if err = binary.Read(f, binary.LittleEndian, u16s); err != nil {
+ return 0, err
+ }
+
+ f32s = make([]float32, len(u16s))
+ for i := range u16s {
+ f32s[i] = float16.Frombits(u16s[i]).Float32()
+ }
+
+ case "BF16":
+ u8s := make([]uint8, st.size)
+ if err = binary.Read(f, binary.LittleEndian, u8s); err != nil {
+ return 0, err
+ }
+
+ f32s = bfloat16.DecodeFloat32(u8s)
+ default:
+ return 0, fmt.Errorf("unknown data type: %s", st.dtype)
+ }
+
+ if st.repacker != nil {
+ f32s, err = st.repacker(st.Name(), f32s, st.Shape())
+ if err != nil {
+ return 0, err
+ }
+ }
+
+ switch st.Kind() {
+ case tensorKindF32:
+ return 0, binary.Write(w, binary.LittleEndian, f32s)
+ case tensorKindF16:
+ f16s := make([]uint16, len(f32s))
+ for i := range f32s {
+ f16s[i] = float16.Fromfloat32(f32s[i]).Bits()
+ }
+
+ return 0, binary.Write(w, binary.LittleEndian, f16s)
+ default:
+ return 0, fmt.Errorf("unknown storage type: %d", st.Kind())
+ }
+}
diff --git a/convert/reader_torch.go b/convert/reader_torch.go
new file mode 100644
index 00000000..1b3e1c9f
--- /dev/null
+++ b/convert/reader_torch.go
@@ -0,0 +1,48 @@
+package convert
+
+import (
+ "io"
+ "io/fs"
+ "strings"
+
+ "github.com/nlpodyssey/gopickle/pytorch"
+ "github.com/nlpodyssey/gopickle/types"
+)
+
+func parseTorch(fsys fs.FS, replacer *strings.Replacer, ps ...string) ([]Tensor, error) {
+ var ts []Tensor
+ for _, p := range ps {
+ pt, err := pytorch.Load(p)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, k := range pt.(*types.Dict).Keys() {
+ t := pt.(*types.Dict).MustGet(k)
+
+ var shape []uint64
+ for dim := range t.(*pytorch.Tensor).Size {
+ shape = append(shape, uint64(dim))
+ }
+
+ ts = append(ts, torch{
+ storage: t.(*pytorch.Tensor).Source,
+ tensorBase: &tensorBase{
+ name: replacer.Replace(k.(string)),
+ shape: shape,
+ },
+ })
+ }
+ }
+
+ return ts, nil
+}
+
+type torch struct {
+ storage pytorch.StorageInterface
+ *tensorBase
+}
+
+func (pt torch) WriteTo(w io.Writer) (int64, error) {
+ return 0, nil
+}
diff --git a/convert/sentencepiece/sentencepiece_model.pb.go b/convert/sentencepiece/sentencepiece_model.pb.go
new file mode 100644
index 00000000..5c8db9bc
--- /dev/null
+++ b/convert/sentencepiece/sentencepiece_model.pb.go
@@ -0,0 +1,1497 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.!
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// protoc-gen-go v1.32.0
+// protoc v4.25.2
+// source: sentencepiece_model.proto
+
+package sentencepiece
+
+import (
+ protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+ protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+ reflect "reflect"
+ sync "sync"
+)
+
+const (
+ // Verify that this generated code is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+ // Verify that runtime/protoimpl is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// Model type. only have UNIGRAM now.
+type TrainerSpec_ModelType int32
+
+const (
+ TrainerSpec_UNIGRAM TrainerSpec_ModelType = 1 // Unigram language model with dynamic algorithm
+ TrainerSpec_BPE TrainerSpec_ModelType = 2 // Byte Pair Encoding
+ TrainerSpec_WORD TrainerSpec_ModelType = 3 // Delimitered by whitespace.
+ TrainerSpec_CHAR TrainerSpec_ModelType = 4 // tokenizes into character sequence
+)
+
+// Enum value maps for TrainerSpec_ModelType.
+var (
+ TrainerSpec_ModelType_name = map[int32]string{
+ 1: "UNIGRAM",
+ 2: "BPE",
+ 3: "WORD",
+ 4: "CHAR",
+ }
+ TrainerSpec_ModelType_value = map[string]int32{
+ "UNIGRAM": 1,
+ "BPE": 2,
+ "WORD": 3,
+ "CHAR": 4,
+ }
+)
+
+func (x TrainerSpec_ModelType) Enum() *TrainerSpec_ModelType {
+ p := new(TrainerSpec_ModelType)
+ *p = x
+ return p
+}
+
+func (x TrainerSpec_ModelType) String() string {
+ return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (TrainerSpec_ModelType) Descriptor() protoreflect.EnumDescriptor {
+ return file_sentencepiece_model_proto_enumTypes[0].Descriptor()
+}
+
+func (TrainerSpec_ModelType) Type() protoreflect.EnumType {
+ return &file_sentencepiece_model_proto_enumTypes[0]
+}
+
+func (x TrainerSpec_ModelType) Number() protoreflect.EnumNumber {
+ return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Do not use.
+func (x *TrainerSpec_ModelType) UnmarshalJSON(b []byte) error {
+ num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b)
+ if err != nil {
+ return err
+ }
+ *x = TrainerSpec_ModelType(num)
+ return nil
+}
+
+// Deprecated: Use TrainerSpec_ModelType.Descriptor instead.
+func (TrainerSpec_ModelType) EnumDescriptor() ([]byte, []int) {
+ return file_sentencepiece_model_proto_rawDescGZIP(), []int{0, 0}
+}
+
+type ModelProto_SentencePiece_Type int32
+
+const (
+ ModelProto_SentencePiece_NORMAL ModelProto_SentencePiece_Type = 1 // normal symbol
+ ModelProto_SentencePiece_UNKNOWN ModelProto_SentencePiece_Type = 2 // unknown symbol. only <unk> for now.
+ ModelProto_SentencePiece_CONTROL ModelProto_SentencePiece_Type = 3 // control symbols. </s>, <s>, <2ja> etc.
+ ModelProto_SentencePiece_USER_DEFINED ModelProto_SentencePiece_Type = 4 // user defined symbols.
+ // Typical usage of USER_DEFINED symbol
+ // is placeholder.
+ ModelProto_SentencePiece_BYTE ModelProto_SentencePiece_Type = 6 // byte symbols. Used when `byte_fallback` is true.
+ ModelProto_SentencePiece_UNUSED ModelProto_SentencePiece_Type = 5 // this piece is not used.
+)
+
+// Enum value maps for ModelProto_SentencePiece_Type.
+var (
+ ModelProto_SentencePiece_Type_name = map[int32]string{
+ 1: "NORMAL",
+ 2: "UNKNOWN",
+ 3: "CONTROL",
+ 4: "USER_DEFINED",
+ 6: "BYTE",
+ 5: "UNUSED",
+ }
+ ModelProto_SentencePiece_Type_value = map[string]int32{
+ "NORMAL": 1,
+ "UNKNOWN": 2,
+ "CONTROL": 3,
+ "USER_DEFINED": 4,
+ "BYTE": 6,
+ "UNUSED": 5,
+ }
+)
+
+func (x ModelProto_SentencePiece_Type) Enum() *ModelProto_SentencePiece_Type {
+ p := new(ModelProto_SentencePiece_Type)
+ *p = x
+ return p
+}
+
+func (x ModelProto_SentencePiece_Type) String() string {
+ return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ModelProto_SentencePiece_Type) Descriptor() protoreflect.EnumDescriptor {
+ return file_sentencepiece_model_proto_enumTypes[1].Descriptor()
+}
+
+func (ModelProto_SentencePiece_Type) Type() protoreflect.EnumType {
+ return &file_sentencepiece_model_proto_enumTypes[1]
+}
+
+func (x ModelProto_SentencePiece_Type) Number() protoreflect.EnumNumber {
+ return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Do not use.
+func (x *ModelProto_SentencePiece_Type) UnmarshalJSON(b []byte) error {
+ num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b)
+ if err != nil {
+ return err
+ }
+ *x = ModelProto_SentencePiece_Type(num)
+ return nil
+}
+
+// Deprecated: Use ModelProto_SentencePiece_Type.Descriptor instead.
+func (ModelProto_SentencePiece_Type) EnumDescriptor() ([]byte, []int) {
+ return file_sentencepiece_model_proto_rawDescGZIP(), []int{3, 0, 0}
+}
+
+// TrainerSpec encodes a various parameters for SentencePiece training.
+// Next id: 55
+type TrainerSpec struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+ extensionFields protoimpl.ExtensionFields
+
+ // /////////////////////////////////////////////////////////////////
+ // General parameters
+ //
+ // Input corpus files.
+ //
+ // Trainer accepts the following two formats:
+ // A) Monolingual: plain text, one sentence per line.
+ // B) Bilingual: TSV, source sentence <tab> target sentence
+ // When bilingual data is passed, shared vocabulary model is built.
+ // Note that the input file must be raw corpus, not a preprocessed corpus.
+ // Trainer only loads the first `input_sentence_size` sentences specified
+ // with this parameter.
+ Input []string `protobuf:"bytes,1,rep,name=input" json:"input,omitempty"`
+ // Input corpus format:
+ // "text": one-sentence-per-line text format (default)
+ // "tsv": sentence <tab> freq
+ InputFormat *string `protobuf:"bytes,7,opt,name=input_format,json=inputFormat" json:"input_format,omitempty"`
+ // Output model file prefix.
+ // <model_prefix>.model and <model_prefix>.vocab are generated.
+ ModelPrefix *string `protobuf:"bytes,2,opt,name=model_prefix,json=modelPrefix" json:"model_prefix,omitempty"`
+ ModelType *TrainerSpec_ModelType `protobuf:"varint,3,opt,name=model_type,json=modelType,enum=sentencepiece.TrainerSpec_ModelType,def=1" json:"model_type,omitempty"`
+ // Vocabulary size. 8k is the default size.
+ VocabSize *int32 `protobuf:"varint,4,opt,name=vocab_size,json=vocabSize,def=8000" json:"vocab_size,omitempty"`
+ // List of the languages this model can accept.
+ // Since the model is language-agnostic, this field is used as a reference.
+ AcceptLanguage []string `protobuf:"bytes,5,rep,name=accept_language,json=acceptLanguage" json:"accept_language,omitempty"`
+ // Size of self-test samples, which are encoded in the model file.
+ SelfTestSampleSize *int32 `protobuf:"varint,6,opt,name=self_test_sample_size,json=selfTestSampleSize,def=0" json:"self_test_sample_size,omitempty"`
+ // Whether to use DP version of sentencepiece. Use it with TSV input format
+ // (requires precomputed word tab counts to work).
+ EnableDifferentialPrivacy *bool `protobuf:"varint,50,opt,name=enable_differential_privacy,json=enableDifferentialPrivacy,def=0" json:"enable_differential_privacy,omitempty"`
+ // Set these parameters if you need DP version of sentencepiece.
+ // std of noise to add.
+ DifferentialPrivacyNoiseLevel *float32 `protobuf:"fixed32,51,opt,name=differential_privacy_noise_level,json=differentialPrivacyNoiseLevel,def=0" json:"differential_privacy_noise_level,omitempty"`
+ // Clipping threshold to apply after adding noise. All the words with
+ // frequency less than this value are dropped.
+ DifferentialPrivacyClippingThreshold *uint64 `protobuf:"varint,52,opt,name=differential_privacy_clipping_threshold,json=differentialPrivacyClippingThreshold,def=0" json:"differential_privacy_clipping_threshold,omitempty"`
+ // /////////////////////////////////////////////////////////////////
+ // Training parameters.
+ //
+ // Uses characters which cover the corpus with the ratio of `chars_coverage`.
+ // This parameter determines the set of basic Alphabet of sentence piece.
+ // 1.0 - `chars_coverage` characters are treated as UNK.
+ // See also required_chars field.
+ CharacterCoverage *float32 `protobuf:"fixed32,10,opt,name=character_coverage,json=characterCoverage,def=0.9995" json:"character_coverage,omitempty"`
+ // Maximum size of sentences the trainer loads from `input` parameter.
+ // Trainer simply loads the `input` files in sequence.
+ // It is better to shuffle the input corpus randomly.
+ InputSentenceSize *uint64 `protobuf:"varint,11,opt,name=input_sentence_size,json=inputSentenceSize,def=0" json:"input_sentence_size,omitempty"`
+ ShuffleInputSentence *bool `protobuf:"varint,19,opt,name=shuffle_input_sentence,json=shuffleInputSentence,def=1" json:"shuffle_input_sentence,omitempty"`
+ // Maximum size of sentences to make seed sentence pieces.
+ // Extended suffix array is constructed to extract frequent
+ // sub-strings from the corpus. This uses 20N working space,
+ // where N is the size of corpus.
+ //
+ // Deprecated: Marked as deprecated in sentencepiece_model.proto.
+ MiningSentenceSize *int32 `protobuf:"varint,12,opt,name=mining_sentence_size,json=miningSentenceSize" json:"mining_sentence_size,omitempty"`
+ // Maximum size of sentences to train sentence pieces.
+ //
+ // Deprecated: Marked as deprecated in sentencepiece_model.proto.
+ TrainingSentenceSize *int32 `protobuf:"varint,13,opt,name=training_sentence_size,json=trainingSentenceSize" json:"training_sentence_size,omitempty"`
+ // The size of seed sentencepieces.
+ // `seed_sentencepiece_size` must be larger than `vocab_size`.
+ SeedSentencepieceSize *int32 `protobuf:"varint,14,opt,name=seed_sentencepiece_size,json=seedSentencepieceSize,def=1000000" json:"seed_sentencepiece_size,omitempty"`
+ // In every EM sub-iterations, keeps top
+ // `shrinking_factor` * `current sentencepieces size` with respect to
+ // the loss of the sentence piece. This value should be smaller than 1.0.
+ ShrinkingFactor *float32 `protobuf:"fixed32,15,opt,name=shrinking_factor,json=shrinkingFactor,def=0.75" json:"shrinking_factor,omitempty"`
+ // The maximum sentence length in byte. The sentences with the length
+ // larger than `max_sentence_length` is simply ignored.
+ // Longer input tends to bring the following risks:
+ // - Overflow during EM training (unigram language model only)
+ // - Performance drop because of O(n log n) cost in BPE.
+ MaxSentenceLength *int32 `protobuf:"varint,18,opt,name=max_sentence_length,json=maxSentenceLength,def=4192" json:"max_sentence_length,omitempty"`
+ // Number of threads in the training.
+ NumThreads *int32 `protobuf:"varint,16,opt,name=num_threads,json=numThreads,def=16" json:"num_threads,omitempty"`
+ // Number of EM sub iterations.
+ NumSubIterations *int32 `protobuf:"varint,17,opt,name=num_sub_iterations,json=numSubIterations,def=2" json:"num_sub_iterations,omitempty"`
+ // /////////////////////////////////////////////////////////////////
+ // SentencePiece parameters which control the shapes of sentence piece.
+ //
+ // Maximum length of sentencepiece.
+ MaxSentencepieceLength *int32 `protobuf:"varint,20,opt,name=max_sentencepiece_length,json=maxSentencepieceLength,def=16" json:"max_sentencepiece_length,omitempty"`
+ // Uses Unicode script to split sentence pieces.
+ // When `split_by_unicode_script` is true, we do not allow sentence piece to
+ // include multiple Unicode scripts, e.g. "F1" is not a valid piece.
+ // Exception: CJ characters (Hiragana/Katakana/Han) are all handled
+ // as one script type, since Japanese word can consist of multiple scripts.
+ // This exception is always applied regardless of the accept-language
+ // parameter.
+ SplitByUnicodeScript *bool `protobuf:"varint,21,opt,name=split_by_unicode_script,json=splitByUnicodeScript,def=1" json:"split_by_unicode_script,omitempty"`
+ // When `split_by_number` is true, put a boundary between number and
+ // non-number transition. If we want to treat "F1" is one token, set this flag
+ // to be false.
+ SplitByNumber *bool `protobuf:"varint,23,opt,name=split_by_number,json=splitByNumber,def=1" json:"split_by_number,omitempty"`
+ // Use a white space to split sentence pieces.
+ // When `split_by_whitespace` is false, we may have the piece containing
+ // a white space in the middle. e.g., "in_the".
+ SplitByWhitespace *bool `protobuf:"varint,22,opt,name=split_by_whitespace,json=splitByWhitespace,def=1" json:"split_by_whitespace,omitempty"`
+ // Adds whitespace symbol (_) as a suffix instead of prefix. e.g., _hello =>
+ // hello_. When `treat_whitespace_as_suffix` is true,
+ // NormalizerSpec::add_dummy_prefix will add the dummy whitespace to the end
+ // of sentence.
+ TreatWhitespaceAsSuffix *bool `protobuf:"varint,24,opt,name=treat_whitespace_as_suffix,json=treatWhitespaceAsSuffix,def=0" json:"treat_whitespace_as_suffix,omitempty"`
+ // Allows pieces that only contain whitespaces instead of appearing only as
+ // prefix or suffix of other pieces.
+ AllowWhitespaceOnlyPieces *bool `protobuf:"varint,26,opt,name=allow_whitespace_only_pieces,json=allowWhitespaceOnlyPieces,def=0" json:"allow_whitespace_only_pieces,omitempty"`
+ // Split all digits (0-9) into separate pieces.
+ SplitDigits *bool `protobuf:"varint,25,opt,name=split_digits,json=splitDigits,def=0" json:"split_digits,omitempty"`
+ // Defines the pre-tokenization delimiter.
+ // When specified, no pieces crossing this delimiter is not included
+ // in the vocab. Then the delimiter string is virtually ignored
+ // during the training. This field can allows constraints on the vocabulary
+ // selection. Note that this field is available on unigram mode.
+ PretokenizationDelimiter *string `protobuf:"bytes,53,opt,name=pretokenization_delimiter,json=pretokenizationDelimiter,def=" json:"pretokenization_delimiter,omitempty"`
+ // /////////////////////////////////////////////////////////////////
+ // Vocabulary management
+ //
+ // Defines control symbols used as an indicator to
+ // change the behavior of the decoder. <s> and </s> are pre-defined.
+ // We can use this field to encode various meta information,
+ // including language indicator in multilingual model.
+ // These symbols are not visible to users, but visible to
+ // the decoder. Note that when the input sentence contains control symbols,
+ // they are not treated as one token, but segmented into normal pieces.
+ // Control symbols must be inserted independently from the segmentation.
+ ControlSymbols []string `protobuf:"bytes,30,rep,name=control_symbols,json=controlSymbols" json:"control_symbols,omitempty"`
+ // Defines user defined symbols.
+ // These symbols are added with extremely high score
+ // so they are always treated as one unique symbol in any context.
+ // Typical usage of user_defined_symbols is placeholder for named entities.
+ UserDefinedSymbols []string `protobuf:"bytes,31,rep,name=user_defined_symbols,json=userDefinedSymbols" json:"user_defined_symbols,omitempty"`
+ // Defines required characters. Each UTF8 character in this string is included
+ // in the character set regardless of character_coverage value. Unlike
+ // user_defined_symbols, these characters have scores based on the frequency
+ // on input sentences, and the model can form subwords using characters
+ // in this field.
+ RequiredChars *string `protobuf:"bytes,36,opt,name=required_chars,json=requiredChars" json:"required_chars,omitempty"`
+ // Decomposes unknown pieces into UTF-8 bytes.
+ ByteFallback *bool `protobuf:"varint,35,opt,name=byte_fallback,json=byteFallback,def=0" json:"byte_fallback,omitempty"`
+ // When creating the vocabulary file, defines whether or not to additionally
+ // output the score for each piece.
+ VocabularyOutputPieceScore *bool `protobuf:"varint,32,opt,name=vocabulary_output_piece_score,json=vocabularyOutputPieceScore,def=1" json:"vocabulary_output_piece_score,omitempty"`
+ // `vocab_size` is treated as hard limit. Crash if
+ // the model can not produce the vocab of size `vocab_size`,
+ // When `hard_vocab_limit` is false, vocab_size is treated
+ // as soft limit. Note that when model_type=char,
+ // always assumes hard_vocab_limit = false.
+ HardVocabLimit *bool `protobuf:"varint,33,opt,name=hard_vocab_limit,json=hardVocabLimit,def=1" json:"hard_vocab_limit,omitempty"`
+ // use all symbols for vocab extraction. This flag is valid
+ // if model type is either CHAR or WORD
+ UseAllVocab *bool `protobuf:"varint,34,opt,name=use_all_vocab,json=useAllVocab,def=0" json:"use_all_vocab,omitempty"`
+ // /////////////////////////////////////////////////////////////////
+ // Reserved special meta tokens.
+ // * -1 is not used.
+ // * unk_id must not be -1.
+ // Id must starts with 0 and be contigous.
+ UnkId *int32 `protobuf:"varint,40,opt,name=unk_id,json=unkId,def=0" json:"unk_id,omitempty"` // <unk>
+ BosId *int32 `protobuf:"varint,41,opt,name=bos_id,json=bosId,def=1" json:"bos_id,omitempty"` // <s>
+ EosId *int32 `protobuf:"varint,42,opt,name=eos_id,json=eosId,def=2" json:"eos_id,omitempty"` // </s>
+ PadId *int32 `protobuf:"varint,43,opt,name=pad_id,json=padId,def=-1" json:"pad_id,omitempty"` // <pad> (padding)
+ UnkPiece *string `protobuf:"bytes,45,opt,name=unk_piece,json=unkPiece,def=<unk>" json:"unk_piece,omitempty"`
+ BosPiece *string `protobuf:"bytes,46,opt,name=bos_piece,json=bosPiece,def=<s>" json:"bos_piece,omitempty"`
+ EosPiece *string `protobuf:"bytes,47,opt,name=eos_piece,json=eosPiece,def=</s>" json:"eos_piece,omitempty"`
+ PadPiece *string `protobuf:"bytes,48,opt,name=pad_piece,json=padPiece,def=<pad>" json:"pad_piece,omitempty"`
+ // Encodes <unk> into U+2047 (DOUBLE QUESTION MARK),
+ // since this character can be useful both for user and
+ // developer. We can easily figure out that <unk> is emitted.
+ UnkSurface *string `protobuf:"bytes,44,opt,name=unk_surface,json=unkSurface,def= ⁇ " json:"unk_surface,omitempty"`
+ // Increase bit depth to allow unigram model training on large
+ // (>10M sentences) corpora. A Side-effect of enabling this flag
+ // is increased memory usage.
+ TrainExtremelyLargeCorpus *bool `protobuf:"varint,49,opt,name=train_extremely_large_corpus,json=trainExtremelyLargeCorpus,def=0" json:"train_extremely_large_corpus,omitempty"`
+ // Path to a seed sentencepieces file, with one tab-separated
+ // seed sentencepiece <tab> frequency per line.
+ SeedSentencepiecesFile *string `protobuf:"bytes,54,opt,name=seed_sentencepieces_file,json=seedSentencepiecesFile,def=" json:"seed_sentencepieces_file,omitempty"`
+}
+
+// Default values for TrainerSpec fields.
+const (
+ Default_TrainerSpec_ModelType = TrainerSpec_UNIGRAM
+ Default_TrainerSpec_VocabSize = int32(8000)
+ Default_TrainerSpec_SelfTestSampleSize = int32(0)
+ Default_TrainerSpec_EnableDifferentialPrivacy = bool(false)
+ Default_TrainerSpec_DifferentialPrivacyNoiseLevel = float32(0)
+ Default_TrainerSpec_DifferentialPrivacyClippingThreshold = uint64(0)
+ Default_TrainerSpec_CharacterCoverage = float32(0.9994999766349792)
+ Default_TrainerSpec_InputSentenceSize = uint64(0)
+ Default_TrainerSpec_ShuffleInputSentence = bool(true)
+ Default_TrainerSpec_SeedSentencepieceSize = int32(1000000)
+ Default_TrainerSpec_ShrinkingFactor = float32(0.75)
+ Default_TrainerSpec_MaxSentenceLength = int32(4192)
+ Default_TrainerSpec_NumThreads = int32(16)
+ Default_TrainerSpec_NumSubIterations = int32(2)
+ Default_TrainerSpec_MaxSentencepieceLength = int32(16)
+ Default_TrainerSpec_SplitByUnicodeScript = bool(true)
+ Default_TrainerSpec_SplitByNumber = bool(true)
+ Default_TrainerSpec_SplitByWhitespace = bool(true)
+ Default_TrainerSpec_TreatWhitespaceAsSuffix = bool(false)
+ Default_TrainerSpec_AllowWhitespaceOnlyPieces = bool(false)
+ Default_TrainerSpec_SplitDigits = bool(false)
+ Default_TrainerSpec_PretokenizationDelimiter = string("")
+ Default_TrainerSpec_ByteFallback = bool(false)
+ Default_TrainerSpec_VocabularyOutputPieceScore = bool(true)
+ Default_TrainerSpec_HardVocabLimit = bool(true)
+ Default_TrainerSpec_UseAllVocab = bool(false)
+ Default_TrainerSpec_UnkId = int32(0)
+ Default_TrainerSpec_BosId = int32(1)
+ Default_TrainerSpec_EosId = int32(2)
+ Default_TrainerSpec_PadId = int32(-1)
+ Default_TrainerSpec_UnkPiece = string("<unk>")
+ Default_TrainerSpec_BosPiece = string("<s>")
+ Default_TrainerSpec_EosPiece = string("</s>")
+ Default_TrainerSpec_PadPiece = string("<pad>")
+ Default_TrainerSpec_UnkSurface = string(" ⁇ ")
+ Default_TrainerSpec_TrainExtremelyLargeCorpus = bool(false)
+ Default_TrainerSpec_SeedSentencepiecesFile = string("")
+)
+
+func (x *TrainerSpec) Reset() {
+ *x = TrainerSpec{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_sentencepiece_model_proto_msgTypes[0]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *TrainerSpec) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*TrainerSpec) ProtoMessage() {}
+
+func (x *TrainerSpec) ProtoReflect() protoreflect.Message {
+ mi := &file_sentencepiece_model_proto_msgTypes[0]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use TrainerSpec.ProtoReflect.Descriptor instead.
+func (*TrainerSpec) Descriptor() ([]byte, []int) {
+ return file_sentencepiece_model_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *TrainerSpec) GetInput() []string {
+ if x != nil {
+ return x.Input
+ }
+ return nil
+}
+
+func (x *TrainerSpec) GetInputFormat() string {
+ if x != nil && x.InputFormat != nil {
+ return *x.InputFormat
+ }
+ return ""
+}
+
+func (x *TrainerSpec) GetModelPrefix() string {
+ if x != nil && x.ModelPrefix != nil {
+ return *x.ModelPrefix
+ }
+ return ""
+}
+
+func (x *TrainerSpec) GetModelType() TrainerSpec_ModelType {
+ if x != nil && x.ModelType != nil {
+ return *x.ModelType
+ }
+ return Default_TrainerSpec_ModelType
+}
+
+func (x *TrainerSpec) GetVocabSize() int32 {
+ if x != nil && x.VocabSize != nil {
+ return *x.VocabSize
+ }
+ return Default_TrainerSpec_VocabSize
+}
+
+func (x *TrainerSpec) GetAcceptLanguage() []string {
+ if x != nil {
+ return x.AcceptLanguage
+ }
+ return nil
+}
+
+func (x *TrainerSpec) GetSelfTestSampleSize() int32 {
+ if x != nil && x.SelfTestSampleSize != nil {
+ return *x.SelfTestSampleSize
+ }
+ return Default_TrainerSpec_SelfTestSampleSize
+}
+
+func (x *TrainerSpec) GetEnableDifferentialPrivacy() bool {
+ if x != nil && x.EnableDifferentialPrivacy != nil {
+ return *x.EnableDifferentialPrivacy
+ }
+ return Default_TrainerSpec_EnableDifferentialPrivacy
+}
+
+func (x *TrainerSpec) GetDifferentialPrivacyNoiseLevel() float32 {
+ if x != nil && x.DifferentialPrivacyNoiseLevel != nil {
+ return *x.DifferentialPrivacyNoiseLevel
+ }
+ return Default_TrainerSpec_DifferentialPrivacyNoiseLevel
+}
+
+func (x *TrainerSpec) GetDifferentialPrivacyClippingThreshold() uint64 {
+ if x != nil && x.DifferentialPrivacyClippingThreshold != nil {
+ return *x.DifferentialPrivacyClippingThreshold
+ }
+ return Default_TrainerSpec_DifferentialPrivacyClippingThreshold
+}
+
+func (x *TrainerSpec) GetCharacterCoverage() float32 {
+ if x != nil && x.CharacterCoverage != nil {
+ return *x.CharacterCoverage
+ }
+ return Default_TrainerSpec_CharacterCoverage
+}
+
+func (x *TrainerSpec) GetInputSentenceSize() uint64 {
+ if x != nil && x.InputSentenceSize != nil {
+ return *x.InputSentenceSize
+ }
+ return Default_TrainerSpec_InputSentenceSize
+}
+
+func (x *TrainerSpec) GetShuffleInputSentence() bool {
+ if x != nil && x.ShuffleInputSentence != nil {
+ return *x.ShuffleInputSentence
+ }
+ return Default_TrainerSpec_ShuffleInputSentence
+}
+
+// Deprecated: Marked as deprecated in sentencepiece_model.proto.
+func (x *TrainerSpec) GetMiningSentenceSize() int32 {
+ if x != nil && x.MiningSentenceSize != nil {
+ return *x.MiningSentenceSize
+ }
+ return 0
+}
+
+// Deprecated: Marked as deprecated in sentencepiece_model.proto.
+func (x *TrainerSpec) GetTrainingSentenceSize() int32 {
+ if x != nil && x.TrainingSentenceSize != nil {
+ return *x.TrainingSentenceSize
+ }
+ return 0
+}
+
+func (x *TrainerSpec) GetSeedSentencepieceSize() int32 {
+ if x != nil && x.SeedSentencepieceSize != nil {
+ return *x.SeedSentencepieceSize
+ }
+ return Default_TrainerSpec_SeedSentencepieceSize
+}
+
+func (x *TrainerSpec) GetShrinkingFactor() float32 {
+ if x != nil && x.ShrinkingFactor != nil {
+ return *x.ShrinkingFactor
+ }
+ return Default_TrainerSpec_ShrinkingFactor
+}
+
+func (x *TrainerSpec) GetMaxSentenceLength() int32 {
+ if x != nil && x.MaxSentenceLength != nil {
+ return *x.MaxSentenceLength
+ }
+ return Default_TrainerSpec_MaxSentenceLength
+}
+
+func (x *TrainerSpec) GetNumThreads() int32 {
+ if x != nil && x.NumThreads != nil {
+ return *x.NumThreads
+ }
+ return Default_TrainerSpec_NumThreads
+}
+
+func (x *TrainerSpec) GetNumSubIterations() int32 {
+ if x != nil && x.NumSubIterations != nil {
+ return *x.NumSubIterations
+ }
+ return Default_TrainerSpec_NumSubIterations
+}
+
+func (x *TrainerSpec) GetMaxSentencepieceLength() int32 {
+ if x != nil && x.MaxSentencepieceLength != nil {
+ return *x.MaxSentencepieceLength
+ }
+ return Default_TrainerSpec_MaxSentencepieceLength
+}
+
+func (x *TrainerSpec) GetSplitByUnicodeScript() bool {
+ if x != nil && x.SplitByUnicodeScript != nil {
+ return *x.SplitByUnicodeScript
+ }
+ return Default_TrainerSpec_SplitByUnicodeScript
+}
+
+func (x *TrainerSpec) GetSplitByNumber() bool {
+ if x != nil && x.SplitByNumber != nil {
+ return *x.SplitByNumber
+ }
+ return Default_TrainerSpec_SplitByNumber
+}
+
+func (x *TrainerSpec) GetSplitByWhitespace() bool {
+ if x != nil && x.SplitByWhitespace != nil {
+ return *x.SplitByWhitespace
+ }
+ return Default_TrainerSpec_SplitByWhitespace
+}
+
+func (x *TrainerSpec) GetTreatWhitespaceAsSuffix() bool {
+ if x != nil && x.TreatWhitespaceAsSuffix != nil {
+ return *x.TreatWhitespaceAsSuffix
+ }
+ return Default_TrainerSpec_TreatWhitespaceAsSuffix
+}
+
+func (x *TrainerSpec) GetAllowWhitespaceOnlyPieces() bool {
+ if x != nil && x.AllowWhitespaceOnlyPieces != nil {
+ return *x.AllowWhitespaceOnlyPieces
+ }
+ return Default_TrainerSpec_AllowWhitespaceOnlyPieces
+}
+
+func (x *TrainerSpec) GetSplitDigits() bool {
+ if x != nil && x.SplitDigits != nil {
+ return *x.SplitDigits
+ }
+ return Default_TrainerSpec_SplitDigits
+}
+
+func (x *TrainerSpec) GetPretokenizationDelimiter() string {
+ if x != nil && x.PretokenizationDelimiter != nil {
+ return *x.PretokenizationDelimiter
+ }
+ return Default_TrainerSpec_PretokenizationDelimiter
+}
+
+func (x *TrainerSpec) GetControlSymbols() []string {
+ if x != nil {
+ return x.ControlSymbols
+ }
+ return nil
+}
+
+func (x *TrainerSpec) GetUserDefinedSymbols() []string {
+ if x != nil {
+ return x.UserDefinedSymbols
+ }
+ return nil
+}
+
+func (x *TrainerSpec) GetRequiredChars() string {
+ if x != nil && x.RequiredChars != nil {
+ return *x.RequiredChars
+ }
+ return ""
+}
+
+func (x *TrainerSpec) GetByteFallback() bool {
+ if x != nil && x.ByteFallback != nil {
+ return *x.ByteFallback
+ }
+ return Default_TrainerSpec_ByteFallback
+}
+
+func (x *TrainerSpec) GetVocabularyOutputPieceScore() bool {
+ if x != nil && x.VocabularyOutputPieceScore != nil {
+ return *x.VocabularyOutputPieceScore
+ }
+ return Default_TrainerSpec_VocabularyOutputPieceScore
+}
+
+func (x *TrainerSpec) GetHardVocabLimit() bool {
+ if x != nil && x.HardVocabLimit != nil {
+ return *x.HardVocabLimit
+ }
+ return Default_TrainerSpec_HardVocabLimit
+}
+
+func (x *TrainerSpec) GetUseAllVocab() bool {
+ if x != nil && x.UseAllVocab != nil {
+ return *x.UseAllVocab
+ }
+ return Default_TrainerSpec_UseAllVocab
+}
+
+func (x *TrainerSpec) GetUnkId() int32 {
+ if x != nil && x.UnkId != nil {
+ return *x.UnkId
+ }
+ return Default_TrainerSpec_UnkId
+}
+
+func (x *TrainerSpec) GetBosId() int32 {
+ if x != nil && x.BosId != nil {
+ return *x.BosId
+ }
+ return Default_TrainerSpec_BosId
+}
+
+func (x *TrainerSpec) GetEosId() int32 {
+ if x != nil && x.EosId != nil {
+ return *x.EosId
+ }
+ return Default_TrainerSpec_EosId
+}
+
+func (x *TrainerSpec) GetPadId() int32 {
+ if x != nil && x.PadId != nil {
+ return *x.PadId
+ }
+ return Default_TrainerSpec_PadId
+}
+
+func (x *TrainerSpec) GetUnkPiece() string {
+ if x != nil && x.UnkPiece != nil {
+ return *x.UnkPiece
+ }
+ return Default_TrainerSpec_UnkPiece
+}
+
+func (x *TrainerSpec) GetBosPiece() string {
+ if x != nil && x.BosPiece != nil {
+ return *x.BosPiece
+ }
+ return Default_TrainerSpec_BosPiece
+}
+
+func (x *TrainerSpec) GetEosPiece() string {
+ if x != nil && x.EosPiece != nil {
+ return *x.EosPiece
+ }
+ return Default_TrainerSpec_EosPiece
+}
+
+func (x *TrainerSpec) GetPadPiece() string {
+ if x != nil && x.PadPiece != nil {
+ return *x.PadPiece
+ }
+ return Default_TrainerSpec_PadPiece
+}
+
+func (x *TrainerSpec) GetUnkSurface() string {
+ if x != nil && x.UnkSurface != nil {
+ return *x.UnkSurface
+ }
+ return Default_TrainerSpec_UnkSurface
+}
+
+func (x *TrainerSpec) GetTrainExtremelyLargeCorpus() bool {
+ if x != nil && x.TrainExtremelyLargeCorpus != nil {
+ return *x.TrainExtremelyLargeCorpus
+ }
+ return Default_TrainerSpec_TrainExtremelyLargeCorpus
+}
+
+func (x *TrainerSpec) GetSeedSentencepiecesFile() string {
+ if x != nil && x.SeedSentencepiecesFile != nil {
+ return *x.SeedSentencepiecesFile
+ }
+ return Default_TrainerSpec_SeedSentencepiecesFile
+}
+
+// NormalizerSpec encodes a various parameters for string normalizaiton
+type NormalizerSpec struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+ extensionFields protoimpl.ExtensionFields
+
+ // name of normalization rule.
+ Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+ // Pre-compiled normalization rule created by
+ // Builder::GetPrecompiledCharsMap() or Builder::CompileCharsMap() method.
+ // Usually this field is set by Builder::GetNormalizerSpec() method.
+ PrecompiledCharsmap []byte `protobuf:"bytes,2,opt,name=precompiled_charsmap,json=precompiledCharsmap" json:"precompiled_charsmap,omitempty"`
+ // Adds dummy whitespace at the beginning of text in order to
+ // treat "world" in "world" and "hello world" in the same way.
+ AddDummyPrefix *bool `protobuf:"varint,3,opt,name=add_dummy_prefix,json=addDummyPrefix,def=1" json:"add_dummy_prefix,omitempty"`
+ // Removes leading, trailing, and duplicate internal whitespace.
+ RemoveExtraWhitespaces *bool `protobuf:"varint,4,opt,name=remove_extra_whitespaces,json=removeExtraWhitespaces,def=1" json:"remove_extra_whitespaces,omitempty"`
+ // Replaces whitespace with meta symbol.
+ // This field must be true to train sentence piece model.
+ EscapeWhitespaces *bool `protobuf:"varint,5,opt,name=escape_whitespaces,json=escapeWhitespaces,def=1" json:"escape_whitespaces,omitempty"`
+ // Custom normalization rule file in TSV format.
+ // https://github.com/google/sentencepiece/blob/master/doc/normalization.md
+ // This field is only used in SentencePieceTrainer::Train() method, which
+ // compiles the rule into the binary rule stored in `precompiled_charsmap`.
+ NormalizationRuleTsv *string `protobuf:"bytes,6,opt,name=normalization_rule_tsv,json=normalizationRuleTsv" json:"normalization_rule_tsv,omitempty"`
+}
+
+// Default values for NormalizerSpec fields.
+const (
+ Default_NormalizerSpec_AddDummyPrefix = bool(true)
+ Default_NormalizerSpec_RemoveExtraWhitespaces = bool(true)
+ Default_NormalizerSpec_EscapeWhitespaces = bool(true)
+)
+
+func (x *NormalizerSpec) Reset() {
+ *x = NormalizerSpec{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_sentencepiece_model_proto_msgTypes[1]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *NormalizerSpec) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*NormalizerSpec) ProtoMessage() {}
+
+func (x *NormalizerSpec) ProtoReflect() protoreflect.Message {
+ mi := &file_sentencepiece_model_proto_msgTypes[1]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use NormalizerSpec.ProtoReflect.Descriptor instead.
+func (*NormalizerSpec) Descriptor() ([]byte, []int) {
+ return file_sentencepiece_model_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *NormalizerSpec) GetName() string {
+ if x != nil && x.Name != nil {
+ return *x.Name
+ }
+ return ""
+}
+
+func (x *NormalizerSpec) GetPrecompiledCharsmap() []byte {
+ if x != nil {
+ return x.PrecompiledCharsmap
+ }
+ return nil
+}
+
+func (x *NormalizerSpec) GetAddDummyPrefix() bool {
+ if x != nil && x.AddDummyPrefix != nil {
+ return *x.AddDummyPrefix
+ }
+ return Default_NormalizerSpec_AddDummyPrefix
+}
+
+func (x *NormalizerSpec) GetRemoveExtraWhitespaces() bool {
+ if x != nil && x.RemoveExtraWhitespaces != nil {
+ return *x.RemoveExtraWhitespaces
+ }
+ return Default_NormalizerSpec_RemoveExtraWhitespaces
+}
+
+func (x *NormalizerSpec) GetEscapeWhitespaces() bool {
+ if x != nil && x.EscapeWhitespaces != nil {
+ return *x.EscapeWhitespaces
+ }
+ return Default_NormalizerSpec_EscapeWhitespaces
+}
+
+func (x *NormalizerSpec) GetNormalizationRuleTsv() string {
+ if x != nil && x.NormalizationRuleTsv != nil {
+ return *x.NormalizationRuleTsv
+ }
+ return ""
+}
+
+// Proto to store samples for self-testing.
+type SelfTestData struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+ extensionFields protoimpl.ExtensionFields
+
+ Samples []*SelfTestData_Sample `protobuf:"bytes,1,rep,name=samples" json:"samples,omitempty"`
+}
+
+func (x *SelfTestData) Reset() {
+ *x = SelfTestData{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_sentencepiece_model_proto_msgTypes[2]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *SelfTestData) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SelfTestData) ProtoMessage() {}
+
+func (x *SelfTestData) ProtoReflect() protoreflect.Message {
+ mi := &file_sentencepiece_model_proto_msgTypes[2]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use SelfTestData.ProtoReflect.Descriptor instead.
+func (*SelfTestData) Descriptor() ([]byte, []int) {
+ return file_sentencepiece_model_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *SelfTestData) GetSamples() []*SelfTestData_Sample {
+ if x != nil {
+ return x.Samples
+ }
+ return nil
+}
+
+// ModelProto stores model parameters.
+// SentencePieceProcessor is supposed to be self-contained.
+// All settings/parameters which may change the behavior must be encoded
+// in ModelProto.
+type ModelProto struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+ extensionFields protoimpl.ExtensionFields
+
+ // Sentence pieces with scores.
+ Pieces []*ModelProto_SentencePiece `protobuf:"bytes,1,rep,name=pieces" json:"pieces,omitempty"`
+ // Spec used to generate this model file.
+ TrainerSpec *TrainerSpec `protobuf:"bytes,2,opt,name=trainer_spec,json=trainerSpec" json:"trainer_spec,omitempty"`
+ // Spec for text normalization.
+ NormalizerSpec *NormalizerSpec `protobuf:"bytes,3,opt,name=normalizer_spec,json=normalizerSpec" json:"normalizer_spec,omitempty"`
+ // Stores sample input and its expected segmentation to verify the model.
+ SelfTestData *SelfTestData `protobuf:"bytes,4,opt,name=self_test_data,json=selfTestData" json:"self_test_data,omitempty"`
+ // Spec for text de-normalization.
+ DenormalizerSpec *NormalizerSpec `protobuf:"bytes,5,opt,name=denormalizer_spec,json=denormalizerSpec" json:"denormalizer_spec,omitempty"`
+}
+
+func (x *ModelProto) Reset() {
+ *x = ModelProto{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_sentencepiece_model_proto_msgTypes[3]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *ModelProto) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ModelProto) ProtoMessage() {}
+
+func (x *ModelProto) ProtoReflect() protoreflect.Message {
+ mi := &file_sentencepiece_model_proto_msgTypes[3]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use ModelProto.ProtoReflect.Descriptor instead.
+func (*ModelProto) Descriptor() ([]byte, []int) {
+ return file_sentencepiece_model_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *ModelProto) GetPieces() []*ModelProto_SentencePiece {
+ if x != nil {
+ return x.Pieces
+ }
+ return nil
+}
+
+func (x *ModelProto) GetTrainerSpec() *TrainerSpec {
+ if x != nil {
+ return x.TrainerSpec
+ }
+ return nil
+}
+
+func (x *ModelProto) GetNormalizerSpec() *NormalizerSpec {
+ if x != nil {
+ return x.NormalizerSpec
+ }
+ return nil
+}
+
+func (x *ModelProto) GetSelfTestData() *SelfTestData {
+ if x != nil {
+ return x.SelfTestData
+ }
+ return nil
+}
+
+func (x *ModelProto) GetDenormalizerSpec() *NormalizerSpec {
+ if x != nil {
+ return x.DenormalizerSpec
+ }
+ return nil
+}
+
+type SelfTestData_Sample struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Input *string `protobuf:"bytes,1,opt,name=input" json:"input,omitempty"`
+ Expected *string `protobuf:"bytes,2,opt,name=expected" json:"expected,omitempty"`
+}
+
+func (x *SelfTestData_Sample) Reset() {
+ *x = SelfTestData_Sample{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_sentencepiece_model_proto_msgTypes[4]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *SelfTestData_Sample) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SelfTestData_Sample) ProtoMessage() {}
+
+func (x *SelfTestData_Sample) ProtoReflect() protoreflect.Message {
+ mi := &file_sentencepiece_model_proto_msgTypes[4]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use SelfTestData_Sample.ProtoReflect.Descriptor instead.
+func (*SelfTestData_Sample) Descriptor() ([]byte, []int) {
+ return file_sentencepiece_model_proto_rawDescGZIP(), []int{2, 0}
+}
+
+func (x *SelfTestData_Sample) GetInput() string {
+ if x != nil && x.Input != nil {
+ return *x.Input
+ }
+ return ""
+}
+
+func (x *SelfTestData_Sample) GetExpected() string {
+ if x != nil && x.Expected != nil {
+ return *x.Expected
+ }
+ return ""
+}
+
+type ModelProto_SentencePiece struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+ extensionFields protoimpl.ExtensionFields
+
+ Piece *string `protobuf:"bytes,1,opt,name=piece" json:"piece,omitempty"` // piece must not be empty.
+ Score *float32 `protobuf:"fixed32,2,opt,name=score" json:"score,omitempty"`
+ Type *ModelProto_SentencePiece_Type `protobuf:"varint,3,opt,name=type,enum=sentencepiece.ModelProto_SentencePiece_Type,def=1" json:"type,omitempty"`
+}
+
+// Default values for ModelProto_SentencePiece fields.
+const (
+ Default_ModelProto_SentencePiece_Type = ModelProto_SentencePiece_NORMAL
+)
+
+func (x *ModelProto_SentencePiece) Reset() {
+ *x = ModelProto_SentencePiece{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_sentencepiece_model_proto_msgTypes[5]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *ModelProto_SentencePiece) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ModelProto_SentencePiece) ProtoMessage() {}
+
+func (x *ModelProto_SentencePiece) ProtoReflect() protoreflect.Message {
+ mi := &file_sentencepiece_model_proto_msgTypes[5]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use ModelProto_SentencePiece.ProtoReflect.Descriptor instead.
+func (*ModelProto_SentencePiece) Descriptor() ([]byte, []int) {
+ return file_sentencepiece_model_proto_rawDescGZIP(), []int{3, 0}
+}
+
+func (x *ModelProto_SentencePiece) GetPiece() string {
+ if x != nil && x.Piece != nil {
+ return *x.Piece
+ }
+ return ""
+}
+
+func (x *ModelProto_SentencePiece) GetScore() float32 {
+ if x != nil && x.Score != nil {
+ return *x.Score
+ }
+ return 0
+}
+
+func (x *ModelProto_SentencePiece) GetType() ModelProto_SentencePiece_Type {
+ if x != nil && x.Type != nil {
+ return *x.Type
+ }
+ return Default_ModelProto_SentencePiece_Type
+}
+
+var File_sentencepiece_model_proto protoreflect.FileDescriptor
+
+var file_sentencepiece_model_proto_rawDesc = []byte{
+ 0x0a, 0x19, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x5f,
+ 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x73, 0x65, 0x6e,
+ 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x22, 0xc6, 0x12, 0x0a, 0x0b, 0x54,
+ 0x72, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e,
+ 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74,
+ 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74,
+ 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x46, 0x6f, 0x72,
+ 0x6d, 0x61, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x70, 0x72, 0x65,
+ 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6d, 0x6f, 0x64, 0x65, 0x6c,
+ 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x4c, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f,
+ 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x73, 0x65, 0x6e,
+ 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x54, 0x72, 0x61, 0x69, 0x6e,
+ 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65,
+ 0x3a, 0x07, 0x55, 0x4e, 0x49, 0x47, 0x52, 0x41, 0x4d, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c,
+ 0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0a, 0x76, 0x6f, 0x63, 0x61, 0x62, 0x5f, 0x73, 0x69,
+ 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x04, 0x38, 0x30, 0x30, 0x30, 0x52, 0x09,
+ 0x76, 0x6f, 0x63, 0x61, 0x62, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x63, 0x63,
+ 0x65, 0x70, 0x74, 0x5f, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x03,
+ 0x28, 0x09, 0x52, 0x0e, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x4c, 0x61, 0x6e, 0x67, 0x75, 0x61,
+ 0x67, 0x65, 0x12, 0x34, 0x0a, 0x15, 0x73, 0x65, 0x6c, 0x66, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x5f,
+ 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28,
+ 0x05, 0x3a, 0x01, 0x30, 0x52, 0x12, 0x73, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74, 0x53, 0x61,
+ 0x6d, 0x70, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x45, 0x0a, 0x1b, 0x65, 0x6e, 0x61, 0x62,
+ 0x6c, 0x65, 0x5f, 0x64, 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f,
+ 0x70, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x18, 0x32, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66,
+ 0x61, 0x6c, 0x73, 0x65, 0x52, 0x19, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x44, 0x69, 0x66, 0x66,
+ 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x50, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x12,
+ 0x4a, 0x0a, 0x20, 0x64, 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f,
+ 0x70, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x5f, 0x6e, 0x6f, 0x69, 0x73, 0x65, 0x5f, 0x6c, 0x65,
+ 0x76, 0x65, 0x6c, 0x18, 0x33, 0x20, 0x01, 0x28, 0x02, 0x3a, 0x01, 0x30, 0x52, 0x1d, 0x64, 0x69,
+ 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x50, 0x72, 0x69, 0x76, 0x61, 0x63,
+ 0x79, 0x4e, 0x6f, 0x69, 0x73, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x58, 0x0a, 0x27, 0x64,
+ 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x70, 0x72, 0x69, 0x76,
+ 0x61, 0x63, 0x79, 0x5f, 0x63, 0x6c, 0x69, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x68, 0x72,
+ 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x34, 0x20, 0x01, 0x28, 0x04, 0x3a, 0x01, 0x30, 0x52,
+ 0x24, 0x64, 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x50, 0x72, 0x69,
+ 0x76, 0x61, 0x63, 0x79, 0x43, 0x6c, 0x69, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x54, 0x68, 0x72, 0x65,
+ 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x35, 0x0a, 0x12, 0x63, 0x68, 0x61, 0x72, 0x61, 0x63, 0x74,
+ 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28,
+ 0x02, 0x3a, 0x06, 0x30, 0x2e, 0x39, 0x39, 0x39, 0x35, 0x52, 0x11, 0x63, 0x68, 0x61, 0x72, 0x61,
+ 0x63, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x12, 0x31, 0x0a, 0x13,
+ 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x73,
+ 0x69, 0x7a, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x3a, 0x01, 0x30, 0x52, 0x11, 0x69, 0x6e,
+ 0x70, 0x75, 0x74, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12,
+ 0x3a, 0x0a, 0x16, 0x73, 0x68, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74,
+ 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x08, 0x3a,
+ 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x14, 0x73, 0x68, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x49, 0x6e,
+ 0x70, 0x75, 0x74, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x14, 0x6d,
+ 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x73,
+ 0x69, 0x7a, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05, 0x42, 0x02, 0x18, 0x01, 0x52, 0x12, 0x6d,
+ 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x53, 0x69, 0x7a,
+ 0x65, 0x12, 0x38, 0x0a, 0x16, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65,
+ 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28,
+ 0x05, 0x42, 0x02, 0x18, 0x01, 0x52, 0x14, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x53,
+ 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x3f, 0x0a, 0x17, 0x73,
+ 0x65, 0x65, 0x64, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63,
+ 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x07, 0x31, 0x30,
+ 0x30, 0x30, 0x30, 0x30, 0x30, 0x52, 0x15, 0x73, 0x65, 0x65, 0x64, 0x53, 0x65, 0x6e, 0x74, 0x65,
+ 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2f, 0x0a, 0x10,
+ 0x73, 0x68, 0x72, 0x69, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72,
+ 0x18, 0x0f, 0x20, 0x01, 0x28, 0x02, 0x3a, 0x04, 0x30, 0x2e, 0x37, 0x35, 0x52, 0x0f, 0x73, 0x68,
+ 0x72, 0x69, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x34, 0x0a,
+ 0x13, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6c, 0x65,
+ 0x6e, 0x67, 0x74, 0x68, 0x18, 0x12, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x04, 0x34, 0x31, 0x39, 0x32,
+ 0x52, 0x11, 0x6d, 0x61, 0x78, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x4c, 0x65, 0x6e,
+ 0x67, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61,
+ 0x64, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x02, 0x31, 0x36, 0x52, 0x0a, 0x6e, 0x75,
+ 0x6d, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x2f, 0x0a, 0x12, 0x6e, 0x75, 0x6d, 0x5f,
+ 0x73, 0x75, 0x62, 0x5f, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x11,
+ 0x20, 0x01, 0x28, 0x05, 0x3a, 0x01, 0x32, 0x52, 0x10, 0x6e, 0x75, 0x6d, 0x53, 0x75, 0x62, 0x49,
+ 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3c, 0x0a, 0x18, 0x6d, 0x61, 0x78,
+ 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x5f, 0x6c,
+ 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x14, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x02, 0x31, 0x36, 0x52,
+ 0x16, 0x6d, 0x61, 0x78, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63,
+ 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x3b, 0x0a, 0x17, 0x73, 0x70, 0x6c, 0x69, 0x74,
+ 0x5f, 0x62, 0x79, 0x5f, 0x75, 0x6e, 0x69, 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x73, 0x63, 0x72, 0x69,
+ 0x70, 0x74, 0x18, 0x15, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x14,
+ 0x73, 0x70, 0x6c, 0x69, 0x74, 0x42, 0x79, 0x55, 0x6e, 0x69, 0x63, 0x6f, 0x64, 0x65, 0x53, 0x63,
+ 0x72, 0x69, 0x70, 0x74, 0x12, 0x2c, 0x0a, 0x0f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, 0x62, 0x79,
+ 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x17, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74,
+ 0x72, 0x75, 0x65, 0x52, 0x0d, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x42, 0x79, 0x4e, 0x75, 0x6d, 0x62,
+ 0x65, 0x72, 0x12, 0x34, 0x0a, 0x13, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, 0x62, 0x79, 0x5f, 0x77,
+ 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x16, 0x20, 0x01, 0x28, 0x08, 0x3a,
+ 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x11, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x42, 0x79, 0x57, 0x68,
+ 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x42, 0x0a, 0x1a, 0x74, 0x72, 0x65, 0x61,
+ 0x74, 0x5f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x61, 0x73, 0x5f,
+ 0x73, 0x75, 0x66, 0x66, 0x69, 0x78, 0x18, 0x18, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61,
+ 0x6c, 0x73, 0x65, 0x52, 0x17, 0x74, 0x72, 0x65, 0x61, 0x74, 0x57, 0x68, 0x69, 0x74, 0x65, 0x73,
+ 0x70, 0x61, 0x63, 0x65, 0x41, 0x73, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x12, 0x46, 0x0a, 0x1c,
+ 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65,
+ 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x18, 0x1a, 0x20, 0x01,
+ 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x19, 0x61, 0x6c, 0x6c, 0x6f, 0x77,
+ 0x57, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x4f, 0x6e, 0x6c, 0x79, 0x50, 0x69,
+ 0x65, 0x63, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x0c, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, 0x64, 0x69,
+ 0x67, 0x69, 0x74, 0x73, 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73,
+ 0x65, 0x52, 0x0b, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x44, 0x69, 0x67, 0x69, 0x74, 0x73, 0x12, 0x3d,
+ 0x0a, 0x19, 0x70, 0x72, 0x65, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f,
+ 0x6e, 0x5f, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x35, 0x20, 0x01, 0x28,
+ 0x09, 0x3a, 0x00, 0x52, 0x18, 0x70, 0x72, 0x65, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x69, 0x7a, 0x61,
+ 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x12, 0x27, 0x0a,
+ 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73,
+ 0x18, 0x1e, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53,
+ 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x64,
+ 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x18, 0x1f,
+ 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x75, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65,
+ 0x64, 0x53, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x71, 0x75,
+ 0x69, 0x72, 0x65, 0x64, 0x5f, 0x63, 0x68, 0x61, 0x72, 0x73, 0x18, 0x24, 0x20, 0x01, 0x28, 0x09,
+ 0x52, 0x0d, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x43, 0x68, 0x61, 0x72, 0x73, 0x12,
+ 0x2a, 0x0a, 0x0d, 0x62, 0x79, 0x74, 0x65, 0x5f, 0x66, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b,
+ 0x18, 0x23, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0c, 0x62,
+ 0x79, 0x74, 0x65, 0x46, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x47, 0x0a, 0x1d, 0x76,
+ 0x6f, 0x63, 0x61, 0x62, 0x75, 0x6c, 0x61, 0x72, 0x79, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74,
+ 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x20, 0x20, 0x01,
+ 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x1a, 0x76, 0x6f, 0x63, 0x61, 0x62, 0x75,
+ 0x6c, 0x61, 0x72, 0x79, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x50, 0x69, 0x65, 0x63, 0x65, 0x53,
+ 0x63, 0x6f, 0x72, 0x65, 0x12, 0x2e, 0x0a, 0x10, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x76, 0x6f, 0x63,
+ 0x61, 0x62, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x21, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04,
+ 0x74, 0x72, 0x75, 0x65, 0x52, 0x0e, 0x68, 0x61, 0x72, 0x64, 0x56, 0x6f, 0x63, 0x61, 0x62, 0x4c,
+ 0x69, 0x6d, 0x69, 0x74, 0x12, 0x29, 0x0a, 0x0d, 0x75, 0x73, 0x65, 0x5f, 0x61, 0x6c, 0x6c, 0x5f,
+ 0x76, 0x6f, 0x63, 0x61, 0x62, 0x18, 0x22, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c,
+ 0x73, 0x65, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x41, 0x6c, 0x6c, 0x56, 0x6f, 0x63, 0x61, 0x62, 0x12,
+ 0x18, 0x0a, 0x06, 0x75, 0x6e, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x28, 0x20, 0x01, 0x28, 0x05, 0x3a,
+ 0x01, 0x30, 0x52, 0x05, 0x75, 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x06, 0x62, 0x6f, 0x73,
+ 0x5f, 0x69, 0x64, 0x18, 0x29, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x01, 0x31, 0x52, 0x05, 0x62, 0x6f,
+ 0x73, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x06, 0x65, 0x6f, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x2a, 0x20,
+ 0x01, 0x28, 0x05, 0x3a, 0x01, 0x32, 0x52, 0x05, 0x65, 0x6f, 0x73, 0x49, 0x64, 0x12, 0x19, 0x0a,
+ 0x06, 0x70, 0x61, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x02, 0x2d,
+ 0x31, 0x52, 0x05, 0x70, 0x61, 0x64, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x09, 0x75, 0x6e, 0x6b, 0x5f,
+ 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x2d, 0x20, 0x01, 0x28, 0x09, 0x3a, 0x05, 0x3c, 0x75, 0x6e,
+ 0x6b, 0x3e, 0x52, 0x08, 0x75, 0x6e, 0x6b, 0x50, 0x69, 0x65, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x09,
+ 0x62, 0x6f, 0x73, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x2e, 0x20, 0x01, 0x28, 0x09, 0x3a,
+ 0x03, 0x3c, 0x73, 0x3e, 0x52, 0x08, 0x62, 0x6f, 0x73, 0x50, 0x69, 0x65, 0x63, 0x65, 0x12, 0x21,
+ 0x0a, 0x09, 0x65, 0x6f, 0x73, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x2f, 0x20, 0x01, 0x28,
+ 0x09, 0x3a, 0x04, 0x3c, 0x2f, 0x73, 0x3e, 0x52, 0x08, 0x65, 0x6f, 0x73, 0x50, 0x69, 0x65, 0x63,
+ 0x65, 0x12, 0x22, 0x0a, 0x09, 0x70, 0x61, 0x64, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x30,
+ 0x20, 0x01, 0x28, 0x09, 0x3a, 0x05, 0x3c, 0x70, 0x61, 0x64, 0x3e, 0x52, 0x08, 0x70, 0x61, 0x64,
+ 0x50, 0x69, 0x65, 0x63, 0x65, 0x12, 0x26, 0x0a, 0x0b, 0x75, 0x6e, 0x6b, 0x5f, 0x73, 0x75, 0x72,
+ 0x66, 0x61, 0x63, 0x65, 0x18, 0x2c, 0x20, 0x01, 0x28, 0x09, 0x3a, 0x05, 0x20, 0xe2, 0x81, 0x87,
+ 0x20, 0x52, 0x0a, 0x75, 0x6e, 0x6b, 0x53, 0x75, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x46, 0x0a,
+ 0x1c, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x65, 0x6d, 0x65, 0x6c, 0x79,
+ 0x5f, 0x6c, 0x61, 0x72, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x72, 0x70, 0x75, 0x73, 0x18, 0x31, 0x20,
+ 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x19, 0x74, 0x72, 0x61, 0x69,
+ 0x6e, 0x45, 0x78, 0x74, 0x72, 0x65, 0x6d, 0x65, 0x6c, 0x79, 0x4c, 0x61, 0x72, 0x67, 0x65, 0x43,
+ 0x6f, 0x72, 0x70, 0x75, 0x73, 0x12, 0x3a, 0x0a, 0x18, 0x73, 0x65, 0x65, 0x64, 0x5f, 0x73, 0x65,
+ 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x5f, 0x66, 0x69, 0x6c,
+ 0x65, 0x18, 0x36, 0x20, 0x01, 0x28, 0x09, 0x3a, 0x00, 0x52, 0x16, 0x73, 0x65, 0x65, 0x64, 0x53,
+ 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x46, 0x69, 0x6c,
+ 0x65, 0x22, 0x35, 0x0a, 0x09, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b,
+ 0x0a, 0x07, 0x55, 0x4e, 0x49, 0x47, 0x52, 0x41, 0x4d, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x42,
+ 0x50, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x4f, 0x52, 0x44, 0x10, 0x03, 0x12, 0x08,
+ 0x0a, 0x04, 0x43, 0x48, 0x41, 0x52, 0x10, 0x04, 0x2a, 0x09, 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80,
+ 0x80, 0x80, 0x02, 0x22, 0xbd, 0x02, 0x0a, 0x0e, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a,
+ 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
+ 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x31, 0x0a, 0x14, 0x70, 0x72,
+ 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x63, 0x68, 0x61, 0x72, 0x73, 0x6d,
+ 0x61, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x13, 0x70, 0x72, 0x65, 0x63, 0x6f, 0x6d,
+ 0x70, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x68, 0x61, 0x72, 0x73, 0x6d, 0x61, 0x70, 0x12, 0x2e, 0x0a,
+ 0x10, 0x61, 0x64, 0x64, 0x5f, 0x64, 0x75, 0x6d, 0x6d, 0x79, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69,
+ 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x0e, 0x61,
+ 0x64, 0x64, 0x44, 0x75, 0x6d, 0x6d, 0x79, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x3e, 0x0a,
+ 0x18, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x77, 0x68,
+ 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x3a,
+ 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x78, 0x74,
+ 0x72, 0x61, 0x57, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x33, 0x0a,
+ 0x12, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x5f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61,
+ 0x63, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52,
+ 0x11, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x57, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63,
+ 0x65, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74,
+ 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x5f, 0x74, 0x73, 0x76, 0x18, 0x06, 0x20, 0x01,
+ 0x28, 0x09, 0x52, 0x14, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f,
+ 0x6e, 0x52, 0x75, 0x6c, 0x65, 0x54, 0x73, 0x76, 0x2a, 0x09, 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80,
+ 0x80, 0x80, 0x02, 0x22, 0x93, 0x01, 0x0a, 0x0c, 0x53, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74,
+ 0x44, 0x61, 0x74, 0x61, 0x12, 0x3c, 0x0a, 0x07, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x18,
+ 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65,
+ 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x53, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74, 0x44, 0x61,
+ 0x74, 0x61, 0x2e, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x07, 0x73, 0x61, 0x6d, 0x70, 0x6c,
+ 0x65, 0x73, 0x1a, 0x3a, 0x0a, 0x06, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05,
+ 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70,
+ 0x75, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02,
+ 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x2a, 0x09,
+ 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x22, 0xd7, 0x04, 0x0a, 0x0a, 0x4d, 0x6f,
+ 0x64, 0x65, 0x6c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x3f, 0x0a, 0x06, 0x70, 0x69, 0x65, 0x63,
+ 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65,
+ 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x50, 0x72,
+ 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x50, 0x69, 0x65, 0x63,
+ 0x65, 0x52, 0x06, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x12, 0x3d, 0x0a, 0x0c, 0x74, 0x72, 0x61,
+ 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
+ 0x1a, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e,
+ 0x54, 0x72, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0b, 0x74, 0x72, 0x61,
+ 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x46, 0x0a, 0x0f, 0x6e, 0x6f, 0x72, 0x6d,
+ 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28,
+ 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63,
+ 0x65, 0x2e, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63,
+ 0x52, 0x0e, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63,
+ 0x12, 0x41, 0x0a, 0x0e, 0x73, 0x65, 0x6c, 0x66, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x64, 0x61,
+ 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65,
+ 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x53, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73,
+ 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x0c, 0x73, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74, 0x44,
+ 0x61, 0x74, 0x61, 0x12, 0x4a, 0x0a, 0x11, 0x64, 0x65, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69,
+ 0x7a, 0x65, 0x72, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d,
+ 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x4e,
+ 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x10, 0x64,
+ 0x65, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x1a,
+ 0xe6, 0x01, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x50, 0x69, 0x65, 0x63,
+ 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+ 0x52, 0x05, 0x70, 0x69, 0x65, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65,
+ 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x48, 0x0a,
+ 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x73, 0x65,
+ 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x65,
+ 0x6c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x50,
+ 0x69, 0x65, 0x63, 0x65, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x3a, 0x06, 0x4e, 0x4f, 0x52, 0x4d, 0x41,
+ 0x4c, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x54, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12,
+ 0x0a, 0x0a, 0x06, 0x4e, 0x4f, 0x52, 0x4d, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x55,
+ 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e, 0x54,
+ 0x52, 0x4f, 0x4c, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x44, 0x45,
+ 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x59, 0x54, 0x45, 0x10,
+ 0x06, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x4e, 0x55, 0x53, 0x45, 0x44, 0x10, 0x05, 0x2a, 0x09, 0x08,
+ 0xc8, 0x01, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x2a, 0x09, 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80,
+ 0x80, 0x80, 0x02, 0x42, 0x13, 0x48, 0x03, 0x5a, 0x0f, 0x2e, 0x2f, 0x73, 0x65, 0x6e, 0x74, 0x65,
+ 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65,
+}
+
+var (
+ file_sentencepiece_model_proto_rawDescOnce sync.Once
+ file_sentencepiece_model_proto_rawDescData = file_sentencepiece_model_proto_rawDesc
+)
+
+func file_sentencepiece_model_proto_rawDescGZIP() []byte {
+ file_sentencepiece_model_proto_rawDescOnce.Do(func() {
+ file_sentencepiece_model_proto_rawDescData = protoimpl.X.CompressGZIP(file_sentencepiece_model_proto_rawDescData)
+ })
+ return file_sentencepiece_model_proto_rawDescData
+}
+
+var file_sentencepiece_model_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
+var file_sentencepiece_model_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
+var file_sentencepiece_model_proto_goTypes = []interface{}{
+ (TrainerSpec_ModelType)(0), // 0: sentencepiece.TrainerSpec.ModelType
+ (ModelProto_SentencePiece_Type)(0), // 1: sentencepiece.ModelProto.SentencePiece.Type
+ (*TrainerSpec)(nil), // 2: sentencepiece.TrainerSpec
+ (*NormalizerSpec)(nil), // 3: sentencepiece.NormalizerSpec
+ (*SelfTestData)(nil), // 4: sentencepiece.SelfTestData
+ (*ModelProto)(nil), // 5: sentencepiece.ModelProto
+ (*SelfTestData_Sample)(nil), // 6: sentencepiece.SelfTestData.Sample
+ (*ModelProto_SentencePiece)(nil), // 7: sentencepiece.ModelProto.SentencePiece
+}
+var file_sentencepiece_model_proto_depIdxs = []int32{
+ 0, // 0: sentencepiece.TrainerSpec.model_type:type_name -> sentencepiece.TrainerSpec.ModelType
+ 6, // 1: sentencepiece.SelfTestData.samples:type_name -> sentencepiece.SelfTestData.Sample
+ 7, // 2: sentencepiece.ModelProto.pieces:type_name -> sentencepiece.ModelProto.SentencePiece
+ 2, // 3: sentencepiece.ModelProto.trainer_spec:type_name -> sentencepiece.TrainerSpec
+ 3, // 4: sentencepiece.ModelProto.normalizer_spec:type_name -> sentencepiece.NormalizerSpec
+ 4, // 5: sentencepiece.ModelProto.self_test_data:type_name -> sentencepiece.SelfTestData
+ 3, // 6: sentencepiece.ModelProto.denormalizer_spec:type_name -> sentencepiece.NormalizerSpec
+ 1, // 7: sentencepiece.ModelProto.SentencePiece.type:type_name -> sentencepiece.ModelProto.SentencePiece.Type
+ 8, // [8:8] is the sub-list for method output_type
+ 8, // [8:8] is the sub-list for method input_type
+ 8, // [8:8] is the sub-list for extension type_name
+ 8, // [8:8] is the sub-list for extension extendee
+ 0, // [0:8] is the sub-list for field type_name
+}
+
+func init() { file_sentencepiece_model_proto_init() }
+func file_sentencepiece_model_proto_init() {
+ if File_sentencepiece_model_proto != nil {
+ return
+ }
+ if !protoimpl.UnsafeEnabled {
+ file_sentencepiece_model_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*TrainerSpec); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ case 3:
+ return &v.extensionFields
+ default:
+ return nil
+ }
+ }
+ file_sentencepiece_model_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*NormalizerSpec); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ case 3:
+ return &v.extensionFields
+ default:
+ return nil
+ }
+ }
+ file_sentencepiece_model_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*SelfTestData); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ case 3:
+ return &v.extensionFields
+ default:
+ return nil
+ }
+ }
+ file_sentencepiece_model_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*ModelProto); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ case 3:
+ return &v.extensionFields
+ default:
+ return nil
+ }
+ }
+ file_sentencepiece_model_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*SelfTestData_Sample); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_sentencepiece_model_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*ModelProto_SentencePiece); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ case 3:
+ return &v.extensionFields
+ default:
+ return nil
+ }
+ }
+ }
+ type x struct{}
+ out := protoimpl.TypeBuilder{
+ File: protoimpl.DescBuilder{
+ GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+ RawDescriptor: file_sentencepiece_model_proto_rawDesc,
+ NumEnums: 2,
+ NumMessages: 6,
+ NumExtensions: 0,
+ NumServices: 0,
+ },
+ GoTypes: file_sentencepiece_model_proto_goTypes,
+ DependencyIndexes: file_sentencepiece_model_proto_depIdxs,
+ EnumInfos: file_sentencepiece_model_proto_enumTypes,
+ MessageInfos: file_sentencepiece_model_proto_msgTypes,
+ }.Build()
+ File_sentencepiece_model_proto = out.File
+ file_sentencepiece_model_proto_rawDesc = nil
+ file_sentencepiece_model_proto_goTypes = nil
+ file_sentencepiece_model_proto_depIdxs = nil
+}
diff --git a/convert/sentencepiece_model.proto b/convert/sentencepiece_model.proto
new file mode 100644
index 00000000..5dc02d6c
--- /dev/null
+++ b/convert/sentencepiece_model.proto
@@ -0,0 +1,333 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.!
+
+syntax = "proto2";
+
+// TODO(taku): Needs to use LITE RUNTIME in OSS release.
+option optimize_for = LITE_RUNTIME;
+option go_package = "./sentencepiece";
+
+package sentencepiece;
+
+// TrainerSpec encodes a various parameters for SentencePiece training.
+// Next id: 55
+message TrainerSpec {
+ ///////////////////////////////////////////////////////////////////
+ // General parameters
+ //
+ // Input corpus files.
+ // Trainer accepts the following two formats:
+ // A) Monolingual: plain text, one sentence per line.
+ // B) Bilingual: TSV, source sentence <tab> target sentence
+ // When bilingual data is passed, shared vocabulary model is built.
+ // Note that the input file must be raw corpus, not a preprocessed corpus.
+ // Trainer only loads the first `input_sentence_size` sentences specified
+ // with this parameter.
+ repeated string input = 1;
+
+ // Input corpus format:
+ // "text": one-sentence-per-line text format (default)
+ // "tsv": sentence <tab> freq
+ optional string input_format = 7;
+
+ // Output model file prefix.
+ // <model_prefix>.model and <model_prefix>.vocab are generated.
+ optional string model_prefix = 2;
+
+ // Model type. only have UNIGRAM now.
+ enum ModelType {
+ UNIGRAM = 1; // Unigram language model with dynamic algorithm
+ BPE = 2; // Byte Pair Encoding
+ WORD = 3; // Delimitered by whitespace.
+ CHAR = 4; // tokenizes into character sequence
+ }
+ optional ModelType model_type = 3 [default = UNIGRAM];
+
+ // Vocabulary size. 8k is the default size.
+ optional int32 vocab_size = 4 [default = 8000];
+
+ // List of the languages this model can accept.
+ // Since the model is language-agnostic, this field is used as a reference.
+ repeated string accept_language = 5;
+
+ // Size of self-test samples, which are encoded in the model file.
+ optional int32 self_test_sample_size = 6 [default = 0];
+
+ // Whether to use DP version of sentencepiece. Use it with TSV input format
+ // (requires precomputed word tab counts to work).
+ optional bool enable_differential_privacy = 50 [default = false];
+ // Set these parameters if you need DP version of sentencepiece.
+ // std of noise to add.
+ optional float differential_privacy_noise_level = 51 [default = 0.0];
+ // Clipping threshold to apply after adding noise. All the words with
+ // frequency less than this value are dropped.
+ optional uint64 differential_privacy_clipping_threshold = 52 [default = 0];
+
+ ///////////////////////////////////////////////////////////////////
+ // Training parameters.
+ //
+ // Uses characters which cover the corpus with the ratio of `chars_coverage`.
+ // This parameter determines the set of basic Alphabet of sentence piece.
+ // 1.0 - `chars_coverage` characters are treated as UNK.
+ // See also required_chars field.
+ optional float character_coverage = 10 [default = 0.9995];
+
+ // Maximum size of sentences the trainer loads from `input` parameter.
+ // Trainer simply loads the `input` files in sequence.
+ // It is better to shuffle the input corpus randomly.
+ optional uint64 input_sentence_size = 11 [default = 0];
+ optional bool shuffle_input_sentence = 19 [default = true];
+
+ // Maximum size of sentences to make seed sentence pieces.
+ // Extended suffix array is constructed to extract frequent
+ // sub-strings from the corpus. This uses 20N working space,
+ // where N is the size of corpus.
+ optional int32 mining_sentence_size = 12 [deprecated = true];
+
+ // Maximum size of sentences to train sentence pieces.
+ optional int32 training_sentence_size = 13 [deprecated = true];
+
+ // The size of seed sentencepieces.
+ // `seed_sentencepiece_size` must be larger than `vocab_size`.
+ optional int32 seed_sentencepiece_size = 14 [default = 1000000];
+
+ // In every EM sub-iterations, keeps top
+ // `shrinking_factor` * `current sentencepieces size` with respect to
+ // the loss of the sentence piece. This value should be smaller than 1.0.
+ optional float shrinking_factor = 15 [default = 0.75];
+
+ // The maximum sentence length in byte. The sentences with the length
+ // larger than `max_sentence_length` is simply ignored.
+ // Longer input tends to bring the following risks:
+ // * Overflow during EM training (unigram language model only)
+ // * Performance drop because of O(n log n) cost in BPE.
+ optional int32 max_sentence_length = 18 [default = 4192];
+
+ // Number of threads in the training.
+ optional int32 num_threads = 16 [default = 16];
+
+ // Number of EM sub iterations.
+ optional int32 num_sub_iterations = 17 [default = 2];
+
+ ///////////////////////////////////////////////////////////////////
+ // SentencePiece parameters which control the shapes of sentence piece.
+ //
+ // Maximum length of sentencepiece.
+ optional int32 max_sentencepiece_length = 20 [default = 16];
+
+ // Uses Unicode script to split sentence pieces.
+ // When `split_by_unicode_script` is true, we do not allow sentence piece to
+ // include multiple Unicode scripts, e.g. "F1" is not a valid piece.
+ // Exception: CJ characters (Hiragana/Katakana/Han) are all handled
+ // as one script type, since Japanese word can consist of multiple scripts.
+ // This exception is always applied regardless of the accept-language
+ // parameter.
+ optional bool split_by_unicode_script = 21 [default = true];
+
+ // When `split_by_number` is true, put a boundary between number and
+ // non-number transition. If we want to treat "F1" is one token, set this flag
+ // to be false.
+ optional bool split_by_number = 23 [default = true];
+
+ // Use a white space to split sentence pieces.
+ // When `split_by_whitespace` is false, we may have the piece containing
+ // a white space in the middle. e.g., "in_the".
+ optional bool split_by_whitespace = 22 [default = true];
+
+ // Adds whitespace symbol (_) as a suffix instead of prefix. e.g., _hello =>
+ // hello_. When `treat_whitespace_as_suffix` is true,
+ // NormalizerSpec::add_dummy_prefix will add the dummy whitespace to the end
+ // of sentence.
+ optional bool treat_whitespace_as_suffix = 24 [default = false];
+
+ // Allows pieces that only contain whitespaces instead of appearing only as
+ // prefix or suffix of other pieces.
+ optional bool allow_whitespace_only_pieces = 26 [default = false];
+
+ // Split all digits (0-9) into separate pieces.
+ optional bool split_digits = 25 [default = false];
+
+ // Defines the pre-tokenization delimiter.
+ // When specified, no pieces crossing this delimiter is not included
+ // in the vocab. Then the delimiter string is virtually ignored
+ // during the training. This field can allows constraints on the vocabulary
+ // selection. Note that this field is available on unigram mode.
+ optional string pretokenization_delimiter = 53 [ default = ""];
+
+ ///////////////////////////////////////////////////////////////////
+ // Vocabulary management
+ //
+ // Defines control symbols used as an indicator to
+ // change the behavior of the decoder. <s> and </s> are pre-defined.
+ // We can use this field to encode various meta information,
+ // including language indicator in multilingual model.
+ // These symbols are not visible to users, but visible to
+ // the decoder. Note that when the input sentence contains control symbols,
+ // they are not treated as one token, but segmented into normal pieces.
+ // Control symbols must be inserted independently from the segmentation.
+ repeated string control_symbols = 30;
+
+ // Defines user defined symbols.
+ // These symbols are added with extremely high score
+ // so they are always treated as one unique symbol in any context.
+ // Typical usage of user_defined_symbols is placeholder for named entities.
+ repeated string user_defined_symbols = 31;
+
+ // Defines required characters. Each UTF8 character in this string is included
+ // in the character set regardless of character_coverage value. Unlike
+ // user_defined_symbols, these characters have scores based on the frequency
+ // on input sentences, and the model can form subwords using characters
+ // in this field.
+ optional string required_chars = 36;
+
+ // Decomposes unknown pieces into UTF-8 bytes.
+ optional bool byte_fallback = 35 [default = false];
+
+ // When creating the vocabulary file, defines whether or not to additionally
+ // output the score for each piece.
+ optional bool vocabulary_output_piece_score = 32 [default = true];
+
+ // `vocab_size` is treated as hard limit. Crash if
+ // the model can not produce the vocab of size `vocab_size`,
+ // When `hard_vocab_limit` is false, vocab_size is treated
+ // as soft limit. Note that when model_type=char,
+ // always assumes hard_vocab_limit = false.
+ optional bool hard_vocab_limit = 33 [default = true];
+
+ // use all symbols for vocab extraction. This flag is valid
+ // if model type is either CHAR or WORD
+ optional bool use_all_vocab = 34 [default = false];
+
+ ///////////////////////////////////////////////////////////////////
+ // Reserved special meta tokens.
+ // * -1 is not used.
+ // * unk_id must not be -1.
+ // Id must starts with 0 and be contigous.
+ optional int32 unk_id = 40 [default = 0]; // <unk>
+ optional int32 bos_id = 41 [default = 1]; // <s>
+ optional int32 eos_id = 42 [default = 2]; // </s>
+ optional int32 pad_id = 43 [default = -1]; // <pad> (padding)
+ optional string unk_piece = 45 [default = "<unk>"];
+ optional string bos_piece = 46 [default = "<s>"];
+ optional string eos_piece = 47 [default = "</s>"];
+ optional string pad_piece = 48 [default = "<pad>"];
+
+ // Encodes <unk> into U+2047 (DOUBLE QUESTION MARK),
+ // since this character can be useful both for user and
+ // developer. We can easily figure out that <unk> is emitted.
+ optional string unk_surface = 44 [default = " \xE2\x81\x87 "];
+
+ // Increase bit depth to allow unigram model training on large
+ // (>10M sentences) corpora. A Side-effect of enabling this flag
+ // is increased memory usage.
+ optional bool train_extremely_large_corpus = 49 [default = false];
+
+ // Path to a seed sentencepieces file, with one tab-separated
+ // seed sentencepiece <tab> frequency per line.
+ optional string seed_sentencepieces_file = 54 [default = ""];
+
+ // Customized extensions: the range of field numbers
+ // are open to third-party extensions.
+ extensions 200 to max;
+}
+
+// NormalizerSpec encodes a various parameters for string normalizaiton
+message NormalizerSpec {
+ // name of normalization rule.
+ optional string name = 1;
+
+ // Pre-compiled normalization rule created by
+ // Builder::GetPrecompiledCharsMap() or Builder::CompileCharsMap() method.
+ // Usually this field is set by Builder::GetNormalizerSpec() method.
+ optional bytes precompiled_charsmap = 2;
+
+ // Adds dummy whitespace at the beginning of text in order to
+ // treat "world" in "world" and "hello world" in the same way.
+ optional bool add_dummy_prefix = 3 [default = true];
+
+ // Removes leading, trailing, and duplicate internal whitespace.
+ optional bool remove_extra_whitespaces = 4 [default = true];
+
+ // Replaces whitespace with meta symbol.
+ // This field must be true to train sentence piece model.
+ optional bool escape_whitespaces = 5 [default = true];
+
+ // Custom normalization rule file in TSV format.
+ // https://github.com/google/sentencepiece/blob/master/doc/normalization.md
+ // This field is only used in SentencePieceTrainer::Train() method, which
+ // compiles the rule into the binary rule stored in `precompiled_charsmap`.
+ optional string normalization_rule_tsv = 6;
+
+ // Customized extensions: the range of field numbers
+ // are open to third-party extensions.
+ extensions 200 to max;
+}
+
+// Proto to store samples for self-testing.
+message SelfTestData {
+ message Sample {
+ optional string input = 1;
+ optional string expected = 2;
+ }
+ repeated Sample samples = 1;
+
+ // Customized extensions: the range of field numbers
+ // are open to third-party extensions.
+ extensions 200 to max;
+}
+
+// ModelProto stores model parameters.
+// SentencePieceProcessor is supposed to be self-contained.
+// All settings/parameters which may change the behavior must be encoded
+// in ModelProto.
+message ModelProto {
+ message SentencePiece {
+ enum Type {
+ NORMAL = 1; // normal symbol
+ UNKNOWN = 2; // unknown symbol. only <unk> for now.
+ CONTROL = 3; // control symbols. </s>, <s>, <2ja> etc.
+ USER_DEFINED = 4; // user defined symbols.
+ // Typical usage of USER_DEFINED symbol
+ // is placeholder.
+ BYTE = 6; // byte symbols. Used when `byte_fallback` is true.
+ UNUSED = 5; // this piece is not used.
+ }
+ optional string piece = 1; // piece must not be empty.
+ optional float score = 2;
+ optional Type type = 3 [default = NORMAL];
+
+ // Customized extensions: the range of field numbers
+ // are open to third-party extensions.
+ extensions 200 to max;
+ }
+
+ // Sentence pieces with scores.
+ repeated SentencePiece pieces = 1;
+
+ // Spec used to generate this model file.
+ optional TrainerSpec trainer_spec = 2;
+
+ // Spec for text normalization.
+ optional NormalizerSpec normalizer_spec = 3;
+
+ // Stores sample input and its expected segmentation to verify the model.
+ optional SelfTestData self_test_data = 4;
+
+ // Spec for text de-normalization.
+ optional NormalizerSpec denormalizer_spec = 5;
+
+ // Customized extensions: the range of field numbers
+ // are open to third-party extensions.
+ extensions 200 to max;
+}
diff --git a/convert/testdata/Meta-Llama-3-8B-Instruct.json b/convert/testdata/Meta-Llama-3-8B-Instruct.json
new file mode 100644
index 00000000..808826bb
--- /dev/null
+++ b/convert/testdata/Meta-Llama-3-8B-Instruct.json
@@ -0,0 +1,313 @@
+{
+ "general.architecture": "llama",
+ "general.file_type": "1",
+ "general.quantization_version": "2",
+ "llama.block_count": "32",
+ "llama.context_length": "8192",
+ "llama.embedding_length": "4096",
+ "llama.feed_forward_length": "14336",
+ "llama.rope.dimension_count": "128",
+ "llama.rope.freq_base": "500000",
+ "llama.vocab_size": "128256",
+ "llama.attention.head_count": "32",
+ "llama.attention.head_count_kv": "8",
+ "llama.attention.layer_norm_rms_epsilon": "1e-05",
+ "tokenizer.ggml.model": "gpt2",
+ "tokenizer.ggml.pre": "llama-bpe",
+ "tokenizer.ggml.bos_token_id": "128000",
+ "tokenizer.ggml.eos_token_id": "128009",
+ "tokenizer.ggml.merges": "d0cbac1fcc9dcf03724b8db5c9bfb593ae1cf68fb9bc72eb1d15274dcbbf618b",
+ "tokenizer.ggml.token_type": "d70a88809fd7da6f1f028622685cd64268a7a922c5d343c96f25b66327358978",
+ "tokenizer.ggml.tokens": "765b529dbcbc42dd202ce657341c63807b51f3b07e09898f6aa6196326865d5a",
+ "token_embd.weight": "b53102a11d9064bbd404833e3464b1b13e08ce73300b442312cccde2f19b2698",
+ "blk.0.attn_norm.weight": "7318df3cca9e8d153ff0a503026a1265e63d20b2a8c1dd7a2769585082b5d1ee",
+ "blk.0.ffn_down.weight": "b950806a1fc722c9fad7fd0b20c3c0a7fb50f14395e1e7663a590bfd62e20900",
+ "blk.0.ffn_gate.weight": "e73e580af6d4f08e060a74a3c25efdf5d3bed99e183d95a5a85ae859014839fd",
+ "blk.0.ffn_up.weight": "c8158af679ef99746da1befb67eebb19489e0bbe6ce7d97e13e348508244e516",
+ "blk.0.ffn_norm.weight": "7ec69c3c31e95e49a3359003b0033f6b9e85561a3e3fd83e7476661ecdd756bb",
+ "blk.0.attn_k.weight": "2732303257bac969b4964e0e32ec08b5a7f5c031bb02bf6ac4467b3ea0ebcf1e",
+ "blk.0.attn_output.weight": "ecda1d43b4ccc91cd5b366d7e7a275353990ac78561a07c83d9c77031aba12dc",
+ "blk.0.attn_q.weight": "569b1f5faf92b6f00910cf7effb2d5862f91038ce5c3b0019fc10e5d79fbd5e1",
+ "blk.0.attn_v.weight": "aa8416c5ef7e32fb54a1f20d6ac651656845d4af240564b397c39bd83e06e3b8",
+ "blk.1.attn_norm.weight": "03327e02862908c2a44b2f52decdb924bf4201f400b46f8037a9cb2e1d7a61ff",
+ "blk.1.ffn_down.weight": "5a83a87603f38c99f8e1e370a2d5f967bb45ac51d881a609304a7811027321e0",
+ "blk.1.ffn_gate.weight": "31da0572c79e655186c721c231376f85e56cdcc6257c28d08c8c5b40d5c22b40",
+ "blk.1.ffn_up.weight": "e0c811d64ca155c8de10a868e72015d43888834804614ee1aa2953129ffbc90f",
+ "blk.1.ffn_norm.weight": "5861f313d6137d6f0f904d423df47fffc6069e224ff746e1b637ac9c7f0af862",
+ "blk.1.attn_k.weight": "5fbbec0acca6457b9416ebdcd90e526885d0224537b7628f6be376a7f275313d",
+ "blk.1.attn_output.weight": "b237c9763fa3f75166a6f70b70f1566e77d0d89dfa164ed1b3137393e90575c3",
+ "blk.1.attn_q.weight": "c0a9cf4a98b4882b16f3eb2b49d933793dcc5357abb246fd3fe3134ed2b12e1c",
+ "blk.1.attn_v.weight": "96867111727200cac1af7865189dd41fd62b47584e5e5f33a91f1d34509cbd40",
+ "blk.2.attn_norm.weight": "f392f8a88ee3a95b1cc19c40dd4ef66317037b0faaa1800f610779e129ee0539",
+ "blk.2.ffn_down.weight": "73823eef46632aedcc8c1cb08a736b6aa97ca97842cd1fdfc5567d8dec459662",
+ "blk.2.ffn_gate.weight": "f4909ae19fc3848b00bb8b9050122e74f8e903b89e22937036f4cc9fea20a718",
+ "blk.2.ffn_up.weight": "16f4904a3d814ea68f00519724fc4943e48444a84c786bda39aa5efc298a7d84",
+ "blk.2.ffn_norm.weight": "e3ccdf56e75cb969f6f69c39caf6daf7c4e70e89e25df0f4d2e4bc60e159aafe",
+ "blk.2.attn_k.weight": "c3beb1e0a11bcf007ef0f0d8f6bdd3082d8b29090cd29597846b5d51e308a8e5",
+ "blk.2.attn_output.weight": "bb9f66c32cff51154fea92933c2cd62549236f8cb1a767f9ef28d3f99809b343",
+ "blk.2.attn_q.weight": "8eba394132eef2a05c5a92d62d2376000f7948448d7a2dc74e6b608203add20d",
+ "blk.2.attn_v.weight": "88f61f77c53567c617db3eef8f30621109a750e679f6784f7911739bd42c2f02",
+ "blk.3.attn_norm.weight": "7b996675b7ca75fa24107b3ebe0788653ede0f49ac83b8659d71ff54d591f81a",
+ "blk.3.ffn_down.weight": "2cb332bc05e4821962fdc9dcbcc7cc12630f32117711b687d18fb53c0bc4fbf4",
+ "blk.3.ffn_gate.weight": "340b387c7f208c8f0a6db904ef8d87c1e84b7d6ad57177abd32d86c8d18b760f",
+ "blk.3.ffn_up.weight": "07484433f8a7ee061c55aa0de2ecc009f769b0617c9c0ec096e9bb2946df9f0e",
+ "blk.3.ffn_norm.weight": "4f1a4ade36b393af341240bc894a2aab09cff7e4d56dc4658445deb107f9371b",
+ "blk.3.attn_k.weight": "483dcd96acb4528df84b9842970994630dbd82b8715ace394aa8b39fcf8d6291",
+ "blk.3.attn_output.weight": "beaff0810687923585642ee11d929cbf3b43dc6f87f30ddb552c222ab57bdbb3",
+ "blk.3.attn_q.weight": "0739355002f6fce520863add697e0ff25fc88215322dc3f993be7bb68dcce7e8",
+ "blk.3.attn_v.weight": "c216d17b6d90ee3e07f82598b8161fae34de2f392dbb0f745b682b578c324767",
+ "blk.4.attn_norm.weight": "91ab405bc4ba15bf63af233f266aa43aaab43789a9e6596e14a357c2ac7df217",
+ "blk.4.ffn_down.weight": "620f34ee75cdc73aecb8949af5fbb0d2437fd81422b6d8eb7acfc52addb9fc68",
+ "blk.4.ffn_gate.weight": "f6feec7bc9acadf35ec22532f8998d8e50f31afedabb19263590dcf8b9a92eee",
+ "blk.4.ffn_up.weight": "4a72af7cd28fd07b038f6cc4406678d120517280236ea85d9e76eff40ab2cc22",
+ "blk.4.ffn_norm.weight": "1805b37b44d5d682bdbd2fadeafb763ee001617d7870848cc487079ee34b21f9",
+ "blk.4.attn_k.weight": "a1e4f9d97cdf4c1b0d177cf00c4e32d1be30c1984a239b3c9bd73f8848888853",
+ "blk.4.attn_output.weight": "a1547e2497c423b0aff0eee71d9300d6fdf4e4986679418b6e637b69a9a6720b",
+ "blk.4.attn_q.weight": "0677483a9264ea6803d03d304d87a54632242cb516e8b76b6e3e8284c2f4de04",
+ "blk.4.attn_v.weight": "02691ba3af344fcc1969428ab0df811ac94aaa2fd91b0dc4ec1ac0a58806980d",
+ "blk.5.attn_norm.weight": "ba9c028335e5c895b87a5bd1448ca429248f9746ed97bdcb8679923206117156",
+ "blk.5.ffn_down.weight": "ccfdc9006acad1940a6bc05042a3947f1066acd671e0bb53b7684e9eea9ef5c9",
+ "blk.5.ffn_gate.weight": "623157679f1e742ccc3807c0b0153ddc8450104de75ec62f1370ec3807c09cf4",
+ "blk.5.ffn_up.weight": "05748804c65091f963729b58b085f58351891cac8a2861f5eae26b06aa60b2a0",
+ "blk.5.ffn_norm.weight": "84bae55af2efc8b8429f09056c8c04990c466dae31cb3f9356038b8957f1b406",
+ "blk.5.attn_k.weight": "8c766180c726b037d587fc52371de6e3307140c52409011609d1225624b6a3eb",
+ "blk.5.attn_output.weight": "490b582b3b1dc151ae55aee8b6743dad6c01fb49e43afefb6e68394b74be3d73",
+ "blk.5.attn_q.weight": "6f7b8ca4d9025ec836a44bbcca46be30c66b471a9fb62943ddff8288b3731409",
+ "blk.5.attn_v.weight": "9f70df3ba00c9e723214b3da83ff435a2163fff5915f75515c9664c05c866c27",
+ "blk.6.attn_norm.weight": "1a4a66613a682df6f061fc7c4d986f9f7e9175b62f0c42fc1ef31db536bd5942",
+ "blk.6.ffn_down.weight": "c56f25e4e49b443dbc82d88311ee63bc1f5002cc67e52f4787fd5f003aedeac1",
+ "blk.6.ffn_gate.weight": "31a5cf1aa9b831a81588d508550f51fc425f9517c43254d4ef7096d38029cf04",
+ "blk.6.ffn_up.weight": "ce135f3a1163e0c9297a615bdbe68a67ead21edce8debbfa9f6e15e6af8d4c94",
+ "blk.6.ffn_norm.weight": "4e328ce0648c94e732bc40501858ef6262ad1161e2e407b0cdcf4813fa9d45d8",
+ "blk.6.attn_k.weight": "1eb1c4c9f9c4c7ff7f5429075e0dc6a7782bed55109fa88df209a817dd8ef960",
+ "blk.6.attn_output.weight": "3d32986b56873b88655ee1edabdd413fdd9ab18b82108c9ce90bdbc2d3a6f3a3",
+ "blk.6.attn_q.weight": "8432f583b3a2809c99c393f9beb077cb0534dd5d247c17108f2986cadc6651f6",
+ "blk.6.attn_v.weight": "5045381513815bb91839dbac8335ffe49bbc7b0008369de7ea97eb676c5e2b36",
+ "blk.7.attn_norm.weight": "3dabd003638ec2499bfc8a48c49eef34276caab4fe76894eb963207848c2fdaf",
+ "blk.7.ffn_down.weight": "194fae858608bdcffd235be59ab119d0b91c8549f864ea06dae69249e099935f",
+ "blk.7.ffn_gate.weight": "00b24c29c30246892bce0791be804a89701d4c1332777e0bcdad5d9d5666604f",
+ "blk.7.ffn_up.weight": "44d7082a5280080c90cef9e19d410391de34f212ca0736377769b8ddd0c82d5e",
+ "blk.7.ffn_norm.weight": "21fe8a7fd6911c64e0d15a788b3b4cb6d71dd6ec51de65f760ee89afbb6ae53e",
+ "blk.7.attn_k.weight": "57a149eec5f6744a9526cd3925ac073f9d12db0fbcb5afe042ef4dc846458c44",
+ "blk.7.attn_output.weight": "0e9c28a3e81a2880251ce5eed77bcb8be8aaa1a51c9cb6de820b47ed83849fc2",
+ "blk.7.attn_q.weight": "15ee75263ee4e2a43eb322bc159ae004bb7d77e3a7e63ee4ddab700430693fff",
+ "blk.7.attn_v.weight": "440aa970bba4bff429fd7b7b1de21f2ad14fb2952b776cfa4acee68d7c6e9b8f",
+ "blk.8.attn_norm.weight": "af5b44825633c42c1ae964c82bb2be6a242d3a751f0a91f1bae4f593e8f5b6ec",
+ "blk.8.ffn_down.weight": "b11c14c76adca94fa200496dd2c10743becb23aab6642443ef1ae6d8710edbc1",
+ "blk.8.ffn_gate.weight": "7bb03d3325bf8637ae2fa1296b0651356515578d46a7c5ca65c7a923d7de27bc",
+ "blk.8.ffn_up.weight": "b956ef0a0669b5a9c9bf3a8da2d1c24f52d331cfb7354f6d7c51bd65be355e30",
+ "blk.8.ffn_norm.weight": "c78c3d748302edfef76f71ea5cb2055c94352122eee8b9b1173779a1814d224e",
+ "blk.8.attn_k.weight": "c0fba6a596ed9c1c32a7055c31a935a8b31e42b77282ee47c1f03ee3bde736b5",
+ "blk.8.attn_output.weight": "83cf9947080c5d8d571f04a842bc3dcfe7bbb0195fb25b346e22635e8649f2d4",
+ "blk.8.attn_q.weight": "47409350a576b333d97b7c877d69f47f46df504f3765102dfc0be9e521c7ecd6",
+ "blk.8.attn_v.weight": "1999dff91404fdcf1ecb34d9eaaaa9244ec7658a74dec8feb7cfd1fddba0347e",
+ "blk.9.attn_norm.weight": "1e6e29d5c3889ab4e1b0a5b9998cba60179b0f1fca133515df49cbc19d092593",
+ "blk.9.ffn_down.weight": "acb898a6490adff592e10b4c62d70edc5941661ee6da44658500e9205357c8e9",
+ "blk.9.ffn_gate.weight": "4cff63013593aadc3ffbaaa6ed70ffdba1224cd43c3644bf6f4162b5ac1ab542",
+ "blk.9.ffn_up.weight": "f985b5a2d6cf4fe32c7256301c3c89b8ad22b59e516342c52da42d8110766a4e",
+ "blk.9.ffn_norm.weight": "0d659c538bc6b21ed0018f107ab674a7424a00a42946c80e07208b479b21918f",
+ "blk.9.attn_k.weight": "f67611d888780d1b38c1c146b361c65310c8183bdf64fd73e2259985c6e8517f",
+ "blk.9.attn_output.weight": "f12ca1fa62a02ddc3f77f798bfb5707e0c50bf18ee0eaa67025521a98355f26b",
+ "blk.9.attn_q.weight": "3865185f4361a645b086ad47b72904c095313fb1c624e511647bf1a7dfc1c476",
+ "blk.9.attn_v.weight": "92125bbfed63544ab56052bd1e4aa453bbf34c795249ee54cde54907c8c6d1d3",
+ "blk.10.attn_norm.weight": "5d6bfbe545bcc2fcb2fc75c68f64b1f4c918badaf53e0156fe2d88aa977b2f94",
+ "blk.10.ffn_down.weight": "1dd9da8b0d2696ab5531fbca8a29c7d67567620a9d3e5fc2a19ec5d7e4c6cc8a",
+ "blk.10.ffn_gate.weight": "6e55e7f014edaebda0ac6819a426221d3b025c27312a2e18cc5806f31e3db226",
+ "blk.10.ffn_up.weight": "d80dde54af5db51241345ee8d64c1972608644f4deeac1e8195dc423bf27474a",
+ "blk.10.ffn_norm.weight": "f6ca65951d58ae3379eee8247bec34ebd0db05674cc9295593573841b8a55df3",
+ "blk.10.attn_k.weight": "b58e350bd6b49aba0fba4e4dd6865de3a2a0651ab865dbf2419b627b53ffc187",
+ "blk.10.attn_output.weight": "6b26a986e12fe66ec286a21d7d5af5eaa1bfe6f2bf502165d270e4497235a54a",
+ "blk.10.attn_q.weight": "3440e0e5b7e0d1e426424ae5a33f4e057be623249e9035ea12e57dbe5d3893c4",
+ "blk.10.attn_v.weight": "ebfadcfe14bcd6dee933053df0a67e12e7a196d5cc45728c1ffb2a2daedd5ca2",
+ "blk.11.attn_norm.weight": "3ed057b9576cd2de84507ef64c7646dc478c651efca4c2024cbe91a4f3fbf0bc",
+ "blk.11.ffn_down.weight": "8ff1c2487d22f5c499761e4eb721418f141f960160d0bab779595a34e4d68898",
+ "blk.11.ffn_gate.weight": "9c74e4507c7e45bf39b7cc7402198cd1dd77e3fff8c625b0413acaeb16efeb9f",
+ "blk.11.ffn_up.weight": "4367158007161d29939e00a322bb6776016e43f648a94f9b08a96a477aae75be",
+ "blk.11.ffn_norm.weight": "1cc0288c1491072121f4c9a0af20be0e13af49895696a3320e4fcac608768de3",
+ "blk.11.attn_k.weight": "066f5b3c144fce1366835e1ebf376f768b333b8ae29f5b478c42d1d0c809c855",
+ "blk.11.attn_output.weight": "e0d9f3d3f2c54aed59c02713ea4fb562799ddbacbe67ca3998dfc887bc44e47b",
+ "blk.11.attn_q.weight": "28d3ecc8a88cb3815e89a7f7a7d043da7a71f702b337a126e4d3a2ac1cd6370f",
+ "blk.11.attn_v.weight": "7c5cdef10ee73bca0a3b9f6ece5f0a0155664e0ce3d8de90ccdccfab5545e5e7",
+ "blk.12.attn_norm.weight": "973b133301a1af760cd7b3a7955371ea0a750808b442deb6adaf7b98482bd0c6",
+ "blk.12.ffn_down.weight": "d6c87b4b4ca03f75546ddd6a9e7fca720585a309188723c1ace8122438d4b200",
+ "blk.12.ffn_gate.weight": "2189a6e0cab1540bd05d6089b922aa8fd694be51255654933c165f302a0c955f",
+ "blk.12.ffn_up.weight": "5affbec19b58d092b9305721e3552481fe2eff51269ea3ed91cda3b9ef84d4df",
+ "blk.12.ffn_norm.weight": "f650fd42a34e950f758b4a130e7b8b1a712b1dcbede0291bb8edde47aaed0ef6",
+ "blk.12.attn_k.weight": "59b1e86f10450a7cc188beefc0856d2dcf44e8d7fdd9cd8859c30ec1ebaf24b6",
+ "blk.12.attn_output.weight": "446b0d36b2f66bd72a2323f4f4e9d85a0f621e9a58872e89a27248d6b1123238",
+ "blk.12.attn_q.weight": "3ed6bfd39f040301ed99fad882d3e569769d594259f9948445bef0e44ec881fb",
+ "blk.12.attn_v.weight": "e73652cd5d0029b1931be3ba9d82508f6696dce5a29d085476a54fb7a2ddbabc",
+ "blk.13.attn_norm.weight": "491b85278c0bd67bd31b9b8a9720902c244bd067e53a4a03641b7c0994782e82",
+ "blk.13.ffn_down.weight": "ad71cc248a85e9ced49307a24a9bfae01d387e979a7689c82ff59998e09741f3",
+ "blk.13.ffn_gate.weight": "0a55984d53971fab97575ee0ef5882013be7fdecfa76e3fbebb5dc85a07a14d4",
+ "blk.13.ffn_up.weight": "378b697b35e2e53c0de98e8e29b73d42ae3ec112ec16129aa5997a9e2f3b5943",
+ "blk.13.ffn_norm.weight": "f8aff2f69ab286210fad45a62b03f8d10b38f96a420d7baadf6b95d7b0b0bcd2",
+ "blk.13.attn_k.weight": "25ceb841afb1034831bea7f4d6a6c578def2ce4d4c412c780ef147dc9a598360",
+ "blk.13.attn_output.weight": "a242b322889c6bdaa14b67a7bab593db39df8eea3721638ef639abbb74d482e3",
+ "blk.13.attn_q.weight": "d80be9945a369439e835c55cfb0e97828b8a66bb7ced534d9059c92487bf20a9",
+ "blk.13.attn_v.weight": "ac33274cf9b67979d9ecdc967a55175afe0c9c4aeeff6391433cd9840c818706",
+ "blk.14.attn_norm.weight": "12a1e1091de5b2da12c9e7c0b1c8e6f09ce2a749733cf7d5240445b8e21cd093",
+ "blk.14.ffn_down.weight": "cfd41965c88266e32bc2dcdadda512499c35519e8686fefb9a7f249ab2291eb5",
+ "blk.14.ffn_gate.weight": "8dcfe774f07a095c7c6cf0a901c9df70d938bad7b5ba347fbc8f694e7603c0d1",
+ "blk.14.ffn_up.weight": "c7995577fe4a72ea0fb17c4a7b6b87b959072bbfdd5edacc6c367d43465809ae",
+ "blk.14.ffn_norm.weight": "81c41ebde41739e7016ffec31d2256217b825dc3cae049a935f5f61a60d22003",
+ "blk.14.attn_k.weight": "fb708bdebe4384f5c4b479c110028554f4d122f166b8091eda7d8d65e6780eb8",
+ "blk.14.attn_output.weight": "f5295caf2dfdc60553dcabe17537a80577e8b153c902247daac058df23542514",
+ "blk.14.attn_q.weight": "c12b7a3601c68c63ab5dc9d2599ebf3f3a10abc2c59d3a2126fffd5818f2763b",
+ "blk.14.attn_v.weight": "1ce968d9149bf0d5e237d52cc6d6433565b4bbf03252a736262bb00a2b34a687",
+ "blk.15.attn_norm.weight": "266fd2c36d7dcefc6b6bb7f1c9374c41f2bab5d6c84a063b6f91c4f682dad3c4",
+ "blk.15.ffn_down.weight": "6154886e9ef0a6cc08ab0d264a35f497e6f0987efdac992ed04e87088bea7801",
+ "blk.15.ffn_gate.weight": "183d9fd3c1b5657840099053d2fd3f72ad953b1de523296159b7761f20491a76",
+ "blk.15.ffn_up.weight": "51546d4498842ae2340ee226a0888d5f61e7d2ca4d052dfa06a77b0451242d3d",
+ "blk.15.ffn_norm.weight": "ef7378091a41a25a5f58bf1bf9d3bc64ea562e7f421e1c232b1f177c30fd3500",
+ "blk.15.attn_k.weight": "8d556ab8d9639324141774999b6eed0e91d7ee645bf3e7a3dcd200b2e7a00751",
+ "blk.15.attn_output.weight": "54aa6ba87def7cbe18b0c6ab3aff5c351cb3b6ca4a0d7b2cd5f75a1312991429",
+ "blk.15.attn_q.weight": "10731b0dc031ea8e0ef37bd7f010e0a78518a10a6df05a8bae48e3148b73ef3e",
+ "blk.15.attn_v.weight": "cbbe50c2ed7224866d3cf9b489c599f3ec41a4ea1aa3181e9f4e87e1fa0cefec",
+ "blk.16.attn_norm.weight": "387058eb39d4b28c04cf1368247417f1faeae8ae79d894c9f293457e0eaa00b0",
+ "blk.16.ffn_down.weight": "2cb26ccee585e933401ad5c82ed36ddacb3289efa0b28f8cf91b020ffbd9c333",
+ "blk.16.ffn_gate.weight": "d745985efb5bab42304e5d509024631efe35f92f2b2ec4931ead6db97ca9727e",
+ "blk.16.ffn_up.weight": "7a67bd195e0642828ca36eb7818149bb70c2c25f82de07e2b5807c520daf540e",
+ "blk.16.ffn_norm.weight": "7cefd061c8182482a89272f8a4e88a954b12609a62716923ca1cb3593b1c1651",
+ "blk.16.attn_k.weight": "d7968a2de67e755b4533e061aaad1cb62f8882af92dcad67f99d6d5112513439",
+ "blk.16.attn_output.weight": "9e9ab5788272ca3394ea89eadbce8c86ecc3fd75b7899184d6191c134ad9aae0",
+ "blk.16.attn_q.weight": "ef81c261b536c1a3a093b33f44cf2d42b86e5aa2d821674f07a0c80e992ed925",
+ "blk.16.attn_v.weight": "aef38e7958301b4a437cbdd2fbae6197f677b09269ec1eaf63188cd5da428d25",
+ "blk.17.attn_norm.weight": "28f6b289f1bc3131041e9f791b7a2a3a48baee0dfea27bf7051ebbb7ed364d80",
+ "blk.17.ffn_down.weight": "1a502829aafc6a9bd6bc81f12573bf8632d5c8c659f0dfb13c8b2411f3b1ec05",
+ "blk.17.ffn_gate.weight": "ddfd8aa0eb98846ebc9afe31366249159f46ae9815199dd70161527ed241ac4d",
+ "blk.17.ffn_up.weight": "4211a3cc247071bd361b30de2131d02382f552855062bf3b3e004c17992e5d09",
+ "blk.17.ffn_norm.weight": "647e5fa99a5b0d232af36d15816539f4d27e60a50a341b00aa88bb6e4474f8b9",
+ "blk.17.attn_k.weight": "d9125ff33a19c502c0f8846433ffc24395048582fc2f463d34a0301a82156f02",
+ "blk.17.attn_output.weight": "3d64fbb1cfef04444827f37c35fd9ad3413eb2165094d339ef89f00503f09de4",
+ "blk.17.attn_q.weight": "e5b29424028f578beca385fd82e29f37adedf3037cd51e5889d5a1ffb0428ca7",
+ "blk.17.attn_v.weight": "1809c5aaf2ac04c5d65539097564ad62796e87d24bb8b9ce5b095561a61d908a",
+ "blk.18.attn_norm.weight": "99daca58d001c627523d3adfbca1d95f04e590382a326866544d57989d5f4835",
+ "blk.18.ffn_down.weight": "84f30231ce6ca0f10227541dfc602d6418c1a210386b0c4926ef1656e7d4635c",
+ "blk.18.ffn_gate.weight": "ca5bbe4468b541740e54f69b9e08fcc8e478c344b70551dab21b1206acfbaadb",
+ "blk.18.ffn_up.weight": "0b3067b9dded31686dcfdc1e247eae3974a28a61ac59e9862758dbfaad64e8f7",
+ "blk.18.ffn_norm.weight": "8154a102232dbc0f90ce77ae5c1ff8f26f8b6e4dcf326e9ec1645749669e7960",
+ "blk.18.attn_k.weight": "25abb26021ccc481471a30e0d4cbeb7e1db29828417ec5136edeb93fecf09ac4",
+ "blk.18.attn_output.weight": "d87d481d9b046b68efa06ccdd4ed8cbf61e692d61114b75b7fad5ed75f5d87b2",
+ "blk.18.attn_q.weight": "cc6400379e15766992ff1293be79dc67682c28e9e15155a78109f4b64653b164",
+ "blk.18.attn_v.weight": "45c75cb1dd496aea3173aafe2575b841dd1d02cbe010b3198099731eb98f531c",
+ "blk.19.attn_norm.weight": "65389efc75297684773284ef8e5f8789a4504b636c9f33b8a32e0ee42499fa72",
+ "blk.19.ffn_down.weight": "4eefab7e939f64a17e4a214ca3c77a6fa110d94f677e2d6401086f70fc538b04",
+ "blk.19.ffn_gate.weight": "f1c0a59cafda66f466ab585b0b8b4861b58abe87a67cea1f6a488492242edfdf",
+ "blk.19.ffn_up.weight": "c42d045eef588db4a0e56960a57e110e1ff92eb8041107d19899165fd3b90f17",
+ "blk.19.ffn_norm.weight": "a8f33eda6d5d62ff5f333ad9771783caff556641f4e7df713451385676f441fa",
+ "blk.19.attn_k.weight": "0bab5d9e9083492bfb05a5a3bb23b79c0e7b99ef6a6644817b4d57d5c453b8a5",
+ "blk.19.attn_output.weight": "c99c551d70eafad0f7aea98fb6f9251635897168eb3895f76abf0d4ea3b3aa6f",
+ "blk.19.attn_q.weight": "c98bde95627c3b54c9443813ca50b4e14f518319681db6bbf7b2332ba26e9a60",
+ "blk.19.attn_v.weight": "ff3a490518cf64904db89ce0dc7d6eb89e870f1440e41883c6b55a221f82de84",
+ "blk.20.ffn_gate.weight": "761f0e317229cafe9d3754048ab038a0a84e9a287b196ab65f633139f2d29aba",
+ "blk.20.attn_k.weight": "45d13439b41066d282e8490a726785abf513605f46c79bd0c840f6419d27e790",
+ "blk.20.attn_output.weight": "a3b958d84b4a097844179b7d55c18fd0e4f319cb15e918c6fde33b68de1bcac6",
+ "blk.20.attn_q.weight": "127ab8e7d8c3f882874904196a02712bab42e6744fde45871b67350609d19f5e",
+ "blk.20.attn_v.weight": "5f0ad2d14a8ae42dd3bbeccfb33295687a14055fa92c54bc946249373c1c9f17",
+ "blk.20.attn_norm.weight": "77300b1755edc8c70089e0f45efa646056b9add7d8568b2324d2f3e62b64971a",
+ "blk.20.ffn_down.weight": "ab93d0e075b42e9017b701a070d561e698050d90aac4b4b9919256fbe50c3204",
+ "blk.20.ffn_up.weight": "4fd6628a07acc57a48d1ef83f81b7d7aa0bce569c1160a99d307284f8821322c",
+ "blk.20.ffn_norm.weight": "2a9e46b9e48e8e55215de56592e1f189530037c1c94a1428e3d6f106c7f26fb2",
+ "blk.21.attn_norm.weight": "4b3b5912c7bc61eb9da8e47d4651f896e85d9e59c4ecaa65df7acf3c21737298",
+ "blk.21.ffn_down.weight": "7146f931663d93b8771cd84405cd4802ea6560d0729b0d6d44588203c095bc53",
+ "blk.21.ffn_gate.weight": "b44ec5d64388fa40b90b3e9976d97a8b6800fa3b97584f32e64b03daffb8601f",
+ "blk.21.ffn_up.weight": "0cf3643fd23c685e17062cd11e116e17ce57a405e5e78953bab94cd62fe48789",
+ "blk.21.ffn_norm.weight": "4ef2cdb53da166df70b39f3e6b17af51848cfa5ea3c27ad6a1ae2a1bb1da1ce9",
+ "blk.21.attn_k.weight": "5d40f32a706f670c19972b14176bf660d5b045e3637b110dbf8d7de4ff32101a",
+ "blk.21.attn_output.weight": "18afaa916752ce16c9653ec0ec7e2fe60be55faa2aa5025d147be184adb75cac",
+ "blk.21.attn_q.weight": "2621daa5f858931514a4b2f0fe8d81cf9b96f541e6af99bfa7539e9bde8e34ee",
+ "blk.21.attn_v.weight": "63226dafc54c899bbce4aa49efceeedd8908e94faa613450fdda91f332b62864",
+ "blk.22.attn_norm.weight": "cf3058daab4d2c04387e7d169d1553bb8e7358eea66285ec067703f6ce62043a",
+ "blk.22.ffn_down.weight": "6a58d5fd220abdbac6cee7ba048abab794731af318f04982c2506df59413d0b3",
+ "blk.22.ffn_gate.weight": "d5614535324b03c7b91727a903b2a72f8d07ad17f7aa8b61ea173cf9b895069e",
+ "blk.22.ffn_up.weight": "ec20da3949566e93f66cabb67f8cd7eab399047ec6ebf5d43edfaf3669b82296",
+ "blk.22.ffn_norm.weight": "84c82f38f53a649972a44466fc476bf764e064ce18de870291edc302f3700e28",
+ "blk.22.attn_k.weight": "a3d2ecc37fde7c201176bb8abadf27f0d8ede9679a6034913e03d9db924fda12",
+ "blk.22.attn_output.weight": "5a3b8bb433f43a387df43dd371bdf80ddfac986dfeaf38e9bac1d7a0ec6628de",
+ "blk.22.attn_q.weight": "3a875cec661b4859f30a8fd2c866811184b25b68c9e36fe2663d299caf8b59c6",
+ "blk.22.attn_v.weight": "8717a83b79035058dcfd3ef6f8e5b36e71d77379e5a239e1899eef8766fb7703",
+ "blk.23.attn_norm.weight": "2b4a68a0a2f023dd646e4755c9bef17c2f631901154afd839edac7ac006ec99c",
+ "blk.23.ffn_down.weight": "29499b1586c6fc4883c9b7a9c8cf388035146b5aecf90c5c4c8c8e082c71e7d7",
+ "blk.23.ffn_gate.weight": "7d6554036d21c587b9b556428054f9c15cbef96d24b257f906fcef4ae38bd9c8",
+ "blk.23.ffn_up.weight": "19761ecb288d6ebd44b681c4535661583b1e19dc29e96d0c007333cd8f00aacf",
+ "blk.23.ffn_norm.weight": "37dc35500790a4ca33807b39cf7af65065e535dc25b9e94f3ed2759f61887ac9",
+ "blk.23.attn_k.weight": "717547d00323817b0cb40a72ec5f8cf42ecd1f9e3e42715c2cc5e38f07fffffe",
+ "blk.23.attn_output.weight": "a24786feb6a905fdf166d7500133757cbe494779d4ebcba9eb03046b319557df",
+ "blk.23.attn_q.weight": "6a2c4a98f138b928d22136efa163562691d3b4ed526d52d46a2fa2694a8f3965",
+ "blk.23.attn_v.weight": "c6e6081eb9c38a7fda023085957b460e9ea321e1fff408b38c2b58595c39979c",
+ "blk.24.attn_norm.weight": "5e6283f891e538670425f3e244b08dc6f96f33dfa4aefa913f8eb17212421850",
+ "blk.24.ffn_down.weight": "e09eb170f389deea0a4a1cbfdb52c12490768a2c60491b7bef8a4c445e2a08f5",
+ "blk.24.ffn_gate.weight": "af29d815cf49a38fc2ebd0bf9b2dd9933d023a29f2d766981acb9a1b53f09117",
+ "blk.24.ffn_up.weight": "36ccd9333426666de9d3088bd4dcdf5b624b09dca9e3a83a22fc0383f2d950fa",
+ "blk.24.ffn_norm.weight": "a88e1692318826db6ac42582d182e51a3c698c655d0e21e04fa086318832d07b",
+ "blk.24.attn_k.weight": "f7d61d6d1225289bcc502e3bbb0168b4584add0253218c1b77ac92ccef9a1c2e",
+ "blk.24.attn_output.weight": "85a1363b3ccc87312094c2195022687c16b0dad7fafb9e80bb4ec474d53c29ac",
+ "blk.24.attn_q.weight": "53482a2c008f42f4fad779ca323addc3712040149dfc12f782417756388a72bb",
+ "blk.24.attn_v.weight": "67498272369af7dd10097c73b07f731b565cfc9a559e711cc0d526389e7b44e2",
+ "blk.25.attn_norm.weight": "98dd617def5cb7825ee4833132ca2da2121245921585e1d9e36b93344adc321b",
+ "blk.25.ffn_down.weight": "7fd477d6c50aed5f424a878dd284343379cffbee8a34c0b6e55100c8305fa13f",
+ "blk.25.ffn_gate.weight": "f892c9806c8ec22e8aa746734ac9213428c534921cf161239e1d249fdb5d1ec0",
+ "blk.25.ffn_up.weight": "528bed14c9bf9762f790525ee40412545221f4321d2a2323fa8e73c58b7643c5",
+ "blk.25.ffn_norm.weight": "ca5831966672e7be6a578feeb631ec3570d3b5afe12860819ccb96e896ffc346",
+ "blk.25.attn_k.weight": "610d3068cc9b20401f0c3a0efea39a279dd9f564fde19baf3403b2ec2319e4c4",
+ "blk.25.attn_output.weight": "798aaf702e53b657265ac3b5e6caf3a0ab515bdadfeb1a3a156b4f3bfba76666",
+ "blk.25.attn_q.weight": "8a7fa25248de83029fb97b51d036a01baebe31fcb4be121ab00dd8b7de209b10",
+ "blk.25.attn_v.weight": "2a53d5e9f8a1218c66958c6388d3b37400a9af7956c785024ca44bfbc3c7d371",
+ "blk.26.attn_norm.weight": "5f44fc043481eb0771f3e6d2420bcbcf73140afb9a9feb8eddb6575452acebee",
+ "blk.26.ffn_down.weight": "944a60a409d0d5b6a851e33c69aca152454b691711a8b96f5bcc488772ab2833",
+ "blk.26.ffn_gate.weight": "2a0ca4abb3de5593e6693d8be69b63d6d1a639855ac8332a75f520353f030c62",
+ "blk.26.ffn_up.weight": "0b1df496163f9ac07bf89375d3eb441b51a81d41b47d769a04a61efc18dbe35b",
+ "blk.26.ffn_norm.weight": "56b8dd046e9be6ea71f7efd80dbd14e7fb1aa020d3cd38e063275f3873fd12f8",
+ "blk.26.attn_k.weight": "b1dabfabb970e6971c7ea6e53c63cf7ef56341e6a2edd9cf177785cad9af2f9a",
+ "blk.26.attn_output.weight": "39532c7e836baad164a655fb97ec5114ea4da37ffba9fdea2684f6e4450e6f84",
+ "blk.26.attn_q.weight": "8f48bf6aaa1252bc149e98af2be1777a5c0d2c3274c6d314171ea9344a41b604",
+ "blk.26.attn_v.weight": "02fb145f7fd905133750e90571effacadddfd3f4966552dc59982ac3900ab8c4",
+ "blk.27.attn_norm.weight": "654d168fc3cab716d91261f5719f180b7d697218401633b4878a759f1b5283f2",
+ "blk.27.ffn_down.weight": "2823272bec3a1c12f02cc4cb24aa4031abd7e9dbe0b02676e2305b21671818f0",
+ "blk.27.ffn_gate.weight": "b1a1d40cd02f97182cac17a79971d1934ee0daf3aa0bf11303568c636e208a64",
+ "blk.27.ffn_up.weight": "ed62ec72a020d070e64eb7b50237b32213944727b5b2427f45d989f50df5fb2a",
+ "blk.27.ffn_norm.weight": "c69649ac65d694b306a905dee8b03b89eec1ed188b1eaaf38f8e29d4b12e38a0",
+ "blk.27.attn_k.weight": "cc57bbf413f1fd227128dc66efc8590c73634cbd6f96d01ec4878b5e7ca6a925",
+ "blk.27.attn_output.weight": "cac407ad02361d53207b3c7e25ceab84dcb4347b8087055162e2efe14d11d84a",
+ "blk.27.attn_q.weight": "0af18e07cee12015761c07c94407024f4f4d77d97bdb24163db0e16669e2cef3",
+ "blk.27.attn_v.weight": "a1d08fbdfa40af773c5adcf93bd68b78a44ed144e3fc6bbeb8af02e937527eb6",
+ "blk.28.attn_norm.weight": "f39a51f814512b040a1082143150e4a49ff730f85cef49d7f77fc79d83e91f40",
+ "blk.28.ffn_down.weight": "74f29ed51055d1c1adb8f0660bbe538a27e016c65650f2d67efc6f1c84fa1b45",
+ "blk.28.ffn_gate.weight": "ae48bb16487ded6781c60aafc0bf738fb4ae15729952906f247d216592ce249a",
+ "blk.28.ffn_up.weight": "543009727718ac22f11ee4b17815f68ea6f15ba1f3e7ed5ecdb755cf6417565b",
+ "blk.28.ffn_norm.weight": "b8f9e54c322079ff20a82b88948cdc2916c22c7db40b9a9ed6d3cbe89efb727e",
+ "blk.28.attn_k.weight": "55d055ba653b728d6e784f9e013786fed07115c9fdf23367e3941386d5e77db8",
+ "blk.28.attn_output.weight": "155101c03ddbf18f4fd0694bfc982f33c7bae25c9b087d6f5273c2bfbffcf2c9",
+ "blk.28.attn_q.weight": "1ed19bfdd22e9c14eca014739982492e9516d411515a8585f65cf754d849e53f",
+ "blk.28.attn_v.weight": "11ba854dd575c025d37256eee9041f6d1bd2b549a083d6409a09bfc1542913f3",
+ "blk.29.attn_norm.weight": "02b0bf5e2fcefd11a153cc988c81ba672682e4844fcf6442423e21a0e10d566d",
+ "blk.29.ffn_down.weight": "594bb692ec2779938721ff4748666ca8370e0e4fe85229503f616438b8884f5f",
+ "blk.29.ffn_gate.weight": "8bedcf47e91dcb2cf4093de56b048ee411faab6ff472f89ab2c9c113a08e6967",
+ "blk.29.ffn_up.weight": "e241a547b5fd6dfca8200b8141e21c1c487a96cbc4e5855f181a7ed1be91b642",
+ "blk.29.ffn_norm.weight": "e63eba5e4c6b288bfd9f15e46e236086456c8b7f1f9c732c0b5de84962a2e7cc",
+ "blk.29.attn_k.weight": "afe5979d5bcf211aebb526620f5974bcb0a2c39c8be71e815575c55d6385e3aa",
+ "blk.29.attn_output.weight": "9c944ed44b124b014906fc240afd3b90aed56bbd9567f2eddfd5b7a685b3cb48",
+ "blk.29.attn_q.weight": "e234e08e5c1bd9245a2edc8d63e9933b6b879f97c01392209cad4f55f05f3ada",
+ "blk.29.attn_v.weight": "5cb8e3e5f954e775c5a5e4de7a9a62b17e9c6931bb0ff0e2f82c4126fd3e1a1c",
+ "blk.30.attn_norm.weight": "a65483ee51a0b214144ec8a14f28ea5437586e9e12ebe342a57d1f8627ee12af",
+ "blk.30.ffn_down.weight": "417959da77ceb33ead4271cbb9428b195196173a893c44e52880a7ec61b4856b",
+ "blk.30.ffn_gate.weight": "a0d503ffcbe45dc927600bb98c9f6082487e65cb577ab545add400d666a87638",
+ "blk.30.ffn_up.weight": "f8ab957b82ffcd10b21303cb5e866209b6fe95f827b1b94e9a949207952d12c0",
+ "blk.30.ffn_norm.weight": "210c7ceb0514a9ef27b5d4d1b3aff6dde43f1af0345a050d71097940e0e73e03",
+ "blk.30.attn_k.weight": "16861b9abcf5a3fe73c93d977ca45a1e6daa65be0fd85c2cff53486ce2033afa",
+ "blk.30.attn_output.weight": "ca541fb2e57e2257118c35784845b0c731278af8db3036ac53d71aa1681fdbdc",
+ "blk.30.attn_q.weight": "f7834917748e26bb456b945e230bc926c228e93696bc01fbc2b134bdeeac71a1",
+ "blk.30.attn_v.weight": "9292783171dbe5eb689d17c9bda11e537f0e9b328fced6986c938d61ed590e81",
+ "blk.31.ffn_gate.weight": "e4766a04bcd8f937ba883c6a144101e546747804ca66c35c97281d6ccb47b566",
+ "blk.31.ffn_up.weight": "cc1e666116f7e6b06736db4aa4b81003c583f54f4d9200bfa48842249940e16a",
+ "blk.31.attn_k.weight": "fc80b57557687504efae7d24265cb7dc39b8f826bb3d897a11783012dbedc44f",
+ "blk.31.attn_output.weight": "215617f50a1f5d9b2250b82f3652b35a9e9aa0ad9ef2b485d73965a14b2b872a",
+ "blk.31.attn_q.weight": "274b4f1dfb0bdec28632705677049fb3e327ce6d9e1f3baaad1560439039982f",
+ "blk.31.attn_v.weight": "e641b8b926f9dfcbbf6b6da1c02555525ac4b1c306d96f20cfbba7d6662c4e56",
+ "blk.31.attn_norm.weight": "b3243c361d4041ddb892ce6862dd5091f57d87357e3c67e177451b85d8baf34d",
+ "blk.31.ffn_down.weight": "0a00cd3ecd5e91624a27f9e239b1de425d5ba3cfff82c256a11a4ad434abf3c2",
+ "blk.31.ffn_norm.weight": "2a0d67ea2bb1303975712243f07273c92fce83baa11b1cd6d8e42e74ea3c810b",
+ "output.weight": "768615f077fb797967844571c58b94d7c399d884d115be3ab4b0154504cae892",
+ "output_norm.weight": "7cc5b7ce10e5082000fa00bfa68af8c7c5da218e59e2c41cf2f1499d40ca229e"
+}
diff --git a/convert/testdata/Meta-Llama-3.1-8B-Instruct.json b/convert/testdata/Meta-Llama-3.1-8B-Instruct.json
new file mode 100644
index 00000000..ad7cd20a
--- /dev/null
+++ b/convert/testdata/Meta-Llama-3.1-8B-Instruct.json
@@ -0,0 +1,3 @@
+{
+ "rope_freqs.weight": "80fd5efb2f729381785b293a091a268cfeceb0079167f6ece9b07070e662b222"
+}
diff --git a/convert/testdata/Mistral-7B-Instruct-v0.2.json b/convert/testdata/Mistral-7B-Instruct-v0.2.json
new file mode 100644
index 00000000..88d447b3
--- /dev/null
+++ b/convert/testdata/Mistral-7B-Instruct-v0.2.json
@@ -0,0 +1,313 @@
+{
+ "general.architecture": "llama",
+ "general.file_type": "1",
+ "general.quantization_version": "2",
+ "llama.block_count": "32",
+ "llama.context_length": "32768",
+ "llama.embedding_length": "4096",
+ "llama.feed_forward_length": "14336",
+ "llama.attention.head_count": "32",
+ "llama.attention.head_count_kv": "8",
+ "llama.attention.layer_norm_rms_epsilon": "1e-05",
+ "llama.rope.dimension_count": "128",
+ "tokenizer.ggml.model": "llama",
+ "tokenizer.ggml.add_bos_token": "true",
+ "tokenizer.ggml.add_eos_token": "false",
+ "tokenizer.ggml.bos_token_id": "1",
+ "tokenizer.ggml.eos_token_id": "2",
+ "tokenizer.ggml.unknown_token_id": "0",
+ "tokenizer.ggml.scores": "e3d3eea80bb41a1213f2d0aa3e8a38581d1f19323be77dbd779c9c7e3b72e676",
+ "tokenizer.ggml.token_type": "6040635e6bd38d98af06698feb75c1802bad35180ee6ae0a503e38c0f60fd71e",
+ "tokenizer.ggml.tokens": "604ac4bfbd019e430d7b6cdf18c6c0cd5b967900601f0307f714ec7773aa5ca6",
+ "token_embd.weight": "cde834ccac5e94324b25cb81b02d27312cac0c551b55a7e1d555d90bf6cb6e81",
+ "blk.0.attn_k.weight": "458bfdd9715c66e017c2447b1ed3c582963a3111479314e664faad8c914f42be",
+ "blk.0.attn_norm.weight": "e1fd60b95f713bae7b7e3ca933c64ae6c9cd1e8d808000204bbfdc19f0ba635b",
+ "blk.0.attn_output.weight": "df13b6a157d9d4f96c53b012b3b9bcd207d0c94144cbd22ae3ec13bb07d6c373",
+ "blk.0.attn_q.weight": "13b4126b4245bf06c915a93317c42b8174e05053535ec99dc576541e4cec7c25",
+ "blk.0.attn_v.weight": "5b1781d3a341214511b27eb4e268674ea3ea829dbdf8ae5a6bb89b3c0b33fafd",
+ "blk.0.ffn_down.weight": "49186f5d8148d316b07458841d13a2e66587f4af69b776188a809591ed9c070d",
+ "blk.0.ffn_gate.weight": "4397e30ece09136f00f4ff84ff49e5241b765a374deb8c5a12e897e2bf73473e",
+ "blk.0.ffn_norm.weight": "43260589aac3850a779bca3f9649f793bbfbe5db538361cb743b3830217f8287",
+ "blk.0.ffn_up.weight": "fd7ac918240a07566f6967527ffca58fcf433a30b78fdd6d84b2136d4ebd9987",
+ "blk.1.attn_k.weight": "209839566c7d235bdc20565a4766378b6ee8553133a5a3315abe8a85baa80712",
+ "blk.1.attn_norm.weight": "58c52986f7c69784ba327cb7f350923420782bee17fa39b1fbd13839d4005357",
+ "blk.1.attn_output.weight": "5067cc628449682665dfcf59b16e58fe2a9d2a81cb099f0fcd42f4f8670c6740",
+ "blk.1.attn_q.weight": "f410f9f0dd5edc09401af597d02e2a4c727f1502ec3ec3898321617b36c6df6b",
+ "blk.1.attn_v.weight": "d40fa49e07c102c0644e130e7909eaa93ed0d54e2edddc0759e721d58a4e4f5e",
+ "blk.1.ffn_down.weight": "594b1eff6ed4defbdd819fabbe2d48764984f08878a860bdb808511d5a25b8db",
+ "blk.1.ffn_gate.weight": "4cda97541e388a5bb607ce4cc8b3db1da7045830a630e7ba4d17807befcff346",
+ "blk.1.ffn_norm.weight": "66c13d7481be65b97aa474735ddc9674f33d512ddda76fa6fb45c7464b09f1ed",
+ "blk.1.ffn_up.weight": "1adc6de288ba4cc1237833ca8b4eb81107149842e38bc452e18e5cfe284338a2",
+ "blk.2.attn_k.weight": "5420423559f236ab22d85a00849f31e0cc6e9c7dd879de724393d8cd2b379153",
+ "blk.2.attn_norm.weight": "495fe1ab40cc52aa054ddd4f0c2d2790f4326c8d103296b1b38f3b1060db2a24",
+ "blk.2.attn_output.weight": "ccb83e7085381f558bfd65588c525ad2671feddcbc3887afb4038ad9c7aac348",
+ "blk.2.attn_q.weight": "2e8f77478392bc93c2a391f2e0f4a173a952bbab88a7aca099c6ee909726409a",
+ "blk.2.attn_v.weight": "d64512590f3b7ebbb9e77c2eb97fbda90b00d45c944f2b174f03a2cb11007567",
+ "blk.2.ffn_down.weight": "1de5084a05dcaa6b1bd926e83517dbe9ebe7fde79235fe56018b3028b1aa6397",
+ "blk.2.ffn_gate.weight": "cbea526b557f49aad8c976973cf367fcd12175b900f551984f498b9e07e4b7fd",
+ "blk.2.ffn_norm.weight": "530aa49b10c7eae08899d143409240deb95dae4e1d5bf78cea3b26393cff3ba1",
+ "blk.2.ffn_up.weight": "13a5fc19b96b4dcc1e9bd01998c8272ebe52034c1933ed123a506b711fae9a5c",
+ "blk.3.attn_k.weight": "1913b63a73305941d8cdc472e7f101c633d3357a78602eac0a4b49a744261075",
+ "blk.3.attn_norm.weight": "9c11bed5ab41f4adbfdae4ead65b525c8f19443e656a8c61ba412a4e1ad1193b",
+ "blk.3.attn_output.weight": "bb0b42c1d34779c5943272ed71f1dbb31ad8edd75f8bcd5c868f88505ac3a610",
+ "blk.3.attn_q.weight": "3461a1fe4e49f5319ea047cae98ccdb46528a3ec23831183fe87610b48c94948",
+ "blk.3.attn_v.weight": "82aa30be6a61526a41fb79bb28a2617416f5909f0477aa9e95e16be9370fcb38",
+ "blk.3.ffn_down.weight": "68521011ae03f5e3b0966127111afa8ee9f2eaeeef8d3a0b86b633e0332e9fbf",
+ "blk.3.ffn_gate.weight": "1e89e26338fd364bb679695968c65106382f15ad55c95cbb5ec9bdfeb766f432",
+ "blk.3.ffn_norm.weight": "c81932529a5a8c417c27b888dbe95fff8b447c2ea5f6f560444ec5d50b93832c",
+ "blk.3.ffn_up.weight": "305021735afd8669afefd713f56137248d5e817e60471a112ad06b7fa07ffe88",
+ "blk.4.attn_k.weight": "cc26ba5c5c28082a79e6abfe61186029e80b145252ca6a7924c437f0bcf2d51b",
+ "blk.4.attn_norm.weight": "302d251fdcc91f7468cf33f80b49484251d8917d7018ad264ab3a85c8ecf9ddd",
+ "blk.4.attn_output.weight": "a012f5bee3520cd4ce51f0076c132ebc3653309f304032ad051aa308f55f36de",
+ "blk.4.attn_q.weight": "3c8d607e447f5ef21e73af71e3c0d32fae16f91f31faae34ff06912cf9cb68fa",
+ "blk.4.attn_v.weight": "49f6c81a634ce46d71c2350206ecbd231b1732af96e4e4e67693c41a07e007d8",
+ "blk.4.ffn_down.weight": "e89504f311a4a34dc819a67b761022f14d71c43df3ead4f892c87aaa8e9f0adf",
+ "blk.4.ffn_gate.weight": "18b22f079a2fbaefe3572eec61fdcd996fd747724e2f0ff4f08cfcb43eb7bfb6",
+ "blk.4.ffn_norm.weight": "22415a492c168a0878912b05c854a631228b01c3ea8842e1d75989ec46c18a65",
+ "blk.4.ffn_up.weight": "f57379eae2874d8853f14ddf0f0fcc4ff1338574d5ed5d7e88331d5fb84f5642",
+ "blk.5.attn_k.weight": "d627af853c40bddf9762ce3988008c1ff17f2686fa8f73a0b5da38010147c316",
+ "blk.5.attn_norm.weight": "9ce01092c7f7f1c3ef72d6b794da12d77aa1f6a24fb96ba1b9bd5a0bcc3e2443",
+ "blk.5.attn_output.weight": "0388da8064c4b6b795ce2d8079e8a36535e82b2c9cf794e38ce8ae460aae726d",
+ "blk.5.attn_q.weight": "039b7ce1c909761fdf475c06cf14cabe5a90199282c89e4dcf460e95a4b6275d",
+ "blk.5.attn_v.weight": "c47bfd8d2496bdb6e00e03b903e15fd0ee806a515094ec257e43cc433147ab7e",
+ "blk.5.ffn_down.weight": "1d62e6708974bae318cbf00a8bf621d9ba0537e549ce4710a536520a8d14168e",
+ "blk.5.ffn_gate.weight": "8b42b1b11c92db19985094cbb50434e3a7c9cfea71ee6f21ea79eae7c49284a5",
+ "blk.5.ffn_norm.weight": "e0bc520f1505e687ec391d632a381d38d8ebcdec19f614a11a2000ab573e8b7b",
+ "blk.5.ffn_up.weight": "8cdcd17d2ea89bb9ab902dbc6bf3f827fa4ee029c6bf19eecbdefd146d8b6f2f",
+ "blk.6.attn_k.weight": "5dc6bcff89794d1756bf57ec665b58622d9352130d31082a6c66e1a079f99932",
+ "blk.6.attn_norm.weight": "13b26008abe0f119b5104b9d78ebd5e797d3cdd68122b93d73a3b4831a54d085",
+ "blk.6.attn_output.weight": "f5a49917ea70c3fb311ccfffbfafa63ab18416a5d55e5429b70ce8bfba57c075",
+ "blk.6.attn_q.weight": "d9c2f652c87dbd09ec3822e12876648fa32e86553ac25afab723b1cd9f8cef90",
+ "blk.6.attn_v.weight": "5ecc5fe67609a35151011cb526f45c56fc0a999079ae0ff37c755ca03c68c555",
+ "blk.6.ffn_down.weight": "0ec125ae0ecb2d9277fdb1b04f17efee94e37d0ae37311057c212ca2db3fe6d1",
+ "blk.6.ffn_gate.weight": "fa4d6d38355ee8aa3b80b476d65ae7e343c9b7770d7b097fc848ee8a6e091d1f",
+ "blk.6.ffn_norm.weight": "30e8f7defc627532e1739dc76d31223d45767391a431f925b63dabe334b0f392",
+ "blk.6.ffn_up.weight": "6b97cc32b290fa9087806b5d65aa6dc1760737730c8c71394cc4f30c2157f9ab",
+ "blk.7.attn_k.weight": "0231cb127cb7c3714cd72b8f39343891d7715a9bab2237ade9e7bc5f4ed2e68a",
+ "blk.7.attn_norm.weight": "7c3187f07eead7d219d98ab2daf87905e88d5f1ace109b6f5fa55dce3914981f",
+ "blk.7.attn_output.weight": "2f30ad972c284ae7c8eb0482053433495ebe8fe9c5ee2c28b4bc4ed1f33050fe",
+ "blk.7.attn_q.weight": "3a2b4b8d61cc9956d304fa9f82a9e65b4bb9fda2196670b16df7e0d8c43eff2c",
+ "blk.7.attn_v.weight": "d2aab97d0dcf0f61dd2f32848f7a8a99c423a4948a660a660a03a546972b8db8",
+ "blk.7.ffn_down.weight": "2270d520468c5549cd30023ff9c452a277058310104c4239a616373fc5a94387",
+ "blk.7.ffn_gate.weight": "4134a3ef71b3eac8f76b6f1a2e58625b3bae48081f175994bc3ed7d8b0d4f2d0",
+ "blk.7.ffn_norm.weight": "42df4abd4b8769b16f3930068f96960af1b061f1aeb7505384f272233b2badff",
+ "blk.7.ffn_up.weight": "c920549054ec16ff8c73a72f5d837cf4e11885e44db57c1c1c584c18fbd7a9a5",
+ "blk.8.attn_k.weight": "01c609bd3bf31ce65688f1f640ee413740e821330134d4ed1877a3065d1527d5",
+ "blk.8.attn_norm.weight": "48857411f769b00290f4e4f2e593e092781fdc2503f80c1e3eeda1b85a20f74d",
+ "blk.8.attn_output.weight": "90fb273f8df83744554bd59236515c16c5a5a698ca3fbedc17cc89ddcee354ff",
+ "blk.8.attn_q.weight": "ade617ac4653c7f00593dbb51837a468afef20a14eaab3780fb96ac3d6714369",
+ "blk.8.attn_v.weight": "c2c37496494864fee5c527d1fe1f88529d31c73f9cbd02ef9b2e9b23611ea50f",
+ "blk.8.ffn_down.weight": "2da58572e9ad79087c03cbb0c23c9ef69f93ec221fd5fe4ed92fb93871d23ffa",
+ "blk.8.ffn_gate.weight": "4483294e628edaa4901708e73e92c917bdd93b780fa01aa74aed57166f2bbf0a",
+ "blk.8.ffn_norm.weight": "c0cbb7a4f8123b62f0c4652a687f3b394802bc32870dc446eefb709e42043a7f",
+ "blk.8.ffn_up.weight": "9eaf8a2060cb9224cd585997cd671866c4051ad885c2c6d9fdc7056c2a5c0d89",
+ "blk.9.attn_k.weight": "5dd36c45fbc9c50fd35c36cd75576288506971eac5c5311d4f5c16ef60099645",
+ "blk.9.attn_norm.weight": "3c8ca64f2f75ed7c8fc1da010c23be787648139a96ca0ef3ad10be7b14942b8d",
+ "blk.9.attn_output.weight": "6277e1f833024f53c409be919ec76d34464a78b278c8f9dbf79e777746e3b995",
+ "blk.9.attn_q.weight": "87352b70d9e328c2d51d59090cf5ea5a046529864a890d0bc8986447a0a5c006",
+ "blk.9.attn_v.weight": "2efdf01161d7a82a9117cc2d87d37dba5ffefcf730781cb94fcc95130e48ff9e",
+ "blk.9.ffn_down.weight": "e7658a2ca984961c7ace16acb679387bedb1fef656b5330bbbf588db19673a75",
+ "blk.9.ffn_gate.weight": "773cd330d4ff5d64be8af00adf2e2722fae4e33fc26bb9d03549f6f4b3b0fe57",
+ "blk.9.ffn_norm.weight": "c8b86cd5c43b332f72060b807091c33a258e5dac01358ff4733b916cd34c9c97",
+ "blk.9.ffn_up.weight": "d8cc3bcff18bd46124ba2aa7caacc71220b44eeef6fccb993b4c6cb53e8f2c3a",
+ "blk.10.attn_k.weight": "964bdf3b4e77b915a216f750ff7b0f2eb1dd6bfa071358aef21010b90111044d",
+ "blk.10.attn_norm.weight": "59ed411d91d14775764eb514acb0895a75a10cbbfbc1c15d453bc50f8046cb7f",
+ "blk.10.attn_output.weight": "4d35a2a44cfe4ac0a83fd3ab0dcf1f5a0bf54cdb3b7be9fc353ed32c8a3eb81c",
+ "blk.10.attn_q.weight": "defff5339450dd881ac352f5c459293f39e07b9619ebd10ed632d79a3f310278",
+ "blk.10.attn_v.weight": "b9803e8d6a54acea58f662d4c0a5c8ebdf986676de7dfe12d4b288937881ce93",
+ "blk.10.ffn_down.weight": "eba856be64e4be20b92fb4639a783454dd92427250759df92a337e39f1971c08",
+ "blk.10.ffn_gate.weight": "2d5c509b066584db4de3632b01234e86edcde35409c5ebce18957dc80fe465e3",
+ "blk.10.ffn_norm.weight": "ecb9a8679945ff0273856624ce435dd250ffe5a440ea0861a5c84f0e4c44d2c6",
+ "blk.10.ffn_up.weight": "e76ec7e993f399af02958778c643aa78368e3067846714165eb5aba9d5f547f5",
+ "blk.11.attn_k.weight": "29c6d1f34bd3ba2f0904e57b32a5bf8dcb2834d439159a33edf234ce0b775677",
+ "blk.11.attn_norm.weight": "b5817b275149cd2abe18a6a10e19854605fc58fd364666744362ceee8cfe49f4",
+ "blk.11.attn_output.weight": "1e05653220e237cbe0cc770033e183c9a0eed5680510997409b16186c6691950",
+ "blk.11.attn_q.weight": "03db725ae669151e4d536e50285b3b047ad097f52475df208ed3e790e31a44be",
+ "blk.11.attn_v.weight": "27cdf1d4e971326c451a4615a0b79a8c7fe9508f9b76c0d52fa01971fc7eb403",
+ "blk.11.ffn_down.weight": "176938cd7c2966094f614cace8ba568b10532e45a0d438f80eccd19b6c2a7f87",
+ "blk.11.ffn_gate.weight": "9782339915dd6fa70013628a01524ee1d01ad8beab04068da7ac6a5ee7603a60",
+ "blk.11.ffn_norm.weight": "8245f6391e3be97811c0ff27f0d8f484ecc82a468a837c893f059745bfcd95eb",
+ "blk.11.ffn_up.weight": "15616ddde096d0d25e906375c548b6de4bd5576d1f6b68eefdc29f14e183af42",
+ "blk.12.attn_k.weight": "66dd21604993edd1b1fe547bcaa06f5bb7e31c9204902d147a227e4badf7feec",
+ "blk.12.attn_norm.weight": "23a69f85dd8a0904b9839cc5d0afcda299b74e82ae2642106224a1c820f2b761",
+ "blk.12.attn_output.weight": "4a98d132e376beb274a39d4ea9b6a1b870ad5c66625439d7ff6f45c229c3ca04",
+ "blk.12.attn_q.weight": "1c6c309d63afcfde32fe37257e300a78e25d01117e33490801107c0e75d1ea66",
+ "blk.12.attn_v.weight": "723d9e4ebe4e2b1974afa01d8f512b52933698fa36717dd47b37b07760c50a10",
+ "blk.12.ffn_down.weight": "00e0fb09e1f1fbbf3803f1dee373eaae7a93756b6e13063ab77f9927bc6f996a",
+ "blk.12.ffn_gate.weight": "89159f7f97aefb1e100107e3ac2d694e1008ad873f79bb953d60c2c1bb22724d",
+ "blk.12.ffn_norm.weight": "5f70aebd0e43a39d6373d8658cc670c13aadd7818831d3d84f761d5f688442f0",
+ "blk.12.ffn_up.weight": "faec21b446f061eb4dca561a3180712724347b77a71eb312e7afe9be9e89fa04",
+ "blk.13.attn_k.weight": "3d440825d19eac3b1753b34d94fee2b3a3cb6636c10b2703ffcf688d3c1eded3",
+ "blk.13.attn_norm.weight": "47b575e57e410738ad13fd3c74bb49c06b3d31030910834ece509cd1a5c6d9be",
+ "blk.13.attn_output.weight": "05436d8e613f4475741c1798a7c371b53d61b229507fa04fe23c504ba1f0e12a",
+ "blk.13.attn_q.weight": "002b5024ce520da41256e3ded5cdc60e5ae07ad9b202cb19d76ab511efd02b1b",
+ "blk.13.attn_v.weight": "c1f2d6763587c50312cee0d7140fa2c7ee326f5b172bc99b2d8946e08329cabd",
+ "blk.13.ffn_down.weight": "b5c4e0d8a3ff96cd76a135e415b89f02d28c28f7f3c16a36af31ef0ab8773da5",
+ "blk.13.ffn_gate.weight": "ae06e9e3d2e1f64c7ad23a4009dc904c2eccd7241f9f91c4974ab2504f116be0",
+ "blk.13.ffn_norm.weight": "e44a22321bcbcb4a3c345b504e939e8071370f54a8cd702fabdb40b97e0d7683",
+ "blk.13.ffn_up.weight": "7e6f366d538e21ad431264b12c011892d0be9dfe4c4da9f730af677f920641ba",
+ "blk.14.attn_k.weight": "95492d6417952ec24b2cab87bceb750fc7e95ac6b1944fc328a3852d980164be",
+ "blk.14.attn_norm.weight": "6b7b09e1c51addcdbb160ea59edf032531421c520ec5645fe1ff9ca4180cef54",
+ "blk.14.attn_output.weight": "75887474e4d72c218e6ab0f69f1bf3ec3dc414d51b36fc59df00cdb23421bb6a",
+ "blk.14.attn_q.weight": "940e33f76e48c21215d19e8a21234c8246d4d084381a7d9806aecb24b071d5bd",
+ "blk.14.attn_v.weight": "c58601cf5a9833f80f7f9a5b2656e8eab5eb133211446ebd48f8be15fed4ebb9",
+ "blk.14.ffn_down.weight": "f9f886e7f9b2a54d717b08947a25a0a93e8c2a5b8bcd5a907c06817c8ee3ac11",
+ "blk.14.ffn_gate.weight": "727ed0ee68594a3f59d704ed3240b6929f083b9c36650fb848d182315737245c",
+ "blk.14.ffn_norm.weight": "bd2471008ff1b2bae9aa26bea019393fb2bbc5b9493b8cec3ebd2c280fca24ca",
+ "blk.14.ffn_up.weight": "b006446769f51e4f93b503c4727deae897bc1fc7f4fad49f85024b63c4548d38",
+ "blk.15.attn_k.weight": "23bb70f9035356624039547a603e46be7d1e4403616eafc2451cc09c5373d522",
+ "blk.15.attn_norm.weight": "718cb371ca052eeb3bfac6ac506abb887df125271821fd171797a7f2d8dd6313",
+ "blk.15.attn_output.weight": "c76a2695a204b43a8e5acfa5720590b5d449a9ad9e082cbe3e80fab5903ea16a",
+ "blk.15.attn_q.weight": "2b3e4037b9e91bdd26d6e8d904cf39f948192dcf09bb6445cb55ca058d4f4626",
+ "blk.15.attn_v.weight": "7c15e89b6acafc8619e86aa9d412f5893ab17843ff2cfaf40eea9637b24910c6",
+ "blk.15.ffn_down.weight": "e16fd4bdc6d1c1209c6b633454df4992870c8cefb2cb0e8c92a7e489e9fb5d19",
+ "blk.15.ffn_gate.weight": "95a46bea366c260337c537fde06b4cbeaeec52484a69c3390bb1d178eb0525c9",
+ "blk.15.ffn_norm.weight": "37730293f704da265dc6d1896b3be00c39c0a41dab07f573af39dc30a481d623",
+ "blk.15.ffn_up.weight": "ba74a199da2d0875d7410824238c4ffafbda3993568812284a72b8800df91f15",
+ "blk.16.attn_k.weight": "f58f79a2a91c9a763adefce0c53a71eb5ce6bd8442f4af554b04b58083bff27e",
+ "blk.16.attn_norm.weight": "0c16e41b95e81978e0e0e3b338e2afe2d297426578cacee94de15df74e94eaad",
+ "blk.16.attn_output.weight": "ead22fc337514e4add49aee19720008558e52090466866e849671953a1fccba4",
+ "blk.16.attn_q.weight": "ef59c4e8fe8918c1add43d7e9c6fb3ef799dd3e1bdd731ec7b6a4a6f97c86048",
+ "blk.16.attn_v.weight": "902e6b84c2b64241470b13e6f412f859f66b4b223bcfb9c15d5cb1106b07ef3b",
+ "blk.16.ffn_down.weight": "2ad6e9eb4d8372c32a554395d460d17cfb02d6dbcb757cc962b6bfa36db4f5ee",
+ "blk.16.ffn_gate.weight": "825b2d50fcce3dbe6a5d8d8a50a95466f83ca4a10343efe67894c20b4628fb15",
+ "blk.16.ffn_norm.weight": "3bf6ac90befb0e17e077c8ea9454a8485a30f89f2d761ec7751b60c90aed1af9",
+ "blk.16.ffn_up.weight": "9fbdd08739b32411f5ab0252174d386bab19eb0b17884862f760429b7d41d78c",
+ "blk.17.attn_k.weight": "4033398718bf3674830ed1b73071ed8482b6dd4ef27f31a6c5fbb998321b6c07",
+ "blk.17.attn_norm.weight": "714f2e8ac9592966a0f1c02ee979eee8f84586405b992e8ee9543e840199ffa1",
+ "blk.17.attn_output.weight": "b6bbb618597d767b8f535117be68f92911e4a71d4eb4d8b5d943444151445ece",
+ "blk.17.attn_q.weight": "b84a0dc00ceb515faa2628125dcec502eed923077b21cfe900a4ff16c2e5f9ed",
+ "blk.17.attn_v.weight": "4387c7d6a17da9cc7a6bca8f4a75618b20407d570792056283a8e93b6ec65f18",
+ "blk.17.ffn_down.weight": "47db95c6f1e12b399c3eaf9ddba261782dd71173dd163b52af96541cf87b5196",
+ "blk.17.ffn_gate.weight": "59abaded0aedfd12f01df81f7a811e84db6a227f51b60abe9a247ca726e87392",
+ "blk.17.ffn_norm.weight": "b7e86445be5c7b722e01ddb98d5c7527ca86cb827ce0354f2c269e0f2558751e",
+ "blk.17.ffn_up.weight": "8e31c293bac649d2f60da4b3fc4a3acdce1111ec6058d8805eeeb242443011de",
+ "blk.18.attn_k.weight": "5ce762ab7b032511c131df81093b587871718c7097f79d8e07d707571f18a47b",
+ "blk.18.attn_norm.weight": "1f52cdc7af1f4dc1f0ef6ad1ad02e18cda32133654e57cfa9c72ada9c0b1d995",
+ "blk.18.attn_output.weight": "6486957f30bf8a88516e25772c6650f98b13923f490a2865a8752e36439d1cfa",
+ "blk.18.attn_q.weight": "93621c8abf69d2ca29c5207180eb628fb2b544d89de6c4a7fb0699be95534899",
+ "blk.18.attn_v.weight": "11604083b5a74828ac1d226af015ad5dc0215a1fdca44fa7131c2163c02d8156",
+ "blk.18.ffn_down.weight": "8f9997feb94385f106915df810239c9753b31efda2bf14bdf18a9fbbeec8233d",
+ "blk.18.ffn_gate.weight": "427c213b3a4e94af703429daf2f65766f70424d8230c123e7e712a18bceb5ecb",
+ "blk.18.ffn_norm.weight": "c45d305c4ea6a54013ba112f12dafaade064a32cf01317373464a3618d8ba44a",
+ "blk.18.ffn_up.weight": "a2811f2e73ac9eb9cce91a21a454e84e230a155244e2cd73f2c12aad3c9b8cfd",
+ "blk.19.attn_k.weight": "b2daed159925eac58c291e2f1e2000beed21002b03c9e1bc7e7a52e22240666c",
+ "blk.19.attn_norm.weight": "6307306ede2ab5bffa1bcac3f8b139354678c0376b1d9f5530c1fcb4268cfeb4",
+ "blk.19.attn_output.weight": "ebb98218b2a9c84d3fb6baeb02c5df264b7ab80d994d1098ba1cd47aa398effe",
+ "blk.19.attn_q.weight": "4f10df2ad09177e7528e9456039b670d07db22940a49417101b725d239c16724",
+ "blk.19.attn_v.weight": "30f1efc5114badaeaafa91fa466dc7fa14b1616db433c6f563ab851f7333a5dd",
+ "blk.19.ffn_down.weight": "be5ec7fe6b48855cd0015b0e430d1b70c620de87a7ff188c7c1afef546d7b6bd",
+ "blk.19.ffn_gate.weight": "10dffea4213881f8a9b583ee0fd370e033756d32255ed15053f794375b9400e9",
+ "blk.19.ffn_norm.weight": "e75cd24ade45dca78fdb0cbcaaa2d4a17d83a5a73dcc94ce0ec2d68fbdb2a881",
+ "blk.19.ffn_up.weight": "63e81bdb951410ffa81bcfba1b94a679ec9ebae59cd1623ce2651ed5d4c78bfd",
+ "blk.20.attn_k.weight": "c2fc5ad39e9bdd45e73c6e54aecc474388d944c4be1ee1921b7fcd035bad02e0",
+ "blk.20.attn_norm.weight": "aaa9169171937bdce20c1f057e94e9252f221cabacf1ced12e11b9586f23d308",
+ "blk.20.attn_output.weight": "a9f4fb496e4bc053e3f6cf2e72e22d4cd2b545ef6c32f7e782c2ef6ebcc21d4b",
+ "blk.20.attn_q.weight": "5a07ac619ed251494170b213921ef3fcc4c2712839da262516d9d5b8ea1ff185",
+ "blk.20.attn_v.weight": "d6689473105d241eacb17f09f06000ee237336916cf5ec4f48271c5b41bcb8e7",
+ "blk.20.ffn_down.weight": "74be38db51df736f26ede7c6b52ea787e385f181cb66231e2cced4556a25c9b8",
+ "blk.20.ffn_gate.weight": "ea91e06dc3d051c0ba0243b5a8bb40edbf254eadfb54fda7247e05cfdd88cbe2",
+ "blk.20.ffn_norm.weight": "5fbd357b3d6f44a7a91e8a4fc246b24303891b7957e0f3c32818ae5dc16ddd8d",
+ "blk.20.ffn_up.weight": "fe3290333e056af4ed12942ac72aeba97a6b562e2db05e79cd35dd07eab5b101",
+ "blk.21.attn_k.weight": "201ec6ee95f06ea5eb80fe86fd07bd016d3ae9ab6abd25d631834414e14a010e",
+ "blk.21.attn_norm.weight": "ea8154f93e06485828475a00b98cc397ac84768dd70e06ecc0c075b5712d7276",
+ "blk.21.attn_output.weight": "9f8af74d531478fd304723fd8e4e01578db598441b80dc7c960cb801dbbc501e",
+ "blk.21.attn_q.weight": "277de9953a8d3cff894ffd06c15ad0ee1407e319df0c1a693d4f45fa9c74ac7f",
+ "blk.21.attn_v.weight": "6bfdc16cfb898909b7788ddd39dd04b928f31d6732772195d53c558004638dca",
+ "blk.21.ffn_down.weight": "173877146cb94801157796ee9e5eecf3f46acb3b5e797f90b83a3fc22395eb30",
+ "blk.21.ffn_gate.weight": "53146713e2ca1be80496024077a028f6b6d749b02e71003c349e113b436f48f4",
+ "blk.21.ffn_norm.weight": "b28b97e18ab20a5c553ba422f7d7f6014f5902f1d62a69abd20d9fe19a5f9462",
+ "blk.21.ffn_up.weight": "5c39d0ac4d602b8ec8909dade93b2efcd6b6d9d84a19b252d76bb66dcfaab87c",
+ "blk.22.attn_k.weight": "01f26272c82917a87a3ccf922fa1d521a952b05de878241b7efe3525b617ac87",
+ "blk.22.attn_norm.weight": "5ffc96249d8873b506e9eb7158bdfd07fa1429e53c1951430ca7505d25f11c76",
+ "blk.22.attn_output.weight": "9c2201569358f720244b9c9497e4da02585a167b1414c8a506b85ad75ba990d0",
+ "blk.22.attn_q.weight": "906036eb4ddf027f6d920f9356a6a2a5e529b96f4e1231a0496d46b4434a5842",
+ "blk.22.attn_v.weight": "30ede8b0d166003a4b8a81fc99437f557719fc36e5c4dd510c9f161f36a47e73",
+ "blk.22.ffn_down.weight": "d04c164beabab30e1837b843e18852260efccfbb9d96a34ddd816e6fb3ba23c5",
+ "blk.22.ffn_gate.weight": "19c889db6b19179f0a62d5981a1506592c65de83760d67afbe00d202202750a8",
+ "blk.22.ffn_norm.weight": "4885eff2d851b32dbd306bd632c725857e6d164f0fa8b3d5857e572e6ef98ee9",
+ "blk.22.ffn_up.weight": "365594d8db8e95cf87cc33ac23947942dc326110175cc8ec5a07b5c7059089a7",
+ "blk.23.attn_k.weight": "badfea1569da0fc6ab817c5727ca3a69b07d9cfd622fb8be5e66678d5b3f7ae2",
+ "blk.23.attn_norm.weight": "8968f78a379ac3ca5458b4ed4251e8d9112aca6d6dd1ef6440b4bb0b380375a4",
+ "blk.23.attn_output.weight": "93e43393c03956287b1fe31e9735ff1cfe84f4ae56b83dbaebe96275e4e11831",
+ "blk.23.attn_q.weight": "aaff73c725a8700ae66bf26ac8869dfe96738eff23a8ff340de2ab53400a5795",
+ "blk.23.attn_v.weight": "3a86a8dcf14a746ed1411f5a7e634064bc4dfd6511c24cfeccfb2c9ebb6b4101",
+ "blk.23.ffn_down.weight": "d4da6f37bd7ef69bb203f7b0dd59f50bce37432c70627e6cf274ab81548af5cf",
+ "blk.23.ffn_gate.weight": "5b6072936c4a693923bb4e3d1473fd45545cb02fc07799aca458ef0449a04061",
+ "blk.23.ffn_norm.weight": "cd76e37025f84773180298ddb15e0d4ba9cfc7d832e19c791049daa47c6d9c10",
+ "blk.23.ffn_up.weight": "cde43b99b83124a13b2e4753d12674b3a61dfb34c04703007ced3e8e2aee1801",
+ "blk.24.attn_k.weight": "457379edc4cce4cbbe107385079019bc922264fdfc7bd1d1ae84343a81460c66",
+ "blk.24.attn_norm.weight": "0ce0dfab2edeede5da419fa7833db78e36222cf25c358d08f3ec664310f031fb",
+ "blk.24.attn_output.weight": "0cf91c2fd40c204d2fd4b9c85b69281e5ad4ea8442972fcd44b5fc8e835ffdf8",
+ "blk.24.attn_q.weight": "87ede30c09eafec6a4e6285674c1bc4637140b168b2da4ed34f36fdb6e176cc9",
+ "blk.24.attn_v.weight": "4c0b078b2798ca35d6d2c2258fe499820d2bc88700654ba4016e4b028f563590",
+ "blk.24.ffn_down.weight": "cdb8540c32b1ab988f984484928d39f6841f2131c1cebe90ad9456737fccbcaf",
+ "blk.24.ffn_gate.weight": "da2e0e913648b5526bd2bbb344038dd067639343aed3b413662b064b0db7556e",
+ "blk.24.ffn_norm.weight": "8940bd781c610d75eb2be63cfc8d869a3af05e53c963dc7fd4c6f653df5a80ab",
+ "blk.24.ffn_up.weight": "90cbac2a58801abe11ed6c24560aa4acb949f79429f2aa8ff129ac05868bb87d",
+ "blk.25.attn_k.weight": "90607131e36998e990ce718ad05cbecd1bcaed010931401ce6baa3b0d93ebce6",
+ "blk.25.attn_norm.weight": "fbf679c85656c04a6cf8fedd5412c1ace22960e6c2d47f2d43997827811fbb97",
+ "blk.25.attn_output.weight": "08412724ee7a2086514406e6f68fb9f622e10bac25b0c373b294709f4b09bd2b",
+ "blk.25.attn_q.weight": "9c1238e98a2747654a0d4371d3e7ea8b979867f609dc42482544f25591e85c7f",
+ "blk.25.attn_v.weight": "a57796a535c6cb09581cbafd6a91dc14adc8cca2a2465a7ffd0aec546cd84074",
+ "blk.25.ffn_down.weight": "f7e34e8a6391b480da08b52640613ccadce268373934b409759743a1735b74d6",
+ "blk.25.ffn_gate.weight": "b8d0b2f4612678b5ce42bd4a683f8024514b75fb5ebf6b22c600811e95582ee4",
+ "blk.25.ffn_norm.weight": "cde1fdba2369d315f3c6940a997c471ec891924e642505db580d732763bd7b75",
+ "blk.25.ffn_up.weight": "72e700c32ac8b9c47559c2222e45888a480b527ea512075423c5dc01678e2bb3",
+ "blk.26.attn_k.weight": "6ac83b3414ae75bf3a9055c32e49d2c40fe611ab21f8444f03d2f465d18122c9",
+ "blk.26.attn_norm.weight": "55f9d6dc9d75973dc75136ecb9d991b4398097ac133070873fb96ec76a6f60bc",
+ "blk.26.attn_output.weight": "ebc4fcbd15b33263e50ed2ad45740867cce15bc90e1216623babcb1820734509",
+ "blk.26.attn_q.weight": "080f057521073e412936fe3fee64fd574c8128fa4a148b879d3e598fe4954581",
+ "blk.26.attn_v.weight": "0fa2830d6746487ac91b243716e4302361f891e4e008eddd14abec47c7809d5e",
+ "blk.26.ffn_down.weight": "cb2ab8af1653adc57111ada49d2825c6995e338c8208455b92de10e580f60f31",
+ "blk.26.ffn_gate.weight": "231ce30966086bce2dc0e0afd34a22a1958cfda7a57c41b3b8e9444c5dfde8a6",
+ "blk.26.ffn_norm.weight": "35d959d25d17b00617590f5d5831bf705c385c51e46297a14375a700effca6af",
+ "blk.26.ffn_up.weight": "367680c8d332538b467d1ef87cfeb36cc5c6af564c5023c5fb50e728e3438287",
+ "blk.27.attn_k.weight": "0bfcb351c6d17aeac5b55a915074fbdf00f11c4bda98babb196ac8804805746b",
+ "blk.27.attn_norm.weight": "5d598a88c2e75ba59dd7ba4fee940bdec92d72038f1286536d2dfb71d008a09c",
+ "blk.27.attn_output.weight": "23a9da7347336479f6a10ded14cb3f46e06b5bd56dc4b0fbc526c688552ec840",
+ "blk.27.attn_q.weight": "b83319dba9055f069208e9c9d66da08bc6874f23e575288fcd81697d1777aa54",
+ "blk.27.attn_v.weight": "36ed34ccb2f36fdf16b2c2dd225a98ea6b7b0e376e7791191136ccd7bd7a4add",
+ "blk.27.ffn_down.weight": "5488e1d3a58c71b5e9ddda430540b4776b268cfe1457cbc1c2622dedd9e4526e",
+ "blk.27.ffn_gate.weight": "4ff48011ee0bac39af704849d9132a2410392c87a509c684f2062f6b76b498fb",
+ "blk.27.ffn_norm.weight": "32afe99675983da3de2961d1b5ca41c98970a356823597fe29e91f6e86abf0e8",
+ "blk.27.ffn_up.weight": "1eae3088a75629571fdbf6a20f141bc2bb2ed3f5ba2b9fd1d949f80695e442a1",
+ "blk.28.attn_k.weight": "c4e80af714962d6f9040d2c09f316f4a1cbc3a2e994e19902d7c653cf3c73dba",
+ "blk.28.attn_norm.weight": "c1ecf85dedc1c83d5d402bb7c94fb8b9c11f1a3e5f64e7680f80912d4a560794",
+ "blk.28.attn_output.weight": "72ba47c061b21f5ebc5213a455eaf6fc49c8f8e04ff9ce37e6ed4921b629161d",
+ "blk.28.attn_q.weight": "c4abc47234307f44b8ca789aa6668e298158fa4b459b2c1e84bd581806591cc1",
+ "blk.28.attn_v.weight": "aeba950799d4950e491ad0fcbe30334e39b8975177990a2cb339031c45ac153c",
+ "blk.28.ffn_down.weight": "4e84ce382a37b994fb8608df451a60040559e3f4f3241c3b3cb8989a3ed50d83",
+ "blk.28.ffn_gate.weight": "04df157acdc8e8534ad60acc2d2a4dd3a7a6610f6382535ec728994fa6f83f83",
+ "blk.28.ffn_norm.weight": "4d0386dae2bd1c1a9d0f9730718333e3a486c3bc6a5c5d482193c75d39832c80",
+ "blk.28.ffn_up.weight": "fec60bb0a3daf182a14bd8311fe6dd1e3fd020c5fc273e2549cdb1a2d6b79b05",
+ "blk.29.attn_k.weight": "b0532a263aa5a4e2a7a80adc83fc5dec974493bd18da7f953e7ebfc3f3a19aae",
+ "blk.29.attn_norm.weight": "593fc3b4000c35b7a59dace09ca1756c08be0105b2edd354a0e1c16c82898859",
+ "blk.29.attn_output.weight": "315b896f9f0cbacd0ca8937384c3a3a227efa908cb8c3a9125ec00c480e32b9b",
+ "blk.29.attn_q.weight": "d482d45386d4ad3394f08e9dff233ee3a70d0427d65c0b8fa05905da7e25ca53",
+ "blk.29.attn_v.weight": "cd3b5a6e2852da796902930a6a84bc87fc6a7c7bf51f8fc23758d12a39013b36",
+ "blk.29.ffn_down.weight": "5b3dba6f9753bd1b1ebcba65ef5373dd62c38e755c44b7231b95d93d45761f89",
+ "blk.29.ffn_gate.weight": "8610d9d2db15c256243ffcca3ffd31786d0ada0af0e7c7aa3fd20524370ab036",
+ "blk.29.ffn_norm.weight": "1a2ef2d38b7ac3e51190b9ccb8b6552ba83ab290e523356a7f851ddb35dedca2",
+ "blk.29.ffn_up.weight": "a5fdd15811bde16dc27677cf1a4c97daab4c28cb12a9530f1a0e573134fdb69c",
+ "blk.30.attn_k.weight": "1efeb0b5f4b45a85cdf47300f892ac77ac1f38000ec3653565d1303d1fb8c743",
+ "blk.30.attn_norm.weight": "c73934c182c7fe80838ec1d0b92f50a583f75f7a3d78d822f009b58ad2c80e65",
+ "blk.30.attn_output.weight": "3a0fd89de2d274614750345d827a9c886a4f97b343a13cdf680390505df596a3",
+ "blk.30.attn_q.weight": "711e113362bdb067db843c66236704eb1cd3fc5f40e3767143e96d510686ef4e",
+ "blk.30.attn_v.weight": "82b12a9a74fd3d91b73cc2e841e2b3f0a5197ccd2998afa17020995f880d2267",
+ "blk.30.ffn_down.weight": "af9f4b1287c0d824ae22d6e335d19e04a70135b835be7caa2435f1d85e931993",
+ "blk.30.ffn_gate.weight": "e2ab3e6f15f5c50fca66c084cb6a57a2b6b82406d65150e82ea0437b93dd9a46",
+ "blk.30.ffn_norm.weight": "c1b9c325c83f00e177386a4d7e769945f2995e60950c4a576c0a2c4ab9703d04",
+ "blk.30.ffn_up.weight": "9b94a21efd419715d82071b490d3b635cf1e8da080620dcc39e5bde976d7e9a6",
+ "blk.31.attn_k.weight": "0db0d82e3ddcc2c06209f5f013e1d72a84a996c40bf00186be485b909cc268e8",
+ "blk.31.attn_norm.weight": "2b8b7239471f57140c5cdfe06bd224a4f6326282f99736e44fba4c7b120ac101",
+ "blk.31.attn_output.weight": "a310b048840cc3ff2be4b84796340e8e2cdf05ec89d14bd3655c109b2bfa9fcd",
+ "blk.31.attn_q.weight": "f45e0cd95645175ea82813455356d171838539bc3f7676d877c698f2af0a0eda",
+ "blk.31.attn_v.weight": "8bde008e809112aa7e7c23e9c3099087bcc557313b01306c87efa0a4a30805ba",
+ "blk.31.ffn_down.weight": "8266fec7e203fbfad7033120861e44984581ff8b6851d01dfb7b81c5d8fa90ec",
+ "blk.31.ffn_gate.weight": "b73bc0aa5baf006d9ef6403104891b8133671b0992398fe038380b67e0d7e2cf",
+ "blk.31.ffn_norm.weight": "9c62cc27a7b6017c1df8ad49bff249a8245e8895c6754f402cd44623fda83268",
+ "blk.31.ffn_up.weight": "5b970a4694ea3171a0167f6e1636d9f00268bc1c9640430ffc35218494884adb",
+ "output.weight": "74fa0ef08c57a30e633e7117b1e9c805f833e2e5e21434bc79ddf9c92c6d7330",
+ "output_norm.weight": "59b8a59fd3fbf39353506116e43e5e76edd0cbf2a2873d869da4cf27a04997c3"
+}
diff --git a/convert/testdata/Mixtral-8x7B-Instruct-v0.1.json b/convert/testdata/Mixtral-8x7B-Instruct-v0.1.json
new file mode 100644
index 00000000..a1596532
--- /dev/null
+++ b/convert/testdata/Mixtral-8x7B-Instruct-v0.1.json
@@ -0,0 +1,348 @@
+{
+ "general.architecture": "llama",
+ "general.file_type": "1",
+ "general.quantization_version": "2",
+ "llama.block_count": "32",
+ "llama.context_length": "32768",
+ "llama.embedding_length": "4096",
+ "llama.feed_forward_length": "14336",
+ "llama.rope.dimension_count": "128",
+ "llama.rope.freq_base": "1e+06",
+ "llama.attention.head_count": "32",
+ "llama.attention.head_count_kv": "8",
+ "llama.attention.layer_norm_rms_epsilon": "1e-05",
+ "llama.expert_count": "8",
+ "llama.expert_used_count": "2",
+ "tokenizer.ggml.model": "llama",
+ "tokenizer.ggml.add_bos_token": "true",
+ "tokenizer.ggml.add_eos_token": "false",
+ "tokenizer.ggml.bos_token_id": "1",
+ "tokenizer.ggml.eos_token_id": "2",
+ "tokenizer.ggml.unknown_token_id": "0",
+ "tokenizer.ggml.scores": "e3d3eea80bb41a1213f2d0aa3e8a38581d1f19323be77dbd779c9c7e3b72e676",
+ "tokenizer.ggml.token_type": "6040635e6bd38d98af06698feb75c1802bad35180ee6ae0a503e38c0f60fd71e",
+ "tokenizer.ggml.tokens": "604ac4bfbd019e430d7b6cdf18c6c0cd5b967900601f0307f714ec7773aa5ca6",
+ "token_embd.weight": "1d1d1d39a867d5a4bfb32792a47247d2638c10c95a6259391d02843583505cc4",
+ "blk.0.ffn_gate_exps.weight": "2e5cd43ac3f26c44f071926ff6c3f239ecc52a34bc9a5b5906d3d4c1bf2fbbfa",
+ "blk.0.ffn_down_exps.weight": "a4dfc7e7c96e7402eb70279601675b956bb7331da8101e63fe5c0a611b6972e5",
+ "blk.0.ffn_up_exps.weight": "2d5d87b378b2319c344ed2c642598b6f7cb6beeb582a8ea51abc9ae690d473c3",
+ "blk.0.ffn_gate_inp.weight": "a46aaf5aba7401ce6e41f158242b4879d34901661f3ede85496cbd0ce79d6314",
+ "blk.0.attn_norm.weight": "3fe37d913bdd2b65076bcdd6efe64a37b0b03cacbb1b80b9f7089068aa35f38c",
+ "blk.0.ffn_norm.weight": "5e14308a3c894734eb204c8f558bdc817e94bbd5b4e9cb4094e91ba388c8f7f2",
+ "blk.0.attn_k.weight": "73d943dcac0911e87bd771f4aa1c901e1bfe1aed293af06e1a67812159859f67",
+ "blk.0.attn_output.weight": "4c5f754c855e262e8d4c94c6fbbb57af06399dc0e170d7d99a1a17fc9aab9227",
+ "blk.0.attn_q.weight": "d6fd7403c873d49c05f6f03208f30d99ad34cb3b71c9990c47334d502a8e4c7b",
+ "blk.0.attn_v.weight": "cf17cf64b2d683bd9de6cebaf60e5c264df6fdc38fe719dde9d54c80334f6366",
+ "blk.1.ffn_gate_inp.weight": "0d524de81cd915816b4e714bf595ad6946a9130b3de731cd89428b2781230809",
+ "blk.1.attn_k.weight": "2ea47f412992b374c70674730fe84700e0c8cce177086ce9b6635e42408964bd",
+ "blk.1.attn_output.weight": "b4b2520794d54113e86c8ff678eacfc62e35be4395a594a6c8c22b4383ebcc0c",
+ "blk.1.attn_q.weight": "5db930c98c4f91f6eab57eb974c72210b158e366d23d6d2890b2759c053bee33",
+ "blk.1.attn_v.weight": "079bdde09668394bf7af9f8bc175017b4f48f0ab64e6dd855a4d7561d1693c0f",
+ "blk.1.ffn_gate_exps.weight": "146a62de19f9ab093deb101f9640534ffc3dc40d69f508be12fc0475d01b0c7a",
+ "blk.1.ffn_down_exps.weight": "949da94a3c0f375160672a979e85f7def284264b10d48d038238aad5f5ece793",
+ "blk.1.ffn_up_exps.weight": "7016a3f467d9e3f2f4b4019579ed86b757469cd367f2b225483305376b4bb3c1",
+ "blk.1.attn_norm.weight": "1614d1e6ed537737275eb888666c7bac533f4eefbe73dec92b591045ca9e1afd",
+ "blk.1.ffn_norm.weight": "405a455fa7d1ec36894652ceb554bbcb09a07fd6405f42741e66dc4a4665c19c",
+ "blk.2.ffn_gate_exps.weight": "90d5003fc7421f44220c0842d43128955e91488f6f785fe570b62d81b719e964",
+ "blk.2.ffn_down_exps.weight": "ecdc2b5a8b504ef0a7833acff47d69b0c1fa9c22126de1bb120ff5e48c3d6e2c",
+ "blk.2.ffn_up_exps.weight": "2cbd9485a32460d315eb50a2f3b00863fd77245bfe885b7565efac1cdb1f191e",
+ "blk.2.ffn_gate_inp.weight": "0d0a17a1a2c7a61f2cca49ecbb479154dc93a870873257bc4f225e7607f2e2c2",
+ "blk.2.attn_norm.weight": "b2e4c5a977f87a6f880896bd73596234c9b83622fa0d7add5892501e3155913c",
+ "blk.2.ffn_norm.weight": "0ab875b4280afa922376cfc7b9aa3f7071c9432ea1254091ce7de3749df0e8e6",
+ "blk.2.attn_k.weight": "bb884af51fb51550acfef54ccf1b58ce8284e587806e6a2f88c8265e1ad05a5e",
+ "blk.2.attn_output.weight": "0f03099ba1ef342ea61af9cd71d028123bbd8b1dd7d7fd9b509aef77815427d9",
+ "blk.2.attn_q.weight": "8fad0d29eb4c9d24e564774ee3316b9eb7a4c4985e4567111d2c836c830f6cf3",
+ "blk.2.attn_v.weight": "fe04c847ff677632401a94e7b6b6fdca60391ab21cb23bd791533115de6303a1",
+ "blk.3.ffn_gate_inp.weight": "29e3aaa724590c070e614af8288939603d2641b0ef11e8c0f476bebb2776673c",
+ "blk.3.attn_k.weight": "231cc5631def10f7f292d8862d6125ff555164cd70480ac76362149fad204497",
+ "blk.3.attn_output.weight": "86467a605c62852e05fda1a7ef43150df2cf715fe59785dbcba09f1c27cfa086",
+ "blk.3.attn_q.weight": "901822402453922225c2d6ac79616691d48217635d5ff7338daa971d5ddee210",
+ "blk.3.attn_v.weight": "27030784f44375720df2f090933645a31a022d3fb3b14573e5ca0b78f44070c1",
+ "blk.3.ffn_gate_exps.weight": "231ba59cc0b988d125d77bf627aa3f04636684870af88f081f3944b48a160d86",
+ "blk.3.ffn_down_exps.weight": "530c3ab44ae4d66e8afa4d10c153ba5dfcdfb7321989a988e62e9d12e7234625",
+ "blk.3.ffn_up_exps.weight": "b85c2d4d9d11332e702b3c0a6610d4f525f9a93e5d12f5c7c55c592c40755e75",
+ "blk.3.attn_norm.weight": "05dbb6d88cfa6b199f9d705ccbda97c0ef13f9ec875c595398a1a42d009a4555",
+ "blk.3.ffn_norm.weight": "6880b1c27d46969ce36fac049c05dc8b89e4bb47dc89df357e32df7e18fc512e",
+ "blk.4.ffn_gate_exps.weight": "a883b4f225b760c5a2f6605dc5e2167ab85bb398c70bf64ceb539fcbd6128dcd",
+ "blk.4.ffn_down_exps.weight": "d291bb656aae77947d4b525e2819bf4112afece53ff31de9dab999af1f65f9c4",
+ "blk.4.ffn_up_exps.weight": "38592afb8ba3dcfb26970f906174f7d3fa62da44fa4be4fc6912a19030ea9164",
+ "blk.4.ffn_gate_inp.weight": "1596cb74e8fd6c3080b937b06468bb397b0dbb661e6d180a6bcbdc43e8bfd0c6",
+ "blk.4.attn_norm.weight": "f90c83c5ff4366281d283384efc941620542b9cfdea160d678dc54a75e33f758",
+ "blk.4.ffn_norm.weight": "d28d8c49d1746b7cc085562d1074905fd14023844de823dc4fb22202bb280790",
+ "blk.4.attn_k.weight": "792bbf412cc357140fdaba543e547a9b2f7582919e307bbd9a80c7d6d8f5f1f9",
+ "blk.4.attn_output.weight": "d98e4a062d2631d9c315f1990d5f6ca9a88e7e0e46387f611ccb0353f876aa12",
+ "blk.4.attn_q.weight": "1a11a55a91d9f748a72176ff6b1c174844df406e00d1b66b9aa64dc6ee4bcd1d",
+ "blk.4.attn_v.weight": "04cb3c02b12a6313c7ac7044513441083d534fb4c5a3f63bbaa58f7edbd2fadb",
+ "blk.5.ffn_gate_inp.weight": "cbd5cdf015d33a2da6703eb74c22fcb97581fb9175435173b6dc4f9e8364320d",
+ "blk.5.attn_k.weight": "4fdf3405e4d657403f5647b51233521310ee984b4b81bbcd901cb3e6ab76b7ff",
+ "blk.5.attn_output.weight": "4a25662c46979a29600ed77e1907cf81fb16ef30e724c155444e54ccb76af481",
+ "blk.5.attn_q.weight": "e2acb30e30b97300039bb20ad0878f05159d5657fa811748a51d5b6fb35d631e",
+ "blk.5.attn_v.weight": "306504b6a26aa123c63dbbed3f4ced0ed2ee8fb6a30bf0093539b817539f5ece",
+ "blk.5.ffn_gate_exps.weight": "7e34df9b9944dbeea5e8565786d3aa6937314a4b87acd4d0874687877c5a39fd",
+ "blk.5.ffn_down_exps.weight": "c4b7a57a42b5ac0a8ae27dcd5cb2646d7a7cc7123126d44a56ab128e85f60b13",
+ "blk.5.ffn_up_exps.weight": "09d47593b6dd6c664a9155bff02fc2eb7ac4a70219a88162d05c802a01d3c6ba",
+ "blk.5.attn_norm.weight": "58804a036d6ac4c1fe357b8b6a97a5c37cae1c2f06ee0086c041d449c1c6ef6a",
+ "blk.5.ffn_norm.weight": "d872dee6789f0826211aa46ca9d0869e3e96bcace9e77d6559a7b6f3e524f3ca",
+ "blk.6.ffn_gate_inp.weight": "fb1eae732e974d6c1d020a5b4ef98c5f33016f984701bcea656f999a99daad66",
+ "blk.6.attn_k.weight": "55e9c59c5051ab5519b3a7962e1b5fa96a3c0251cb6200dc2f177885ad2de470",
+ "blk.6.attn_output.weight": "f3c834a8d0027370350e2b6294d95434d31432e57be6313b013c15a56303d61c",
+ "blk.6.attn_q.weight": "efaefe5f11c2140dc7cb532b0832c2a0b363a165cbda21f00fadae77efca377b",
+ "blk.6.attn_v.weight": "900bd734d75616d846a90a121c97e081c956a3d1ab012f66dd0bc62c43e1ec3c",
+ "blk.6.ffn_gate_exps.weight": "312a99661b1468fcaed2474621116f1681432755e973f3ee79d01912974fd424",
+ "blk.6.ffn_down_exps.weight": "ac9cd7db67a2ef0d2b5def86873673d05e48d49d147dd944469dbb8e2d4c46f6",
+ "blk.6.ffn_up_exps.weight": "57613e7e09579400a1a09fee4445acfbfe83f2f327fdf317877787d96ada6b84",
+ "blk.6.attn_norm.weight": "0e8801e09885c633bc01a9a5b85d4e878d30158a4eb41a937dc5b760ebd044cb",
+ "blk.6.ffn_norm.weight": "b8c58062ac93072f878446b0e7f958c737aa47fb769fc3a8f593133d12db2dd1",
+ "blk.7.ffn_gate_exps.weight": "1ef611732ff13edfa8d30981ed9dac00c15ceba9fc012ed0b199e9280a849948",
+ "blk.7.ffn_down_exps.weight": "856c6811945c7b0fa461ca17811cfa43436b4cdf5326bad23cbc30883486d7cc",
+ "blk.7.ffn_up_exps.weight": "6725e3e33994302ee13fa5ec163631ce2dcaa08aadde8fc166c2265d4561c5c5",
+ "blk.7.ffn_gate_inp.weight": "36b49d7f80c1003dc392b2c1b9960cd49889dd69e77b26b9e4b13d01f3d0a32a",
+ "blk.7.attn_norm.weight": "7a0ec49acc5e20ee71c6f80ca02f4f1e564c485e0ae0621309e7c2eb0c616cf0",
+ "blk.7.ffn_norm.weight": "eeae035c39ab6e64bc06a4baa1bf6e50d4c8b8797cb0ad8abd48be86974802c0",
+ "blk.7.attn_k.weight": "e8f78c1def01a7a38d2d9bf7becb17755e28fefe4927856f7890fbee52840187",
+ "blk.7.attn_output.weight": "5367f05ac3bb49ef8745ba5902e1bdd4442415a3ebff2c7e1a3918d7be6fe948",
+ "blk.7.attn_q.weight": "37c95fc5acc55a4f6e5f02cab9be60e4fe54c08b65f98f4455741b4aa542ff4e",
+ "blk.7.attn_v.weight": "c89f1343486ba55814233511e94090f7365662a8a4214aa4c278cdadc79196c2",
+ "blk.8.ffn_gate_inp.weight": "4e239afe8c7afb8de3a005757c887cf14b1622ca2d224227591cb0e5301f4c17",
+ "blk.8.attn_k.weight": "2ad0229f30fdcc1e85ce64e00d8f75902238294844a81d5af43e14ba75c02983",
+ "blk.8.attn_output.weight": "2e44a4722acb3b521b81d0b910f8ca2f6c286d874a92ddd02150566454061699",
+ "blk.8.attn_q.weight": "1cd2b09cb2f43e08de776b5f7eac197a5a6d4ffdfd52b21baa36319450147bd0",
+ "blk.8.attn_v.weight": "5a22c57ebfd33ac500cbcfd321d5b5b1783f8728801db6f3f8bed51c7183e4db",
+ "blk.8.ffn_gate_exps.weight": "91063fe56cb4f3ff3b41052bb5046fcf8ef61516a603ee90aab893a9d68c15a7",
+ "blk.8.ffn_down_exps.weight": "d4c3abc8f1d1b462f67f70bd8f404b3fcf45dceeaa8527fa120527254c383c90",
+ "blk.8.ffn_up_exps.weight": "76a1a1f08ec577716a2e7027b45293e9205751126424f1bebe1de89c78f087d5",
+ "blk.8.attn_norm.weight": "f980d774da39eb76c52358afac3e38cb4c81cb323deaabbe5c41822e3f17a98e",
+ "blk.8.ffn_norm.weight": "1c937658cf90f1a85db9a5f26e077730fdd4b694607dbeeb825c5fb2bc407e0b",
+ "blk.9.ffn_gate_exps.weight": "a2532471ecb7896d5c78e5a34e10cfaf4125265e1595166c8d0d0dfbe2a3187f",
+ "blk.9.ffn_down_exps.weight": "b47921a28412d48fee450b8b9d97cee42344a2e69f06d407fd9523d7adf13333",
+ "blk.9.ffn_up_exps.weight": "7c461bd1b2a73b439cff6a10d94afa01e8b06f7e6f09d9a6f28e3876aef48bce",
+ "blk.9.ffn_gate_inp.weight": "1648dfb08b5c06d7953a5a97ecb764995fae9487fb729a1c867023b2538149d0",
+ "blk.9.attn_norm.weight": "8635db0f299882a63b7cfcd1d4259c9e53fab22c31d3d054de36b1001380b31b",
+ "blk.9.ffn_norm.weight": "f9309aa323062d174c463613afef9b0a33501b510bfaa58a8e0e866d12ffef3c",
+ "blk.9.attn_k.weight": "dfe62030441e947a588512d18d9c6e4ed72c2f71c227d622c095e4263b23dadf",
+ "blk.9.attn_output.weight": "1977beb75c6349c50ba7dd3865d7c0a9c5c5ddc854413147b0eec98ac4fda351",
+ "blk.9.attn_q.weight": "eb132596719605cd6bd1782487f121994629e115190edd69240b12af66e734f5",
+ "blk.9.attn_v.weight": "9e708f15d332d7c5187b0693b1a977eb30a2fa10bf7df48ed9d7537c0aa6ed99",
+ "blk.10.ffn_gate_inp.weight": "97503a5d166c1925f9b65c0eed980753d411714d66896f3d0fad5286c7aba702",
+ "blk.10.attn_k.weight": "1ebdd222336bd25b48df1b138cdbe09021c4a5562ea7cb78cadd1255d2be3a39",
+ "blk.10.attn_output.weight": "5e98faa38e9d514b9057e1c8342c509cbe1083defd518e506f6bad89117d1f5a",
+ "blk.10.attn_q.weight": "3323a26c87d936d1dd87c577d0b763459fced726679612c874b3de5fc6d969c5",
+ "blk.10.attn_v.weight": "d5fa73cb56aca388e205f44455e4b4f676fdc12ed7fac4542fbb3b41ecea59ad",
+ "blk.10.ffn_gate_exps.weight": "225021b53782800906cd13b70be3a4161e8b300b97f984a959ccad6a6e8adcbd",
+ "blk.10.ffn_down_exps.weight": "f08eb91526bd22f5fd0402fe925d6141cdbb308a1ced0330858d0c85c71f5ef3",
+ "blk.10.ffn_up_exps.weight": "a9f688350c3b53eaada5103b5848bd9a3d7d6b327a70fa16c24bf28ece933eac",
+ "blk.10.attn_norm.weight": "5ba426c9dfc79805015ccd76cd1068b0ad3bb7a8453e14bb1d35486f122d8f95",
+ "blk.10.ffn_norm.weight": "98891d6acbc3986b2581b7a3af9f5946a392d9188972c6a8b15d4e745a4f2482",
+ "blk.11.ffn_gate_inp.weight": "b2365a60566e7dace892e1cb0e62eb73ce387352601723e847052b34874feaa6",
+ "blk.11.attn_k.weight": "0efbc1d1430505543ff71532a4fcda821aeac616ef6c1dca40e00d4f2ff70bea",
+ "blk.11.attn_output.weight": "3d5bd4d9a41236f30d4293edb9ae27beaa113ffb31b4fbfadff3a4c370dfd3e6",
+ "blk.11.attn_q.weight": "aa11e9db14dd9c77951511443077c2a1a78070753d7bd3d9811038473f69e325",
+ "blk.11.attn_v.weight": "5adc567f377aa11d1763d35f50e53fb2896a8b03b623ac36acc45efa2486d512",
+ "blk.11.ffn_gate_exps.weight": "71d07d982aabfab9eed3c733d49c20f023bf475368fc71db5084d91beadc4b47",
+ "blk.11.ffn_down_exps.weight": "9a06e61461e48b3925a9f7d9cca634d048c8b62163d7bc5c43e35899f959319e",
+ "blk.11.ffn_up_exps.weight": "bc05494d0dcec61021b3ac0c5bc1bf502736cadf48224e213bc139d562699a89",
+ "blk.11.attn_norm.weight": "a5758a10bdd0404ae1470e8e9db903985d4d07f60553c5001a5e7b660d4f7ada",
+ "blk.11.ffn_norm.weight": "814ae037563aad3771787316bec4806c95bf6f5991dd6474b4b1e5cc13dc18ee",
+ "blk.12.ffn_gate_exps.weight": "3a68b831ba1606fb9ef6dffed4732032447ecef23ea563ff4e79317586c7eb49",
+ "blk.12.ffn_down_exps.weight": "268b25e13f4b7beab08686e83705a41b21d15251809ee4784526f78a580da829",
+ "blk.12.ffn_up_exps.weight": "9105751a5b5b42ca2614d0456f24f779d2e2ac8cdff0f96842aa7ae2b70f341e",
+ "blk.12.ffn_gate_inp.weight": "d0de1558cc1d458c5c504f63ddc59785c323df7330474bb0644c346104b40a3a",
+ "blk.12.attn_norm.weight": "859a4c8113678e2e202d10299850e0cfb52eb11ea50bcbf4fe3ff39bdd394154",
+ "blk.12.ffn_norm.weight": "7fbf4c459c1760218877e9ee3f5ad49e960956a4369bcfe96c143f04ff9ddf97",
+ "blk.12.attn_k.weight": "0a7e254fdf3730a57372b6ff421a613eabaea68cdefd64800857941411318374",
+ "blk.12.attn_output.weight": "ceb763fc15d88af149d8fb78e82db2b7dab3aeae584af8cf7611a12356a397e5",
+ "blk.12.attn_q.weight": "a43402d23c46cb2d3cb3c2a98c81b19d10026b7e6742370fed6b2880b6e049b5",
+ "blk.12.attn_v.weight": "3bc24f2c0480ce91ef72993ee8f1cf962f7359e12183424583ffa1246bf3db52",
+ "blk.13.ffn_gate_inp.weight": "a6d68c82bfe66d8bab68f980f5f18268a9e2c0cd6b8832ed39010e0de198ae05",
+ "blk.13.attn_k.weight": "0166c39546b37dc2e01b2b396ba43e183f797dd04eaa51a6d103d8b58ee4bace",
+ "blk.13.attn_output.weight": "2ce5eb198deab9557475a58b69b11e9874b547e05c23f223c6e42fa35ddca069",
+ "blk.13.attn_q.weight": "745c1bbdf434284a7fae98f45e821c076dd9c2a2467dba6a9d8cf0041e419dbc",
+ "blk.13.attn_v.weight": "9ece68d5ac64d1421ea7aa32e1cff9cc1fecf5175f4c4da858dd31d8633e3337",
+ "blk.13.ffn_gate_exps.weight": "ccfdcb4670b131689de12d396a010b5ea737795cf5c15a14a304d720b3c7c899",
+ "blk.13.ffn_down_exps.weight": "8b8fb328664764f1aaa5cbdec336d5654e981e965a02ef622bde5f07ea1c164d",
+ "blk.13.ffn_up_exps.weight": "d2ace0236c2fb3365fdc85499d676a7f65813c48e5085348b1df1799922766ec",
+ "blk.13.attn_norm.weight": "1ed29d7d89ce52d7cb4d57e895ff7115430466e917136c049c385c030ed44e9c",
+ "blk.13.ffn_norm.weight": "a194fc542597a4dcfdfaec5e3cba2a2b2b21b21edfc87c39c0d7f7651355bc4d",
+ "blk.14.ffn_gate_exps.weight": "a625e3574e5e740e7f8e2f9c40390f2f382c720aab5b10534e298002dd8d1fb9",
+ "blk.14.ffn_down_exps.weight": "bc366f015b83c865946afd74c8a884943e0ea2c671314a0b7bb72f21a44d2f78",
+ "blk.14.ffn_up_exps.weight": "ee3199bf2086de77b49f57f487676be8ee70e102a2fb5a5ef8ddbbc28a9eff41",
+ "blk.14.ffn_gate_inp.weight": "2b437870c850fa2e2044d032bb02908af634356e37466fdae260b933e48ee8b4",
+ "blk.14.attn_norm.weight": "cd8344d193a1cbd42bd898e17f4bcb1ca0b2918420fbdafa9249a6f2b7f4ae06",
+ "blk.14.ffn_norm.weight": "70eec40374e558fed5b07257283cf36342b6b0129285a00007deb59c32c9f7c8",
+ "blk.14.attn_k.weight": "4053bdb507e0543d724b632570bac86b31707696d90a0db44c49b2a082e0d599",
+ "blk.14.attn_output.weight": "0182632cb0e06a07241b8293d25d109fbc1862e1e337d435f908e8681e2eb1ab",
+ "blk.14.attn_q.weight": "ffc7794a4c1b6f793c842dba969435330a7a80b9212e457b4b2ac33e68b41241",
+ "blk.14.attn_v.weight": "6411805292d528e61bbaad8f9aab9dd073529a17946c057fb06864fad9cf3211",
+ "blk.15.ffn_gate_inp.weight": "77d0744567c76e6abb67f81ba9c715b2b544841186d5b948309571eff213bafb",
+ "blk.15.attn_k.weight": "1f7957954ea4c6521c257b35a360e868ffa02bdb3de91f146d5e06bb4a545c98",
+ "blk.15.attn_output.weight": "d7809d36bd8d3342240c46fd87bcc7f9821a222f48d9a95e45ae50460265d3cf",
+ "blk.15.attn_q.weight": "25f509313ae4d8401b871904059f472a26f5714e7c791c725de77a1a522c976e",
+ "blk.15.attn_v.weight": "96fedf5a591fc0f020e6de10fd72ff12b3ef9cf70cd21dabaa0d3e7b06f54e73",
+ "blk.15.ffn_gate_exps.weight": "8f950d976b2fd9a3d213b84123cf114c1377efde9352767fb2ddee89e177c8ef",
+ "blk.15.ffn_down_exps.weight": "6fd09d1557bb94b06efbd4f6a1ca4be532a202ba290e9315bc8da3d12a5c4c4a",
+ "blk.15.ffn_up_exps.weight": "cbeb59ae7b0266a928dc7e3a6e70a9330b92f9ee1b17ee1ed91022108204a33c",
+ "blk.15.attn_norm.weight": "2005330911ac2edc7b6d27aca021c67d30d16eb632e49b1a13f30fdb2717aed0",
+ "blk.15.ffn_norm.weight": "0e9198f3b548eb78acc8961f2b3350d238d26cec110933ba753a8cf0035c501c",
+ "blk.16.ffn_gate_inp.weight": "a41d1f99d739c8b150c3945b6949763988d0c6a4c5a2b5855592ca1a48ed23d5",
+ "blk.16.attn_k.weight": "b624e2ec88c2d3047f60530fb87e72cb4a5e655a9663f6f3e9b09e5ad32cddaa",
+ "blk.16.attn_output.weight": "687759ea75e45108526ffc1573d6fdf084728079bfc2dc89b9979e76280f43c4",
+ "blk.16.attn_q.weight": "beff3a45c7e9ec82ffc6d3c701126be28654d10aabd747d03441210491fd31b6",
+ "blk.16.attn_v.weight": "43a349b13f0b9d040cacecd942bcb168c030fef8c75c987d59a4fce6c14e855b",
+ "blk.16.ffn_gate_exps.weight": "793406d6c13d727c82bb7b692ca98d65ca975baee69fc57be5378d77c5a19b62",
+ "blk.16.ffn_down_exps.weight": "9bad3dd150d0230404b7f886ac7ff8803225757e813f195cdb26bad245243b4d",
+ "blk.16.ffn_up_exps.weight": "7449d663023fea3496475bf0a9c1de7272ad0ce9adcb3265e8e424badaa674dc",
+ "blk.16.attn_norm.weight": "a424ce34c195a401df1ce37ac4f2794e8a6720b1ee8acb21428e2b68c65e0125",
+ "blk.16.ffn_norm.weight": "405a68bb8e16e1064df2de55ca3cd9ceddda1d9fc0af007a9bd7cad4b2676248",
+ "blk.17.ffn_gate_exps.weight": "97c6e5321491ca5dc039ee88da0eb0e78f347372785411809af84b3298cb19dd",
+ "blk.17.ffn_down_exps.weight": "1617ac19788a1be19bac69277408761e6bdf5719d63a8c7fea14d41cc27641b5",
+ "blk.17.ffn_up_exps.weight": "4ead1c365f112581c10610ea3f63d2a1474311d2503d2060fed4b458ef337f5d",
+ "blk.17.ffn_gate_inp.weight": "ed4b3393f2523f2b5e0fc7680a1caa2842e605728a529b5af68a7fa8d7abf940",
+ "blk.17.attn_norm.weight": "beac17ef86a7fb2b5840cc72f7a95a5e3d6bd24e7fa698e0b0ebb9bdac45c561",
+ "blk.17.ffn_norm.weight": "81cb58ec6d6dc02a0b4ede10adc336dc865fa76f982d4eab0e4a37b40f5b0fac",
+ "blk.17.attn_k.weight": "eab569e5ea8c8b05e5a6a209fba031129453c2e28181eee3e736b3b04b36bbec",
+ "blk.17.attn_output.weight": "f85b70f01438ce8fe5d10599b113f30bf18dee2bbae0657d3eba295870001db3",
+ "blk.17.attn_q.weight": "887ceebfbf6a2b94b43d2df4439ac3a5bbc29311d4b28addc04d525546032047",
+ "blk.17.attn_v.weight": "2df9414d65014c06a93da22ba3a668be7b83e2e8008e98d7771f7dfebed98298",
+ "blk.18.ffn_gate_inp.weight": "9b07741a0950fc667e5fd25937e33bc22e1f764f80eb4ff3119f005327ae0f6e",
+ "blk.18.attn_k.weight": "8649598dbb63938744c39bcda5ce8c31773e29c573be8d4d2c114f5030f8d3e8",
+ "blk.18.attn_output.weight": "f8e391adb92622298ca834d5d1eda48b69c3b1c51c5a584ef6c54a725c298d75",
+ "blk.18.attn_q.weight": "84bf8708a2eed618f48f69c178ed7dd11fa4c468102376e72e910ebd037d131f",
+ "blk.18.attn_v.weight": "31db3cd773f09548c2c1b1eac2718e46364a7810970fe9c433fad9d8de5397eb",
+ "blk.18.ffn_gate_exps.weight": "be2a2ba378002f1b61f86c273a69eede9b93786d5ce96b4fee1861f730dca4c4",
+ "blk.18.ffn_down_exps.weight": "d35196159e37705db50a5343e3989f7335477f1a4add67ef42ad64a638cd07ae",
+ "blk.18.ffn_up_exps.weight": "c6ceedd86e97913a6dcadc838e7abb762d629fb8dd55f15cf02fd9bd66d2ba78",
+ "blk.18.attn_norm.weight": "41f0b1ad83d6e3cb9fbe0d27878c2e7ad4a351b9f554a6bc9117c01745cdf6e5",
+ "blk.18.ffn_norm.weight": "96646204bd0d82f25dc77faba4dbd86b1332e449313e6684e00122da8be99057",
+ "blk.19.ffn_gate_exps.weight": "c6eb7f61e7938bda0492dbc05e51e8f631c99224fe18e99861fc4fc53ba9e9ff",
+ "blk.19.ffn_down_exps.weight": "4384803da3a3a3d44120d7dd192fe2c9bbd9a1a0cb492dbec1fdd7565230f1e8",
+ "blk.19.ffn_up_exps.weight": "22d73de2fbb8bb0f1bd2caf17fad8a355c47d914143f7f6e6d0128f66f074a60",
+ "blk.19.ffn_gate_inp.weight": "9a0cc4a2301a5634022fbce41189021bf0d1a961792d2d9330fd35556d18e5bd",
+ "blk.19.attn_norm.weight": "c5cc56ec5df9a1f7d5ad71fbda49f1433132e58895d45cb44c73420bd61ebd6b",
+ "blk.19.ffn_norm.weight": "77e17de741742ef2482fc7872fd423c8e3c1454dc4d2be89ee939084b6d78bc0",
+ "blk.19.attn_k.weight": "a92ea36ce2e3569656306aeefb835ccd5d1b03b33a86e0d3d030644cc923b813",
+ "blk.19.attn_output.weight": "5e2a912b37855f84ea964907a1a86d609cbdd79efa0c93c3e8e2fc07caf7c226",
+ "blk.19.attn_q.weight": "4ef3a5913292ac3c1a6fd3e9e53d011021f2b41d0276cf849706d1ca925cf7a7",
+ "blk.19.attn_v.weight": "42981b75b68ae852cee638b5433605c147da4392aaa6d7a06e756115b0171f39",
+ "blk.20.ffn_gate_inp.weight": "71381b9879a7c80b9f7b475abc0aa31b8cd71ccc00856ebe89764a2acb9df2dc",
+ "blk.20.attn_k.weight": "1928b7ebc054eb3967929ed6fb446314d5352f4aaf8b475ce55c6345019f2ea4",
+ "blk.20.attn_output.weight": "6071ecd9ca91af0d2ba93fef4a1a56f3b243dd70f862a21a2d164d56f386043b",
+ "blk.20.attn_q.weight": "002e95042a40f36ceed5829e3d0c8072e5f5e4ee86a089e2902b2348fed24dd5",
+ "blk.20.attn_v.weight": "42f509cdb1c0e298f89f896e349be86952c5168e49b3f83bb17badbcb7596d57",
+ "blk.20.ffn_gate_exps.weight": "a684a3ffe4b0a57c819a5fa9cb3521de223f392732927271e97ce925b6e33765",
+ "blk.20.ffn_down_exps.weight": "e3081a7bc7ba750d8a4886bc8ca4f231b55db4ca082b54b4106c7531964725cb",
+ "blk.20.ffn_up_exps.weight": "fad0fd5eca36ab154788da28be8ec25bb5d6db06c9d133db89e96df358a2f6a2",
+ "blk.20.attn_norm.weight": "c3e3f2429715ae95e884ef1246b0b461b23c5cc0ed08beecf70a14cddd184820",
+ "blk.20.ffn_norm.weight": "ff31f609dda65ca496b0584fabea6550e42edd05ebf229812aa6b7bb5ede15e6",
+ "blk.21.ffn_gate_exps.weight": "366f09ef0ecfb86808eb3296cc9abdb957951d27f6533c03f1422b54061da660",
+ "blk.21.ffn_down_exps.weight": "3fc495947d27fcca7fc0893c8a96e5d48ba27b2c8c58f8fcfb8dcfcd5539741c",
+ "blk.21.ffn_up_exps.weight": "6713ed51410bcc8283cbb001c4ad784098f25701e8021f4fa4f411e186859c4a",
+ "blk.21.ffn_gate_inp.weight": "6d4c92c01ec801647134d907bf1108878156df266a6107abc10526332b328b93",
+ "blk.21.attn_norm.weight": "27605719ae2df24f4f2e85a730927cab20367631612cb501631f6bbf38eb1209",
+ "blk.21.ffn_norm.weight": "ca80ee8177db185b15a4a378c1cb6f7143c76546a7f1726bda23f329323d4ffa",
+ "blk.21.attn_k.weight": "9e49f743d4a5bda9b4bd9c40c2ca37cdae5aec7e54cb193897ac8b4945ada14d",
+ "blk.21.attn_output.weight": "ab923540879753feaed152f5950f69cdd83d8f2413ca873f5f038b63ab0aea12",
+ "blk.21.attn_q.weight": "62617fc3f1c9d2aa672a4d91a121c7a91b92d145b65e75f0b06b4bb7c825dc36",
+ "blk.21.attn_v.weight": "15f8b2e72f8e8e992f2f6b3e93238a9d7be7bd6136f91c9d04b4b4cd0cd60369",
+ "blk.22.ffn_gate_inp.weight": "3ddb1773d9257b68add7a2a4e94dad25ed926803e02707863dd742ab9b2dc179",
+ "blk.22.attn_k.weight": "680e45a9e8d5feddee5266e119dc053bf80718fa9af1cf6803e6f493b265f1eb",
+ "blk.22.attn_output.weight": "0d5fae3402fb2c5aa3a860010e3973fc8e3168d1015f7a76b7b2964681693206",
+ "blk.22.attn_q.weight": "eee7e3d426ab533bd18d62c9aa142eedbde394bed07db58313e0fccc82a23237",
+ "blk.22.attn_v.weight": "26b5be1fe3c2b6824c5a648a3e4bdf17691904526fca158fbc3ebb627b67e2f4",
+ "blk.22.ffn_gate_exps.weight": "32ab7a7735313d60f6a75229b1aeee940b6aee176c9648536bf5921b0dc2929a",
+ "blk.22.ffn_down_exps.weight": "67590808f6a67777d3eb7976c31fe616d388b98fecbb12253b72d1241d70753f",
+ "blk.22.ffn_up_exps.weight": "fc245c0183e6d90829ff5e71a4ec93e4860b3d4c1a17b9dda2fb64f5f5c9ed32",
+ "blk.22.attn_norm.weight": "128e99d206d4d6724758ec97468af767fa0aea592149c324b731659c1e74a1a8",
+ "blk.22.ffn_norm.weight": "e45f498033f0cffa15da0eff2c47b4472e43fcf8921729fc4eeb2e3a6b3c78e2",
+ "blk.23.ffn_gate_inp.weight": "d63e686f5325fbc89fa242c2c52a3b8ff54f867dca914c9ae6eea13e9d6f46e5",
+ "blk.23.attn_k.weight": "f71f5a577f46ea12b1818f3a5ff4b85ddc45f9a2afb0fa2e041d71a3e31c6779",
+ "blk.23.attn_output.weight": "92b13563c1e0eac0d748fb67b235dfd7a64c8f16e2dafb316885744582e23b4b",
+ "blk.23.attn_q.weight": "2f9b9c35dc4f912f3f51c06e2d68f417b51a0de0a84aac530a64f9d3d7b0a2dd",
+ "blk.23.attn_v.weight": "268e40813806e74a5c364b19556d087bf8374e76e7b6fcf55c381eb7da13ccd1",
+ "blk.23.ffn_gate_exps.weight": "12f857e7a7ce228afac34d99b602c8d6fe96984f2a21118f459a58cb767ee65e",
+ "blk.23.ffn_down_exps.weight": "cdb082c16599c3bb36a28066dcc122d9529b54fa91b6cf0153437ec960a5e16d",
+ "blk.23.ffn_up_exps.weight": "f4b99f6f44d7b8b5a305894e88633bf5938fc1f6303a2b2092399da9c8b64d7c",
+ "blk.23.attn_norm.weight": "a691392210383915916b4d3886d5e4d56e7855e27e37e414fbd73bf66b3712e6",
+ "blk.23.ffn_norm.weight": "0c3dc72f667e5ae19b69bfa9f2bd2a01a57681f89ef9527bad4eb0d8c7b70da8",
+ "blk.24.ffn_gate_exps.weight": "86baca2a3157994df7fd8ced5e08436d5c1810dc29c0715637c36de723e0e7d1",
+ "blk.24.ffn_down_exps.weight": "ac5d559562b35c34993e34b071f66d15c65be5907797078c2d2a49aba54e3192",
+ "blk.24.ffn_up_exps.weight": "fce0a099cf09777f44fbab3606ceb75f7fae6f0b80725f9e871654b8cdf9262a",
+ "blk.24.ffn_gate_inp.weight": "e7c6800c0cfc56b565b2d35ad6f1dbfdb70dd0b05b338bc8da2286ffc3678d79",
+ "blk.24.attn_norm.weight": "dc6cc18ec52d102d015153c4a1132f9d7a504e29cbdec81c5edbf3b9e65815e1",
+ "blk.24.ffn_norm.weight": "480d5a1397af5e0e657f1e67d20ec0cdef5724e71246a326843321b87ffabd33",
+ "blk.24.attn_k.weight": "338c0597954a9b95a782545b2fe36469553e73f86ae2d2b5697767b28e1c7daa",
+ "blk.24.attn_output.weight": "a77d23b79933c67e52f1eef7f83a3dff4f767ce0bbcc39572f8cec4acd457643",
+ "blk.24.attn_q.weight": "45c9478593002be1998e96e70668aafa2dd3972380fbc1df12fb05c24ba959e0",
+ "blk.24.attn_v.weight": "515729420885408a6a9614bc27cda393ed907521318d14d21335d39a3eff0b61",
+ "blk.25.ffn_gate_inp.weight": "aae4ac40e9ab3925241f9d784b54b38851d9bc999a6c3bc03fc3f17c9b28a67c",
+ "blk.25.attn_k.weight": "4ab4808d02396c35b00b426f536015673b71c17ae6cd55bbc2e6bfe7a4c59d0c",
+ "blk.25.attn_output.weight": "1990bb982b77e0c947cd1a8ef0b36227ee1259e6dbbc2829e5c136edf88675eb",
+ "blk.25.attn_q.weight": "a1490f3048e8c0ec8784f8550c43adf5cc8d0f2f90131c934713fe4b1b015bd7",
+ "blk.25.attn_v.weight": "f15e53c6d45b3b6f58808fa968425d65e0b26b7f9b268127a77abb1227c67431",
+ "blk.25.ffn_gate_exps.weight": "656662447ff54f56ee80f78a1b9483f7efdc40f7375d0cd8a9c72ccf21f77e7b",
+ "blk.25.ffn_down_exps.weight": "db06f101bccbaef19cced0f6c185166e18202465f4a42cddfd535fbe5cbabb4a",
+ "blk.25.ffn_up_exps.weight": "584a7b02456f27fe1d8d3c7ccd21d426b6ea887795a3ed77f704596a1e3841d7",
+ "blk.25.attn_norm.weight": "8f0f3597982930fd237e9d609776c64f2b909a455b21678f83a7ebd4bbb83e64",
+ "blk.25.ffn_norm.weight": "3e7079c32582afba0c55e032f254adc18d2997705eec860185e9a6dd3d82f07e",
+ "blk.26.ffn_gate_exps.weight": "e70341691b583b86489812b29b77aa41eb658b1865733d6118da54c66e3bfcc6",
+ "blk.26.ffn_down_exps.weight": "5c1b812d11dfb064af816ced5ab6463bf9722eefdfc341b8a93705d5038fd781",
+ "blk.26.ffn_up_exps.weight": "e18118362ae54ef7432781c83884f9fb230a9d934e342aabeda8822ea5f71fb6",
+ "blk.26.ffn_gate_inp.weight": "cd1c5f6710166b9567c6b74c97b2348b191c60aa860958c6bc264ab095261dff",
+ "blk.26.attn_norm.weight": "71d087531af2520bda2e676c489e8529cef5db8aeea1eec0a937a8b4f2fa2e54",
+ "blk.26.ffn_norm.weight": "7f704e936fda28eb5c2cc339f0f6a5f78170b5aa43c01265b21668870d819c82",
+ "blk.26.attn_k.weight": "1cc62a0ce0ae251275d898c52c4a9fba5995fca10955d2011d10dd1a59e1afb8",
+ "blk.26.attn_output.weight": "636e881b1505f9cef656a4be98bec6a4765321d51f9bf1dac8933397cf44b765",
+ "blk.26.attn_q.weight": "89a3c4d202d7d6adebb9e0c1bcfd8b775f6456386f1be25e86e43acc949c1e16",
+ "blk.26.attn_v.weight": "ff2cc963b597cdf1a21703f3e7022af3bb4c65a34a19e19d9309a7c5e198b5bd",
+ "blk.27.ffn_gate_inp.weight": "6150139498fefe380bb99d11e72028da47a15ecb73dfc5b2774f726f4bed8f9e",
+ "blk.27.attn_k.weight": "f286eb9e5c56c7b801a497aedc40158c2a27877d7f9fb59b3fc67834798902d2",
+ "blk.27.attn_output.weight": "5dc3d3a05f9f7729509147fd09c16fb53f85f520cdab5cb69abf4bae3fd460c7",
+ "blk.27.attn_q.weight": "8462e40f86b24251960d6f35a9ea99b8793a01937faf1aec2859f2e5395dbb61",
+ "blk.27.attn_v.weight": "bac1a99e38e25953f8315f7212eb9777dc216cadb09b959977885ae62724ceca",
+ "blk.27.ffn_gate_exps.weight": "6a15eca7f0f6ecfd93db2e55c63875348ec4a78c4ff643ec46df9e958c0101e4",
+ "blk.27.ffn_down_exps.weight": "2e1c91247c4359e2073a8e5f26fd7f6426da7be3ed5bc65dcfff701f0a5022b2",
+ "blk.27.ffn_up_exps.weight": "65d6f5c553c9332085eae4aeadf25090b5d7768212ea7b08ed698102c21b29a1",
+ "blk.27.attn_norm.weight": "7fab8ae63ec8e91ce625cd130ab96d8427dad3a7413bb21b25ec5f408c5b9f5a",
+ "blk.27.ffn_norm.weight": "532720546b0fdcd423a02ca6e3e9d8aacb84b1b3e8269968f88a47fe2a69bab4",
+ "blk.28.ffn_gate_inp.weight": "a305ea58d98962d9dcf0c53ad2389b7acc8936fb35a0e3fc9410e7767cd49dea",
+ "blk.28.attn_k.weight": "8315e8a2e4f78dfdf36d4fc18fffc74bc95fe42c3ae4f9af2b6c874612c0f71b",
+ "blk.28.attn_output.weight": "9b5fdedd32d39ef46a22cca7cd5355d7b93bd07ea305f466a8aad6ca5a4f3778",
+ "blk.28.attn_q.weight": "4e8fb96997c30e231c437130f410d7c91d541a816f6c568b5f3bfdb4b8dece74",
+ "blk.28.attn_v.weight": "1fec739cf3bd7b4913f72ca358d4cf31391c304de44ac0ae31ecb825beaa7cfd",
+ "blk.28.ffn_gate_exps.weight": "9f259789d535e09268266b9a8020f32d6a6779966c909d91d3a10574f06238a2",
+ "blk.28.ffn_down_exps.weight": "516d3f8abaedb01b9916a4b67d4672159769138ef2850158bc1b32c41e31f0e8",
+ "blk.28.ffn_up_exps.weight": "f2f1d88d2c31ed588806fb5ad981d68f5134d7284c4fc022fd018de2eef437fc",
+ "blk.28.attn_norm.weight": "960fd005598deadaebd969996f4367a9dbfad90539a863674fe95730935acc64",
+ "blk.28.ffn_norm.weight": "e1993b37ced93d4049e9af2c47b0d9207d8f7e6f2cc3a52f57bef30bc806d805",
+ "blk.29.ffn_gate_exps.weight": "58927146338f443513337476b3cd30e6341742f096c2beb5890d400f10121298",
+ "blk.29.ffn_down_exps.weight": "03a3386e4f0b75a28c5608e23b2de8f0de25f21954e4aa7fc343431bde9db07e",
+ "blk.29.ffn_up_exps.weight": "6916b7490a7ae7b04a5d81cc1e7ac9b20c483434f3b186b12d87fe176bf1567b",
+ "blk.29.ffn_gate_inp.weight": "98e710e467a3d567abe4ce29d78b8e8dc033148762290c0c5e1ae4d78efd8c78",
+ "blk.29.attn_norm.weight": "4e64cb307d37be20d55f38c94faf7e451d11df5e60df347906cbaf9c5441be71",
+ "blk.29.ffn_norm.weight": "696c23a52f742679bd44440d687a4c44b4302d57f1e9dc5610d23374336187e7",
+ "blk.29.attn_k.weight": "e85253652fd6120c623634ba66b725bf7cd491318b54ccdad2c7df8851d64c0a",
+ "blk.29.attn_output.weight": "4f650a71efb150d1f24cd4d114d4187bf570ac424da3b92ea6455abdf1aea705",
+ "blk.29.attn_q.weight": "69fa7da901026ebcbbbc848455b425458b7e3295007d7fc093acf4b38e2166ea",
+ "blk.29.attn_v.weight": "17e2e7590b317b21f106de546aafd955579703d1e95d6aea044ee72ec3a514c9",
+ "blk.30.ffn_gate_inp.weight": "3a03284b4aa60d59d4a2ec86253469b61fc656372afca427cb77a5332fbcc62c",
+ "blk.30.attn_k.weight": "d518cfd0db9708e769eb1399e87ee49357dc54d5afdbac3d4c0ca46c64e789eb",
+ "blk.30.attn_output.weight": "9b44378714d784c5ef9ab604359091baca4e0ec222afa139b7f840eaefb371fd",
+ "blk.30.attn_q.weight": "cbb95365bbfbcad0c9cd99b4eebb5a5d32de68ce08e4063b5ec3e792b7548044",
+ "blk.30.attn_v.weight": "e7985c04fe1740e35a9598f43b67b0922b4fc2d00b68a92a9f917b82c3248de1",
+ "blk.30.ffn_gate_exps.weight": "8ac4bbd07935d98f895ba94dc174e5ad5046c3c222b53729d60f987c05e7eb70",
+ "blk.30.ffn_down_exps.weight": "dd672cc71e82abf05064a18121b8e55fe1a4f19bc1d7cb9a142f4add54bc336e",
+ "blk.30.ffn_up_exps.weight": "12282f664a2a12aa25e2deac58946108715ebb978bafed5274cef24569107646",
+ "blk.30.attn_norm.weight": "1a33458fee054c6c9c896a4bb0a4e1fbfa0293b2408c7dd2b81d692e966e7273",
+ "blk.30.ffn_norm.weight": "311e33b68051f507f1478ed8f2693fddb846170ddb7285a91be43f795c2ce31e",
+ "blk.31.ffn_gate_exps.weight": "8af43d9867a51cd8392fb48b981b0ceee0ae979c491c07d711b3b56b5162c786",
+ "blk.31.ffn_down_exps.weight": "5579cb7758c1600b19d1f540deffe081b575962e37437b3b2efb2fb0a2924e40",
+ "blk.31.ffn_up_exps.weight": "f2e7c005276b3a001fb40753f027fa10b4d5a346f43cf4b4bbdeec6e74e1cf6a",
+ "blk.31.ffn_gate_inp.weight": "89885dc0e30b6b16a90c0331d7fa3174671e941364e8102d934f02132237e61b",
+ "blk.31.attn_norm.weight": "99e4e9bf86a9edf8c404153a7e8a82324ba79da462622196e2faba161bd95172",
+ "blk.31.ffn_norm.weight": "55335997cf6de781bf332b943de96ff4646966b05d9fee86b76ea897e27b6ca7",
+ "blk.31.attn_k.weight": "cee570762b78da6316b637892cc4b080e40f57af5551ffb1866b9a8e80e96628",
+ "blk.31.attn_output.weight": "fa321ff55ec7819ead7b819fd45215262f39744569765ba2113c989c03588802",
+ "blk.31.attn_q.weight": "9e2c409b878f8a2a1436874abf428fceb1c534b21f9ad4dd6f532b8a469007f0",
+ "blk.31.attn_v.weight": "a845d0be68ba537b4a775bfba4d897faf7c82a811a2612b0b7420cc4f3574cb8",
+ "output.weight": "16101cbb74b54cda9ebc07ca3c762e3263a56efb3cc011156184b95807d7cf13",
+ "output_norm.weight": "d7aa61585baedd60157aafe157930785742c55989c288573566a971b02423564"
+}
diff --git a/convert/testdata/Phi-3-mini-128k-instruct.json b/convert/testdata/Phi-3-mini-128k-instruct.json
new file mode 100644
index 00000000..19296f5a
--- /dev/null
+++ b/convert/testdata/Phi-3-mini-128k-instruct.json
@@ -0,0 +1,225 @@
+{
+ "general.architecture": "phi3",
+ "general.file_type": "1",
+ "general.quantization_version": "2",
+ "phi3.block_count": "32",
+ "phi3.context_length": "131072",
+ "phi3.embedding_length": "3072",
+ "phi3.feed_forward_length": "8192",
+ "phi3.rope.scaling.original_context_length": "4096",
+ "phi3.rope.dimension_count": "96",
+ "phi3.rope.freq_base": "10000",
+ "phi3.rope.scaling.attn_factor": "1.1902381",
+ "phi3.attention.head_count": "32",
+ "phi3.attention.head_count_kv": "32",
+ "phi3.attention.layer_norm_rms_epsilon": "1e-05",
+ "phi3.attention.sliding_window": "262144",
+ "tokenizer.ggml.model": "llama",
+ "tokenizer.ggml.pre": "default",
+ "tokenizer.ggml.add_bos_token": "false",
+ "tokenizer.ggml.add_eos_token": "false",
+ "tokenizer.ggml.bos_token_id": "1",
+ "tokenizer.ggml.eos_token_id": "32000",
+ "tokenizer.ggml.unknown_token_id": "0",
+ "tokenizer.ggml.padding_token_id": "32000",
+ "tokenizer.ggml.scores": "6e37bcde2adc7e350e87c496eddd7a2124329c1dc66c5bf3ad3997253e4f7a62",
+ "tokenizer.ggml.token_type": "b6ecf55ec64ee67d87750bdb8d757a2c58bf78377e9f4219f5689a6c4dea57ce",
+ "tokenizer.ggml.tokens": "d168da3ddd3eee820916945fcb9baf24dd3cde42f606cffa2d19e7c8a8743918",
+ "blk.0.attn_norm.weight": "216aeb2c9e0c271f899e1ef2a63cceeb8f41e97642e84fada54b1d3c1c11cf25",
+ "blk.0.attn_output.weight": "b597d56f7188ffc1fafc273fadc59d41738cffd677ae98c61a62c3285b3a3099",
+ "blk.0.attn_qkv.weight": "d28a6b44e13f59be5483e4be2bedb544e346168d720aca27f47d1a5a722be91e",
+ "blk.0.ffn_down.weight": "4a691370e5a61fcbbf540fbcbf4c0f1d15dec0364528c0e916d0744f6262b63b",
+ "blk.0.ffn_norm.weight": "0c00af2b4a3128bec64a0cbb1084b042fdbe13d9ad0d03bd577f9449dfead338",
+ "blk.0.ffn_up.weight": "b32b52f790c1c083bfb8a3126dc1111cfeeb28dc8c584a930a1e5334cb176bf4",
+ "blk.1.attn_norm.weight": "68748011503c6c029e8e69a84a8e5a89338f378769627b6dbf7f93d715c292e1",
+ "blk.1.attn_output.weight": "2267344add13b048ca59e4377c86dc512be8046a57156901fa32a20fa74e4ee0",
+ "blk.1.attn_qkv.weight": "9109d2e3d7a2eacfda5226587b8be124a3bf44b972da7ebb17aa15795897eacc",
+ "blk.1.ffn_down.weight": "d675df4df4dd039c0c339ad6445d39eddd2004db6bf35bed6314c7497245a633",
+ "blk.1.ffn_norm.weight": "3b5767ae977bc8baaa06b06efdbea193b6b3ba605ce76d77a76ce317e935500c",
+ "blk.1.ffn_up.weight": "80dfd6d9d234b00334c89b8e0a02f81899c2efd377321c34ba5ba51a5f61b5ff",
+ "blk.2.attn_norm.weight": "6a6743b057e5088f145bc179e92c9bfb41163e7295d7b81c62e23dd89d2b59c4",
+ "blk.2.attn_output.weight": "bc5491ea54e0db81462d7d9b7d25cbdda380c2db8de041bd1c4ab7b76a1d19c3",
+ "blk.2.attn_qkv.weight": "a61287a9852e2f5aca9c100b471d98398b2913a3497c743de3c70ec9ddd7087f",
+ "blk.2.ffn_down.weight": "4fddcc382c8dceeab027fe43d8d44e67edb5e8ce4b9a1b7f773c87770380ade1",
+ "blk.2.ffn_norm.weight": "07e05f82b3f63f711db3b684ca79aed25c0657917e66f88af47348a82065c227",
+ "blk.2.ffn_up.weight": "4835a682ef1826c12df01ae7663fc45f9c82bc8e64b665f13fb7da8e201ec0fb",
+ "blk.3.attn_norm.weight": "f22aba7c03999ba7136f39cda747a39715e498699dc1716cd97fc5dfc58d1b1c",
+ "blk.3.attn_output.weight": "53b579855366fd786c5126b2b30aac4d583ca7bda56833c4865f5cadb5c18c6d",
+ "blk.3.attn_qkv.weight": "bb56aba78158123140fcea59c69ac562ca208f6d3086819417cdad8c50f333ad",
+ "blk.3.ffn_down.weight": "97280897a7cd86db2830c004bccc5bc094f50e293baded0189159a2019145a6e",
+ "blk.3.ffn_norm.weight": "10a8c99f8b57a960e8e0a1133c4a26f9148403d1b9bff2eff114917de996f3b5",
+ "blk.3.ffn_up.weight": "7324046c915e75d621b2043597a245a428d8eea31869135e6257a861491d8dcc",
+ "blk.4.attn_norm.weight": "507d8e164de94646edbfe33def8e8fbf7c9a6ee3fbaedb5000f72d9f51ec5e36",
+ "blk.4.attn_output.weight": "bbb3429e6efa98c150e0fdbf48c16180cbf0d0cbc1b3c253c6c319d78f4593a2",
+ "blk.4.attn_qkv.weight": "b95ee5be0786d3901273d806c339fe6c20e6bfffd2a20672a9f56af80921e8ab",
+ "blk.4.ffn_down.weight": "806bbf91df92a5a22bd5aa1ffb7fc2869f7293ffc7704771c290ecc583b27975",
+ "blk.4.ffn_norm.weight": "cfc2930a81df7aee3a5e7f726a15c1182233e868bf0d9d37f6b6ae6d8c15c234",
+ "blk.4.ffn_up.weight": "c3390c69533de2c8424e8069323ccc5d0c4543111535da04cf2c7d26745576aa",
+ "blk.5.attn_norm.weight": "0d71c4fbcefabbd021569442853d2fe90668b19409ae2805a718a829ca60beab",
+ "blk.5.attn_output.weight": "10ebd93629112bf2df5c30dd0953a4a5e9020306768283181ed426934d47e14f",
+ "blk.5.attn_qkv.weight": "5cb05633369f12d4b00e0ff787736bd846856682115720ebc6cce05270c334f6",
+ "blk.5.ffn_down.weight": "e28bcc5094212eafc7476dbc5b7a520d25b79578cbf4229d698e2655956a80ad",
+ "blk.5.ffn_norm.weight": "b6f2c4cf9f34bb4d59989f96165c14a67dc1e266ad0a6d0fcc49f1add929e6ff",
+ "blk.5.ffn_up.weight": "0f9ef99423cc07ebedc0e9cfa95809f2d7108d910bb4ef97ebc0b0309c440750",
+ "blk.6.attn_norm.weight": "b3edcc47a42218234f7564d7470611b49401a41ae8cd42123f86557c69f5d7f2",
+ "blk.6.attn_output.weight": "eb9b7d257b388bb5b8fe0515e5c6873317239cb94cda236e4b6ada2a6c57c65c",
+ "blk.6.attn_qkv.weight": "eb968081f478c52f07bd9c2761741e982dba33cc4eeadeea3557d391b9ac2106",
+ "blk.6.ffn_down.weight": "1b8588bb7463206290322695577dcfced300895d6e6f4b26966c53a9ae2f0f84",
+ "blk.6.ffn_norm.weight": "1219c04b7770983c77814200eefe743f46d15328ea2b12711e44f8103eab08d3",
+ "blk.6.ffn_up.weight": "197ef287239fec47c55677f0fbb66eaf0644f775bc382de843971730721394f6",
+ "blk.7.attn_norm.weight": "b630ad08c80d564ed1c024384818e9fd3f22a36cd7a14aa96e7e2759a8285099",
+ "blk.7.attn_output.weight": "970255aa750828a47d6b9d399f9612b5bf25aefe7dadbcba41fc416d0d4067c1",
+ "blk.7.attn_qkv.weight": "ebb157c880293e6de8d629f263ba8853ed1dbdc02c311d43432bb8cfbb310739",
+ "blk.7.ffn_down.weight": "24bcd4db4cba844c89f878b81843c373dbbc0675e889d32c5b12e63384a7b670",
+ "blk.7.ffn_norm.weight": "b9c6f71001808ee873ce7db8056e4b53fb4cccec8b7f0f312899b575fae39d39",
+ "blk.7.ffn_up.weight": "979f1828d227455c26015a2a11afe9dd05f2bb97a8ba6b38c8dab3f50e627401",
+ "blk.8.attn_norm.weight": "4e8e347e3775010b7112ee630f2f4f2383be7ff64e6ca6154b9b22566552eaa6",
+ "blk.8.attn_output.weight": "65a44babf44a435a1829945211b3168f9ec78ac3cb7a049a733e93d11f0d6659",
+ "blk.8.attn_qkv.weight": "343ed07671da400b040812a4058482fa38284b5d9af9becfed07417fe26ce747",
+ "blk.8.ffn_down.weight": "7fb7e073e3c2c503c4e9d60efa0988fed7398d900cc003695fe3fffd3e188b82",
+ "blk.8.ffn_norm.weight": "b07c1f655d8593e3892a2cf73f8a0c19ce8e5cb613fafbe7cbd430da8ce4c57d",
+ "blk.8.ffn_up.weight": "8b26e14de54b3fdc2e2d3ea41720f9d9c236a93688c3b7fd7bf43f5fbb327c9b",
+ "blk.9.attn_norm.weight": "46394d408a8e316916177e6aa261de32e137a82d729c0b1800b072f0c38c39b6",
+ "blk.9.attn_output.weight": "d57f3d46107947a7073373a0b35d6ecf7759b5df15406f4a3590a60666af6b16",
+ "blk.9.attn_qkv.weight": "14bb8ace8c5453148f4b536e9f4279c813f31136716947256f5cca333448639c",
+ "blk.9.ffn_down.weight": "2b8d98e2b5ed68338f6e4de43bf7de0c4858cc69103cd5177725f7444eec7694",
+ "blk.9.ffn_norm.weight": "41a499dfd418cc4c6b8c12313f673f7e2cd4a3f9c4065eb6c4feb5eed02fb542",
+ "blk.9.ffn_up.weight": "143aab7533a64b17fbe201490a6f674bc7f0bd370c094500b2e100419073d1c2",
+ "blk.10.attn_norm.weight": "ebb670aafd36816a794347287269d8f1a5b19c1e3c0a1e38023bc19fdba9b073",
+ "blk.10.attn_output.weight": "b5d65bbc0ed5e49fdd9d754bc18163cd042a285024d0cf6f954c503bc8c877cb",
+ "blk.10.attn_qkv.weight": "f06b15bac88da798fa34a62b03eaac0dbe8b846020516603c387541f2d8dd672",
+ "blk.10.ffn_down.weight": "fb091fcd1b4de25d1bea94d1755e255cb02914a030d23e3a234e57b8d46bde6e",
+ "blk.10.ffn_norm.weight": "eb347bdf9c40414af87e13a8e72e40b31f004b50f7cb366f1a219ced60a61355",
+ "blk.10.ffn_up.weight": "ed2d52fc881a173f404fe8a1067862c9856d6c3e0d2e90a330a7aa394e3f84d1",
+ "blk.11.attn_norm.weight": "64e252603cf010a0e502ca39fdf8d0a196a79aec67c0d2bb9213fc0cb80c47d4",
+ "blk.11.attn_output.weight": "228e33e21c69f52efc74fdfc831bc9af271e44b2a29a3dced1d64e667ce36eb5",
+ "blk.11.attn_qkv.weight": "ab9ce6d4ef9e42ee0da3f20a7708a3bbc5e79e967b05fa86ba946a05e2eb63eb",
+ "blk.11.ffn_down.weight": "0ca133b7835c98dc77c25d64e4eb7873778bdb5e4d22d8b80f920f46865b43bd",
+ "blk.11.ffn_norm.weight": "02455741a0dfd161c79aa1ecc381901721f229fdcda5615622a629631fb61cfd",
+ "blk.11.ffn_up.weight": "9fecdcc099fbb8e23c6b1ea9294702a027f4a58d265543ec5e7be79b8f63b354",
+ "blk.12.attn_norm.weight": "783bb459911b1b3609a9b2bdfe272f1670add73b5471da738e07ac47e2e07dfd",
+ "blk.12.attn_output.weight": "1e1a914c9e48b857206ac5a1f7cead994bc1ea91d5d4fff8c834d73f2e38ef5d",
+ "blk.12.attn_qkv.weight": "5953e7185ccb87fb4dae8f9426ec86315d4c7794326e8ab59b3a95d4af2189f0",
+ "blk.12.ffn_down.weight": "a3eecf0f394f86e2cfb48a5940a5c50ca86d71883b2f79fcc642a935fabce0d4",
+ "blk.12.ffn_norm.weight": "0a4272e41373c23bd72f10d2d82930aa3a1480aac75832bfbf01cebf0b86b6a4",
+ "blk.12.ffn_up.weight": "06f42776de3a7ceac3025f26a7a8bd20e062233cce2bdaa2183470dc4b30b87d",
+ "blk.13.attn_norm.weight": "5915da60fb03e201fa649faba780e5fdf1c761c262b206e5415cf83181f65780",
+ "blk.13.attn_output.weight": "4dbf6eab074fa3835fd32bd631a8208e511037d5056d2fd3015735cca7674ef7",
+ "blk.13.attn_qkv.weight": "d3d8339a1c4782d9e73d77fdebe154d3c5b83ac40c9175b3e91a4977d08f876b",
+ "blk.13.ffn_down.weight": "de6772b46a55e1fd42b007637dfbf68b6598e5d5b61622da0935002e1e192d3a",
+ "blk.13.ffn_norm.weight": "5a640ea3b8c7be49c95a58a2327e10d8e8d9d142504bde5c8091613e5b961d7a",
+ "blk.13.ffn_up.weight": "f35e3545e4bd3531b2e843b5efd31dee0c13c807ee6386e65473ba67bbec30d0",
+ "blk.14.attn_norm.weight": "9b34986450b7c98b4927e81e61a816f9e84b1addc7c14926402100037aad6678",
+ "blk.14.attn_output.weight": "155d52efb23d366016d861a251d4d1f4a0c13699188c50d50dba016a0d8bfcd9",
+ "blk.14.attn_qkv.weight": "8e1415084e1f33c73a777f19e752489f4dd312cca047733e5ea643cd4a955e04",
+ "blk.14.ffn_down.weight": "a2a142226b94baa01ccb65bdea2b7418e49085c1d9c3c63e544e3112c58a25da",
+ "blk.14.ffn_norm.weight": "8aecfd9b0ae6affaea31a80c5c9a4a14b31deaa0db7bd8f6da2a64d23447921c",
+ "blk.14.ffn_up.weight": "0c1407237b8c1bd02f193346b5681926fe698a5055eac6a7450451b0f991707c",
+ "blk.15.attn_norm.weight": "e037bd19880bfa83d983200fb0c7866f8ad16c3ff5cc4b4f3a37ca7373870ff6",
+ "blk.15.attn_output.weight": "045fe4fc95cc129a1b92771b179c11b12845c4c088786c607f17bd98857e68e1",
+ "blk.15.attn_qkv.weight": "7621b7559705cab1d4dea1c69f76dbf9dc1c8837a203b656f484703b9c1b70ce",
+ "blk.15.ffn_down.weight": "7e5ac20e290bc60761e1cd972354fde225b7fa861048d44d9a0dd9b046d55f58",
+ "blk.15.ffn_norm.weight": "b6d830d88f1db1825687973c8c2b1a24c6fa84f07af8d0e3ef9c86009baca0b2",
+ "blk.15.ffn_up.weight": "dcda0957cd04fc45476774dba2bbf9aa89d6b05d5ca7b10ae6f73ad2c49b1cd3",
+ "blk.16.attn_norm.weight": "4ee9b70ba15cb2a08240f93990e90f5068c48fceb481f8e2186bec8b7214eb3f",
+ "blk.16.attn_output.weight": "315cfe5536658d2498192b2980eade15b2c9a4ff220e4011911457b1727fa103",
+ "blk.16.attn_qkv.weight": "3c8122e3ad637583b9dcde8ff3a323267d3014bb1f0f9771e5322260ca9ecc8d",
+ "blk.16.ffn_down.weight": "3b5fbebd5ee2b86cad96fb8a9b45a8770d08f82c1c8b74d7061e866f7020a18d",
+ "blk.16.ffn_norm.weight": "ffab69f20bda372de6e5878f0539163e2fc6ba113621ded95705fc3b1465c9f0",
+ "blk.16.ffn_up.weight": "0935ea3d258da42d6258406365f39f58ddaabfe97ea5977580db3635188f24a1",
+ "blk.17.attn_norm.weight": "f030441733f3d147b4a06a1eb4aeb8465c7c24d9c53bf4c48fe7e134d3629803",
+ "blk.17.attn_output.weight": "07a955ef09e8dc766ac0df647d0b2c69f23c4c69a7137654b4aad80303ed0eda",
+ "blk.17.attn_qkv.weight": "1c10688061e21e2fe12ad0cb54bf03895c1f83c3b0df743a42f548b52cbca1b2",
+ "blk.17.ffn_down.weight": "ebb9cc9836f41d88fdae2aa9a4355514e4edaec8d1577ffeb947a35204e77f52",
+ "blk.17.ffn_norm.weight": "50aff44f6528b13db5389f2ddcdb7676244947610bd7ffbff3f881c968c2a0d4",
+ "blk.17.ffn_up.weight": "d716537949582be33bde6b02e38f5a70081c9642a9fb05a61312126718b8d148",
+ "blk.18.attn_norm.weight": "0ea695c4e53d637902f46663a6ee42adc493c36794476acc7dbddaa05b13840d",
+ "blk.18.attn_output.weight": "5fd35b500221a612eb4f4bddf0e9b6b7db4d7733032a75f8802fb2d884647c2e",
+ "blk.18.attn_qkv.weight": "b0da37fd030fe69581f990bf23bfd35467a1bbe558af6de7c0924f6b72e92317",
+ "blk.18.ffn_down.weight": "b355c33f44b328f4bb977567de8f7544db4b005d7a8fbded658518ecf3c5a153",
+ "blk.18.ffn_norm.weight": "58b3fe9094079989a86e0387143259e1cc35952d24dc3df290c4ba6df44f5c51",
+ "blk.18.ffn_up.weight": "2ce530954c342c30ed2ead5353f931960bfae1d278868504c0efb973560fabbe",
+ "blk.19.attn_norm.weight": "533e9aed66feea8f0392aa81f9e293240e1f009a5334253915fb60c2749b615d",
+ "blk.19.attn_output.weight": "84f2d00f98a4113a779d3b5d1c3e7c914eb47784d3ab13b290367c124c2994aa",
+ "blk.19.attn_qkv.weight": "fbe6b9f53b07fa7537d3b3d452d20a9bc666f9fd41ec2091dd28bc2f70fc668f",
+ "blk.19.ffn_down.weight": "b30199e098c8bb3f890183d8b18471e80b62b604729b277ad62488dd71e1206b",
+ "blk.19.ffn_norm.weight": "c81373e41cd340b7badb19f9517c77c4250b4eb9a02dc758b8b49b652487d7ff",
+ "blk.19.ffn_up.weight": "5a5cb083ca7725720e3a890f7fa46354760e8007a8188849a092e305694a75e3",
+ "blk.20.attn_norm.weight": "4953091b4477e354357a8e743ba0a1900633e52f1599ee082a0c9b0b2b5cd978",
+ "blk.20.attn_output.weight": "62d54f7749cd6856097b2632066a322b0296df915fe66f382c5b5981be0d4f23",
+ "blk.20.attn_qkv.weight": "406de9e35b0729ebe902d7a47905cc7fb29a921431ed35dbef0c03e5690a1329",
+ "blk.20.ffn_down.weight": "62fb678b0d1261e19a4903a2b347d67afcc8acff01feb33a687a35a2d1e6f9a5",
+ "blk.20.ffn_norm.weight": "cd9d36b7e71e55c8925b97bb09c28219f182626bcff094878ae39c3db887a14b",
+ "blk.20.ffn_up.weight": "b9276771d79d3e932e73ccc520c3f8476342b9ef312ed2ee1e0da822e6e3ad18",
+ "blk.21.attn_norm.weight": "66d8c8a35e13ce9c2a0e75b670150e2c31484a55c2316df46075312196178ed3",
+ "blk.21.attn_output.weight": "12ab46c9382648f9b3350fdd92a6be6352743d62d6b520d7e2024e0c838588f5",
+ "blk.21.attn_qkv.weight": "a7909676ee1675ca23cd29a5fdd226df8dd9d68f94c6c9bbb51dd9fd38504008",
+ "blk.21.ffn_down.weight": "6fb317279c6542e82f97d5a12a60fac1bd0fa0405154f9fbe265e2fe39bd49cc",
+ "blk.21.ffn_norm.weight": "c0f703eb3ff161b5ba4490d87d8684b8a6c47a8f433e12f418333b9db439010a",
+ "blk.21.ffn_up.weight": "6dbdb80ef0c35e364bbce12d40d5e74c7963c7b55d58d9579567a07ffce7b863",
+ "blk.22.attn_norm.weight": "f94237433bf03d675cb2f655b81ca91a1ce2447bc6b00b13d6b0ccfe2d411eff",
+ "blk.22.attn_output.weight": "e821f95995ce497c01e63ca64f737713b1b65f11df1903e51d444aa516f33f71",
+ "blk.22.attn_qkv.weight": "1b0f717c73afb5eb4c82a1708c4e85c969e8a2a8770d9ddb78b1870a2d8a781e",
+ "blk.22.ffn_down.weight": "0f33f7a3cdc685484be99aa0c03642b0b20850a27d1fddbe054b13a9382f3ccb",
+ "blk.22.ffn_norm.weight": "9df285cf211ddd7df2b36a50489af574755c7d4d98b29a05cd04566ae613c8dc",
+ "blk.22.ffn_up.weight": "63ac300e1efb34041dd0136cf43ea622fac6f0caccce1cd9262f5e08d2cf179c",
+ "blk.23.attn_norm.weight": "5f72d9e88689b4027b28f5f8f26cd3abb03635ceea7ec98a4c91a9fc691f6707",
+ "blk.23.attn_output.weight": "6ecf04ff61125c5fc768f8656497152149373daf321ee9c957e8f7245a1184d1",
+ "blk.23.attn_qkv.weight": "a9d9978806724c2959f2cf386c233831f08e1e933dbf2b32665e788d9d512ea4",
+ "blk.23.ffn_down.weight": "72c7d17886a3da17fa0daa456aa5e877b2ef5b8b403182b870d9ca5ca9c70347",
+ "blk.23.ffn_norm.weight": "971e4b712e3025a13419b5b57d674b5e4ab7f18f74b57b9afc4671623da90c4b",
+ "blk.23.ffn_up.weight": "df2b5c7dbd5834545b815073af0c7355b065124e6d6f0fee78d8fa5b2076dc3e",
+ "blk.24.attn_norm.weight": "c41957c4a79ad3b16f6e11daec1c7f530b9f3f4b618e1e4367c3b67787ac4ab6",
+ "blk.24.attn_output.weight": "ef7d61f5fc88ac6f31bf60cb5f4d2d6b8df42d38825807112361a7224b0dee3b",
+ "blk.24.attn_qkv.weight": "3e6a58fe7d49c90bb6971efbad3371c32256881173ea5aee4b0c296cb206490f",
+ "blk.24.ffn_down.weight": "f43619144047de42fed81dfa495f1815d3cb771330e574043e2b67620819292c",
+ "blk.24.ffn_norm.weight": "5501d4a2a98c8ca6b42e77b53b221dbc08f530f6a067256d787534ec6fe028bd",
+ "blk.24.ffn_up.weight": "d64c8b0e509e2b1118f6000176f8956cacecdbb200c7e95ed93fb78b6e26c84a",
+ "blk.25.attn_norm.weight": "502fa3c302d371f61c5791f4615b73018ffb1daa09b6499b227116581244c5d4",
+ "blk.25.attn_output.weight": "ad8391d4e9c980856f2547aa945b2b6a407a6382158dc1ddd4f08d94ecc24be6",
+ "blk.25.attn_qkv.weight": "42e8983780d4a01a02c54ad23d4df21eea437f119a10af5a9c12a76a42d308c1",
+ "blk.25.ffn_down.weight": "302dd010d4e0ab4eeaee89090409ea0dddeeeed3236415eb8f97c942497eea91",
+ "blk.25.ffn_norm.weight": "fb34c1ee5bca96986c08834df0a0c047ba041c1123ac1f563e9d64312bf82d6a",
+ "blk.25.ffn_up.weight": "10739a8de156816d93c92b935386540bfa976bdbef204f0312960f6fc657582f",
+ "blk.26.attn_norm.weight": "7036c711609128c4e55968ff3681d3043338879a5737efd6c2ac9e1a2a61f1a0",
+ "blk.26.attn_output.weight": "db5db45dead5cb911fa01da59832f121b7c18b2d167bf53741c40819f24d346c",
+ "blk.26.attn_qkv.weight": "cae34c6b7f82ed14348d5ed30a79919c383737c1694a9cb9c0de609d3b0c1d0a",
+ "blk.26.ffn_down.weight": "491ec3a4da9b4f49f8ebc6be658ce397a9b801ae9fb35e82177e47808c65e5d0",
+ "blk.26.ffn_norm.weight": "fd7059d75d7f0e5288511ddeeb0f772eb3cae3ccfe4226b877015834edc3c386",
+ "blk.26.ffn_up.weight": "ea1ee1274c56458ce056d2205e5bb6e5422ce4cb0ad58006b8141749b97a0c39",
+ "blk.27.attn_norm.weight": "cc362c9a937609265052cd38544af17a1a7448cea086d4c801139e1fc865832d",
+ "blk.27.attn_output.weight": "ba757a81dabde9cb1b069d1bb616fe79649a1724f756567ec61caed1304fe6cf",
+ "blk.27.attn_qkv.weight": "1ab8d7d02d87756c12c2275636823aa5ede3d683178225c4cac4bd892c319bd4",
+ "blk.27.ffn_down.weight": "deb1c711c8a66acf4dcd2d088e1548f8e08f296f755e4067d6557fa55afde88c",
+ "blk.27.ffn_norm.weight": "fc6242d8cb8a4a37a8ddb7e41e7e60a63d4a89edf36acb35df052f10b9c91ece",
+ "blk.27.ffn_up.weight": "8df39b09c4801f343aca78f2918a1f6db78c8c55e591eda4c69eadb74c26e180",
+ "blk.28.attn_norm.weight": "75b539308f77e3cefdc6d98484d8b5cbf0538f0c2869a77b7373a145a18bc850",
+ "blk.28.attn_output.weight": "ae128940eb60a6d2e121762ef4b3e9dcf9eb3e105b249507fa7f12de0e19822c",
+ "blk.28.attn_qkv.weight": "bdda781c288e9326c240e33905f8e621b6a2ad902e620739d34f93fcd6f933de",
+ "blk.28.ffn_down.weight": "f1d6e6d1c286b1138bfd7e53fe477f399ae93bc2c04e35416f84218ed7247965",
+ "blk.28.ffn_norm.weight": "3f837ce82c8b9bde0d61d08b6f5fe5574886ea5328dbdc53f2929f18da8b4087",
+ "blk.28.ffn_up.weight": "2af027002e31d1b6cfedbdb30a2b9d7213f3aa691167c353913adfd48fda31e4",
+ "blk.29.attn_norm.weight": "61e8003b5329462ffe0fe172f2b160260de006aed858332d49d75504b6b6aa7a",
+ "blk.29.attn_output.weight": "ca44542a72a37476dc73dbdcc01f5b7497cb3ebc4ea230a55c9634ccd8e56ad4",
+ "blk.29.attn_qkv.weight": "abb3d9d6abe57872ae3daa51935d43264093ded5ce63b49d1e280ee5758be0e4",
+ "blk.29.ffn_down.weight": "6764b895fce881df097489c263446f0106de36217997660c15984b3ee22a5a06",
+ "blk.29.ffn_norm.weight": "89e03e9a33fc0e6e31ba9f0c2bd7c5734a118c5602bb90148793e08a80e8d0ae",
+ "blk.29.ffn_up.weight": "fa7ad57a84954f4121653152efed1a871d8adb20a1ea9086e3e849ce359d7d2e",
+ "blk.30.attn_norm.weight": "91a697aca1e42af54f806a20211031c3369e8d0bd58df1b0147fe24954e1f5a4",
+ "blk.30.attn_output.weight": "36063fcf766c89ac75be56f688cc63cefe5f2c733fbf4378ea9956ad386fa148",
+ "blk.30.attn_qkv.weight": "2cacd1161f1121a2c0b979930134f4666f73fb8d7237b3b0659ae091b15955a6",
+ "blk.30.ffn_down.weight": "9f3fcb6217100595850c05dc98f9ab2a263afdb6ab28df2fcb08aeff512057d7",
+ "blk.30.ffn_norm.weight": "6c600bc1fc7de39d4f8917b81fc7d1d5ed2a9b56492234c13a4bd6028c30d880",
+ "blk.30.ffn_up.weight": "73cabd1bb011956b2689ea3338bb76642ef3a57c197377d666d2ab5f56317668",
+ "blk.31.attn_norm.weight": "72d3e1cc771380645fa75a899858c95f39857a4f3f1ed60fe1578df383b8bc53",
+ "blk.31.attn_output.weight": "40089cdd29994dc19a1d89fa15902a89cfeca3540f12dc9bf4d00ef82506e456",
+ "blk.31.attn_qkv.weight": "1d0bb40e9258071ae14290a53c619a8e331dda07354d2a02ef45766c029ae5e4",
+ "blk.31.ffn_down.weight": "8defa0e06335b793fa8be03883f0a322d6c5b33f52c69c943c35c60d16e42c0a",
+ "blk.31.ffn_norm.weight": "33c55d9d0c496ccfb130361fe131649346e098abaaac39c0519507e5d846721d",
+ "blk.31.ffn_up.weight": "599f6503f61c692c1f82001973d35119f9688db5e6be9d9c298411491c93f09b",
+ "output.weight": "14b8dc662bfa3308ebb2e102c562d8e52c15670e538f20f3216a9c310ca9dd41",
+ "output_norm.weight": "7f2294ba94ce65681df6c7ddd8698799199b9d77dc83c10bdad5c3999f0fdb82",
+ "rope_factors_long.weight": "e34d378664e354652c38f47d10dafb0498ccc2fb042d39ff7fef768146fff22b",
+ "rope_factors_short.weight": "9379146a4988f373d362fe47b06c75e7fe7c54aa4dc9558758df79b7a87471fd",
+ "token_embd.weight": "19a03c1fb5ac0baee93b0a7d8b0f26e9a9b011e229b694afc50ebfc13d84f8bf"
+}
diff --git a/convert/testdata/all-MiniLM-L6-v2.json b/convert/testdata/all-MiniLM-L6-v2.json
new file mode 100644
index 00000000..15c8f039
--- /dev/null
+++ b/convert/testdata/all-MiniLM-L6-v2.json
@@ -0,0 +1,124 @@
+{
+ "general.architecture": "bert",
+ "general.file_type": "1",
+ "general.quantization_version": "2",
+ "bert.attention.causal": "false",
+ "bert.attention.head_count": "12",
+ "bert.attention.layer_norm_epsilon": "1e-12",
+ "bert.block_count": "6",
+ "bert.context_length": "512",
+ "bert.embedding_length": "384",
+ "bert.feed_forward_length": "1536",
+ "bert.pooling_type": "1",
+ "tokenizer.ggml.model": "bert",
+ "tokenizer.ggml.padding_token_id": "0",
+ "tokenizer.ggml.unknown_token_id": "100",
+ "tokenizer.ggml.cls_token_id": "101",
+ "tokenizer.ggml.seperator_token_id": "102",
+ "tokenizer.ggml.mask_token_id": "103",
+ "tokenizer.ggml.token_type_count": "2",
+ "tokenizer.ggml.scores": "6db964fe67338aca57790481a390121ff3dd643eebe49f7dd308029ad99abb6f",
+ "tokenizer.ggml.token_type": "98d247c5404b6b18f05f133b92dd56edf6efefefac326794b00d7b351f6c5aa1",
+ "tokenizer.ggml.tokens": "9efe405e229a45ff9916f54c475d151d2200cd2ab0006f347abfb069cf096c86",
+ "token_embd.weight": "8c1ee80a9ea4f65aa385ba30112010068af3d209bebc6e149d3d4589c2cd0a5a",
+ "position_embd.weight": "6c516f0b1c4e2388ab90394dd80ad69e4e4509b890982fc3408108ae66210eb6",
+ "token_types.weight": "f879f8e422ed211948f28b560d3c5e17aae7993f063b51196a28cf5c0fb3da21",
+ "token_embd_norm.weight": "75076e095d717aab96f8b6beeee503c27940d9a76f2b891a0e3de72f8a6043e4",
+ "token_embd_norm.bias": "298735285ffe944e1bf03e5d35c7280326b85cf121bde9874f1af5dc51ab939d",
+ "blk.0.attn_q.weight": "ab0923ce4c1549175112dcdfcc860fe30137f991e03ea6857fb5993670adaf6c",
+ "blk.0.attn_q.bias": "a3ec29551dabf976e1d34256b8ab5ab7b758f3ed9742c3cafdbd984d5441df62",
+ "blk.0.attn_k.weight": "4c1038a6d035c3e9ffed7fa672b614627814752503755fbad0cfb76a41ad71ba",
+ "blk.0.attn_k.bias": "e0363930eb588d91816aa3d230bb03b6e2551c165117b80b8d60397413819ef9",
+ "blk.0.attn_v.weight": "425e2e53e3f00ce98d29c3e6a161eb55d3e6ae0d96fdb9f6242d1c4fd6eef4b3",
+ "blk.0.attn_v.bias": "6579173a1e65ee124fbd0bd53cbdca4225515b4f2c5f18fb1bfd000f5978f9bb",
+ "blk.0.attn_output.weight": "a6d70a08cd7164de5d12af65d86d657c3db35aaecde778b2b3fda9193c4c9802",
+ "blk.0.attn_output.bias": "2b8d12c4f9a9c5bfaa29c597839568f6e0525cb41eeaf64ddeb6bd84dfeb9701",
+ "blk.0.attn_output_norm.weight": "bbe6e502a473228b525aeed26cc31b7db123ad63bdc5a6eebac6ea70b8b51d62",
+ "blk.0.attn_output_norm.bias": "36eaacaf0007c5c62daea97aab0115390c0682914f78482e37eb76885f4b7a50",
+ "blk.0.ffn_up.weight": "24654561c76ce387d125759ba843f06b904ef721fcceaeff6ccc62180a48e874",
+ "blk.0.ffn_up.bias": "fd3f0126aa1d95768fa60eb6f4ab8a2763cfcb7e5405f35b92353031d86f4d34",
+ "blk.0.ffn_down.weight": "97a829763a6a5bf3329ceb4d39c424ba4787d61653a5b0bbd1f84782e4d4e0ca",
+ "blk.0.ffn_down.bias": "7aa980c30ae8b4ee7f69df28808dbf5c431f56ccc4a80340f644a0419f16c054",
+ "blk.0.layer_output_norm.weight": "ef30dad4c2a083ae1ff5039a2a6cda60ecc89bf1e486a6f8c0d15f50589603f8",
+ "blk.0.layer_output_norm.bias": "8b1b77e67568b1bce43fc476de1b177c53ff688d66beb66995e8eb3dc290da8a",
+ "blk.1.attn_q.weight": "284331622a1f6f9b87ccee4f652bd66a394ca493c4d93be4d1844e4f6159ad10",
+ "blk.1.attn_q.bias": "e24ebd4860330e08f6bfdd077a82db0bee33f4c8846cf1db26327a34754c7069",
+ "blk.1.attn_k.weight": "729dd0d555544b5bd0f7580b3c8b384256b974605f0e7487b95f295aa032997d",
+ "blk.1.attn_k.bias": "2aa51a828a858f35473f54477583fea54ce2ccc34ea60fbd1d228fbe9bca827f",
+ "blk.1.attn_v.weight": "6be304671cc311d5ca5c103f2b51467ee800c589bc5b8101e09ff5aed1f68c21",
+ "blk.1.attn_v.bias": "43bcbab78a8819e07f723bc9e5b737b71e87a7594f15234e882b63e327a64199",
+ "blk.1.attn_output.weight": "15ec8a1a12b26c9976445308a09f748ab0e4bef0f583d13ab08c3129f8738d73",
+ "blk.1.attn_output.bias": "dac2146f4baa6ed16f6c0dc7443831fb7ec79bedcceafd80d1a4b628a1bb072d",
+ "blk.1.attn_output_norm.weight": "d2151eb33bffac536787a4c9a5d2b31c7a80b17c4611877842a3cce2cd6e98d8",
+ "blk.1.attn_output_norm.bias": "31e1b779716dafb855d2cf5631ee168a0ccf372eb9c6ea6091f66fa97a9b9d2d",
+ "blk.1.ffn_up.weight": "a57547fc3fc3b77406f5cdcb0c87af9bc184701f175c39c1f35297826fce3cc7",
+ "blk.1.ffn_up.bias": "123be6d541d086202913c75d878c54d59a749f3af7b58f7ef9eb9e7c62a24c9a",
+ "blk.1.ffn_down.weight": "cfdb79788377e5cbded8790cd41b9e66c397ecab75474071fcd7cf32d30f9613",
+ "blk.1.ffn_down.bias": "bcb58315519a573097960891c9ae41cf4c685ab78c3e0e77471471758a7eae88",
+ "blk.1.layer_output_norm.weight": "819b554271452bfb1d84c2603b90377b2e41a0ac1e3aa8b417ccf9dce63375bd",
+ "blk.1.layer_output_norm.bias": "47a3433ac27f5ce8947fb38dd491f3706df4ef6adb0ddf74612bf0f54b19e164",
+ "blk.2.attn_q.weight": "1557a9ea852b1880551f7290e00aded4f35e6c4180fdcbed1b0039bf805f639e",
+ "blk.2.attn_q.bias": "c3bfe5f3066f655fd36b055530997b59ff33ef013563aaeb3cb8ff07dabd59a9",
+ "blk.2.attn_k.weight": "cfd08eb69c61ae2f9f14f9b7ff5c5394ca264b1a9f3d48156677f90dd1766289",
+ "blk.2.attn_k.bias": "9b839bc0e79974a0b3f5d1895972bc6f5c9a1bc16052e1af786e6a530758152d",
+ "blk.2.attn_v.weight": "02b26b1208480eaeeb00e7b4cf8b690006ca14759357fc44ed4a2a8924ead993",
+ "blk.2.attn_v.bias": "e7e6f0089fded1659a867ab736c220d9653ea7da6b1b94baf5c8d30a748b63ab",
+ "blk.2.attn_output.weight": "a1db121c7d33806b349cadd050300a57db49fdc91224fd07c9ac43bf4299dc79",
+ "blk.2.attn_output.bias": "7675128b6a92555cd955c820311e91e9417d31f48848f45d047b4100c62148b3",
+ "blk.2.attn_output_norm.weight": "5b4595e0fbcba67a700c4331adf746d2fba3546364a4db5607ae241947bb1a21",
+ "blk.2.attn_output_norm.bias": "7b8e16826ea30e5a2ba0b02e0095a901775981a296e98819625320e983060d08",
+ "blk.2.ffn_up.weight": "a0d815d946ac07a65095c4ae4df77b818845e6d97795c7d82f55e689d944db59",
+ "blk.2.ffn_up.bias": "ce37c0a4174d6bf773ded7bd016ede627ad3bdb8bc99b9992a18dc8e8898f252",
+ "blk.2.ffn_down.weight": "f6231d2a25426fbd45b9f1160aa484220eb227ceef0348c4a6a6de890606e5ef",
+ "blk.2.ffn_down.bias": "429e00556e8dc63a785238b309b9d83738500c1ef6d736fe6526ad88ea496d27",
+ "blk.2.layer_output_norm.weight": "651457a573adf3f7dd9ee5dfe1c8e89389e94443993aab77ec6a0b05aa621e35",
+ "blk.2.layer_output_norm.bias": "41fbbeda7fd89b0cef5f945ae44011c316982390401d6f75ba8c6d365e185247",
+ "blk.3.attn_q.weight": "95a43f32949d2cb8d22815bb27a44abfc6665ba96221af817dfe058cb6ca72c6",
+ "blk.3.attn_q.bias": "f4e34385e75d8108b6b3bd336106e2133a8c9be0cc343dfe5dc48c32a823c7cb",
+ "blk.3.attn_k.weight": "6b892da6a17d4d3265265a15f695864a31813ee8c8e710ae9bc9e1adbc6c9a18",
+ "blk.3.attn_k.bias": "40b8067b641a56014cee42548240aa8930820958b1933004892b5f04fbaef39e",
+ "blk.3.attn_v.weight": "9fcd5922319dd2a461082a5ce040c1dfe65d87d70ca6547dd0b46eeecc3eeb2b",
+ "blk.3.attn_v.bias": "b528c56212e66931fdbe267ac327a9c2f87cd03baff3ea719e30afe681da15f1",
+ "blk.3.attn_output.weight": "e3b178c1b03981e75510e0d277af23ea59cc404b5394e61bd32291825719b502",
+ "blk.3.attn_output.bias": "712c84d39a6a5a9c06a09da8fd9939ba0d5525524a4bba61ea4de09b48f45cae",
+ "blk.3.attn_output_norm.weight": "d1ffac88e675592ff72f8a617be32b4a381d443b2f8f2645dbe44a1e5745aac0",
+ "blk.3.attn_output_norm.bias": "ea31a1c73146234c50e0e43f485c458413714867b8e2703af66482f7db2d6c40",
+ "blk.3.ffn_up.weight": "4ef4f3b9a1ea6ab2ef2eb6e8b008e06a44790d099d97482a05a51e39a29afac0",
+ "blk.3.ffn_up.bias": "06a4296dda16f452675c51f108079fe7722552d6521c737d97734943818b9a2b",
+ "blk.3.ffn_down.weight": "f114b2bebe392c7d80433bb880c6730293aa4561b0b0370dcdaf7472daebd847",
+ "blk.3.ffn_down.bias": "2c8e67831d28a3bf613fc7912ae3259b63d72abcaf4d30efd8800758400158de",
+ "blk.3.layer_output_norm.weight": "a1dfeb7b5a51dd56447312ca41e2ad2f361a3ea12ddc355127f5f4219fb0a482",
+ "blk.3.layer_output_norm.bias": "1ed630021b25c6c6fc93fd32988b9907df966d4982a93081f639aac3044618ab",
+ "blk.4.attn_q.weight": "b5fae4c1f9a5f33a2a2e816ac0c01c25f422e4efdd59ef1ed93da2610e5370fc",
+ "blk.4.attn_q.bias": "c2e376524ea98ac3b10d9eee19ecb1b1e261fa5149efe0232844c923dfb428fb",
+ "blk.4.attn_k.weight": "a4632f5ebf9321d9d08f9112a4e5dda2efe5671df4a4e67fee24845f5b14af16",
+ "blk.4.attn_k.bias": "a9a02ffb8b8b4f6dfe487a7e0341f1d5318c9d2b793a688f34cb1b22fc66ef60",
+ "blk.4.attn_v.weight": "10ad8deb81d9fa093b1e5c0f24ea82aa7df43e6aca49e260fcbea56eab8cc86a",
+ "blk.4.attn_v.bias": "7326813e181e021130bd33ac136293fcffccce2d1d8cb59041e5b13a8cceacf6",
+ "blk.4.attn_output.weight": "c92573088c7437c2b3cda51490e152c27fb19e5468df591eabba5a49d5398d44",
+ "blk.4.attn_output.bias": "14e10b419e5859af1eb685af5c330aee67048cd704dcead9217840c6f5393222",
+ "blk.4.attn_output_norm.weight": "02b6831c0e0fb0edbc579a92812a1dd972cb15d14fcd382d4427c5a7b300ac44",
+ "blk.4.attn_output_norm.bias": "7eed5cd503bb6bb6ceb1bc8b07cc077903a4f14fb8b9d6cdf39644815ecf1374",
+ "blk.4.ffn_up.weight": "8d0c91d62e74d6431321116a37cf3339e630bd50ba164d3304fc4fe8dd831223",
+ "blk.4.ffn_up.bias": "d325f07f73c005a273c484c7be8e7abb4d6e8a5c4fd093f5869133b97629d017",
+ "blk.4.ffn_down.weight": "7ba7bd81143f40537b84f938e403e19f30e4928625eb371de052b9025beb4d21",
+ "blk.4.ffn_down.bias": "2853d9c2a75288214a4bf4907dc19d04d01926f4913d302b1aa7bdbfcce0f7a1",
+ "blk.4.layer_output_norm.weight": "a4ed1885fa77b90fed5300c355ef0aa0c876a8c747151d9d790939d464d57d4f",
+ "blk.4.layer_output_norm.bias": "62142a81e813a9e636333b2b805d6bc3b17c5e7cd4b15adce1ada6bc9a32563c",
+ "blk.5.attn_q.weight": "afc1dff080a72c3daad01384b1448d476aaf789871017c8ff8e144788887995d",
+ "blk.5.attn_q.bias": "748a820371c1d4f872c84545b36358d239c35bf6c99e2812c237d88c3292763b",
+ "blk.5.attn_k.weight": "59e30c1ed8acd2cbb01de5f62e7804015b9ecf98ba157d98cab016344639eda5",
+ "blk.5.attn_k.bias": "f839520078f9e589496e982e86d0126c7aa14196047339abffcf49a696229f77",
+ "blk.5.attn_v.weight": "3e21fb874e21b90308e1f46af034a3c32d3eba1628d62ae5f2246d6af5818923",
+ "blk.5.attn_v.bias": "5cd4852bf95c1444d10d756750f6bf49f842c0b39e9953c7f408bb67c325ac8c",
+ "blk.5.attn_output.weight": "636ce6a7752895f204b9d01ba0aedd9a294f908b42f372c22a16d9dd590d7471",
+ "blk.5.attn_output.bias": "82d924d4b0d2b94f2bbff91619216d6967a3541ce9b1531a6a60457a67b5d219",
+ "blk.5.attn_output_norm.weight": "5e7bd0a8d3396080f3360d7c4700bf094a06216431bd014c4479eef72ecf4271",
+ "blk.5.attn_output_norm.bias": "66c6de5edda5466d029c6753780be81ccd4218bf8bc00680000e0f06856ab712",
+ "blk.5.ffn_up.weight": "5bbf6e7ea380e216e33f8bee06d25f2265359d3876a300e92bc6e41d48e33430",
+ "blk.5.ffn_up.bias": "9d795388bb36fb33ad3a37fea3ccb4937838e02800a608fb47d363cd06b47370",
+ "blk.5.ffn_down.weight": "2fd628974e7f075479dd227b46fbd48ae8d3ca34d735b36f391ac06410730368",
+ "blk.5.ffn_down.bias": "cd213ba9eaa75fa541648097fbe9c96e58077e6c3ad6ad2fb1f21f8350f44291",
+ "blk.5.layer_output_norm.weight": "159a9df41d15b7022d136f86a2a2631c4635f9816e957472217077b522bcf52a",
+ "blk.5.layer_output_norm.bias": "24c1f27ffd1eb4e5be7e3a2909943e6f0980635d761fa1efdd0c19645da23766"
+}
diff --git a/convert/testdata/gemma-2-2b-it.json b/convert/testdata/gemma-2-2b-it.json
new file mode 100644
index 00000000..8f34e667
--- /dev/null
+++ b/convert/testdata/gemma-2-2b-it.json
@@ -0,0 +1,312 @@
+{
+ "general.architecture": "gemma2",
+ "general.file_type": "1",
+ "general.quantization_version": "2",
+ "gemma2.block_count": "26",
+ "gemma2.context_length": "8192",
+ "gemma2.embedding_length": "2304",
+ "gemma2.feed_forward_length": "9216",
+ "gemma2.attention.head_count": "8",
+ "gemma2.attention.head_count_kv": "4",
+ "gemma2.attention.key_length": "256",
+ "gemma2.attention.value_length": "256",
+ "gemma2.attention.layer_norm_rms_epsilon": "1e-06",
+ "tokenizer.ggml.model": "llama",
+ "tokenizer.ggml.add_bos_token": "true",
+ "tokenizer.ggml.add_eos_token": "false",
+ "tokenizer.ggml.bos_token_id": "2",
+ "tokenizer.ggml.eos_token_id": "1",
+ "tokenizer.ggml.padding_token_id": "0",
+ "tokenizer.ggml.unknown_token_id": "3",
+ "tokenizer.ggml.scores": "0872465d173867d755d3ee728f882b9dc2057a0bfd596fe1e3d131522f1250d8",
+ "tokenizer.ggml.token_type": "8d40143b3477df77beea4139420335ede458bf5e14102f01b0170197b55da8d8",
+ "tokenizer.ggml.tokens": "c6e66de1841f04de8b8d236d461ab720a4c9b9b5414dc293a09c6e10eab45fda",
+ "token_embd.weight": "64a9d30707e659e2e673656d71f5aef7a9fb9fd83bb9a77558dfc5abbe218a05",
+ "blk.0.attn_k.weight": "d8b4437c5edb3cddf6af9987038e1bb2b191c4f0fce0e160d2abace717f5d5d7",
+ "blk.0.attn_norm.weight": "1eb73e3f7aa8e502f6ca31cd19efbb8e4fd9a89692e13e48ac8205545a7fa7e8",
+ "blk.0.attn_output.weight": "39e7b78e57d356a22dd89ce1c4d7163b970712ba756545e1703f97866cd2192e",
+ "blk.0.attn_q.weight": "795058e23b6109febd9d55c89e1eebe6af0714ec8c56fd86a160876a6135ffe8",
+ "blk.0.attn_v.weight": "0cd6e583d1887c020472e961bbb113fe5a0d23ae2f1c2c876fc366cdb7692b52",
+ "blk.0.ffn_down.weight": "51eb4d962189e945a84e94e0dc1aad3f8f90cc1a11e18029670afcd0ea0acb1b",
+ "blk.0.ffn_gate.weight": "9811a29b8ad48432925897ab21dfcb13c5cbd372aeccbbefca9b7866883b4ce3",
+ "blk.0.ffn_norm.weight": "92cbf4652ef503c1de5b10f2be00b3fcf00100980cb3baa8f3013a8d8bf3d851",
+ "blk.0.ffn_up.weight": "af87de21746879483ed1b374cdd76b19ba11ca2b6dbb1beba98efdf3be3e8077",
+ "blk.0.post_attention_norm.weight": "32e135f1f258ffe407018899e39af1725d59d66d60022b9a21575ba160e0357a",
+ "blk.0.post_ffw_norm.weight": "ba286f5ac11b07fbc986173708c66f1920427be5a6d108af38fa0a837c1c8eb6",
+ "blk.1.attn_k.weight": "51584435552051f7fade76beca582b3f7190cf7fc07adcf527c2774d4b1c3901",
+ "blk.1.attn_norm.weight": "6833104c7fbf35a7e799ae56c262b97fffa14789642aee14381b25acd21ed80a",
+ "blk.1.attn_output.weight": "14c39481369087bf292ac9a3ab2ef166f9fe376a9f90c246653213ef264febdc",
+ "blk.1.attn_q.weight": "443f64ae2229f857c69d6bebb7800b685786cb77884c3ae19d4286aeed081325",
+ "blk.1.attn_v.weight": "0df482de2038f1e4c8a7733ac0ddb69ad90759dab5968b942af0155588de4c4a",
+ "blk.1.ffn_down.weight": "66f30763a8bbbcaea609a0087ed75fadb5e771c06378dd2cea94cf17e492e8cf",
+ "blk.1.ffn_gate.weight": "a7151bff00a545fa18b2c92dcd2a14572ccf9beb957a6c494f1374e8ebe174c9",
+ "blk.1.ffn_norm.weight": "e197d71ea11b5276bc0167d2663b88089b3ff42b47ba91e85f6c5d95f6306435",
+ "blk.1.ffn_up.weight": "57c182e0b14cccd1350d388f0c616991702e74281db54637451b70f4ccc24f9b",
+ "blk.1.post_attention_norm.weight": "3c56f837168d784c2d8bac247c130bdca6610c095c8da4558c536ccad7605609",
+ "blk.1.post_ffw_norm.weight": "d2a51d320fd01069dd7ccaa7082f16a7faeb671885607d7900b10a89c354d0fa",
+ "blk.2.attn_k.weight": "bc103c818192de7ce36caaf89dc117be4df13fb902e0bd9a23c64edace5df9b6",
+ "blk.2.attn_norm.weight": "0f2503aa126083a5d6ac72481be1ef66c6014705b573682b35bd864e4749a3d5",
+ "blk.2.attn_output.weight": "05fcd4a1226e482f91803a266f72caca887a93e63c2d2ba5611ab3c68d38743a",
+ "blk.2.attn_q.weight": "6a10b5c2fd423d1e4c4fd60fa8c154a0159b6b2501ea79cae2ef19f45a674e5e",
+ "blk.2.attn_v.weight": "3cf891945a1f8ae7cc908a5c6b729ff5b70f4436c5ffdbf245cc0ed4cc19cd1b",
+ "blk.2.ffn_down.weight": "ea204fd04e0d2fc728a9861a459216bbfec629c152004ba625f52cd8837bd51e",
+ "blk.2.ffn_gate.weight": "3a3518729f1b8b64a82b8792f33987db5418fdb094be0263c68f146a5c38de54",
+ "blk.2.ffn_norm.weight": "754ede678b725de41a34b82f0edf7688b5c065be7c0d46df6f7ad9430d986884",
+ "blk.2.ffn_up.weight": "ffdcb88439f5828ffbd9fc844b03ff91637b790b9838097258cc3ae75935720c",
+ "blk.2.post_attention_norm.weight": "4b3f53b7ba26e8c36b2dfda3b7e5fc4b1065257cefdea235fc7df9af130ac2fd",
+ "blk.2.post_ffw_norm.weight": "e550369e26b8485e2b54ad34b34bc98af5494287dcc513c2c39cf1eaa5b89d07",
+ "blk.3.attn_k.weight": "89f24ea450e37d9e95757651a83205c085d81b354ee9489dd6310a391d8409f3",
+ "blk.3.attn_norm.weight": "24e2ea662b7cb822b4ca5cd61bc17f2709f406d990ec3b4a0dac1cc112db45cf",
+ "blk.3.attn_output.weight": "ac4dad69473c6e3fac56669212cadd8c34ecc5973d945972e974d94805334967",
+ "blk.3.attn_q.weight": "b6a9c9a7d4722b9096631c65de62228dfddca6e26edfe6af7fce01e116ef0f4c",
+ "blk.3.attn_v.weight": "f272a960a40093942309bc342a379984cbacec2d7bc64428db3f64e6b1887ed4",
+ "blk.3.ffn_down.weight": "c0188ba50d8228805982029c277fc0e87aa57473b8363037c648f6d006ff828a",
+ "blk.3.ffn_gate.weight": "a04aec1561ee6c0fbb18c3db49dc62fb533619cf697fd548cbf2279761aaec3b",
+ "blk.3.ffn_norm.weight": "bc053837d44087ec05eb5d9458357b2a5be787789b19cdbbdc694b57697f99a6",
+ "blk.3.ffn_up.weight": "b3ce8b274f20796d3b1a7c08ba27a919066f9de89a782faa544c4a8d6bea1382",
+ "blk.3.post_attention_norm.weight": "9c922dee7a7df5667289e2788e60170238239cee2dfdbbd9e435763f9f416718",
+ "blk.3.post_ffw_norm.weight": "b682544ac953ad2e0b49027ed8916f2e9d1aba5d1587bb4127ac703570c7a03a",
+ "blk.4.attn_k.weight": "143b0cbb4b787b95c2b6212374410e32173ccef2adb914908a2f89a7916de512",
+ "blk.4.attn_norm.weight": "5668f60491b780273745192662d02c9a92a4f692b29d16aa0bbc7413fec4f85b",
+ "blk.4.attn_output.weight": "b9f2bdb68be1e0cf66dd19f8fa2afb105910ad2ef394864cb32cea8f8944e0d5",
+ "blk.4.attn_q.weight": "ddcf1343dafbc2dfcd0b8741225af22fe4b54b2becce29240bd01c34265d126c",
+ "blk.4.attn_v.weight": "6dc7074366e7ed52d9f48c594dcc85bef738e096276cb99d28228c89eecc5b9c",
+ "blk.4.ffn_down.weight": "30334ffc59ce343cf2a1b973174acb7722823463adc07e19a99bd0f404bc9906",
+ "blk.4.ffn_gate.weight": "890f7c8af208d63b28db52c4b8c16c2288a382d87ff5a6a6d6b0a5b3bf27e6cd",
+ "blk.4.ffn_norm.weight": "ff0316cc7847221eb86a90c1ab441d4ee61553d410c66414a7755021b3b12448",
+ "blk.4.ffn_up.weight": "6af97d113f91564c636734f215e25ee602d48eb045458f300b3ec7582be0f41d",
+ "blk.4.post_attention_norm.weight": "69438f231e105e68216b078bdeb35a7cdc8b12c4e2845e18ecf4c8d361d6a321",
+ "blk.4.post_ffw_norm.weight": "0fd535da78bcf2b32c95b05b2b83dc49817393765be90d8cc1ed3d56f47b68ec",
+ "blk.5.attn_k.weight": "0166eb3c6d20dcf3d3c169e94caa8dee057535bb525e29f698fb6f8844f18a6c",
+ "blk.5.attn_norm.weight": "a7808f27f164023d5cde2be00fc23cac6c71aa0ddeb60bc23e12411b80087672",
+ "blk.5.attn_output.weight": "8b65b2027a0842b68c5308f91d6a31de9599d794157d77df8418b19f9e0d9334",
+ "blk.5.attn_q.weight": "966bc626ef2c2394d872087a41c126bb1b67d1d5f6de920204ef5e5b16c34003",
+ "blk.5.attn_v.weight": "9a362aef3f4437fbf0ef6e1ba785f3329c3db2960f93fe36547d2795e9c254ea",
+ "blk.5.ffn_down.weight": "63e53541d34197720c06f297aa8142ac6b6eec002c7987b296f26e8b1400f931",
+ "blk.5.ffn_gate.weight": "d9591fdd32f783e0fc26e20d5d587ee8971ac8ae2e4c818c6eac1c125c7c7f37",
+ "blk.5.ffn_norm.weight": "677334cc60ecce3a7f4ab3acda15d359353d7358872f614ad8914e3780e9fc6e",
+ "blk.5.ffn_up.weight": "a63764110e1c655ffbd55af0669b2dfe4cc29d0e198d33a8e5426461b08a85f7",
+ "blk.5.post_attention_norm.weight": "c55499f859b2c0a7f5cabceaae47309a5ad38bc29d0f4a8db81f1357023162a9",
+ "blk.5.post_ffw_norm.weight": "82752754665f842418f3e302cb5f43d1e0504dcd124c4b8ddb77018b2c793837",
+ "blk.6.attn_k.weight": "e20a5f0d6c807273c8d491439566b428497ac02097cf0aa55e33748c28e14be6",
+ "blk.6.attn_norm.weight": "2c6ba42fd3c73d72073ced03a32dd28d70a89ed9bbbc8fea1ba03a7ade951e6c",
+ "blk.6.attn_output.weight": "4de7c5c2f4a133a266e17ed8c14c52959466b54cc7ab9e19f789a33b4850f284",
+ "blk.6.attn_q.weight": "56462d921800e6b8cd2213fef04c4ff16d728905cb2f4c58e966d0a053a3b0ae",
+ "blk.6.attn_v.weight": "b758dcbff769d6240c2245ede1dbc62c4170a67c77458e866312589220fe29af",
+ "blk.6.ffn_down.weight": "582247fb3c2bf687cbe9413fe18d18ad47bef4b65df7d78905e10335c6134764",
+ "blk.6.ffn_gate.weight": "3035444d5286aefb7a6d04e55bc27e1fac7cf895cd5be02319a431b8e047b4ae",
+ "blk.6.ffn_norm.weight": "e582d24c66e01b96faa20ce6adfda3d8583b11e809bff89969927398175e369a",
+ "blk.6.ffn_up.weight": "6f4b7bbfedeacf61a4866ae0616c4ba6c9e856662e8f00ae6aaec7f52c53e7b4",
+ "blk.6.post_attention_norm.weight": "8fe51b50bd677d21586aecab0b565c4bf9fa68ad50bfe366f45e8fea3c657ca8",
+ "blk.6.post_ffw_norm.weight": "81ba3cb4c2bf5c546b86855b7a885d3fafededc67eb3a35cd3598b03c9e26e65",
+ "blk.7.attn_k.weight": "2e044179cdcae0946708c86bfea7aa0391e1f7e2a09b33fca035d384cc3ca758",
+ "blk.7.attn_norm.weight": "94b48c546b046803c60e75a3acb17a356b710735989938021b565f68df9b4985",
+ "blk.7.attn_output.weight": "65709b4ad7a581f4d75793d39d4032a359f6bcc0c3835205242a0b99e5b66824",
+ "blk.7.attn_q.weight": "8ded993c95d1f7caf201ceb6fa035cd6ed6d351b50b999fa9355dfee9486cb5b",
+ "blk.7.attn_v.weight": "c92d5e2d2d48397542bc03bea25bf39154075e66c5bb1ead85188505aa04ae91",
+ "blk.7.ffn_down.weight": "e8ba8fb57208805ef1dc23cd7c86e9a2d1fb7c52c3940d292cd5bb2eb24b3fac",
+ "blk.7.ffn_gate.weight": "f0f06d6a2e06c5ac252083bc61d05c814e6289d3f4e4a87d2f06918254c02c36",
+ "blk.7.ffn_norm.weight": "ebf8ef775f72624148e09d68a4332187a7a5020c521fe0623da1cd3485ad33e0",
+ "blk.7.ffn_up.weight": "a554adc4fc7122c247c77670e169916ba1794c787b5be30a2b36705138f1f746",
+ "blk.7.post_attention_norm.weight": "3aa6bc21d85c3a0c12b964e82b12feaedfdd13130c3cd2229228e24e0967ebdf",
+ "blk.7.post_ffw_norm.weight": "508bc7b19ee8ff08f0007c890133a462fc57c7e72b16ee8f6dd64def264ef876",
+ "blk.8.attn_k.weight": "363c8e74056642fe9e7c2f3f9769d57319cd3fa0a6022810189ab8d894322885",
+ "blk.8.attn_norm.weight": "685b49a1f1acb169f4df0bdd8e3de6943f3033cebad14b898a72000595610d92",
+ "blk.8.attn_output.weight": "7bde571e4efef1c6a6143f0526721dfb59e0a0ea0e1a3616a322b2eb937efa48",
+ "blk.8.attn_q.weight": "fc993dbc1074c28a0e1d85e5ab2f4ea6a9c6c1affe7ee56027000a275daed9b6",
+ "blk.8.attn_v.weight": "281e8791d3aef9b3864f1cb054da0ae0c2fef4ce0a58b1bad8bc136b2fa0f62b",
+ "blk.8.ffn_down.weight": "b1164a2578a7f87ed99c2bbc76c5dfbbbc6a1a803605391acc3f320fc989ffd7",
+ "blk.8.ffn_gate.weight": "6b39a3b3aaaa79aee61416b54d62160b9258042650e61c6b47bc77c2dd17daf3",
+ "blk.8.ffn_norm.weight": "17ea1362c72da27f12bc936500492035bdef3fd8f940cb12b57f37d42ba8ecb1",
+ "blk.8.ffn_up.weight": "bc3a7c47afc440d2bdf8fbe9ddf2c9220467472c60c8b4ded8c0f181470ec96c",
+ "blk.8.post_attention_norm.weight": "5c506204e00411ef9c8b4134d40eedcc19fffe68dd0af7d7cc49dcabf2dfac7e",
+ "blk.8.post_ffw_norm.weight": "002faec235c3678864e2901eed275ce4e9dc229164a91c9cd4c965142ba62305",
+ "blk.9.attn_k.weight": "0bab39d8c237f1b6d0010db40467142625a9e6f2e0e4c49a56c12b41e4e0b1fa",
+ "blk.9.attn_norm.weight": "de5f38e873b17f07aa7598831b89cc1cae2c9bc3eb2e042ee9af059d2563e84e",
+ "blk.9.attn_output.weight": "8a8184702c25a62df9ff309c0c7badc8587208523b2be3e8fa90ce7080573e6f",
+ "blk.9.attn_q.weight": "7c961b2431b09ddf95377acd07201cb91bf13d9cd3ae0f2c25c7d6a0358d9f50",
+ "blk.9.attn_v.weight": "e22d240cb4743067033e659cbf210ebe2ebbab3e1dea6ccbe5eaa982382ca038",
+ "blk.9.ffn_down.weight": "a426f81210f03d6ad53277416e1fdcdf37d8065e4817613edaf6c67a343426be",
+ "blk.9.ffn_gate.weight": "a82eba825cb77b8e64f85ff99ede2fc71bc9b01751eeb17e9e6c246ee12ea62e",
+ "blk.9.ffn_norm.weight": "1a97f9b1302a3a326d534c5c3fed2db6db0ae45fd0edd381a3e4fc1c75d81030",
+ "blk.9.ffn_up.weight": "5f20bac2bbf03bb42adb92fbf99561651e1edda57e0b61935ac7f6c08c0ed7cb",
+ "blk.9.post_attention_norm.weight": "9f9866d13988e1946b1e1c80d9374a92a6e3be33748f8eaed3e126d1e1a4c796",
+ "blk.9.post_ffw_norm.weight": "a6896dbf698db4dbbe5dbf12417d4fd80e9cad0c539c858892ec0aa5b046bb58",
+ "blk.10.attn_k.weight": "ca8446e5d21ecd4e6a70dca8d321be480be4fba94d70cba065205436feb44270",
+ "blk.10.attn_norm.weight": "4f41fe290e8f21f63b82151b6cce94bf7318d121468816b0c58af0ff7c1658ab",
+ "blk.10.attn_output.weight": "c626d2e9681c5c941bbde43dddfae1a8d4986bf2be4470857bc8e8bd7f869044",
+ "blk.10.attn_q.weight": "1e61b210a13a429977325cf15d781ab77d604cfa862f4270329cbd94237d5835",
+ "blk.10.attn_v.weight": "8ff8d3e3f058ec3b35ada1057f2ed59c06494d0e0be6a8dc3ff9edf9f0e1a115",
+ "blk.10.ffn_down.weight": "bcebc04219f8081a5f483e58103c0ddbbbc631a0a54fd6dd9d55778e041f70ee",
+ "blk.10.ffn_gate.weight": "7a23a1e620ef871384ddf9611ccdcfb893fbf013cc203ac8e72f745420f1eea0",
+ "blk.10.ffn_norm.weight": "e3a375e43c349a1c6c66c22328e513cc1af3137fe839e43dc8e9be2f65914fd7",
+ "blk.10.ffn_up.weight": "5d182e7c94369194fca5f19cbbe668a999911e57f3d363bc7fb6088428700cb9",
+ "blk.10.post_attention_norm.weight": "b841c6308296e8984f3c5f549c6e3a242f4b3e19141e1f54cc08de9c46759c09",
+ "blk.10.post_ffw_norm.weight": "9d66fa05b5c940208f634f5053d809094c99a2a10a1d1e8847c8281fbd99fb49",
+ "blk.11.attn_k.weight": "14adf24ebb2bb17b336ca81cec3e690fd854782f4440ca6c66cc1d7e7bf1c850",
+ "blk.11.attn_norm.weight": "2d2213f311f50414702b5b34f22aafb9d9a0b6787243e7578562583dc40ad195",
+ "blk.11.attn_output.weight": "de1f14cc2a7fff00cf11b229f0576999205f17b9536e97abc9d6de3cc79a7884",
+ "blk.11.attn_q.weight": "2bcc5c147524003109ece0be08b89ac8b25baa71416ffa76573c6c052ffc6eea",
+ "blk.11.attn_v.weight": "2e6ab8573070c22dc1e0d7aebe4d52123226dacf7822dcce06fadbb38fb036a4",
+ "blk.11.ffn_down.weight": "1b86902f4e36868421e5228b9445051f8290b292df22a6d1af836dcecc1f25c3",
+ "blk.11.ffn_gate.weight": "e756e8081bd0a16aea4a9ef5076ad102113524f7a3d50a3a77aaa7f7938b63e8",
+ "blk.11.ffn_norm.weight": "6913887267be227cf9d1991a3dd8db2e7e74bb9b5fbdfcb9ac954fd7d7b95b3b",
+ "blk.11.ffn_up.weight": "619a3ac0609ebdf42c3fb2b6e4b1db48df79e6dd8418d7ab8f1bbff13d8a6a50",
+ "blk.11.post_attention_norm.weight": "e4b4ba92cef7b6a78407e8ab1b0307d47dac6c3df7b6817e28038317ff662d7e",
+ "blk.11.post_ffw_norm.weight": "40aceeec58cb855f0c158c9cc217168fcd5d0e735567d587217b1d78df17bc5f",
+ "blk.12.attn_k.weight": "c54c5a4d4892522022d1aa2204cfc624f0b4042caa536e678967316293fe5cb1",
+ "blk.12.attn_norm.weight": "7cd2ef58298569ffdf244d9b390f3917245276c8206e5780af5f96d8c0bbb446",
+ "blk.12.attn_output.weight": "85495ef9cc8b3deb21f741bde463ff6493acae2be51f02ecdeef952cbdec3375",
+ "blk.12.attn_q.weight": "d19383f83fd119bfb8c0280c9515705c11d8e7d502019fcf8f49efeef0d106d0",
+ "blk.12.attn_v.weight": "869ac669ba49531d9128892a0e27cef15de508ff40cdf80cc1681dde50d09204",
+ "blk.12.ffn_down.weight": "578f39f8f9fc2f09138afc884a952d7cc3a9a31de4216acd10e88e19e0b75f8c",
+ "blk.12.ffn_gate.weight": "e29a0186bc6c4a0720246306e922d3a83f777dadcf4ac80bad468287031cc8b5",
+ "blk.12.ffn_norm.weight": "e1ee95c6584b5cb57fcf1db8ce2bcc03aff91eb389238c094a61c00dde93d1f2",
+ "blk.12.ffn_up.weight": "2a826f06d7cdfb3edc6ae250ff44363ef77a2a9cdf96313e23a331b99ebfa17d",
+ "blk.12.post_attention_norm.weight": "4bafc7699b948d5cbc0d3e09b418b06c6abc4651a61ada9609d9a2f21c7e5607",
+ "blk.12.post_ffw_norm.weight": "bbb8c34a7176bb1a49f9fe2bacca0bd26b673d52c0835b2e90fa11f2962f077f",
+ "blk.13.attn_k.weight": "ffeefccfe8255d1b694382012ff4134eee5fec9d9491c8d0ff0a13832d1a37e8",
+ "blk.13.attn_norm.weight": "35713726529e3887c4135a88e86e8a4d7270ba5b9f2d1ab462622fbf40a7cdce",
+ "blk.13.attn_output.weight": "0d60b7c5cd71190a9ef4b873b0f516be15447c32d83914db2794b14592b0b460",
+ "blk.13.attn_q.weight": "8296069e65bef794cefc61257fc65789b3cb22955e30f3df129205e5041b2222",
+ "blk.13.attn_v.weight": "ca0f4ab9d16a748fc643a5c0c7a19826a811bf2a4e7316a8c935d4bf0ce8abc6",
+ "blk.13.ffn_down.weight": "d5514e0c8e7b3ed1cbcc1605eb5be1733b6ab3514cf8a0508fc72f7d05ed8bcb",
+ "blk.13.ffn_gate.weight": "8108e517a82e08a3aefbbd267bfa50a1668f92a76273280ce8a6bc1f6dd61521",
+ "blk.13.ffn_norm.weight": "5fcb6132d2134bf1f835b904a99820fa501dbc57d2224129f7098bf3cabc1d36",
+ "blk.13.ffn_up.weight": "6d744b7cd390a3cae3aa350dd379b81246acd056a2259996b6aaadece8465ccc",
+ "blk.13.post_attention_norm.weight": "e08b14698912509790e9575b8676971fbb0a4d82d719367e3756c0d0c4ab8cc0",
+ "blk.13.post_ffw_norm.weight": "2b196e4450fc5f1e7367b2cf7fe33a15fe919fbcdd861d11002346f16e980535",
+ "blk.14.attn_k.weight": "120e5f48d7268dfd9ab5f4bc9cc57a7cec63ea9635f56b80d435eb22936e9483",
+ "blk.14.attn_norm.weight": "146367bcce4db72cc894419a2e0145a6f533507dd68e4739c10ee480308c401f",
+ "blk.14.attn_output.weight": "720fa0165e756876c5cb6ad9e2780dd910390933f3f8849e5add5da04266650b",
+ "blk.14.attn_q.weight": "f5183466f56219ca1aca52d8b82c2d966a4198fea40fdd6b39f4d8b06ca2a6dd",
+ "blk.14.attn_v.weight": "24f8ea3d5512cd37c43c8329cb0da0c90d1895aef763ac2dcee3fe5157ec50a2",
+ "blk.14.ffn_down.weight": "e29960965b384ae5ab3d898a4dbaa8fddd28fa0e477ac28bcac49dec12a5ac67",
+ "blk.14.ffn_gate.weight": "6d0d6a74bfe9692e8f8eedff0fc34fc4fa1c8687794f35f2e2b033ab2d7510b8",
+ "blk.14.ffn_norm.weight": "f7036c1a9a71e046c9d2af16e9218fda5dbb0f7241ab44747abed1f0f9d602ca",
+ "blk.14.ffn_up.weight": "7d69ea1424007ffc9c12247dd0308c616e93ac02a59ec341cfa48f92d6ce3b10",
+ "blk.14.post_attention_norm.weight": "65b9712834d9445d4236bec362f3fb795c20d60c541b3dc6dbb7914d9b493e41",
+ "blk.14.post_ffw_norm.weight": "9c6a8da2e4e437d5cfdf3b9097e9f8b64bf07946a048badec20f4d374613f38f",
+ "blk.15.attn_k.weight": "864bc618303a0e4ee67fb1d5e751de61e936cd51e96669dd86f8cd08f2305045",
+ "blk.15.attn_norm.weight": "f9f4187da6eeadc2fc5921d8fe669741697d16c13d71e4aaeb73b82f50dc577e",
+ "blk.15.attn_output.weight": "ce2419a0b097036b2a31f2f4ad731d5814bcc2ef4c511786e24471e5eefd273b",
+ "blk.15.attn_q.weight": "9539db5a970d11ebe99722d1e13fcd635e250033630811efe583d2f97778e4a9",
+ "blk.15.attn_v.weight": "1c834b48ccd88adaeabb7d8bcb6be0bcd6d5ac1354ce88fc28f19a1a96b81ab3",
+ "blk.15.ffn_down.weight": "bc1f97a65dde6fa2c1e5397afb612266944b343f2eaa868b635ddd25829f8a42",
+ "blk.15.ffn_gate.weight": "1b14529d57056b79037f6cb5008132e62cc35992353b38dda59572274623103b",
+ "blk.15.ffn_norm.weight": "9af77458de9ee55c66f93865759f9c2c398557f94f3fa8fa6af30543d7339cde",
+ "blk.15.ffn_up.weight": "41d524a26b61a9595816b4fd53cf57ef50a702e4ef32933ff6136dca9136a267",
+ "blk.15.post_attention_norm.weight": "c60a03cd0e63a7db5c80015e58e9b97ba2208caa19f66a6fef5c4447eca900ce",
+ "blk.15.post_ffw_norm.weight": "34f7f9f96769215bbc3d17084df091864aef96a6645b7d0b3b7d9bd92f1a4b0b",
+ "blk.16.attn_k.weight": "7e27240d9f3a8c6cf0f4a980113d43234f514eadc3e3e1792b86efb29ffb1a6d",
+ "blk.16.attn_norm.weight": "af798acc0899282a30448edec48223b3e8efda177090273e612d8eca5e377301",
+ "blk.16.attn_output.weight": "79df39a3709d3d53e84146291e0944a7a653d06705293d9ccb5648dceadb432c",
+ "blk.16.attn_q.weight": "db58a1c3b83ad294804e5fd7321005719e200659173466df5a52a182b80b7165",
+ "blk.16.attn_v.weight": "2af6d48cbaeb225b5c1a704f76abd89c8ab1521417695b112b4dcc2cbd39b74d",
+ "blk.16.ffn_down.weight": "fc1c813eb5e7da3d6194569d6cb21602fc6eff2dc8e1b0eb753f2d5df148189c",
+ "blk.16.ffn_gate.weight": "7a80bcbc42464bd55df4814a6edbd7b5c153e0428323bbe49de55e2d2add33e7",
+ "blk.16.ffn_norm.weight": "2041685ee926d30f3f2ae4ec35b5688f1cd834167a6359a7d4057eac804c58b2",
+ "blk.16.ffn_up.weight": "8da4b718973ac1d43b928829bc45e062fd101984d6c98dd825bd7c5d08ebfbe3",
+ "blk.16.post_attention_norm.weight": "975c48fe680a6167438a106140a8872eee7765191f152d80e3b8ddf47693e095",
+ "blk.16.post_ffw_norm.weight": "4de2d4d483acfe4fc77860ea929025df2f4e15c10729413f36a18c94eaa6d689",
+ "blk.17.attn_k.weight": "f937e61f0af8c4cd98ee742648eb60e02e579683e21d421071295a3b70aebaad",
+ "blk.17.attn_norm.weight": "c3270583ed28b7e423f5b170c59113234f258169b93a867d9274f4c10b7cb115",
+ "blk.17.attn_output.weight": "b8c1150e81e685e539a5dcf2c19047a24eba2b281fabe166674b1d71ef4612ea",
+ "blk.17.attn_q.weight": "c255100ae2011e7dc7e3bf3bc3ccd96d859fbb98581cae993d7b82c1ba8e8b39",
+ "blk.17.attn_v.weight": "5830bb0a555984c6485348067f70b5d22ae337c011aa9248dac2ff4c95944551",
+ "blk.17.ffn_down.weight": "8ff9a7cccaa3776434a9d895aae4fb5c36c736bf2ec98784226b4c234940fbb0",
+ "blk.17.ffn_gate.weight": "1b52876739712831c272911533da206f407b46034a1a4ae8a88c1f96b6bd5747",
+ "blk.17.ffn_norm.weight": "d0e16ba5e87c91b545334e022058c7d03849665c3b1a6298771b656531366b66",
+ "blk.17.ffn_up.weight": "4dd6211d01dbebbe21052708eddc242b082a58b5f18ed16479e17987c1d3432e",
+ "blk.17.post_attention_norm.weight": "6f49c775c7417dade77ba8268a0f8441c1e5ec28b5d7e4dc5ed07a04d04600c8",
+ "blk.17.post_ffw_norm.weight": "b91a0bb2e6679e9c9be06ad323adae441d00a3d673efb19d7c4954be2aa84b27",
+ "blk.18.attn_k.weight": "22b565ace1b4da8b33865a58625be1d90beea9891f29686a69fa9cf7c93217db",
+ "blk.18.attn_norm.weight": "3e0160d7063c8753de65d2356a66648e47d921efdc5c917efb8209892120f8db",
+ "blk.18.attn_output.weight": "e3180f0bb4ca90b31e9b08158db38e332de62dfbaefe34aa94cc316409331e09",
+ "blk.18.attn_q.weight": "f3a5a83614c3ba7ea41cdd5b1b0819a241ee2a951a381ce4a9e001d3f700ed8f",
+ "blk.18.attn_v.weight": "f3350a5984fb951fc738adcf78147e6d812ff1c576670c460cafc99c253c1654",
+ "blk.18.ffn_down.weight": "9e9d09b13a33525e14bdaee6efc65c551ac7cf7680e534b940ab122a3a7c1ac9",
+ "blk.18.ffn_gate.weight": "ebaec8b4b578a2e8d815baac12f1675c208f80c68074d5a18288a2e1a60680ee",
+ "blk.18.ffn_norm.weight": "33e7687c53a242f2f8dc7093a491c97b18d4a5a8c14d183f02bd586a770f05aa",
+ "blk.18.ffn_up.weight": "78a1816662378ce56cc870e705174492781897b3afd2d4d97a51f10f2f2987c1",
+ "blk.18.post_attention_norm.weight": "a58dde3f12df3e94cbc27d87c8ea86f89af8a388a506446ff6758f05399b05fc",
+ "blk.18.post_ffw_norm.weight": "cebf90cc143577d483cca27b032dfd82031ee59bdf17c0e2cf60a0a3ad5bf996",
+ "blk.19.attn_k.weight": "4683375d0599ac9e2232196aae1e90af13a14cae26e865465de5c8e257bb2055",
+ "blk.19.attn_norm.weight": "f3eba936bfb1814bbcb0a1d62739eb66daac839df8c9c836fe0e94860df88525",
+ "blk.19.attn_output.weight": "51c0f01d38a9dcfe9bdbc4643576fab164c1d9e4b7168b7695c0ee55e6965667",
+ "blk.19.attn_q.weight": "28d15b69b8416f2e7ddc88fe381cb1e2ef2ad705fb1c268139ba96498cc74848",
+ "blk.19.attn_v.weight": "6860f1cd720638e63a981fa2c0b4db900129826bcb9823c9ddf9fb8b1b9f3383",
+ "blk.19.ffn_down.weight": "bc7f2d7827ee01c2dd41401c7b3b1700ad3a4ff620e8bb734f92630d342dcc7f",
+ "blk.19.ffn_gate.weight": "54d03ef69ba373fc410fbca8f1e34a565d58e4296d9a035ff7e48340b9c848e7",
+ "blk.19.ffn_norm.weight": "9178fc796a340ee6e8128ca74c0cb6203d1adbed6927af4e5ac7863da57affc7",
+ "blk.19.ffn_up.weight": "a77bd708026c6e83ad5c79c223278e74621bcf74a9641c7818d96b595daaad20",
+ "blk.19.post_attention_norm.weight": "ae94aa26f4c411bf9496a6fd4a6df64ee589ee1ae9a04b531d45acc95721e582",
+ "blk.19.post_ffw_norm.weight": "9ad210700edeef12133bdcff04bf1c7f62b49f6f4a9ba483c7cdc59857c24a5c",
+ "blk.20.attn_k.weight": "e35bce1e9f4a7a09ef34721f57ea38cfca68c272f52d923fe50af8308f66cfaa",
+ "blk.20.attn_norm.weight": "644800f6926fd34f233795c4dec1151a295d2138ca8cac33e3e48167d26f8b41",
+ "blk.20.attn_output.weight": "8d3758cd236471741e1ad66c0710cb79077dc8c7a3a292d35bc551c0c5abe627",
+ "blk.20.attn_q.weight": "c333b1f0f6f956b5d73891df10b1a0321e55fc31c40d623a24e1f52caa6a998b",
+ "blk.20.attn_v.weight": "8562b418d0c4868a050fb19fa3fcaf50a8cf1c669f537d666c80c7b3a04714e1",
+ "blk.20.ffn_down.weight": "97efb608ac44cc804198faec3ee66eafe56ced6b7ca5359700c6f1df75b7205e",
+ "blk.20.ffn_gate.weight": "5c61151d86f28415c73c73d90ec088c646cbe5c1640197caf58eb501ba7db293",
+ "blk.20.ffn_norm.weight": "24bbe0a701afd4bbeea65b3edde712b3cbb2281043bbc43dbf250582453116ed",
+ "blk.20.ffn_up.weight": "e170cf68e249566aa99eb6f6b265679bf9a5a6b76830ba24e7e130c2515910c4",
+ "blk.20.post_attention_norm.weight": "e092d751cfe20dbf2d348358f3b38397bd83e4ed94d6bbaa6bbaddcd902b2ac4",
+ "blk.20.post_ffw_norm.weight": "219a18a47dcba76e669e4322223a5a9227bd3db1de3fbd3d3cfb22e54a783c5a",
+ "blk.21.attn_k.weight": "c3a095ebddb42c63824f1c98da65263dc88e4d790a26aa1632840b44f5cc7cb1",
+ "blk.21.attn_norm.weight": "ef8bbaded5fbc45ad9cf3985ae02174524e7090fe6362811124f942ef643bec7",
+ "blk.21.attn_output.weight": "668f018aba72baac6252aa3ad58569ddd55ab751a0dd8d7bcc9fb9b6efb4bf53",
+ "blk.21.attn_q.weight": "e759c65663089f3bbbd51847934c185e680c82f1249065d5d487da638e519e6d",
+ "blk.21.attn_v.weight": "2ff57762686cf9ba1f5a6be76503454b97556ce67f4ac98254bd0562231197ba",
+ "blk.21.ffn_down.weight": "3fd106556fb721b1c28ae3f4026bc83eb1b08ed910f2ba5f466c6b5f327d91cb",
+ "blk.21.ffn_gate.weight": "338022d882f4b6619e8054a6fb909696fa3eef3013cf69b65c3cacdfc5b9e42c",
+ "blk.21.ffn_norm.weight": "1e77660c23a3f9653ee721a863d1960f773d87437cabc4dc0a6e17ee3d4e5e44",
+ "blk.21.ffn_up.weight": "7d31b20fbc2e6eba8f350f170069dc36f0cb12f68fbc4206ec5022a74085ebcb",
+ "blk.21.post_attention_norm.weight": "9638bae8d8bdcd7ed68da282979cd84a07c41ff9cabcaea94ebc846a1803db23",
+ "blk.21.post_ffw_norm.weight": "d622ef11115fe0cbe04b727d5a3b6371e7f39bf08c8d5eb9bc6da52e3f3cfb9d",
+ "blk.22.attn_k.weight": "5c321cb29deffbe57de200dd206a62005f1e80acb86c4fd2349dd44c8d3594fd",
+ "blk.22.attn_norm.weight": "198d949705d7170a331d75889d8c7500c3635254dac2cc6aa4dc35d556584536",
+ "blk.22.attn_output.weight": "19805cd5d7025b457e5d41d70db8b3fd63c2dd0e4a94d3ef1704d50ef4e749e8",
+ "blk.22.attn_q.weight": "177836cd583fc87405975ddc21ebfebdaa090a0363799664c72caa3da851ae2c",
+ "blk.22.attn_v.weight": "fea255692483e30d0108f9e4e250eb3ed7dbda8d83f499b06519b8c223ae6096",
+ "blk.22.ffn_down.weight": "00cb8939f03e5817d6d412de8cf2c923c9568d5493e382cec7faf5718fb034eb",
+ "blk.22.ffn_gate.weight": "b0591065b91281b2fbd8a9567f3568d40479f680e1f0a29e27ae213f37642489",
+ "blk.22.ffn_norm.weight": "96b5c5d0737c2ceb8fc869f54adb9e5f46e28cb7b177c40f49fa926b923c00f8",
+ "blk.22.ffn_up.weight": "81f472185b24344ab0594ea8246cc6e200e0dc1cab4943e74fbe4ca19d5a9701",
+ "blk.22.post_attention_norm.weight": "27fa9aa6260aa3071e0391e1a1d49322dcb6e8072315b8a9b7064087108dbd06",
+ "blk.22.post_ffw_norm.weight": "f37e1dcd7f643d9545675ffe9dc527a11eba86eb204989c2f44f636b266d896a",
+ "blk.23.attn_k.weight": "5d82f36658a56c3f94d0bb2d61f65509c966fa6568f81812e0d3e338b380ef8c",
+ "blk.23.attn_norm.weight": "b7983f88d9cad88bc88a528923e6da592ad20e699965b223ebc10840fe1f4fec",
+ "blk.23.attn_output.weight": "59f97f80f430d71606aab0158a195aed29ccd3405e6c0a5c41c809be8eb01898",
+ "blk.23.attn_q.weight": "53ac4789fe958919cc02ea4222bcd64c0ea1b4baa54304bff46635bdf42f7490",
+ "blk.23.attn_v.weight": "ec8abe09b9e84dbb52c7a068094657c6d3c62fe551ba8d7c3a3f23da622e9756",
+ "blk.23.ffn_down.weight": "3cf547eccb1b82aa64f208cee9682d7f558ca84e0aead7d9d3d1420d90f3d992",
+ "blk.23.ffn_gate.weight": "366aa2486d911ba81eb519119e13807deacf7e9908bc1975a2a63e00d6b10124",
+ "blk.23.ffn_norm.weight": "6d1d4a4af34bb7dc090ac87d6457d398c3e0fb68bd2e2b60b099dc318b6cfac3",
+ "blk.23.ffn_up.weight": "53f76692e253f5d2420b3f200c731b9f3b7a83e379920b4a067c729b4674aa4d",
+ "blk.23.post_attention_norm.weight": "7c952fa0efa76b3f048c8c4c9e8dcb5e3724d231327eda6423a34d3f3d3367de",
+ "blk.23.post_ffw_norm.weight": "7ab188cfe61f0a91b40309a0ab6bfa99f19d0ff2a37b6ac10e5f0c7f44eb5270",
+ "blk.24.attn_k.weight": "225798792f9bfdd10eff0505ebe61e0aad0209c17b431f6044ee7968ffe8c198",
+ "blk.24.attn_norm.weight": "635e3c1ebf5219bbebfc40ef164bc32d2b726ef595a94da64ac524ae878e2915",
+ "blk.24.attn_output.weight": "482f5bb2db8d9ed22b253d9a3296333b239efe698e5992e5d77e7e12dc2a5cf5",
+ "blk.24.attn_q.weight": "43805bbccddb65d58fffc4be9b5c374d4e1df1395ec1e1ffb4bcff03e98d5adb",
+ "blk.24.attn_v.weight": "fa741af54b4a3b1775d32f59134756090c5df2e7345a12a2d8db94fe289667a7",
+ "blk.24.ffn_down.weight": "83c6351e3162626b276f524a57836144625c2556dbe321b57cbd8fd486a68fab",
+ "blk.24.ffn_gate.weight": "fbe66be0d84d12cea5176cc7eaef64382ffc7324cd9d6266a3342dc43442f2ac",
+ "blk.24.ffn_norm.weight": "77c1445a8639ad24938bdf0280233eea2362d47391421833dfa72ec756dfc1e8",
+ "blk.24.ffn_up.weight": "78235ac729ee23c1cf1ae543751e3af32776d8808cee6e529c2a625a1f027654",
+ "blk.24.post_attention_norm.weight": "161f71b6d07628d43e4ae51a4c9088ec6ca2db123a17986a14505d83fdd04dad",
+ "blk.24.post_ffw_norm.weight": "cf1ba692aa683368b02ac413e69b2521b98c69a5274eacbb54165b53bf38a8b2",
+ "blk.25.attn_k.weight": "057a56bd8c8d2b41608d1f71faa3052902152ddf85e47669ad950c1c3e77c33f",
+ "blk.25.attn_norm.weight": "b7179fe02c334da556ddcf6c1b502245639a728c4cbba8b552d8e1df4565ee9d",
+ "blk.25.attn_output.weight": "4fed8b05b08a0ff75ffd022701bbeb52f17b23d09332a1ddcba737244bd0d3b0",
+ "blk.25.attn_q.weight": "c52e99f5d38bf7538d6106a0bbf38ac6dc6296bca9a3f849afa384ea67b4af01",
+ "blk.25.attn_v.weight": "c49c23d8e1cfa6a8eb971eb69942204890c6d7d830dc8774c84b108a80598912",
+ "blk.25.ffn_down.weight": "c08d4dc8412b19fdc870c164b83c341b236ec6fe7bb4a9bcfe0dc100faa20286",
+ "blk.25.ffn_gate.weight": "1a4cb3f36735d59181721471452807903006539e5e1b5ceb4f72d1d7ae134127",
+ "blk.25.ffn_norm.weight": "8fd6bd0dcec5198761525a36992a57c9ec5e9da60a22092839a84ae8c4e87f26",
+ "blk.25.ffn_up.weight": "3a00f39bdd5f31dc5e3b281d2002e1ac4f2475d49a0ac1d7720a25b377dcd04a",
+ "blk.25.post_attention_norm.weight": "e5f31a648612c859b6d21c9ee426e87a86cb1973dfdd86276c767371d9cef5ad",
+ "blk.25.post_ffw_norm.weight": "553c3bd774922c99c2384380a142d019881d30dbf0fe3bf9430dabfb3f6cbd33",
+ "output_norm.weight": "49445c4585ab0a8135717a0bdb1cda4a062a030177d0119561d91542aec5744b"
+}
diff --git a/convert/testdata/gemma-2-9b-it.json b/convert/testdata/gemma-2-9b-it.json
new file mode 100644
index 00000000..90cdbee4
--- /dev/null
+++ b/convert/testdata/gemma-2-9b-it.json
@@ -0,0 +1,6 @@
+{
+ "general.architecture": "gemma2",
+ "gemma2.attention.sliding_window": "4096",
+ "gemma2.attn_logit_softcapping": "50",
+ "gemma2.final_logit_softcapping": "30"
+}
diff --git a/convert/testdata/gemma-2b-it.json b/convert/testdata/gemma-2b-it.json
new file mode 100644
index 00000000..0482f1e1
--- /dev/null
+++ b/convert/testdata/gemma-2b-it.json
@@ -0,0 +1,188 @@
+{
+ "general.architecture": "gemma",
+ "general.file_type": "1",
+ "general.quantization_version": "2",
+ "gemma.block_count": "18",
+ "gemma.context_length": "8192",
+ "gemma.embedding_length": "2048",
+ "gemma.feed_forward_length": "16384",
+ "gemma.attention.head_count": "8",
+ "gemma.attention.head_count_kv": "1",
+ "gemma.attention.key_length": "256",
+ "gemma.attention.value_length": "256",
+ "gemma.attention.layer_norm_rms_epsilon": "1e-06",
+ "tokenizer.ggml.model": "llama",
+ "tokenizer.ggml.add_bos_token": "true",
+ "tokenizer.ggml.add_eos_token": "false",
+ "tokenizer.ggml.bos_token_id": "2",
+ "tokenizer.ggml.eos_token_id": "1",
+ "tokenizer.ggml.padding_token_id": "0",
+ "tokenizer.ggml.unknown_token_id": "3",
+ "tokenizer.ggml.scores": "0872465d173867d755d3ee728f882b9dc2057a0bfd596fe1e3d131522f1250d8",
+ "tokenizer.ggml.token_type": "485e40bf3d715a4764818fc097d6a2a41db872d82ee714bc500872a3437ff48d",
+ "tokenizer.ggml.tokens": "c6e66de1841f04de8b8d236d461ab720a4c9b9b5414dc293a09c6e10eab45fda",
+ "token_embd.weight": "17b87ab2c01c80657855a5413d0457b4a041afaeda0cc785080e44e2f04acf07",
+ "blk.0.attn_k.weight": "28ac0da05754ad2714ae95da28a5ad191192140b30b8fd22d108d4700c9d989f",
+ "blk.0.attn_norm.weight": "3f9d5675d1ab0eb8a816719dac9fab81f2e95c52be02c34263339acbc087febb",
+ "blk.0.attn_output.weight": "703295c2c63990ff896778685c678f145298886f680f3ed5dc2a7ad54c293265",
+ "blk.0.attn_q.weight": "69c2d0e4870e9d722a190d356203c9605575a16863466c3d1747966ef1cf5791",
+ "blk.0.attn_v.weight": "95219c9c07b5ffe9a9a01e456d845eef2b11f4fc12c93dbbba479db395444c13",
+ "blk.0.ffn_down.weight": "a2feb5eb3d572c57c5bafbf0ab506862df1160fe40965dcfe4b9fd855c08bed7",
+ "blk.0.ffn_gate.weight": "fcca072c445c31f4dc4d5dfaa785b1bdf7271342442099b74fd17268b5829fbf",
+ "blk.0.ffn_norm.weight": "7621f95dbd245cade6fffd6b08797d69d8e3954e960f0b5551b90d967ab95448",
+ "blk.0.ffn_up.weight": "14a9bcdd451403c67136391e1b6e53b3b1830f00199bd911dbcc56d8749c14f4",
+ "blk.1.attn_k.weight": "c70f73c5df20579cb44d971164b48b5f0d8d5abdb38b381e7a8b880ba12aa406",
+ "blk.1.attn_norm.weight": "88b6b91f93a1ef83425a7c7dc2a2fbd3b22704a04c64a80061df376ac8c33626",
+ "blk.1.attn_output.weight": "f031a537490c452be3b3bb51e6b7949a636405756e160976a1c070a792ea00ee",
+ "blk.1.attn_q.weight": "bdb23214b1cf9cfd30f863a0a5868e52c6809d93b7e8f44df096a94204d9896a",
+ "blk.1.attn_v.weight": "e9bbc0b05f2c872fb1403f8f938cd1612b502229ee401f12593b1164c61acc00",
+ "blk.1.ffn_down.weight": "5ff53811038b661a7b8f2bfdf213bebfb185ec1a6060b662f063714f33584d79",
+ "blk.1.ffn_gate.weight": "205085c8c951a5c7543b1495183cd96028fb49f67464b3e9862a2693a6077a33",
+ "blk.1.ffn_norm.weight": "798f354fc85afce9625f5d10093a585a966831698a0560e6c9b97ce659eb4b22",
+ "blk.1.ffn_up.weight": "db92dc5684cb6e90940e13f4d1da555ed20ba4f8cab1e990ddfd7553e2e91315",
+ "blk.2.attn_k.weight": "ef5ce360c4eed6d00d03ca4761e0f8e4b0af4509978468314be14f3d46621044",
+ "blk.2.attn_norm.weight": "6dadbc05dbd0d3fabb4216affa60a3de1378a82d2859dc90b338cbe70f50d455",
+ "blk.2.attn_output.weight": "6bbf87a966f691bbfd7c8d25629aa4e6710107bd431a667434861febb391edc5",
+ "blk.2.attn_q.weight": "4e575c09ae2de417ce9057ce8b073680e860a24aae13a472b68f101b760752e5",
+ "blk.2.attn_v.weight": "cd33f7f01141e9439afdaf2ea1aaced9feaa335e32a58daa136ebd555d4d96f4",
+ "blk.2.ffn_down.weight": "b970ff1b0b6494165defe2fbfa1d31425766ed71e64de9ec4e66ac3955c8bc5f",
+ "blk.2.ffn_gate.weight": "dbb3e1360402e0e369b101995bb686b73f95d4a7673f061be85d64d15dfb0061",
+ "blk.2.ffn_norm.weight": "bfb7980105d8ac9647710454f57a5cdac50598a0f6f4884e16f1d94b00844687",
+ "blk.2.ffn_up.weight": "50ef89339b275a438b664686f6227dd9b6e43853ed6856ec9e33ef4bbd90bda1",
+ "blk.3.attn_k.weight": "be942ea98151434eebcd2c1da4b00e0146152fe524a530689b1fd491cb833d21",
+ "blk.3.attn_norm.weight": "0df2f218daf609c289fb7c60c5f375fa99c0d4e04381ad5a494a19144edd8e20",
+ "blk.3.attn_output.weight": "c2184aaf86aa2cb8f47be49f60b165834e97205f39c6ee1dfd19fd4411a156ce",
+ "blk.3.attn_q.weight": "4f86e2a0a4221c1c84ff9c409ac89893cb95d7208cf65bf1e98e24e01125f991",
+ "blk.3.attn_v.weight": "abfdb8a60c349dadde641d1afc9542025e24fbf41a3238bfa9675e0b1f1e4b68",
+ "blk.3.ffn_down.weight": "58821a8d87008d47d122427911c6fad5272aca70c448bbae223256a74bacd07e",
+ "blk.3.ffn_gate.weight": "776e051f1a0ddd5c4934e69186683a75ca9a3c8c0f61911bba321fed1dd287d2",
+ "blk.3.ffn_norm.weight": "7f380f29335e28be90bfcfae6f6d69fdf5751211b36d2dd62aa5541ed113e4f2",
+ "blk.3.ffn_up.weight": "fc5ae8d488894cbd4951059675468d227da27871d26e925c9941863841c097ee",
+ "blk.4.attn_k.weight": "14833b078cc4c5137bdd5fdc0538047974ca147a99b0282e1b144440c78bc1db",
+ "blk.4.attn_norm.weight": "0a69957d4a15599fb80ad4753558020804925221457d9a5052926754d3768065",
+ "blk.4.attn_output.weight": "887a49b6130fb6297cf10767207c3dd97191b2cf63723449af9c27bca8dbeda0",
+ "blk.4.attn_q.weight": "51fd577b76764824dd6f0d4891c137ebe4736f591b5ca2793c5fff2be49abbde",
+ "blk.4.attn_v.weight": "1a623c43cf9c509d1b7ea0d1a5c04d0af4809665f9f9e93b7d6dba8c5df178fa",
+ "blk.4.ffn_down.weight": "5d61e8856d8941d2b1fd138116d015f63840d0fa1e31e20e20a5ceca1536ceec",
+ "blk.4.ffn_gate.weight": "06640f7273764f8ca5df7e386547417916b6cd7d565a8343153113239a94b0a1",
+ "blk.4.ffn_norm.weight": "91a6c6c41b894228e361435ecbc5058dca34d4911a23da5b56de219299c964d3",
+ "blk.4.ffn_up.weight": "d016dac1055e36d6a10b6317e57f98a904709ea892ef3194342f4d2f6326561e",
+ "blk.5.attn_k.weight": "987146afe124131500808cc0da33c06d207433656d41df6e6d8c99118a83bac5",
+ "blk.5.attn_norm.weight": "6b354938966f2608a2fb8d0f5b363ed0d8b0967c2ec8d0abd5c625b413042ded",
+ "blk.5.attn_output.weight": "cdcbfe02c6ff79d5326882b017a02099f5af71beedf6b1b3eb4de01e3a844536",
+ "blk.5.attn_q.weight": "b910d0cff781d3efb42eab0a302f46f286b2de717079175680d5b42bf8c309c8",
+ "blk.5.attn_v.weight": "66d3a279f747412f9f4b0e8abad44540c122ab2e811a7ee74c1f33bc36caade9",
+ "blk.5.ffn_down.weight": "c9b0efd2212981f16d956d8571f054b68780ad01f4917033647e359b557a4653",
+ "blk.5.ffn_gate.weight": "fe96b94109ca141c01f6a04788e20783019ca6ec334aa1f3134810bdb499e557",
+ "blk.5.ffn_norm.weight": "aa7b016e832e7055a36c6e20de58ea1936f995f390401fff1c5fc65906064e49",
+ "blk.5.ffn_up.weight": "555ce27c4873d3375394f38ad3b45e3d8848f9d5642dc1602383d0f0a33c2a14",
+ "blk.6.attn_k.weight": "88280d461db324c4f36475ce396793063e61a27283ec64511b0480890fb5b3b4",
+ "blk.6.attn_norm.weight": "af8f460c411f660d33196286d208f1845fd5a2b45f7b56549a4df31e7515447a",
+ "blk.6.attn_output.weight": "dd9996fb0a256e8375ad3917705258a33fce006bcea0f536caae420a77974d8b",
+ "blk.6.attn_q.weight": "7a4841541191e037cfb9b07930c4d8cab451809658b182f0ada6ccde9615c003",
+ "blk.6.attn_v.weight": "ae81e6a592b64d701a9d40233e986039a56cba8d8d24f61aea93c6393cf3078a",
+ "blk.6.ffn_down.weight": "622dd1ce1706355cbc659a8ab2c4509678ffe0f3ad34258e5e25ed2a5d951bcd",
+ "blk.6.ffn_gate.weight": "8389a735c0bd5591010f8ced9805a2a12c749f6df0d3c18ad4d05c2a302e7168",
+ "blk.6.ffn_norm.weight": "621f5346400382474d61358397bd58fb1459b07c53e376e4bca15e08b3f9b3fb",
+ "blk.6.ffn_up.weight": "8d834e4c42f13c251dfee36cf89e12f1bd400680d00d5c2e6cac0459e9ce2f7f",
+ "blk.7.attn_k.weight": "8bd0412de65a3e64901ef8fe6a28c95e116bf39dc9aa22f0126b9d36688e5ea7",
+ "blk.7.attn_norm.weight": "056d8e56be4e87d6dc6f900762f0dc6fde07bfdc50dd85bfc510415e2bba3f3d",
+ "blk.7.attn_output.weight": "27972eda51da53d416ff95aed78149a2c5a287b47d2cd46f2f544ca692ecb3bb",
+ "blk.7.attn_q.weight": "41eca977b9371f7932800c11a9c45b931310196919e2a0651b847703b180fc7f",
+ "blk.7.attn_v.weight": "13c74fd7e07f08883a09fb070a1fe5bbdd2341b4cb8d1cac07c4b637049b5774",
+ "blk.7.ffn_down.weight": "9e75db42468800849a9a7da603d0072c5e86c8ed2b4d8b20a312a51fb86a7a10",
+ "blk.7.ffn_gate.weight": "db6bdc3117f910088aaf7db51f2da63ea5bd933de36af5599c215bfb26f7db2b",
+ "blk.7.ffn_norm.weight": "48bb82b49bfc8679a1e77f282ee182d952db7a3c11be7ef9a102ee2ddd8011e2",
+ "blk.7.ffn_up.weight": "feebea87175817a0f3585ec0af09dc873d94c203581ae97a712eb356d3b49efe",
+ "blk.8.attn_k.weight": "d5640ad71b6af68d88e17bf8e7fc26c907d2262605457a84247dd9afc2884d69",
+ "blk.8.attn_norm.weight": "75b850c481a69083ae09d0207ba7317b37c735a39fcf5fef5400e6c84fb1257f",
+ "blk.8.attn_output.weight": "cbd669dbdea2bdd90f9f0cc97566b3dffff3c56cecb4f47290ceef30da83b2d6",
+ "blk.8.attn_q.weight": "9edcb63087a431bac361822497e6ecdaa06d9ea4a1a754e36da7ba9f8db81c7c",
+ "blk.8.attn_v.weight": "3fb72c2c4f95a83626aa3e30062f9450b09ab37c7871e229f18bbc5cf744633c",
+ "blk.8.ffn_down.weight": "bd69d2c9172974fff154441b237b4787fb53b2d185325442d5048130ef5bc4ef",
+ "blk.8.ffn_gate.weight": "d04689c80553edd011d1cbaa5d570fffa7fa91e88b66cf1352d89ab60b72f908",
+ "blk.8.ffn_norm.weight": "e49984183b735b7f2c4e4730c289eed9394056d2e283a00fd83ea0915df31a73",
+ "blk.8.ffn_up.weight": "8fe62a1ce8e847e567add6c6f6bf2922bc467495b5eb4c116b3cb85b85b3b211",
+ "blk.9.attn_k.weight": "d90904959e5004cf0d6e729c6bff18cc33c094798b802473c1ec55ab8d276183",
+ "blk.9.attn_norm.weight": "79277f290cc07411115d8fa138045edf4a17b3416ab2145409cbe8ab829fd4ee",
+ "blk.9.attn_output.weight": "5a21bf2e1f09a81405025f96d4153ffb630158e17269cff8ffff935c38ceb1a7",
+ "blk.9.attn_q.weight": "51b1d0febc3b350945be4504f55afa4347517bde0f710e1a4b88e6b17e71e7c7",
+ "blk.9.attn_v.weight": "aab7e1db0a8b50a03036356791ffce736ab010d15674c96eaef8049d80076054",
+ "blk.9.ffn_down.weight": "cbf43ec84becb40c9359a181ab0e641fd7faae7d34b549501f7cfb7afdc3d764",
+ "blk.9.ffn_gate.weight": "dce0e8661c778327bed7f03b6790d26710764188aed9dc746e6e05863891fa57",
+ "blk.9.ffn_norm.weight": "6d41642104f995c77bf31122b13237caebda3e7fcccb1367ce91db36b015e923",
+ "blk.9.ffn_up.weight": "82fe4c67bf24e7b2d6f6e05f7b1234c2bf90c3932951091a9066211b8e15ecbb",
+ "blk.10.attn_k.weight": "f6a9ed8fd8d3229b5d03175c413ffc56a07f2ce7236271986361dd3d8993f9aa",
+ "blk.10.attn_norm.weight": "cebbef89f0326ca8e02df3867a571e4d61c20c2a12f295f98ae590d62bc86010",
+ "blk.10.attn_output.weight": "34f5efb86accb4f06347d83a32558ea8eab3039d128969161a741ebacbb656ff",
+ "blk.10.attn_q.weight": "1e0efe27df2d5d50f7157253ba2cfd436d6781c3dc78ca176d0c16a210b5b763",
+ "blk.10.attn_v.weight": "8f085bf50a2b0f83cd6cdda3c8ef5a9e204a36348ed95871aac725d1f68640cf",
+ "blk.10.ffn_down.weight": "bf3b3cb4cace435809ac7b4cc933f20853af12f1f272d3dcefe7f19c0f203b8b",
+ "blk.10.ffn_gate.weight": "d3df7a1413b1c5adf1a1dcda9e5225a15c89874bae53bb6137ad1ea42fca2d34",
+ "blk.10.ffn_norm.weight": "a1da603b0480471b5ed8e862148cecd5fed918f8304d6933ab0bdb25b8d2fb8f",
+ "blk.10.ffn_up.weight": "bffbba605922e972dc47dda88a0b4659aa52236c76e5fe861a949e6d9a367492",
+ "blk.11.attn_k.weight": "9f31c63d66cd32c29b1eb8bb829d0c8525ce2ae936e0eefdaab6335a2d12a3df",
+ "blk.11.attn_norm.weight": "0bde1a266d8b2e8f202bb7e2e88b19147ca83021901f6d3cae77a4df5548c754",
+ "blk.11.attn_output.weight": "e10725c7cf746ed4a7e472cf7aea6cb564e5db6a1d5197adc980d650a387ccea",
+ "blk.11.attn_q.weight": "05ee758a7d065802630f8c65dca424364c1c8825e389aa33f9405c45e8a50cce",
+ "blk.11.attn_v.weight": "0c3ae7090f11775d24c51120db6e305db6aff706493e7ee123dcab74485ba789",
+ "blk.11.ffn_down.weight": "7ba40b8e12c09c5fb2006b77a771cb01ce894e88a3b3e1877f927a5b89c91709",
+ "blk.11.ffn_gate.weight": "db76388a023b98097972d354ba1c6a5e26efdeb1c596b9c28bf2cd8f6596975e",
+ "blk.11.ffn_norm.weight": "a38c3ae1b89a68ddc7b72c99c5b28be7fe3787c4fad9904d0c43d64eaf00c474",
+ "blk.11.ffn_up.weight": "13c8142f9cf1eddc658babf978daf3515c4ccc45f849f3e7e3930aa18a8480a0",
+ "blk.12.attn_k.weight": "f03241c36ac87cb57429a2ef22186b8d7d0b590a8b173beb01fa13d93772f3b1",
+ "blk.12.attn_norm.weight": "4568f654e6d65104d586e7c16ba960c83428698ce103022b7e0be15e2884e13b",
+ "blk.12.attn_output.weight": "04867603f82f91e41306e09b33ecda0104b3ee4834061f2c0bbdc8da33c72509",
+ "blk.12.attn_q.weight": "70fe04b9a8e08b6100cc8d6b58bf4cbbad15ca1de82d63baca5d352ba6c4cbae",
+ "blk.12.attn_v.weight": "15cb28db61a86c98687991d7e611bc92a1fcc6007f3432149cfb5fe518a4f65e",
+ "blk.12.ffn_down.weight": "6d10c790a4e3dc44c2dc36d96251ae97cdf30a4fa04d4c43e31bfbd038e6a7b7",
+ "blk.12.ffn_gate.weight": "3462a2d8f6b4743b25e24da51b90018ac2858d05ac7e582bcb69063cfdac1104",
+ "blk.12.ffn_norm.weight": "1f96392c1faa34e34ae5dea55a6a86c5aa4c79758952075d53d28de89dd88456",
+ "blk.12.ffn_up.weight": "d22eacc612a7411953d948483c5fb201e11722955ee0754da866e7bec578ac6d",
+ "blk.13.attn_k.weight": "5864977e6b733ea942647d6feed5c76156c48c200649c22e4e11b9e5860e57f3",
+ "blk.13.attn_norm.weight": "87e053535144723db4145aa5402acc54331b7696752d852bb9fc542ff33f0fb5",
+ "blk.13.attn_output.weight": "078145f5ad83f8b14f97a869346f7fd1583b24d1e3edadaa95d3da4242973f8f",
+ "blk.13.attn_q.weight": "3b8caf35504cbc4d1a7dd6e011a95760703b7f71e2218b030b1254f811362dd7",
+ "blk.13.attn_v.weight": "4fdf8365a603e043e5b40c4a21c84ac167f9be62794178f9d8a608dfe5653bf9",
+ "blk.13.ffn_down.weight": "a07d3abbfcacf48ba028df2cab895be32cc15022d23389a745286e79c1b1d1fd",
+ "blk.13.ffn_gate.weight": "1d2ab39666aa2909acc96787432a3ed13b19d25170f74665fadff9b17bbaffb1",
+ "blk.13.ffn_norm.weight": "4f2e809fda5f3eadf52578ee50e0ba36e53be91e55dce418c12dfe595f5f18e7",
+ "blk.13.ffn_up.weight": "8783d2720c2c37ca176a5801e0b3ef1f9cc9cf3ef1cd37af423aaf6b2a27e2bd",
+ "blk.14.attn_k.weight": "ce9428e2b55d43ae0c6690dbd56182f99adc427694ba8236b405cc8ea5035e86",
+ "blk.14.attn_norm.weight": "6abb35f9db8251d6ae954bda147c6ada2371b0574d11702e828f3c6ac99b7cc0",
+ "blk.14.attn_output.weight": "fe3880916d0ceb5bff672c88bbefb7060a545be609bf049beb2024b38221836d",
+ "blk.14.attn_q.weight": "7c8ad81be6f4a350931fd108b5f7c9e366e8c26ef62d1d85ffef5dca8fd893f8",
+ "blk.14.attn_v.weight": "e4bdedffacbebe38567a0734dfd67db90e911d9a9669fcde9a7c4ad8a0066c52",
+ "blk.14.ffn_down.weight": "ef6694dff1e05820aac0cd2b22f39ac7788b4967afc9250775575554c66aab2c",
+ "blk.14.ffn_gate.weight": "db63c4179e2db704bc505e2b4696e055b593e295a1b7c4c586fc793bdd5aab19",
+ "blk.14.ffn_norm.weight": "2796a62d832a9710148f95d533320492a33e712b2e5218659c548705bd11684d",
+ "blk.14.ffn_up.weight": "3f78c78d8c2d54df45f799d4ff902316628af296834afe4ceed63d4a324ff03e",
+ "blk.15.attn_k.weight": "6e810ee3859e07695645ee0c9a5efc7962668984a5f0a9325f47e462743b447c",
+ "blk.15.attn_norm.weight": "0956b576ae96db0b28cb09f761f801cfd9281432284664f0fe181c8d9c55d1ec",
+ "blk.15.attn_output.weight": "03a17f7e94208177aace5cc41b7f54670ba57873b7274ff6e23caf58cce110ca",
+ "blk.15.attn_q.weight": "b8edafe7d2216a6f8b4ae4905a906475490e6ea418f6e1d3cec563dbdc6fab91",
+ "blk.15.attn_v.weight": "f8ae8cae0f4cfa34a459824eba57350c3c248104ba5607e7d9dc7d7c39aaf4a6",
+ "blk.15.ffn_down.weight": "8d02eb439da852246d2ca67e9b7b6de0b090b80744355e64728a23e41926505b",
+ "blk.15.ffn_gate.weight": "ed5bf361c67db8731f186b775826f21c33bdb521111fd2d922539719a770239f",
+ "blk.15.ffn_norm.weight": "5942ca3c73209ac9a0c8bfd9b4aab7f7be7aee9aa12d9c35833493b44af76767",
+ "blk.15.ffn_up.weight": "f4bebf4ad99ec5f911327dec347be6c595814885309c7bc5647ce28c7f4d1cf5",
+ "blk.16.attn_k.weight": "756a534c19364448e0958b8948fe33891c6ccda0fbb4dfa2024e1f532a87804b",
+ "blk.16.attn_norm.weight": "386b7b9e4e6509f6af9c022d942b6c6c6cc136aeed8751ecb037c74d7c4bfb93",
+ "blk.16.attn_output.weight": "3ba1a766a25830b84d7c22178203635f9c5624caad290bc5e5d73da5d5e7a2ec",
+ "blk.16.attn_q.weight": "d39b0c91e1fda7685d50a0f7cc8d18c44b5bdc90a142c7fda0bc329cca1afa74",
+ "blk.16.attn_v.weight": "98b33fcb0ee3483cff1b06ecb44d7b7ffb4d34c268248e4d73dfdf82b2065b2f",
+ "blk.16.ffn_down.weight": "14006f5e4acb2f9416271ae562e299359cd2585739c7fc77ccbca54495563948",
+ "blk.16.ffn_gate.weight": "12f8abae2d301d8f88bedb6af98b1daecc7b0b8d05148594f931f30958d77aca",
+ "blk.16.ffn_norm.weight": "129a15a046ee96d06de288bd43c80f77a6b0fb3a159c7367154c6e4aaf362672",
+ "blk.16.ffn_up.weight": "b4a5911a45f3871ef1d4efb7dc7108645a564b70f818eccf45beebef2e844ee9",
+ "blk.17.attn_k.weight": "5e1bfcff0146ebdde3817b656952892eb671e14e75afc92fa53f84f8eecbec4c",
+ "blk.17.attn_norm.weight": "60bc988fab7c4b29ee9de599df41a8de00caa94fcd74677da011fac82f60f465",
+ "blk.17.attn_output.weight": "ba49b40d6a0b5685f749c24b0edbed3adc44dbe13b5d5e5fa1e56169fc746555",
+ "blk.17.attn_q.weight": "82bb415d24efcd14d03ace03f907bb70db6a204c76a0bdd1892e0fba165db87d",
+ "blk.17.attn_v.weight": "73dbe54beb91a899884e275ea81ffc5187a20cb7d5b68d5c299b783096999d94",
+ "blk.17.ffn_down.weight": "7c086166241e0664f8963fd1ca4ed74c737abfb2525ec20f8435821ff50158f3",
+ "blk.17.ffn_gate.weight": "51a32f78244d42a539f619c5ce661db9e6cf41636280a826d439b5444edcd28c",
+ "blk.17.ffn_norm.weight": "c4bb247fccd1ecc84875028af63dd20aaf5cbd17eb94a9bc36679c09285dccab",
+ "blk.17.ffn_up.weight": "b5886182790bc6fbadd63de9bc4ffee416f3b69a66280d197ab8c18edf769abf",
+ "output_norm.weight": "481f3097d0a20412e35b3a739b1b958487bcd41ff67744baa3c9acbddd2ee4d4"
+}
diff --git a/convert/tokenizer.go b/convert/tokenizer.go
new file mode 100644
index 00000000..14d6ba66
--- /dev/null
+++ b/convert/tokenizer.go
@@ -0,0 +1,278 @@
+package convert
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/fs"
+ "log/slog"
+ "os"
+ "slices"
+
+ "golang.org/x/exp/maps"
+)
+
+const (
+ _ int32 = iota
+ tokenTypeNormal
+ tokenTypeUnknown
+ tokenTypeControl
+ tokenTypeUserDefined
+ tokenTypeUnused
+ tokenTypeByte
+)
+
+type Tokenizer struct {
+ *Vocabulary
+ SpecialVocabulary []*SpecialVocabulary
+ Merges []string
+
+ Pre string
+ Template string
+}
+
+func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error) {
+ v, err := parseVocabulary(fsys)
+ if err != nil {
+ return nil, err
+ }
+
+ t := &Tokenizer{
+ Vocabulary: v,
+ Pre: "default",
+ }
+
+ addedTokens := make(map[string]token)
+ if f, err := fsys.Open("tokenizer.json"); errors.Is(err, os.ErrNotExist) {
+ } else if err != nil {
+ return nil, err
+ } else {
+ defer f.Close()
+
+ var tt tokenizer
+ if err := json.NewDecoder(f).Decode(&tt); err != nil {
+ return nil, err
+ }
+
+ for _, t := range tt.AddedTokens {
+ addedTokens[t.Content] = t
+ }
+
+ t.Merges = tt.Model.Merges
+
+ sha256sum := sha256.New()
+ for _, pt := range tt.PreTokenizer.PreTokenizers {
+ switch pt.Type {
+ case "Split":
+ if pt.Pattern.Regex != "" {
+ // create a checksum of all Split pretokenizers which should be sufficient
+ // to identify the pretokenizer
+ sha256sum.Write([]byte(pt.Pattern.Regex))
+ }
+ }
+ }
+
+ switch digest := hex.EncodeToString(sha256sum.Sum(nil)); digest {
+ case "d98f9631be1e9607a9848c26c1f9eac1aa9fc21ac6ba82a2fc0741af9780a48f":
+ t.Pre = "llama-bpe"
+ case "03df5c5863ad70781dcfdef491ead25140f895fe8010964be0daefe27be32b02":
+ t.Pre = "deepseek-llm"
+ case "21cde974d587f0d54dc8d56b183cc1e6239600172035c68fbd6d4b9f8da0576e":
+ t.Pre = "deepseek-coder"
+ case "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855":
+ // noop, empty pretokenizer
+ default:
+ slog.Warn("unknown pretokenizer, using default", "digest", digest)
+ }
+ }
+
+ if f, err := fsys.Open("tokenizer_config.json"); errors.Is(err, os.ErrNotExist) {
+ } else if err != nil {
+ return nil, err
+ } else {
+ defer f.Close()
+
+ var p map[string]json.RawMessage
+ if err := json.NewDecoder(f).Decode(&p); err != nil {
+ return nil, err
+ }
+
+ if template, ok := p["chat_template"]; ok {
+ var s []struct {
+ Name string `json:"name"`
+ Template string `json:"template"`
+ }
+ if err := json.Unmarshal(template, &t.Template); err == nil {
+ // noop
+ } else if err := json.Unmarshal(template, &s); err == nil {
+ for _, e := range s {
+ if e.Name == "default" {
+ t.Template = e.Template
+ break
+ }
+ }
+ } else {
+ return nil, fmt.Errorf("invalid chat_template: %w", err)
+ }
+ }
+
+ for _, st := range specialTokenTypes {
+ sv := SpecialVocabulary{Type: st}
+ if bts, ok := p[fmt.Sprintf("add_%s_token", st)]; ok {
+ if err := json.Unmarshal(bts, &sv.AddToken); err != nil {
+ return nil, err
+ }
+ }
+
+ if bts, ok := p[fmt.Sprintf("%s_token", st)]; ok {
+ var content string
+ if err := json.Unmarshal(bts, &content); err != nil {
+ var mm map[string]any
+ if err := json.Unmarshal(bts, &mm); err != nil {
+ continue
+ }
+
+ content, ok = mm["content"].(string)
+ if !ok {
+ continue
+ }
+ }
+
+ sv.Content = content
+ }
+
+ if id, ok := addedTokens[sv.Content]; ok {
+ sv.ID = id.ID
+ t.SpecialVocabulary = append(t.SpecialVocabulary, &sv)
+ }
+ }
+ }
+
+ return t, nil
+}
+
+type tokenizer struct {
+ AddedTokens []token `json:"added_tokens"`
+ Model struct {
+ Type string `json:"type"`
+ Vocab map[string]int `json:"vocab"`
+ Merges []string `json:"merges"`
+ } `json:"model"`
+
+ PreTokenizer struct {
+ PreTokenizers []struct {
+ Type string `json:"type"`
+ Pattern struct {
+ Regex string `json:"Regex"`
+ } `json:"pattern"`
+ } `json:"pretokenizers"`
+ } `json:"pre_tokenizer"`
+}
+
+type token struct {
+ ID int `json:"id"`
+ Content string `json:"content"`
+ Special bool `json:"special"`
+ UserDefined bool
+}
+
+type Vocabulary struct {
+ Model string
+ Tokens []string
+ Scores []float32
+ Types []int32
+}
+
+func parseVocabularyFromTokenizer(fsys fs.FS) (*Vocabulary, error) {
+ f, err := fsys.Open("tokenizer.json")
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ var t tokenizer
+ if err := json.NewDecoder(f).Decode(&t); err != nil {
+ return nil, err
+ }
+
+ tokens := make(map[int]token, len(t.Model.Vocab))
+ for k, v := range t.Model.Vocab {
+ tokens[v] = token{
+ ID: v,
+ Content: k,
+ }
+ }
+
+ for _, token := range t.AddedTokens {
+ token.UserDefined = true
+ tokens[token.ID] = token
+ }
+
+ keys := maps.Keys(tokens)
+ slices.Sort(keys)
+
+ v := Vocabulary{Model: "gpt2"}
+ for _, k := range keys {
+ token := tokens[k]
+ v.Tokens = append(v.Tokens, token.Content)
+ v.Scores = append(v.Scores, float32(token.ID))
+
+ switch {
+ case token.Special:
+ v.Types = append(v.Types, tokenTypeControl)
+ case token.UserDefined:
+ v.Types = append(v.Types, tokenTypeUserDefined)
+ default:
+ v.Types = append(v.Types, tokenTypeNormal)
+ }
+ }
+
+ return &v, nil
+}
+
+func parseVocabulary(fsys fs.FS) (*Vocabulary, error) {
+ patterns := []struct {
+ Pattern string
+ Func func(fs.FS) (*Vocabulary, error)
+ }{
+ {"tokenizer.model", parseSentencePiece},
+ {"tokenizer.json", parseVocabularyFromTokenizer},
+ }
+
+ for _, pattern := range patterns {
+ if _, err := fs.Stat(fsys, pattern.Pattern); errors.Is(err, os.ErrNotExist) {
+ continue
+ } else if err != nil {
+ return nil, err
+ }
+
+ return pattern.Func(fsys)
+ }
+
+ return nil, errors.New("unknown tokenizer format")
+}
+
+type SpecialVocabulary struct {
+ Type string
+ ID int
+ Content string
+ AddToken bool
+}
+
+func (sv SpecialVocabulary) Key() string {
+ switch t := sv.Type; t {
+ case "bos", "eos", "cls", "mask":
+ return t
+ case "unk":
+ return "unknown"
+ case "sep":
+ //nolint:misspell // this is an upstream typo
+ return "seperator"
+ case "pad":
+ return "padding"
+ }
+
+ panic("unknown special vocabulary type")
+}
diff --git a/convert/tokenizer_spm.go b/convert/tokenizer_spm.go
new file mode 100644
index 00000000..5e506087
--- /dev/null
+++ b/convert/tokenizer_spm.go
@@ -0,0 +1,113 @@
+package convert
+
+import (
+ "cmp"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "slices"
+
+ "google.golang.org/protobuf/proto"
+
+ "github.com/ollama/ollama/convert/sentencepiece"
+)
+
+func parseSentencePiece(fsys fs.FS) (*Vocabulary, error) {
+ ast, err := parseAdditionalSpecialTokens(fsys)
+ if err != nil {
+ return nil, err
+ }
+
+ bts, err := fs.ReadFile(fsys, "tokenizer.model")
+ if err != nil {
+ return nil, err
+ }
+
+ var spm sentencepiece.ModelProto
+ if err := proto.Unmarshal(bts, &spm); err != nil {
+ return nil, err
+ }
+
+ v := Vocabulary{Model: "llama"}
+ for _, piece := range spm.GetPieces() {
+ v.Tokens = append(v.Tokens, piece.GetPiece())
+ v.Scores = append(v.Scores, piece.GetScore())
+
+ switch t := piece.GetType(); t {
+ case sentencepiece.ModelProto_SentencePiece_UNKNOWN,
+ sentencepiece.ModelProto_SentencePiece_CONTROL,
+ sentencepiece.ModelProto_SentencePiece_UNUSED,
+ sentencepiece.ModelProto_SentencePiece_BYTE:
+ v.Types = append(v.Types, int32(t))
+ default:
+ tt := int32(sentencepiece.ModelProto_SentencePiece_NORMAL)
+ if slices.Contains(ast, piece.GetPiece()) {
+ tt = int32(sentencepiece.ModelProto_SentencePiece_CONTROL)
+ }
+
+ v.Types = append(v.Types, tt)
+ }
+ }
+
+ f, err := fsys.Open("added_tokens.json")
+ if errors.Is(err, os.ErrNotExist) {
+ return &v, nil
+ } else if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ var atm map[string]int
+ if err := json.NewDecoder(f).Decode(&atm); err != nil {
+ return nil, err
+ }
+
+ type t struct {
+ id int
+ content string
+ }
+
+ var ts []t
+ for content, id := range atm {
+ ts = append(ts, t{id, content})
+ }
+
+ slices.SortFunc(ts, func(i, j t) int {
+ return cmp.Compare(i.id, j.id)
+ })
+
+ n := len(v.Tokens)
+ for i, t := range ts {
+ if t.id != i+n {
+ return nil, fmt.Errorf("invalid token id: %d", t.id)
+ }
+
+ v.Tokens = append(v.Tokens, t.content)
+ v.Scores = append(v.Scores, -1000.0)
+ v.Types = append(v.Types, tokenTypeUserDefined)
+ }
+
+ return &v, nil
+}
+
+func parseAdditionalSpecialTokens(fsys fs.FS) ([]string, error) {
+ f, err := fsys.Open("special_tokens_map.json")
+ if errors.Is(err, os.ErrNotExist) {
+ return nil, nil
+ } else if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ var m struct {
+ AdditionalSpecialTokens []string `json:"additional_special_tokens"`
+ }
+
+ if err := json.NewDecoder(f).Decode(&m); err != nil {
+ return nil, err
+ }
+
+ return m.AdditionalSpecialTokens, nil
+}
diff --git a/convert/tokenizer_test.go b/convert/tokenizer_test.go
new file mode 100644
index 00000000..d9550e09
--- /dev/null
+++ b/convert/tokenizer_test.go
@@ -0,0 +1,208 @@
+package convert
+
+import (
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func createTokenizerFS(t *testing.T, dir string, files map[string]io.Reader) fs.FS {
+ t.Helper()
+
+ for k, v := range files {
+ if err := func() error {
+ f, err := os.Create(filepath.Join(dir, k))
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ if _, err := io.Copy(f, v); err != nil {
+ return err
+ }
+
+ return nil
+ }(); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ }
+
+ return os.DirFS(dir)
+}
+
+func TestParseTokenizer(t *testing.T) {
+ cases := []struct {
+ name string
+ fsys fs.FS
+ specialTokenTypes []string
+ want *Tokenizer
+ }{
+ {
+ name: "string chat template",
+ fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{
+ "tokenizer.json": strings.NewReader(`{}`),
+ "tokenizer_config.json": strings.NewReader(`{
+ "chat_template": "<default template>"
+ }`),
+ }),
+ want: &Tokenizer{
+ Vocabulary: &Vocabulary{Model: "gpt2"},
+ Pre: "default",
+ Template: "<default template>",
+ },
+ },
+ {
+ name: "list chat template",
+ fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{
+ "tokenizer.json": strings.NewReader(`{}`),
+ "tokenizer_config.json": strings.NewReader(`{
+ "chat_template": [
+ {
+ "name": "default",
+ "template": "<default template>"
+ },
+ {
+ "name": "tools",
+ "template": "<tools template>"
+ }
+ ]
+ }`),
+ }),
+ want: &Tokenizer{
+ Vocabulary: &Vocabulary{Model: "gpt2"},
+ Pre: "default",
+ Template: "<default template>",
+ },
+ },
+ {
+ name: "added tokens",
+ fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{
+ "tokenizer.json": strings.NewReader(`{
+ "added_tokens": [
+ {
+ "id": 999,
+ "content": "<unused999>",
+ "special": false
+ }
+ ]
+ }`),
+ }),
+ want: &Tokenizer{
+ Vocabulary: &Vocabulary{
+ Model: "gpt2",
+ Tokens: []string{"<unused999>"},
+ Scores: []float32{999},
+ Types: []int32{4},
+ },
+ Pre: "default",
+ },
+ },
+ {
+ name: "added tokens overlap vocab",
+ fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{
+ "tokenizer.json": strings.NewReader(`{
+ "added_tokens": [
+ {
+ "id": 0,
+ "content": "<pad>",
+ "special": true
+ }
+ ],
+ "model": {
+ "vocab": {
+ "<pad>": 0
+ }
+ }
+ }`),
+ }),
+ want: &Tokenizer{
+ Vocabulary: &Vocabulary{
+ Model: "gpt2",
+ Tokens: []string{"<pad>"},
+ Scores: []float32{0},
+ Types: []int32{3},
+ },
+ Pre: "default",
+ },
+ },
+ {
+ name: "special token types",
+ fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{
+ "tokenizer.json": strings.NewReader(`{
+ "added_tokens": [
+ {
+ "id": 0,
+ "content": "<pad>",
+ "special": true
+ },
+ {
+ "id": 1,
+ "content": "<eos>",
+ "special": true
+ },
+ {
+ "id": 2,
+ "content": "<bos>",
+ "special": true
+ },
+ {
+ "id": 3,
+ "content": "<unk>",
+ "special": true
+ }
+ ],
+ "model": {
+ "vocab": {
+ "<pad>": 0,
+ "<eos>": 1,
+ "<bos>": 2,
+ "<unk>": 3
+ }
+ }
+ }`),
+ "tokenizer_config.json": strings.NewReader(`{
+ "add_bos_token": true,
+ "add_eos_token": false,
+ "bos_token": "<bos>",
+ "eos_token": "<eos>",
+ "pad_token": "<pad>",
+ "unk_token": "<unk>"
+ }`),
+ }),
+ specialTokenTypes: []string{"pad", "eos", "bos", "unk"},
+ want: &Tokenizer{
+ Vocabulary: &Vocabulary{
+ Model: "gpt2",
+ Tokens: []string{"<pad>", "<eos>", "<bos>", "<unk>"},
+ Scores: []float32{0, 1, 2, 3},
+ Types: []int32{3, 3, 3, 3},
+ },
+ SpecialVocabulary: []*SpecialVocabulary{
+ {Type: "pad", Content: "<pad>", ID: 0, AddToken: false},
+ {Type: "eos", Content: "<eos>", ID: 1, AddToken: false},
+ {Type: "bos", Content: "<bos>", ID: 2, AddToken: true},
+ {Type: "unk", Content: "<unk>", ID: 3, AddToken: false},
+ },
+ Pre: "default",
+ },
+ },
+ }
+
+ for _, tt := range cases {
+ t.Run(tt.name, func(t *testing.T) {
+ tokenizer, err := parseTokenizer(tt.fsys, tt.specialTokenTypes)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+
+ if diff := cmp.Diff(tt.want, tokenizer); diff != "" {
+ t.Errorf("unexpected tokenizer (-want +got):\n%s", diff)
+ }
+ })
+ }
+}
diff --git a/docs/README.md b/docs/README.md
index dde3d1f9..b6221041 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,25 +1,21 @@
# Documentation
-To get started, see the project's **[quicktart](../README.md#quickstart)**.
+### Getting Started
+* [Quickstart](../README.md#quickstart)
+* [Examples](../examples)
+* [Importing models](./import.md)
+* [Linux Documentation](./linux.md)
+* [Windows Documentation](./windows.md)
+* [Docker Documentation](./docker.md)
-Ollama is a tool for running AI models on your hardware. Many users will choose to use the Command Line Interface (CLI) to work with Ollama. Learn more about all the commands in the CLI in the **[Main Readme](../README.md)**.
+### Reference
-Use the RESTful API using any language, including Python, JavaScript, Typescript, Go, Rust, and many more. Learn more about using the API in the **[API Documentation](./api.md)**.
+* [API Reference](./api.md)
+* [Modelfile Reference](./modelfile.md)
+* [OpenAI Compatibility](./openai.md)
-Create new models or modify models already in the library using the Modelfile. Learn more about the Modelfile syntax in the **[Modelfile Documentation](./modelfile.md)**.
+### Resources
-Import models using source model weights found on Hugging Face and similar sites by referring to the **[Import Documentation](./import.md)**.
-
-Installing on Linux in most cases is easy using the script on Ollama.ai. To get more detail about the install, including CUDA drivers, see the **[Linux Documentation](./linux.md)**.
-
-Many of our users like the flexibility of using our official Docker Image. Learn more about using Docker with Ollama using the **[Docker Documentation](./docker.md)**.
-
-It is easy to install on Linux and Mac, but many users will choose to build Ollama on their own. To do this, refer to the **[Development Documentation](./development.md)**.
-
-If encountering a problem with Ollama, the best place to start is the logs. Find more information about them here in the **[Troubleshooting Guide](./troubleshooting.md)**.
-
-Finally for all the questions that don't fit anywhere else, there is the **[FAQ](./faq.md)**
-
-[Tutorials](./tutorials.md) apply the documentation to tasks.
-
-For working code examples of using Ollama, see [Examples](../examples).
\ No newline at end of file
+* [Troubleshooting Guide](./troubleshooting.md)
+* [FAQ](./faq.md)
+* [Development guide](./development.md)
diff --git a/docs/api.md b/docs/api.md
index d93303d8..fe2eb82c 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -12,12 +12,13 @@
- [Pull a Model](#pull-a-model)
- [Push a Model](#push-a-model)
- [Generate Embeddings](#generate-embeddings)
+- [List Running Models](#list-running-models)
## Conventions
### Model names
-Model names follow a `model:tag` format, where `model` can have an optional namespace such as `example/model`. Some examples are `orca-mini:3b-q4_1` and `llama2:70b`. The tag is optional and, if not provided, will default to `latest`. The tag is used to identify a specific version.
+Model names follow a `model:tag` format, where `model` can have an optional namespace such as `example/model`. Some examples are `orca-mini:3b-q4_1` and `llama3:70b`. The tag is optional and, if not provided, will default to `latest`. The tag is used to identify a specific version.
### Durations
@@ -25,7 +26,7 @@ All durations are returned in nanoseconds.
### Streaming responses
-Certain endpoints stream responses as JSON objects and can optional return non-streamed responses.
+Certain endpoints stream responses as JSON objects. Streaming can be disabled by providing `{"stream": false}` for these endpoints.
## Generate a completion
@@ -39,6 +40,7 @@ Generate a response for a given prompt with a provided model. This is a streamin
- `model`: (required) the [model name](#model-names)
- `prompt`: the prompt to generate a response for
+- `suffix`: the text after the model response
- `images`: (optional) a list of base64-encoded images (for multimodal models such as `llava`)
Advanced parameters (optional):
@@ -46,16 +48,18 @@ Advanced parameters (optional):
- `format`: the format to return a response in. Currently the only accepted value is `json`
- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
- `system`: system message to (overrides what is defined in the `Modelfile`)
-- `template`: the full prompt or prompt template (overrides what is defined in the `Modelfile`)
+- `template`: the prompt template to use (overrides what is defined in the `Modelfile`)
- `context`: the context parameter returned from a previous request to `/generate`, this can be used to keep a short conversational memory
- `stream`: if `false` the response will be returned as a single response object, rather than a stream of objects
-- `raw`: if `true` no formatting will be applied to the prompt. You may choose to use the `raw` parameter if you are specifying a full templated prompt in your request to the API.
+- `raw`: if `true` no formatting will be applied to the prompt. You may choose to use the `raw` parameter if you are specifying a full templated prompt in your request to the API
+- `keep_alive`: controls how long the model will stay loaded into memory following the request (default: `5m`)
#### JSON mode
-Enable JSON mode by setting the `format` parameter to `json`. This will structure the response as a valid JSON object. See the JSON mode [example](#generate-request-json-mode) below.
+Enable JSON mode by setting the `format` parameter to `json`. This will structure the response as a valid JSON object. See the JSON mode [example](#request-json-mode) below.
-> Note: it's important to instruct the model to use JSON in the `prompt`. Otherwise, the model may generate large amounts whitespace.
+> [!IMPORTANT]
+> It's important to instruct the model to use JSON in the `prompt`. Otherwise, the model may generate large amounts whitespace.
### Examples
@@ -65,7 +69,7 @@ Enable JSON mode by setting the `format` parameter to `json`. This will structur
```shell
curl http://localhost:11434/api/generate -d '{
- "model": "llama2",
+ "model": "llama3.2",
"prompt": "Why is the sky blue?"
}'
```
@@ -76,7 +80,7 @@ A stream of JSON objects is returned:
```json
{
- "model": "llama2",
+ "model": "llama3.2",
"created_at": "2023-08-04T08:52:19.385406455-07:00",
"response": "The",
"done": false
@@ -89,16 +93,16 @@ The final response in the stream also includes additional data about the generat
- `load_duration`: time spent in nanoseconds loading the model
- `prompt_eval_count`: number of tokens in the prompt
- `prompt_eval_duration`: time spent in nanoseconds evaluating the prompt
-- `eval_count`: number of tokens the response
+- `eval_count`: number of tokens in the response
- `eval_duration`: time in nanoseconds spent generating the response
- `context`: an encoding of the conversation used in this response, this can be sent in the next request to keep a conversational memory
- `response`: empty if the response was streamed, if not streamed, this will contain the full response
-To calculate how fast the response is generated in tokens per second (token/s), divide `eval_count` / `eval_duration`.
+To calculate how fast the response is generated in tokens per second (token/s), divide `eval_count` / `eval_duration` * `10^9`.
```json
{
- "model": "llama2",
+ "model": "llama3.2",
"created_at": "2023-08-04T19:22:45.499127Z",
"response": "",
"done": true,
@@ -120,7 +124,7 @@ A response can be received in one reply when streaming is off.
```shell
curl http://localhost:11434/api/generate -d '{
- "model": "llama2",
+ "model": "llama3.2",
"prompt": "Why is the sky blue?",
"stream": false
}'
@@ -132,7 +136,7 @@ If `stream` is set to `false`, the response will be a single JSON object:
```json
{
- "model": "llama2",
+ "model": "llama3.2",
"created_at": "2023-08-04T19:22:45.499127Z",
"response": "The sky is blue because it is the color of the sky.",
"done": true,
@@ -146,15 +150,51 @@ If `stream` is set to `false`, the response will be a single JSON object:
}
```
+#### Request (with suffix)
+
+##### Request
+
+```shell
+curl http://localhost:11434/api/generate -d '{
+ "model": "codellama:code",
+ "prompt": "def compute_gcd(a, b):",
+ "suffix": " return result",
+ "options": {
+ "temperature": 0
+ },
+ "stream": false
+}'
+```
+
+##### Response
+
+```json
+{
+ "model": "codellama:code",
+ "created_at": "2024-07-22T20:47:51.147561Z",
+ "response": "\n if a == 0:\n return b\n else:\n return compute_gcd(b % a, a)\n\ndef compute_lcm(a, b):\n result = (a * b) / compute_gcd(a, b)\n",
+ "done": true,
+ "done_reason": "stop",
+ "context": [...],
+ "total_duration": 1162761250,
+ "load_duration": 6683708,
+ "prompt_eval_count": 17,
+ "prompt_eval_duration": 201222000,
+ "eval_count": 63,
+ "eval_duration": 953997000
+}
+```
+
#### Request (JSON mode)
+> [!IMPORTANT]
> When `format` is set to `json`, the output will always be a well-formed JSON object. It's important to also instruct the model to respond in JSON.
##### Request
```shell
curl http://localhost:11434/api/generate -d '{
- "model": "llama2",
+ "model": "llama3.2",
"prompt": "What color is the sky at different times of the day? Respond using JSON",
"format": "json",
"stream": false
@@ -165,7 +205,7 @@ curl http://localhost:11434/api/generate -d '{
```json
{
- "model": "llama2",
+ "model": "llama3.2",
"created_at": "2023-11-09T21:07:55.186497Z",
"response": "{\n\"morning\": {\n\"color\": \"blue\"\n},\n\"noon\": {\n\"color\": \"blue-gray\"\n},\n\"afternoon\": {\n\"color\": \"warm gray\"\n},\n\"evening\": {\n\"color\": \"orange\"\n}\n}\n",
"done": true,
@@ -246,6 +286,22 @@ curl http://localhost:11434/api/generate -d '{
}'
```
+#### Request (Reproducible outputs)
+
+For reproducible outputs, set `seed` to a number:
+
+##### Request
+
+```shell
+curl http://localhost:11434/api/generate -d '{
+ "model": "mistral",
+ "prompt": "Why is the sky blue?",
+ "options": {
+ "seed": 123
+ }
+}'
+```
+
##### Response
```json
@@ -271,7 +327,7 @@ If you want to set custom options for the model at runtime rather than in the Mo
```shell
curl http://localhost:11434/api/generate -d '{
- "model": "llama2",
+ "model": "llama3.2",
"prompt": "Why is the sky blue?",
"stream": false,
"options": {
@@ -280,6 +336,7 @@ curl http://localhost:11434/api/generate -d '{
"num_predict": 100,
"top_k": 20,
"top_p": 0.9,
+ "min_p": 0.0,
"tfs_z": 0.5,
"typical_p": 0.7,
"repeat_last_n": 33,
@@ -295,7 +352,6 @@ curl http://localhost:11434/api/generate -d '{
"numa": false,
"num_ctx": 1024,
"num_batch": 2,
- "num_gqa": 1,
"num_gpu": 1,
"main_gpu": 0,
"low_vram": false,
@@ -303,9 +359,6 @@ curl http://localhost:11434/api/generate -d '{
"vocab_only": false,
"use_mmap": true,
"use_mlock": false,
- "embedding_only": false,
- "rope_frequency_base": 1.1,
- "rope_frequency_scale": 0.8,
"num_thread": 8
}
}'
@@ -315,7 +368,7 @@ curl http://localhost:11434/api/generate -d '{
```json
{
- "model": "llama2",
+ "model": "llama3.2",
"created_at": "2023-08-04T19:22:45.499127Z",
"response": "The sky is blue because it is the color of the sky.",
"done": true,
@@ -337,7 +390,7 @@ If an empty prompt is provided, the model will be loaded into memory.
```shell
curl http://localhost:11434/api/generate -d '{
- "model": "llama2"
+ "model": "llama3.2"
}'
```
@@ -347,13 +400,40 @@ A single JSON object is returned:
```json
{
- "model": "llama2",
+ "model": "llama3.2",
"created_at": "2023-12-18T19:52:07.071755Z",
"response": "",
"done": true
}
```
+#### Unload a model
+
+If an empty prompt is provided and the `keep_alive` parameter is set to `0`, a model will be unloaded from memory.
+
+##### Request
+
+```shell
+curl http://localhost:11434/api/generate -d '{
+ "model": "llama3.2",
+ "keep_alive": 0
+}'
+```
+
+##### Response
+
+A single JSON object is returned:
+
+```json
+{
+ "model": "llama3.2",
+ "created_at": "2024-09-12T03:54:03.516566Z",
+ "response": "",
+ "done": true,
+ "done_reason": "unload"
+}
+```
+
## Generate a chat completion
```shell
@@ -366,19 +446,21 @@ Generate the next message in a chat with a provided model. This is a streaming e
- `model`: (required) the [model name](#model-names)
- `messages`: the messages of the chat, this can be used to keep a chat memory
+- `tools`: tools for the model to use if supported. Requires `stream` to be set to `false`
The `message` object has the following fields:
-- `role`: the role of the message, either `system`, `user` or `assistant`
+- `role`: the role of the message, either `system`, `user`, `assistant`, or `tool`
- `content`: the content of the message
- `images` (optional): a list of images to include in the message (for multimodal models such as `llava`)
+- `tool_calls` (optional): a list of tools the model wants to use
Advanced parameters (optional):
- `format`: the format to return a response in. Currently the only accepted value is `json`
- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
-- `template`: the full prompt or prompt template (overrides what is defined in the `Modelfile`)
- `stream`: if `false` the response will be returned as a single response object, rather than a stream of objects
+- `keep_alive`: controls how long the model will stay loaded into memory following the request (default: `5m`)
### Examples
@@ -390,7 +472,7 @@ Send a chat message with a streaming response.
```shell
curl http://localhost:11434/api/chat -d '{
- "model": "llama2",
+ "model": "llama3.2",
"messages": [
{
"role": "user",
@@ -406,10 +488,10 @@ A stream of JSON objects is returned:
```json
{
- "model": "llama2",
+ "model": "llama3.2",
"created_at": "2023-08-04T08:52:19.385406455-07:00",
"message": {
- "role": "assisant",
+ "role": "assistant",
"content": "The",
"images": null
},
@@ -421,7 +503,7 @@ Final response:
```json
{
- "model": "llama2",
+ "model": "llama3.2",
"created_at": "2023-08-04T19:22:45.499127Z",
"done": true,
"total_duration": 4883583458,
@@ -439,7 +521,7 @@ Final response:
```shell
curl http://localhost:11434/api/chat -d '{
- "model": "llama2",
+ "model": "llama3.2",
"messages": [
{
"role": "user",
@@ -454,7 +536,7 @@ curl http://localhost:11434/api/chat -d '{
```json
{
- "model": "registry.ollama.ai/library/llama2:latest",
+ "model": "llama3.2",
"created_at": "2023-12-12T14:13:43.416799Z",
"message": {
"role": "assistant",
@@ -478,7 +560,7 @@ Send a chat message with a conversation history. You can use this same approach
```shell
curl http://localhost:11434/api/chat -d '{
- "model": "llama2",
+ "model": "llama3.2",
"messages": [
{
"role": "user",
@@ -502,10 +584,10 @@ A stream of JSON objects is returned:
```json
{
- "model": "llama2",
+ "model": "llama3.2",
"created_at": "2023-08-04T08:52:19.385406455-07:00",
"message": {
- "role": "assisant",
+ "role": "assistant",
"content": "The"
},
"done": false
@@ -516,7 +598,7 @@ Final response:
```json
{
- "model": "llama2",
+ "model": "llama3.2",
"created_at": "2023-08-04T19:22:45.499127Z",
"done": true,
"total_duration": 8113331500,
@@ -532,7 +614,7 @@ Final response:
##### Request
-Send a chat message with a conversation history.
+Send a chat message with images. The images should be provided as an array, with the individual images encoded in Base64.
```shell
curl http://localhost:11434/api/chat -d '{
@@ -542,7 +624,7 @@ curl http://localhost:11434/api/chat -d '{
"role": "user",
"content": "what is in this image?",
"images": ["iVBORw0KGgoAAAANSUhEUgAAAG0AAABmCAYAAADBPx+VAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAA3VSURBVHgB7Z27r0zdG8fX743i1bi1ikMoFMQloXRpKFFIqI7LH4BEQ+NWIkjQuSWCRIEoULk0gsK1kCBI0IhrQVT7tz/7zZo888yz1r7MnDl7z5xvsjkzs2fP3uu71nNfa7lkAsm7d++Sffv2JbNmzUqcc8m0adOSzZs3Z+/XES4ZckAWJEGWPiCxjsQNLWmQsWjRIpMseaxcuTKpG/7HP27I8P79e7dq1ars/yL4/v27S0ejqwv+cUOGEGGpKHR37tzJCEpHV9tnT58+dXXCJDdECBE2Ojrqjh071hpNECjx4cMHVycM1Uhbv359B2F79+51586daxN/+pyRkRFXKyRDAqxEp4yMlDDzXG1NPnnyJKkThoK0VFd1ELZu3TrzXKxKfW7dMBQ6bcuWLW2v0VlHjx41z717927ba22U9APcw7Nnz1oGEPeL3m3p2mTAYYnFmMOMXybPPXv2bNIPpFZr1NHn4HMw0KRBjg9NuRw95s8PEcz/6DZELQd/09C9QGq5RsmSRybqkwHGjh07OsJSsYYm3ijPpyHzoiacg35MLdDSIS/O1yM778jOTwYUkKNHWUzUWaOsylE00MyI0fcnOwIdjvtNdW/HZwNLGg+sR1kMepSNJXmIwxBZiG8tDTpEZzKg0GItNsosY8USkxDhD0Rinuiko2gfL/RbiD2LZAjU9zKQJj8RDR0vJBR1/Phx9+PHj9Z7REF4nTZkxzX4LCXHrV271qXkBAPGfP/atWvu/PnzHe4C97F48eIsRLZ9+3a3f/9+87dwP1JxaF7/3r17ba+5l4EcaVo0lj3SBq5kGTJSQmLWMjgYNei2GPT1MuMqGTDEFHzeQSP2wi/jGnkmPJ/nhccs44jvDAxpVcxnq0F6eT8h4ni/iIWpR5lPyA6ETkNXoSukvpJAD3AsXLiwpZs49+fPn5ke4j10TqYvegSfn0OnafC+Tv9ooA/JPkgQysqQNBzagXY55nO/oa1F7qvIPWkRL12WRpMWUvpVDYmxAPehxWSe8ZEXL20sadYIozfmNch4QJPAfeJgW3rNsnzphBKNJM2KKODo1rVOMRYik5ETy3ix4qWNI81qAAirizgMIc+yhTytx0JWZuNI03qsrgWlGtwjoS9XwgUhWGyhUaRZZQNNIEwCiXD16tXcAHUs79co0vSD8rrJCIW98pzvxpAWyyo3HYwqS0+H0BjStClcZJT5coMm6D2LOF8TolGJtK9fvyZpyiC5ePFi9nc/oJU4eiEP0jVoAnHa9wyJycITMP78+eMeP37sXrx44d6+fdt6f82aNdkx1pg9e3Zb5W+RSRE+n+VjksQWifvVaTKFhn5O8my63K8Qabdv33b379/PiAP//vuvW7BggZszZ072/+TJk91YgkafPn166zXB1rQHFvouAWHq9z3SEevSUerqCn2/dDCeta2jxYbr69evk4MHDyY7d+7MjhMnTiTPnz9Pfv/+nfQT2ggpO2dMF8cghuoM7Ygj5iWCqRlGFml0QC/ftGmTmzt3rmsaKDsgBSPh0/8yPeLLBihLkOKJc0jp8H8vUzcxIA1k6QJ/c78tWEyj5P3o4u9+jywNPdJi5rAH9x0KHcl4Hg570eQp3+vHXGyrmEeigzQsQsjavXt38ujRo44LQuDDhw+TW7duRS1HGgMxhNXHgflaNTOsHyKvHK5Ijo2jbFjJBQK9YwFd6RVMzfgRBmEfP37suBBm/p49e1qjEP2mwTViNRo0VJWH1deMXcNK08uUjVUu7s/zRaL+oLNxz1bpANco4npUgX4G2eFbpDFyQoQxojBCpEGSytmOH8qrH5Q9vuzD6ofQylkCUmh8DBAr+q8JCyVNtWQIidKQE9wNtLSQnS4jDSsxNHogzFuQBw4cyM61UKVsjfr3ooBkPSqqQHesUPWVtzi9/vQi1T+rJj7WiTz4Pt/l3LxUkr5P2VYZaZ4URpsE+st/dujQoaBBYokbrz/8TJNQYLSonrPS9kUaSkPeZyj1AWSj+d+VBoy1pIWVNed8P0Ll/ee5HdGRhrHhR5GGN0r4LGZBaj8oFDJitBTJzIZgFcmU0Y8ytWMZMzJOaXUSrUs5RxKnrxmbb5YXO9VGUhtpXldhEUogFr3IzIsvlpmdosVcGVGXFWp2oU9kLFL3dEkSz6NHEY1sjSRdIuDFWEhd8KxFqsRi1uM/nz9/zpxnwlESONdg6dKlbsaMGS4EHFHtjFIDHwKOo46l4TxSuxgDzi+rE2jg+BaFruOX4HXa0Nnf1lwAPufZeF8/r6zD97WK2qFnGjBxTw5qNGPxT+5T/r7/7RawFC3j4vTp09koCxkeHjqbHJqArmH5UrFKKksnxrK7FuRIs8STfBZv+luugXZ2pR/pP9Ois4z+TiMzUUkUjD0iEi1fzX8GmXyuxUBRcaUfykV0YZnlJGKQpOiGB76x5GeWkWWJc3mOrK6S7xdND+W5N6XyaRgtWJFe13GkaZnKOsYqGdOVVVbGupsyA/l7emTLHi7vwTdirNEt0qxnzAvBFcnQF16xh/TMpUuXHDowhlA9vQVraQhkudRdzOnK+04ZSP3DUhVSP61YsaLtd/ks7ZgtPcXqPqEafHkdqa84X6aCeL7YWlv6edGFHb+ZFICPlljHhg0bKuk0CSvVznWsotRu433alNdFrqG45ejoaPCaUkWERpLXjzFL2Rpllp7PJU2a/v7Ab8N05/9t27Z16KUqoFGsxnI9EosS2niSYg9SpU6B4JgTrvVW1flt1sT+0ADIJU2maXzcUTraGCRaL1Wp9rUMk16PMom8QhruxzvZIegJjFU7LLCePfS8uaQdPny4jTTL0dbee5mYokQsXTIWNY46kuMbnt8Kmec+LGWtOVIl9cT1rCB0V8WqkjAsRwta93TbwNYoGKsUSChN44lgBNCoHLHzquYKrU6qZ8lolCIN0Rh6cP0Q3U6I6IXILYOQI513hJaSKAorFpuHXJNfVlpRtmYBk1Su1obZr5dnKAO+L10Hrj3WZW+E3qh6IszE37F6EB+68mGpvKm4eb9bFrlzrok7fvr0Kfv727dvWRmdVTJHw0qiiCUSZ6wCK+7XL/AcsgNyL74DQQ730sv78Su7+t/A36MdY0sW5o40ahslXr58aZ5HtZB8GH64m9EmMZ7FpYw4T6QnrZfgenrhFxaSiSGXtPnz57e9TkNZLvTjeqhr734CNtrK41L40sUQckmj1lGKQ0rC37x544r8eNXRpnVE3ZZY7zXo8NomiO0ZUCj2uHz58rbXoZ6gc0uA+F6ZeKS/jhRDUq8MKrTho9fEkihMmhxtBI1DxKFY9XLpVcSkfoi8JGnToZO5sU5aiDQIW716ddt7ZLYtMQlhECdBGXZZMWldY5BHm5xgAroWj4C0hbYkSc/jBmggIrXJWlZM6pSETsEPGqZOndr2uuuR5rF169a2HoHPdurUKZM4CO1WTPqaDaAd+GFGKdIQkxAn9RuEWcTRyN2KSUgiSgF5aWzPTeA/lN5rZubMmR2bE4SIC4nJoltgAV/dVefZm72AtctUCJU2CMJ327hxY9t7EHbkyJFseq+EJSY16RPo3Dkq1kkr7+q0bNmyDuLQcZBEPYmHVdOBiJyIlrRDq41YPWfXOxUysi5fvtyaj+2BpcnsUV/oSoEMOk2CQGlr4ckhBwaetBhjCwH0ZHtJROPJkyc7UjcYLDjmrH7ADTEBXFfOYmB0k9oYBOjJ8b4aOYSe7QkKcYhFlq3QYLQhSidNmtS2RATwy8YOM3EQJsUjKiaWZ+vZToUQgzhkHXudb/PW5YMHD9yZM2faPsMwoc7RciYJXbGuBqJ1UIGKKLv915jsvgtJxCZDubdXr165mzdvtr1Hz5LONA8jrUwKPqsmVesKa49S3Q4WxmRPUEYdTjgiUcfUwLx589ySJUva3oMkP6IYddq6HMS4o55xBJBUeRjzfa4Zdeg56QZ43LhxoyPo7Lf1kNt7oO8wWAbNwaYjIv5lhyS7kRf96dvm5Jah8vfvX3flyhX35cuX6HfzFHOToS1H4BenCaHvO8pr8iDuwoUL7tevX+b5ZdbBair0xkFIlFDlW4ZknEClsp/TzXyAKVOmmHWFVSbDNw1l1+4f90U6IY/q4V27dpnE9bJ+v87QEydjqx/UamVVPRG+mwkNTYN+9tjkwzEx+atCm/X9WvWtDtAb68Wy9LXa1UmvCDDIpPkyOQ5ZwSzJ4jMrvFcr0rSjOUh+GcT4LSg5ugkW1Io0/SCDQBojh0hPlaJdah+tkVYrnTZowP8iq1F1TgMBBauufyB33x1v+NWFYmT5KmppgHC+NkAgbmRkpD3yn9QIseXymoTQFGQmIOKTxiZIWpvAatenVqRVXf2nTrAWMsPnKrMZHz6bJq5jvce6QK8J1cQNgKxlJapMPdZSR64/UivS9NztpkVEdKcrs5alhhWP9NeqlfWopzhZScI6QxseegZRGeg5a8C3Re1Mfl1ScP36ddcUaMuv24iOJtz7sbUjTS4qBvKmstYJoUauiuD3k5qhyr7QdUHMeCgLa1Ear9NquemdXgmum4fvJ6w1lqsuDhNrg1qSpleJK7K3TF0Q2jSd94uSZ60kK1e3qyVpQK6PVWXp2/FC3mp6jBhKKOiY2h3gtUV64TWM6wDETRPLDfSakXmH3w8g9Jlug8ZtTt4kVF0kLUYYmCCtD/DrQ5YhMGbA9L3ucdjh0y8kOHW5gU/VEEmJTcL4Pz/f7mgoAbYkAAAAAElFTkSuQmCC"]
- },
+ }
]
}'
```
@@ -568,6 +650,177 @@ curl http://localhost:11434/api/chat -d '{
}
```
+#### Chat request (Reproducible outputs)
+
+##### Request
+
+```shell
+curl http://localhost:11434/api/chat -d '{
+ "model": "llama3.2",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Hello!"
+ }
+ ],
+ "options": {
+ "seed": 101,
+ "temperature": 0
+ }
+}'
+```
+
+##### Response
+
+```json
+{
+ "model": "llama3.2",
+ "created_at": "2023-12-12T14:13:43.416799Z",
+ "message": {
+ "role": "assistant",
+ "content": "Hello! How are you today?"
+ },
+ "done": true,
+ "total_duration": 5191566416,
+ "load_duration": 2154458,
+ "prompt_eval_count": 26,
+ "prompt_eval_duration": 383809000,
+ "eval_count": 298,
+ "eval_duration": 4799921000
+}
+```
+
+#### Chat request (with tools)
+
+##### Request
+
+```
+curl http://localhost:11434/api/chat -d '{
+ "model": "llama3.2",
+ "messages": [
+ {
+ "role": "user",
+ "content": "What is the weather today in Paris?"
+ }
+ ],
+ "stream": false,
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_current_weather",
+ "description": "Get the current weather for a location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The location to get the weather for, e.g. San Francisco, CA"
+ },
+ "format": {
+ "type": "string",
+ "description": "The format to return the weather in, e.g. 'celsius' or 'fahrenheit'",
+ "enum": ["celsius", "fahrenheit"]
+ }
+ },
+ "required": ["location", "format"]
+ }
+ }
+ }
+ ]
+}'
+```
+
+##### Response
+
+```json
+{
+ "model": "llama3.2",
+ "created_at": "2024-07-22T20:33:28.123648Z",
+ "message": {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "function": {
+ "name": "get_current_weather",
+ "arguments": {
+ "format": "celsius",
+ "location": "Paris, FR"
+ }
+ }
+ }
+ ]
+ },
+ "done_reason": "stop",
+ "done": true,
+ "total_duration": 885095291,
+ "load_duration": 3753500,
+ "prompt_eval_count": 122,
+ "prompt_eval_duration": 328493000,
+ "eval_count": 33,
+ "eval_duration": 552222000
+}
+```
+
+#### Load a model
+
+If the messages array is empty, the model will be loaded into memory.
+
+##### Request
+
+```
+curl http://localhost:11434/api/chat -d '{
+ "model": "llama3.2",
+ "messages": []
+}'
+```
+
+##### Response
+```json
+{
+ "model": "llama3.2",
+ "created_at":"2024-09-12T21:17:29.110811Z",
+ "message": {
+ "role": "assistant",
+ "content": ""
+ },
+ "done_reason": "load",
+ "done": true
+}
+```
+
+#### Unload a model
+
+If the messages array is empty and the `keep_alive` parameter is set to `0`, a model will be unloaded from memory.
+
+##### Request
+
+```
+curl http://localhost:11434/api/chat -d '{
+ "model": "llama3.2",
+ "messages": [],
+ "keep_alive": 0
+}'
+```
+
+##### Response
+
+A single JSON object is returned:
+
+```json
+{
+ "model": "llama3.2",
+ "created_at":"2024-09-12T21:33:17.547535Z",
+ "message": {
+ "role": "assistant",
+ "content": ""
+ },
+ "done_reason": "unload",
+ "done": true
+}
+```
+
## Create a Model
```shell
@@ -594,7 +847,7 @@ Create a new model from a `Modelfile`.
```shell
curl http://localhost:11434/api/create -d '{
"name": "mario",
- "modelfile": "FROM llama2\nSYSTEM You are mario from Super Mario Bros."
+ "modelfile": "FROM llama3\nSYSTEM You are mario from Super Mario Bros."
}'
```
@@ -701,7 +954,7 @@ A single JSON object will be returned.
}
},
{
- "name": "llama2:latest",
+ "name": "llama3:latest",
"modified_at": "2023-12-07T09:32:18.757212583-08:00",
"size": 3825819519,
"digest": "fe938a131f40e6f6d40083c9f0f430a515233eb2edaa6d72eb85c50d64f2300e",
@@ -723,11 +976,12 @@ A single JSON object will be returned.
POST /api/show
```
-Show information about a model including details, modelfile, template, parameters, license, and system prompt.
+Show information about a model including details, modelfile, template, parameters, license, system prompt.
### Parameters
- `name`: name of the model to show
+- `verbose`: (optional) if set to `true`, returns full data for verbose response fields
### Examples
@@ -735,7 +989,7 @@ Show information about a model including details, modelfile, template, parameter
```shell
curl http://localhost:11434/api/show -d '{
- "name": "llama2"
+ "name": "llama3.2"
}'
```
@@ -743,15 +997,41 @@ curl http://localhost:11434/api/show -d '{
```json
{
- "modelfile": "# Modelfile generated by \"ollama show\"\n# To build a new Modelfile based on this one, replace the FROM line with:\n# FROM llava:latest\n\nFROM /Users/matt/.ollama/models/blobs/sha256:200765e1283640ffbd013184bf496e261032fa75b99498a9613be4e94d63ad52\nTEMPLATE \"\"\"{{ .System }}\nUSER: {{ .Prompt }}\nASSSISTANT: \"\"\"\nPARAMETER num_ctx 4096\nPARAMETER stop \"\u003c/s\u003e\"\nPARAMETER stop \"USER:\"\nPARAMETER stop \"ASSSISTANT:\"",
- "parameters": "num_ctx 4096\nstop \u003c/s\u003e\nstop USER:\nstop ASSSISTANT:",
- "template": "{{ .System }}\nUSER: {{ .Prompt }}\nASSSISTANT: ",
+ "modelfile": "# Modelfile generated by \"ollama show\"\n# To build a new Modelfile based on this one, replace the FROM line with:\n# FROM llava:latest\n\nFROM /Users/matt/.ollama/models/blobs/sha256:200765e1283640ffbd013184bf496e261032fa75b99498a9613be4e94d63ad52\nTEMPLATE \"\"\"{{ .System }}\nUSER: {{ .Prompt }}\nASSISTANT: \"\"\"\nPARAMETER num_ctx 4096\nPARAMETER stop \"\u003c/s\u003e\"\nPARAMETER stop \"USER:\"\nPARAMETER stop \"ASSISTANT:\"",
+ "parameters": "num_keep 24\nstop \"<|start_header_id|>\"\nstop \"<|end_header_id|>\"\nstop \"<|eot_id|>\"",
+ "template": "{{ if .System }}<|start_header_id|>system<|end_header_id|>\n\n{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>\n\n{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>\n\n{{ .Response }}<|eot_id|>",
"details": {
+ "parent_model": "",
"format": "gguf",
"family": "llama",
- "families": ["llama", "clip"],
- "parameter_size": "7B",
+ "families": [
+ "llama"
+ ],
+ "parameter_size": "8.0B",
"quantization_level": "Q4_0"
+ },
+ "model_info": {
+ "general.architecture": "llama",
+ "general.file_type": 2,
+ "general.parameter_count": 8030261248,
+ "general.quantization_version": 2,
+ "llama.attention.head_count": 32,
+ "llama.attention.head_count_kv": 8,
+ "llama.attention.layer_norm_rms_epsilon": 0.00001,
+ "llama.block_count": 32,
+ "llama.context_length": 8192,
+ "llama.embedding_length": 4096,
+ "llama.feed_forward_length": 14336,
+ "llama.rope.dimension_count": 128,
+ "llama.rope.freq_base": 500000,
+ "llama.vocab_size": 128256,
+ "tokenizer.ggml.bos_token_id": 128000,
+ "tokenizer.ggml.eos_token_id": 128009,
+ "tokenizer.ggml.merges": [], // populates if `verbose=true`
+ "tokenizer.ggml.model": "gpt2",
+ "tokenizer.ggml.pre": "llama-bpe",
+ "tokenizer.ggml.token_type": [], // populates if `verbose=true`
+ "tokenizer.ggml.tokens": [] // populates if `verbose=true`
}
}
```
@@ -770,8 +1050,8 @@ Copy a model. Creates a model with another name from an existing model.
```shell
curl http://localhost:11434/api/copy -d '{
- "source": "llama2",
- "destination": "llama2-backup"
+ "source": "llama3.2",
+ "destination": "llama3-backup"
}'
```
@@ -797,7 +1077,7 @@ Delete a model and its data.
```shell
curl -X DELETE http://localhost:11434/api/delete -d '{
- "name": "llama2:13b"
+ "name": "llama3:13b"
}'
```
@@ -825,7 +1105,7 @@ Download a model from the ollama library. Cancelled pulls are resumed from where
```shell
curl http://localhost:11434/api/pull -d '{
- "name": "llama2"
+ "name": "llama3.2"
}'
```
@@ -944,6 +1224,121 @@ If `stream` is set to `false`, then the response is a single JSON object:
## Generate Embeddings
+```shell
+POST /api/embed
+```
+
+Generate embeddings from a model
+
+### Parameters
+
+- `model`: name of model to generate embeddings from
+- `input`: text or list of text to generate embeddings for
+
+Advanced parameters:
+
+- `truncate`: truncates the end of each input to fit within context length. Returns error if `false` and context length is exceeded. Defaults to `true`
+- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
+- `keep_alive`: controls how long the model will stay loaded into memory following the request (default: `5m`)
+
+### Examples
+
+#### Request
+
+```shell
+curl http://localhost:11434/api/embed -d '{
+ "model": "all-minilm",
+ "input": "Why is the sky blue?"
+}'
+```
+
+#### Response
+
+```json
+{
+ "model": "all-minilm",
+ "embeddings": [[
+ 0.010071029, -0.0017594862, 0.05007221, 0.04692972, 0.054916814,
+ 0.008599704, 0.105441414, -0.025878139, 0.12958129, 0.031952348
+ ]],
+ "total_duration": 14143917,
+ "load_duration": 1019500,
+ "prompt_eval_count": 8
+}
+```
+
+#### Request (Multiple input)
+
+```shell
+curl http://localhost:11434/api/embed -d '{
+ "model": "all-minilm",
+ "input": ["Why is the sky blue?", "Why is the grass green?"]
+}'
+```
+
+#### Response
+
+```json
+{
+ "model": "all-minilm",
+ "embeddings": [[
+ 0.010071029, -0.0017594862, 0.05007221, 0.04692972, 0.054916814,
+ 0.008599704, 0.105441414, -0.025878139, 0.12958129, 0.031952348
+ ],[
+ -0.0098027075, 0.06042469, 0.025257962, -0.006364387, 0.07272725,
+ 0.017194884, 0.09032035, -0.051705178, 0.09951512, 0.09072481
+ ]]
+}
+```
+
+## List Running Models
+```shell
+GET /api/ps
+```
+
+List models that are currently loaded into memory.
+
+#### Examples
+
+### Request
+
+```shell
+curl http://localhost:11434/api/ps
+```
+
+#### Response
+
+A single JSON object will be returned.
+
+```json
+{
+ "models": [
+ {
+ "name": "mistral:latest",
+ "model": "mistral:latest",
+ "size": 5137025024,
+ "digest": "2ae6f6dd7a3dd734790bbbf58b8909a606e0e7e97e94b7604e0aa7ae4490e6d8",
+ "details": {
+ "parent_model": "",
+ "format": "gguf",
+ "family": "llama",
+ "families": [
+ "llama"
+ ],
+ "parameter_size": "7.2B",
+ "quantization_level": "Q4_0"
+ },
+ "expires_at": "2024-06-04T14:38:31.83753-07:00",
+ "size_vram": 5137025024
+ }
+ ]
+}
+```
+
+## Generate Embedding
+
+> Note: this endpoint has been superseded by `/api/embed`
+
```shell
POST /api/embeddings
```
@@ -958,6 +1353,7 @@ Generate embeddings from a model
Advanced parameters:
- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
+- `keep_alive`: controls how long the model will stay loaded into memory following the request (default: `5m`)
### Examples
@@ -965,7 +1361,7 @@ Advanced parameters:
```shell
curl http://localhost:11434/api/embeddings -d '{
- "model": "llama2",
+ "model": "all-minilm",
"prompt": "Here is an article about llamas..."
}'
```
diff --git a/docs/development.md b/docs/development.md
index 594787d7..732aa5ff 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -1,15 +1,16 @@
# Development
-- Install cmake or (optionally, required tools for GPUs)
-- run `go generate ./...`
-- run `go build .`
+> [!IMPORTANT]
+> The `llm` package that loads and runs models is being updated to use a new [Go runner](#transition-to-go-runner): this should only impact a small set of PRs however it does change how the project is built.
Install required tools:
- cmake version 3.24 or higher
-- go version 1.20 or higher
+- go version 1.22 or higher
- gcc version 11.4.0 or higher
+### MacOS
+
```bash
brew install go cmake gcc
```
@@ -17,7 +18,11 @@ brew install go cmake gcc
Optionally enable debugging and more verbose logging:
```bash
+# At build time
export CGO_CFLAGS="-g"
+
+# At runtime
+export OLLAMA_DEBUG=1
```
Get the required libraries and build the native LLM code:
@@ -38,37 +43,308 @@ Now you can run `ollama`:
./ollama
```
-## Building on Linux with GPU support
+### Linux
+#### Linux CUDA (NVIDIA)
-### Linux/Windows CUDA (NVIDIA)
-*Your operating system distribution may already have packages for NVIDIA CUDA. Distro packages are often preferable, but instructions are distro-specific. Please consult distro-specific docs for dependencies if available!*
+_Your operating system distribution may already have packages for NVIDIA CUDA. Distro packages are often preferable, but instructions are distro-specific. Please consult distro-specific docs for dependencies if available!_
+
+Install `cmake` and `golang` as well as [NVIDIA CUDA](https://developer.nvidia.com/cuda-downloads)
+development and runtime packages.
+
+Typically the build scripts will auto-detect CUDA, however, if your Linux distro
+or installation approach uses unusual paths, you can specify the location by
+specifying an environment variable `CUDA_LIB_DIR` to the location of the shared
+libraries, and `CUDACXX` to the location of the nvcc compiler. You can customize
+a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "50;60;70")
-Install `cmake` and `golang` as well as [NVIDIA CUDA](https://developer.nvidia.com/cuda-downloads) development and runtime packages.
Then generate dependencies:
+
```
go generate ./...
```
+
Then build the binary:
+
```
go build .
```
-### Linux ROCm (AMD)
-*Your operating system distribution may already have packages for AMD ROCm and CLBlast. Distro packages are often preferable, but instructions are distro-specific. Please consult distro-specific docs for dependencies if available!*
+#### Linux ROCm (AMD)
+
+_Your operating system distribution may already have packages for AMD ROCm and CLBlast. Distro packages are often preferable, but instructions are distro-specific. Please consult distro-specific docs for dependencies if available!_
+
+Install [CLBlast](https://github.com/CNugteren/CLBlast/blob/master/doc/installation.md) and [ROCm](https://rocm.docs.amd.com/en/latest/) development packages first, as well as `cmake` and `golang`.
+
+Typically the build scripts will auto-detect ROCm, however, if your Linux distro
+or installation approach uses unusual paths, you can specify the location by
+specifying an environment variable `ROCM_PATH` to the location of the ROCm
+install (typically `/opt/rocm`), and `CLBlast_DIR` to the location of the
+CLBlast install (typically `/usr/lib/cmake/CLBlast`). You can also customize
+the AMD GPU targets by setting AMDGPU_TARGETS (e.g. `AMDGPU_TARGETS="gfx1101;gfx1102"`)
-Install [CLBlast](https://github.com/CNugteren/CLBlast/blob/master/doc/installation.md) and [ROCm](https://rocm.docs.amd.com/en/latest/deploy/linux/quick_start.html) developement packages first, as well as `cmake` and `golang`.
-Adjust the paths below (correct for Arch) as appropriate for your distributions install locations and generate dependencies:
```
-CLBlast_DIR=/usr/lib/cmake/CLBlast ROCM_PATH=/opt/rocm go generate ./...
+go generate ./...
```
+
Then build the binary:
+
```
go build .
```
-ROCm requires elevated privileges to access the GPU at runtime. On most distros you can add your user account to the `render` group, or run as root.
+ROCm requires elevated privileges to access the GPU at runtime. On most distros you can add your user account to the `render` group, or run as root.
-## Containerized Build
+#### Advanced CPU Settings
-If you have Docker available, you can build linux binaries with `./scripts/build_linux.sh` which has the CUDA and ROCm dependencies included.
\ No newline at end of file
+By default, running `go generate ./...` will compile a few different variations
+of the LLM library based on common CPU families and vector math capabilities,
+including a lowest-common-denominator which should run on almost any 64 bit CPU
+somewhat slowly. At runtime, Ollama will auto-detect the optimal variation to
+load. If you would like to build a CPU-based build customized for your
+processor, you can set `OLLAMA_CUSTOM_CPU_DEFS` to the llama.cpp flags you would
+like to use. For example, to compile an optimized binary for an Intel i9-9880H,
+you might use:
+
+```
+OLLAMA_CUSTOM_CPU_DEFS="-DGGML_AVX=on -DGGML_AVX2=on -DGGML_F16C=on -DGGML_FMA=on" go generate ./...
+go build .
+```
+
+#### Containerized Linux Build
+
+If you have Docker available, you can build linux binaries with `./scripts/build_linux.sh` which has the CUDA and ROCm dependencies included. The resulting binary is placed in `./dist`
+
+### Windows
+
+Note: The Windows build for Ollama is still under development.
+
+First, install required tools:
+
+- MSVC toolchain - C/C++ and cmake as minimal requirements
+- Go version 1.22 or higher
+- MinGW (pick one variant) with GCC.
+ - [MinGW-w64](https://www.mingw-w64.org/)
+ - [MSYS2](https://www.msys2.org/)
+- The `ThreadJob` Powershell module: `Install-Module -Name ThreadJob -Scope CurrentUser`
+
+Then, build the `ollama` binary:
+
+```powershell
+$env:CGO_ENABLED="1"
+go generate ./...
+go build .
+```
+
+#### Windows CUDA (NVIDIA)
+
+In addition to the common Windows development tools described above, install CUDA after installing MSVC.
+
+- [NVIDIA CUDA](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html)
+
+
+#### Windows ROCm (AMD Radeon)
+
+In addition to the common Windows development tools described above, install AMDs HIP package after installing MSVC.
+
+- [AMD HIP](https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html)
+- [Strawberry Perl](https://strawberryperl.com/)
+
+Lastly, add `ninja.exe` included with MSVC to the system path (e.g. `C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja`).
+
+#### Windows arm64
+
+The default `Developer PowerShell for VS 2022` may default to x86 which is not what you want. To ensure you get an arm64 development environment, start a plain PowerShell terminal and run:
+
+```powershell
+import-module 'C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\Tools\\Microsoft.VisualStudio.DevShell.dll'
+Enter-VsDevShell -Arch arm64 -vsinstallpath 'C:\\Program Files\\Microsoft Visual Studio\\2022\\Community' -skipautomaticlocation
+```
+
+You can confirm with `write-host $env:VSCMD_ARG_TGT_ARCH`
+
+Follow the instructions at https://www.msys2.org/wiki/arm64/ to set up an arm64 msys2 environment. Ollama requires gcc and mingw32-make to compile, which is not currently available on Windows arm64, but a gcc compatibility adapter is available via `mingw-w64-clang-aarch64-gcc-compat`. At a minimum you will need to install the following:
+
+```
+pacman -S mingw-w64-clang-aarch64-clang mingw-w64-clang-aarch64-gcc-compat mingw-w64-clang-aarch64-make make
+```
+
+You will need to ensure your PATH includes go, cmake, gcc and clang mingw32-make to build ollama from source. (typically `C:\msys64\clangarm64\bin\`)
+
+
+## Transition to Go runner
+
+The Ollama team is working on moving to a new Go based runner that loads and runs models in a subprocess to replace the previous code under `ext_server`. During this transition period, this new Go runner is "opt in" at build time, and requires using a different approach to build.
+
+After the transition to use the Go server exclusively, both `make` and `go generate` will build the Go runner.
+
+Install required tools:
+
+- go version 1.22 or higher
+- gcc version 11.4.0 or higher
+
+
+### MacOS
+
+[Download Go](https://go.dev/dl/)
+
+Optionally enable debugging and more verbose logging:
+
+```bash
+# At build time
+export CGO_CFLAGS="-g"
+
+# At runtime
+export OLLAMA_DEBUG=1
+```
+
+Get the required libraries and build the native LLM code: (Adjust the job count based on your number of processors for a faster build)
+
+```bash
+make -C llama -j 5
+```
+
+Then build ollama:
+
+```bash
+go build .
+```
+
+Now you can run `ollama`:
+
+```bash
+./ollama
+```
+
+#### Xcode 15 warnings
+
+If you are using Xcode newer than version 14, you may see a warning during `go build` about `ld: warning: ignoring duplicate libraries: '-lobjc'` due to Golang issue https://github.com/golang/go/issues/67799 which can be safely ignored. You can suppress the warning with `export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries"`
+
+### Linux
+
+#### Linux CUDA (NVIDIA)
+
+_Your operating system distribution may already have packages for NVIDIA CUDA. Distro packages are often preferable, but instructions are distro-specific. Please consult distro-specific docs for dependencies if available!_
+
+Install `make`, `gcc` and `golang` as well as [NVIDIA CUDA](https://developer.nvidia.com/cuda-downloads)
+development and runtime packages.
+
+Typically the build scripts will auto-detect CUDA, however, if your Linux distro
+or installation approach uses unusual paths, you can specify the location by
+specifying an environment variable `CUDA_LIB_DIR` to the location of the shared
+libraries, and `CUDACXX` to the location of the nvcc compiler. You can customize
+a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "50;60;70")
+
+Then generate dependencies: (Adjust the job count based on your number of processors for a faster build)
+
+```
+make -C llama -j 5
+```
+
+Then build the binary:
+
+```
+go build .
+```
+
+#### Linux ROCm (AMD)
+
+_Your operating system distribution may already have packages for AMD ROCm and CLBlast. Distro packages are often preferable, but instructions are distro-specific. Please consult distro-specific docs for dependencies if available!_
+
+Install [CLBlast](https://github.com/CNugteren/CLBlast/blob/master/doc/installation.md) and [ROCm](https://rocm.docs.amd.com/en/latest/) development packages first, as well as `make`, `gcc`, and `golang`.
+
+Typically the build scripts will auto-detect ROCm, however, if your Linux distro
+or installation approach uses unusual paths, you can specify the location by
+specifying an environment variable `ROCM_PATH` to the location of the ROCm
+install (typically `/opt/rocm`), and `CLBlast_DIR` to the location of the
+CLBlast install (typically `/usr/lib/cmake/CLBlast`). You can also customize
+the AMD GPU targets by setting AMDGPU_TARGETS (e.g. `AMDGPU_TARGETS="gfx1101;gfx1102"`)
+
+Then generate dependencies: (Adjust the job count based on your number of processors for a faster build)
+
+```
+make -C llama -j 5
+```
+
+Then build the binary:
+
+```
+go build .
+```
+
+ROCm requires elevated privileges to access the GPU at runtime. On most distros you can add your user account to the `render` group, or run as root.
+
+#### Advanced CPU Settings
+
+By default, running `make` will compile a few different variations
+of the LLM library based on common CPU families and vector math capabilities,
+including a lowest-common-denominator which should run on almost any 64 bit CPU
+somewhat slowly. At runtime, Ollama will auto-detect the optimal variation to
+load.
+
+Custom CPU settings are not currently supported in the new Go server build but will be added back after we complete the transition.
+
+#### Containerized Linux Build
+
+If you have Docker available, you can build linux binaries with `OLLAMA_NEW_RUNNERS=1 ./scripts/build_linux.sh` which has the CUDA and ROCm dependencies included. The resulting binary is placed in `./dist`
+
+### Windows
+
+The following tools are required as a minimal development environment to build CPU inference support.
+
+- Go version 1.22 or higher
+ - https://go.dev/dl/
+- Git
+ - https://git-scm.com/download/win
+- GCC and Make. There are multiple options on how to go about installing these tools on Windows. We have verified the following, but others may work as well:
+ - [MSYS2](https://www.msys2.org/)
+ - After installing, from an MSYS2 terminal, run `pacman -S mingw-w64-ucrt-x86_64-gcc make` to install the required tools
+ - Assuming you used the default install prefix for msys2 above, add `c:\msys64\ucrt64\bin` and `c:\msys64\usr\bin` to your environment variable `PATH` where you will perform the build steps below (e.g. system-wide, account-level, powershell, cmd, etc.)
+
+Then, build the `ollama` binary:
+
+```powershell
+$env:CGO_ENABLED="1"
+make -C llama -j 8
+go build .
+```
+
+#### GPU Support
+
+The GPU tools require the Microsoft native build tools. To build either CUDA or ROCm, you must first install MSVC via Visual Studio:
+
+- Make sure to select `Desktop development with C++` as a Workload during the Visual Studio install
+- You must complete the Visual Studio install and run it once **BEFORE** installing CUDA or ROCm for the tools to properly register
+- Add the location of the **64 bit (x64)** compiler (`cl.exe`) to your `PATH`
+- Note: the default Developer Shell may configure the 32 bit (x86) compiler which will lead to build failures. Ollama requires a 64 bit toolchain.
+
+#### Windows CUDA (NVIDIA)
+
+In addition to the common Windows development tools and MSVC described above:
+
+- [NVIDIA CUDA](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html)
+
+#### Windows ROCm (AMD Radeon)
+
+In addition to the common Windows development tools and MSVC described above:
+
+- [AMD HIP](https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html)
+
+#### Windows arm64
+
+The default `Developer PowerShell for VS 2022` may default to x86 which is not what you want. To ensure you get an arm64 development environment, start a plain PowerShell terminal and run:
+
+```powershell
+import-module 'C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\Tools\\Microsoft.VisualStudio.DevShell.dll'
+Enter-VsDevShell -Arch arm64 -vsinstallpath 'C:\\Program Files\\Microsoft Visual Studio\\2022\\Community' -skipautomaticlocation
+```
+
+You can confirm with `write-host $env:VSCMD_ARG_TGT_ARCH`
+
+Follow the instructions at https://www.msys2.org/wiki/arm64/ to set up an arm64 msys2 environment. Ollama requires gcc and mingw32-make to compile, which is not currently available on Windows arm64, but a gcc compatibility adapter is available via `mingw-w64-clang-aarch64-gcc-compat`. At a minimum you will need to install the following:
+
+```
+pacman -S mingw-w64-clang-aarch64-clang mingw-w64-clang-aarch64-gcc-compat mingw-w64-clang-aarch64-make make
+```
+
+You will need to ensure your PATH includes go, cmake, gcc and clang mingw32-make to build ollama from source. (typically `C:\msys64\clangarm64\bin\`)
diff --git a/docs/docker.md b/docs/docker.md
new file mode 100644
index 00000000..9c758c38
--- /dev/null
+++ b/docs/docker.md
@@ -0,0 +1,71 @@
+# Ollama Docker image
+
+### CPU only
+
+```bash
+docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
+```
+
+### Nvidia GPU
+Install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#installation).
+
+#### Install with Apt
+1. Configure the repository
+```bash
+curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
+ | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
+ | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
+ | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
+sudo apt-get update
+```
+2. Install the NVIDIA Container Toolkit packages
+```bash
+sudo apt-get install -y nvidia-container-toolkit
+```
+
+#### Install with Yum or Dnf
+1. Configure the repository
+
+```bash
+curl -s -L https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo \
+ | sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo
+```
+
+2. Install the NVIDIA Container Toolkit packages
+
+```bash
+sudo yum install -y nvidia-container-toolkit
+```
+
+#### Configure Docker to use Nvidia driver
+```
+sudo nvidia-ctk runtime configure --runtime=docker
+sudo systemctl restart docker
+```
+
+#### Start the container
+
+```bash
+docker run -d --gpus=all -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
+```
+
+### AMD GPU
+
+To run Ollama using Docker with AMD GPUs, use the `rocm` tag and the following command:
+
+```
+docker run -d --device /dev/kfd --device /dev/dri -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama:rocm
+```
+
+### Run model locally
+
+Now you can run a model:
+
+```
+docker exec -it ollama ollama run llama3.2
+```
+
+### Try different models
+
+More models can be found on the [Ollama library](https://ollama.com/library).
diff --git a/docs/faq.md b/docs/faq.md
index d39e207c..0dbbb3ff 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -2,81 +2,119 @@
## How can I upgrade Ollama?
-To upgrade Ollama, run the installation process again. On the Mac, click the Ollama icon in the menubar and choose the restart option if an update is available.
+Ollama on macOS and Windows will automatically download updates. Click on the taskbar or menubar item and then click "Restart to update" to apply the update. Updates can also be installed by downloading the latest version [manually](https://ollama.com/download/).
+
+On Linux, re-run the install script:
+
+```shell
+curl -fsSL https://ollama.com/install.sh | sh
+```
## How can I view the logs?
Review the [Troubleshooting](./troubleshooting.md) docs for more about using logs.
-## How do I use Ollama server environment variables on Mac
+## Is my GPU compatible with Ollama?
-On macOS, Ollama runs in the background and is managed by the menubar app. If adding environment variables, Ollama will need to be run manually.
+Please refer to the [GPU docs](./gpu.md).
-1. Click the menubar icon for Ollama and choose **Quit Ollama**.
-2. Open a new terminal window and run the following command (this example uses `OLLAMA_HOST` with an IP address of `123.1.1.1`):
+## How can I specify the context window size?
- ```bash
- OLLAMA_HOST=123.1.1.1 ollama serve
- ```
+By default, Ollama uses a context window size of 2048 tokens.
-## How do I use Ollama server environment variables on Linux?
+To change this when using `ollama run`, use `/set parameter`:
-If Ollama is installed with the install script, a systemd service was created, running as the Ollama user. To add an environment variable, such as OLLAMA_HOST, follow these steps:
+```
+/set parameter num_ctx 4096
+```
-1. Create a `systemd` drop-in directory and add a config file. This is only needed once.
+When using the API, specify the `num_ctx` parameter:
- ```bash
- mkdir -p /etc/systemd/system/ollama.service.d
- echo '[Service]' >>/etc/systemd/system/ollama.service.d/environment.conf
- ```
+```shell
+curl http://localhost:11434/api/generate -d '{
+ "model": "llama3.2",
+ "prompt": "Why is the sky blue?",
+ "options": {
+ "num_ctx": 4096
+ }
+}'
+```
-2. For each environment variable, add it to the config file:
+## How can I tell if my model was loaded onto the GPU?
- ```bash
- echo 'Environment="OLLAMA_HOST=0.0.0.0:11434"' >>/etc/systemd/system/ollama.service.d/environment.conf
- ```
+Use the `ollama ps` command to see what models are currently loaded into memory.
-3. Reload `systemd` and restart Ollama:
+```shell
+ollama ps
+NAME ID SIZE PROCESSOR UNTIL
+llama3:70b bcfb190ca3a7 42 GB 100% GPU 4 minutes from now
+```
+
+The `Processor` column will show which memory the model was loaded in to:
+* `100% GPU` means the model was loaded entirely into the GPU
+* `100% CPU` means the model was loaded entirely in system memory
+* `48%/52% CPU/GPU` means the model was loaded partially onto both the GPU and into system memory
+
+## How do I configure Ollama server?
+
+Ollama server can be configured with environment variables.
+
+### Setting environment variables on Mac
+
+If Ollama is run as a macOS application, environment variables should be set using `launchctl`:
+
+1. For each environment variable, call `launchctl setenv`.
+
+ ```bash
+ launchctl setenv OLLAMA_HOST "0.0.0.0"
+ ```
+
+2. Restart Ollama application.
+
+### Setting environment variables on Linux
+
+If Ollama is run as a systemd service, environment variables should be set using `systemctl`:
+
+1. Edit the systemd service by calling `systemctl edit ollama.service`. This will open an editor.
+
+2. For each environment variable, add a line `Environment` under section `[Service]`:
+
+ ```ini
+ [Service]
+ Environment="OLLAMA_HOST=0.0.0.0"
+ ```
+
+3. Save and exit.
+
+4. Reload `systemd` and restart Ollama:
```bash
systemctl daemon-reload
systemctl restart ollama
```
-## How can I expose Ollama on my network?
+### Setting environment variables on Windows
-Ollama binds to 127.0.0.1 port 11434 by default. Change the bind address with the `OLLAMA_HOST` environment variable. Refer to the section above for how to use environment variables on your platform.
+On Windows, Ollama inherits your user and system environment variables.
-## How can I allow additional web origins to access Ollama?
+1. First Quit Ollama by clicking on it in the task bar.
-Ollama allows cross-origin requests from `127.0.0.1` and `0.0.0.0` by default. Add additional origins with the `OLLAMA_ORIGINS` environment variable. For example, to add all ports on 192.168.1.1 and https://example.com, use:
+2. Start the Settings (Windows 11) or Control Panel (Windows 10) application and search for _environment variables_.
-```shell
-OLLAMA_ORIGINS=http://192.168.1.1:*,https://example.com
-```
+3. Click on _Edit environment variables for your account_.
-Refer to the section above for how to use environment variables on your platform.
+4. Edit or create a new variable for your user account for `OLLAMA_HOST`, `OLLAMA_MODELS`, etc.
-## Where are models stored?
+5. Click OK/Apply to save.
-- macOS: `~/.ollama/models`.
-- Linux: `/usr/share/ollama/.ollama/models`
-
-## How do I set them to a different location?
-
-If a different directory needs to be used, set the environment variable `OLLAMA_MODELS` to the chosen directory. Refer to the section above for how to use environment variables on your platform.
-
-## Does Ollama send my prompts and answers back to Ollama.ai to use in any way?
-
-No, Ollama runs entirely locally, and conversation data will never leave your machine.
-
-## How can I use Ollama in Visual Studio Code?
-
-There is already a large collection of plugins available for VSCode as well as other editors that leverage Ollama. See the list of [extensions & plugins](https://github.com/jmorganca/ollama#extensions--plugins) at the bottom of the main repository readme.
+6. Start the Ollama application from the Windows Start menu.
## How do I use Ollama behind a proxy?
-Ollama is compatible with proxy servers if `HTTP_PROXY` or `HTTPS_PROXY` are configured. When using either variables, ensure it is set where `ollama serve` can access the values. When using `HTTPS_PROXY`, ensure the proxy certificate is installed as a system certificate. Refer to the section above for how to use environment variables on your platform.
+Ollama pulls models from the Internet and may require a proxy server to access the models. Use `HTTPS_PROXY` to redirect outbound requests through the proxy. Ensure the proxy certificate is installed as a system certificate. Refer to the section above for how to use environment variables on your platform.
+
+> [!NOTE]
+> Avoid setting `HTTP_PROXY`. Ollama does not use HTTP for model pulls, only HTTPS. Setting `HTTP_PROXY` may interrupt client connections to the server.
### How do I use Ollama behind a proxy in Docker?
@@ -99,6 +137,71 @@ docker build -t ollama-with-ca .
docker run -d -e HTTPS_PROXY=https://my.proxy.example.com -p 11434:11434 ollama-with-ca
```
+## Does Ollama send my prompts and answers back to ollama.com?
+
+No. Ollama runs locally, and conversation data does not leave your machine.
+
+## How can I expose Ollama on my network?
+
+Ollama binds 127.0.0.1 port 11434 by default. Change the bind address with the `OLLAMA_HOST` environment variable.
+
+Refer to the section [above](#how-do-i-configure-ollama-server) for how to set environment variables on your platform.
+
+## How can I use Ollama with a proxy server?
+
+Ollama runs an HTTP server and can be exposed using a proxy server such as Nginx. To do so, configure the proxy to forward requests and optionally set required headers (if not exposing Ollama on the network). For example, with Nginx:
+
+```
+server {
+ listen 80;
+ server_name example.com; # Replace with your domain or IP
+ location / {
+ proxy_pass http://localhost:11434;
+ proxy_set_header Host localhost:11434;
+ }
+}
+```
+
+## How can I use Ollama with ngrok?
+
+Ollama can be accessed using a range of tools for tunneling tools. For example with Ngrok:
+
+```shell
+ngrok http 11434 --host-header="localhost:11434"
+```
+
+## How can I use Ollama with Cloudflare Tunnel?
+
+To use Ollama with Cloudflare Tunnel, use the `--url` and `--http-host-header` flags:
+
+```shell
+cloudflared tunnel --url http://localhost:11434 --http-host-header="localhost:11434"
+```
+
+## How can I allow additional web origins to access Ollama?
+
+Ollama allows cross-origin requests from `127.0.0.1` and `0.0.0.0` by default. Additional origins can be configured with `OLLAMA_ORIGINS`.
+
+Refer to the section [above](#how-do-i-configure-ollama-server) for how to set environment variables on your platform.
+
+## Where are models stored?
+
+- macOS: `~/.ollama/models`
+- Linux: `/usr/share/ollama/.ollama/models`
+- Windows: `C:\Users\%username%\.ollama\models`
+
+### How do I set them to a different location?
+
+If a different directory needs to be used, set the environment variable `OLLAMA_MODELS` to the chosen directory.
+
+> Note: on Linux using the standard installer, the `ollama` user needs read and write access to the specified directory. To assign the directory to the `ollama` user run `sudo chown -R ollama:ollama <directory>`.
+
+Refer to the section [above](#how-do-i-configure-ollama-server) for how to set environment variables on your platform.
+
+## How can I use Ollama in Visual Studio Code?
+
+There is already a large collection of plugins available for VSCode as well as other editors that leverage Ollama. See the list of [extensions & plugins](https://github.com/ollama/ollama#extensions--plugins) at the bottom of the main repository readme.
+
## How do I use Ollama with GPU acceleration in Docker?
The Ollama Docker container can be configured with GPU acceleration in Linux or Windows (with WSL2). This requires the [nvidia-container-toolkit](https://github.com/NVIDIA/nvidia-container-toolkit). See [ollama/ollama](https://hub.docker.com/r/ollama/ollama) for more details.
@@ -112,3 +215,74 @@ This can impact both installing Ollama, as well as downloading models.
Open `Control Panel > Networking and Internet > View network status and tasks` and click on `Change adapter settings` on the left panel. Find the `vEthernel (WSL)` adapter, right click and select `Properties`.
Click on `Configure` and open the `Advanced` tab. Search through each of the properties until you find `Large Send Offload Version 2 (IPv4)` and `Large Send Offload Version 2 (IPv6)`. *Disable* both of these
properties.
+
+## How can I preload a model into Ollama to get faster response times?
+
+If you are using the API you can preload a model by sending the Ollama server an empty request. This works with both the `/api/generate` and `/api/chat` API endpoints.
+
+To preload the mistral model using the generate endpoint, use:
+```shell
+curl http://localhost:11434/api/generate -d '{"model": "mistral"}'
+```
+
+To use the chat completions endpoint, use:
+```shell
+curl http://localhost:11434/api/chat -d '{"model": "mistral"}'
+```
+
+To preload a model using the CLI, use the command:
+```shell
+ollama run llama3.2 ""
+```
+
+## How do I keep a model loaded in memory or make it unload immediately?
+
+By default models are kept in memory for 5 minutes before being unloaded. This allows for quicker response times if you're making numerous requests to the LLM. If you want to immediately unload a model from memory, use the `ollama stop` command:
+
+```shell
+ollama stop llama3.2
+```
+
+If you're using the API, use the `keep_alive` parameter with the `/api/generate` and `/api/chat` endpoints to set the amount of time that a model stays in memory. The `keep_alive` parameter can be set to:
+* a duration string (such as "10m" or "24h")
+* a number in seconds (such as 3600)
+* any negative number which will keep the model loaded in memory (e.g. -1 or "-1m")
+* '0' which will unload the model immediately after generating a response
+
+For example, to preload a model and leave it in memory use:
+```shell
+curl http://localhost:11434/api/generate -d '{"model": "llama3.2", "keep_alive": -1}'
+```
+
+To unload the model and free up memory use:
+```shell
+curl http://localhost:11434/api/generate -d '{"model": "llama3.2", "keep_alive": 0}'
+```
+
+Alternatively, you can change the amount of time all models are loaded into memory by setting the `OLLAMA_KEEP_ALIVE` environment variable when starting the Ollama server. The `OLLAMA_KEEP_ALIVE` variable uses the same parameter types as the `keep_alive` parameter types mentioned above. Refer to the section explaining [how to configure the Ollama server](#how-do-i-configure-ollama-server) to correctly set the environment variable.
+
+The `keep_alive` API parameter with the `/api/generate` and `/api/chat` API endpoints will override the `OLLAMA_KEEP_ALIVE` setting.
+
+## How do I manage the maximum number of requests the Ollama server can queue?
+
+If too many requests are sent to the server, it will respond with a 503 error indicating the server is overloaded. You can adjust how many requests may be queue by setting `OLLAMA_MAX_QUEUE`.
+
+## How does Ollama handle concurrent requests?
+
+Ollama supports two levels of concurrent processing. If your system has sufficient available memory (system memory when using CPU inference, or VRAM for GPU inference) then multiple models can be loaded at the same time. For a given model, if there is sufficient available memory when the model is loaded, it is configured to allow parallel request processing.
+
+If there is insufficient available memory to load a new model request while one or more models are already loaded, all new requests will be queued until the new model can be loaded. As prior models become idle, one or more will be unloaded to make room for the new model. Queued requests will be processed in order. When using GPU inference new models must be able to completely fit in VRAM to allow concurrent model loads.
+
+Parallel request processing for a given model results in increasing the context size by the number of parallel requests. For example, a 2K context with 4 parallel requests will result in an 8K context and additional memory allocation.
+
+The following server settings may be used to adjust how Ollama handles concurrent requests on most platforms:
+
+- `OLLAMA_MAX_LOADED_MODELS` - The maximum number of models that can be loaded concurrently provided they fit in available memory. The default is 3 * the number of GPUs or 3 for CPU inference.
+- `OLLAMA_NUM_PARALLEL` - The maximum number of parallel requests each model will process at the same time. The default will auto-select either 4 or 1 based on available memory.
+- `OLLAMA_MAX_QUEUE` - The maximum number of requests Ollama will queue when busy before rejecting additional requests. The default is 512
+
+Note: Windows with Radeon GPUs currently default to 1 model maximum due to limitations in ROCm v5.7 for available VRAM reporting. Once ROCm v6.2 is available, Windows Radeon will follow the defaults above. You may enable concurrent model loads on Radeon on Windows, but ensure you don't load more models than will fit into your GPUs VRAM.
+
+## How does Ollama load models on multiple GPUs?
+
+Installing multiple GPUs of the same brand can be a great way to increase your available VRAM to load larger models. When you load a new model, Ollama evaluates the required VRAM for the model against what is currently available. If the model will entirely fit on any single GPU, Ollama will load the model on that GPU. This typically provides the best performance as it reduces the amount of data transfering across the PCI bus during inference. If the model does not fit entirely on one GPU, then it will be spread across all the available GPUs.
diff --git a/docs/gpu.md b/docs/gpu.md
new file mode 100644
index 00000000..2913a2e2
--- /dev/null
+++ b/docs/gpu.md
@@ -0,0 +1,113 @@
+# GPU
+## Nvidia
+Ollama supports Nvidia GPUs with compute capability 5.0+.
+
+Check your compute compatibility to see if your card is supported:
+[https://developer.nvidia.com/cuda-gpus](https://developer.nvidia.com/cuda-gpus)
+
+| Compute Capability | Family | Cards |
+| ------------------ | ------------------- | ----------------------------------------------------------------------------------------------------------- |
+| 9.0 | NVIDIA | `H100` |
+| 8.9 | GeForce RTX 40xx | `RTX 4090` `RTX 4080 SUPER` `RTX 4080` `RTX 4070 Ti SUPER` `RTX 4070 Ti` `RTX 4070 SUPER` `RTX 4070` `RTX 4060 Ti` `RTX 4060` |
+| | NVIDIA Professional | `L4` `L40` `RTX 6000` |
+| 8.6 | GeForce RTX 30xx | `RTX 3090 Ti` `RTX 3090` `RTX 3080 Ti` `RTX 3080` `RTX 3070 Ti` `RTX 3070` `RTX 3060 Ti` `RTX 3060` `RTX 3050 Ti` `RTX 3050` |
+| | NVIDIA Professional | `A40` `RTX A6000` `RTX A5000` `RTX A4000` `RTX A3000` `RTX A2000` `A10` `A16` `A2` |
+| 8.0 | NVIDIA | `A100` `A30` |
+| 7.5 | GeForce GTX/RTX | `GTX 1650 Ti` `TITAN RTX` `RTX 2080 Ti` `RTX 2080` `RTX 2070` `RTX 2060` |
+| | NVIDIA Professional | `T4` `RTX 5000` `RTX 4000` `RTX 3000` `T2000` `T1200` `T1000` `T600` `T500` |
+| | Quadro | `RTX 8000` `RTX 6000` `RTX 5000` `RTX 4000` |
+| 7.0 | NVIDIA | `TITAN V` `V100` `Quadro GV100` |
+| 6.1 | NVIDIA TITAN | `TITAN Xp` `TITAN X` |
+| | GeForce GTX | `GTX 1080 Ti` `GTX 1080` `GTX 1070 Ti` `GTX 1070` `GTX 1060` `GTX 1050 Ti` `GTX 1050` |
+| | Quadro | `P6000` `P5200` `P4200` `P3200` `P5000` `P4000` `P3000` `P2200` `P2000` `P1000` `P620` `P600` `P500` `P520` |
+| | Tesla | `P40` `P4` |
+| 6.0 | NVIDIA | `Tesla P100` `Quadro GP100` |
+| 5.2 | GeForce GTX | `GTX TITAN X` `GTX 980 Ti` `GTX 980` `GTX 970` `GTX 960` `GTX 950` |
+| | Quadro | `M6000 24GB` `M6000` `M5000` `M5500M` `M4000` `M2200` `M2000` `M620` |
+| | Tesla | `M60` `M40` |
+| 5.0 | GeForce GTX | `GTX 750 Ti` `GTX 750` `NVS 810` |
+| | Quadro | `K2200` `K1200` `K620` `M1200` `M520` `M5000M` `M4000M` `M3000M` `M2000M` `M1000M` `K620M` `M600M` `M500M` |
+
+
+### GPU Selection
+
+If you have multiple NVIDIA GPUs in your system and want to limit Ollama to use
+a subset, you can set `CUDA_VISIBLE_DEVICES` to a comma separated list of GPUs.
+Numeric IDs may be used, however ordering may vary, so UUIDs are more reliable.
+You can discover the UUID of your GPUs by running `nvidia-smi -L` If you want to
+ignore the GPUs and force CPU usage, use an invalid GPU ID (e.g., "-1")
+
+### Laptop Suspend Resume
+
+On linux, after a suspend/resume cycle, sometimes Ollama will fail to discover
+your NVIDIA GPU, and fallback to running on the CPU. You can workaround this
+driver bug by reloading the NVIDIA UVM driver with `sudo rmmod nvidia_uvm &&
+sudo modprobe nvidia_uvm`
+
+## AMD Radeon
+Ollama supports the following AMD GPUs:
+
+### Linux Support
+| Family | Cards and accelerators |
+| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
+| AMD Radeon RX | `7900 XTX` `7900 XT` `7900 GRE` `7800 XT` `7700 XT` `7600 XT` `7600` `6950 XT` `6900 XTX` `6900XT` `6800 XT` `6800` `Vega 64` `Vega 56` |
+| AMD Radeon PRO | `W7900` `W7800` `W7700` `W7600` `W7500` `W6900X` `W6800X Duo` `W6800X` `W6800` `V620` `V420` `V340` `V320` `Vega II Duo` `Vega II` `VII` `SSG` |
+| AMD Instinct | `MI300X` `MI300A` `MI300` `MI250X` `MI250` `MI210` `MI200` `MI100` `MI60` `MI50` |
+
+### Windows Support
+With ROCm v6.1, the following GPUs are supported on Windows.
+
+| Family | Cards and accelerators |
+| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
+| AMD Radeon RX | `7900 XTX` `7900 XT` `7900 GRE` `7800 XT` `7700 XT` `7600 XT` `7600` `6950 XT` `6900 XTX` `6900XT` `6800 XT` `6800` |
+| AMD Radeon PRO | `W7900` `W7800` `W7700` `W7600` `W7500` `W6900X` `W6800X Duo` `W6800X` `W6800` `V620` |
+
+
+### Overrides on Linux
+Ollama leverages the AMD ROCm library, which does not support all AMD GPUs. In
+some cases you can force the system to try to use a similar LLVM target that is
+close. For example The Radeon RX 5400 is `gfx1034` (also known as 10.3.4)
+however, ROCm does not currently support this target. The closest support is
+`gfx1030`. You can use the environment variable `HSA_OVERRIDE_GFX_VERSION` with
+`x.y.z` syntax. So for example, to force the system to run on the RX 5400, you
+would set `HSA_OVERRIDE_GFX_VERSION="10.3.0"` as an environment variable for the
+server. If you have an unsupported AMD GPU you can experiment using the list of
+supported types below.
+
+At this time, the known supported GPU types on linux are the following LLVM Targets.
+This table shows some example GPUs that map to these LLVM targets:
+| **LLVM Target** | **An Example GPU** |
+|-----------------|---------------------|
+| gfx900 | Radeon RX Vega 56 |
+| gfx906 | Radeon Instinct MI50 |
+| gfx908 | Radeon Instinct MI100 |
+| gfx90a | Radeon Instinct MI210 |
+| gfx940 | Radeon Instinct MI300 |
+| gfx941 | |
+| gfx942 | |
+| gfx1030 | Radeon PRO V620 |
+| gfx1100 | Radeon PRO W7900 |
+| gfx1101 | Radeon PRO W7700 |
+| gfx1102 | Radeon RX 7600 |
+
+AMD is working on enhancing ROCm v6 to broaden support for families of GPUs in a
+future release which should increase support for more GPUs.
+
+Reach out on [Discord](https://discord.gg/ollama) or file an
+[issue](https://github.com/ollama/ollama/issues) for additional help.
+
+### GPU Selection
+
+If you have multiple AMD GPUs in your system and want to limit Ollama to use a
+subset, you can set `HIP_VISIBLE_DEVICES` to a comma separated list of GPUs.
+You can see the list of devices with `rocminfo`. If you want to ignore the GPUs
+and force CPU usage, use an invalid GPU ID (e.g., "-1")
+
+### Container Permission
+
+In some Linux distributions, SELinux can prevent containers from
+accessing the AMD GPU devices. On the host system you can run
+`sudo setsebool container_use_devices=1` to allow containers to use devices.
+
+### Metal (Apple GPUs)
+Ollama supports GPU acceleration on Apple devices via the Metal API.
diff --git a/docs/images/ollama-keys.png b/docs/images/ollama-keys.png
new file mode 100644
index 00000000..942079a8
Binary files /dev/null and b/docs/images/ollama-keys.png differ
diff --git a/docs/images/signup.png b/docs/images/signup.png
new file mode 100644
index 00000000..e80bb4e7
Binary files /dev/null and b/docs/images/signup.png differ
diff --git a/docs/import.md b/docs/import.md
index be50a953..2346886f 100644
--- a/docs/import.md
+++ b/docs/import.md
@@ -1,194 +1,186 @@
-# Import a model
+# Importing a model
-This guide walks through importing a GGUF, PyTorch or Safetensors model.
+## Table of Contents
-## Importing (GGUF)
+ * [Importing a Safetensors adapter](#Importing-a-fine-tuned-adapter-from-Safetensors-weights)
+ * [Importing a Safetensors model](#Importing-a-model-from-Safetensors-weights)
+ * [Importing a GGUF file](#Importing-a-GGUF-based-model-or-adapter)
+ * [Sharing models on ollama.com](#Sharing-your-model-on-ollamacom)
-### Step 1: Write a `Modelfile`
+## Importing a fine tuned adapter from Safetensors weights
-Start by creating a `Modelfile`. This file is the blueprint for your model, specifying weights, parameters, prompt templates and more.
+First, create a `Modelfile` with a `FROM` command pointing at the base model you used for fine tuning, and an `ADAPTER` command which points to the directory with your Safetensors adapter:
-```
-FROM ./mistral-7b-v0.1.Q4_0.gguf
+```dockerfile
+FROM <base model name>
+ADAPTER /path/to/safetensors/adapter/directory
```
-(Optional) many chat models require a prompt template in order to answer correctly. A default prompt template can be specified with the `TEMPLATE` instruction in the `Modelfile`:
+Make sure that you use the same base model in the `FROM` command as you used to create the adapter otherwise you will get erratic results. Most frameworks use different quantization methods, so it's best to use non-quantized (i.e. non-QLoRA) adapters. If your adapter is in the same directory as your `Modelfile`, use `ADAPTER .` to specify the adapter path.
-```
-FROM ./q4_0.bin
-TEMPLATE "[INST] {{ .Prompt }} [/INST]"
+Now run `ollama create` from the directory where the `Modelfile` was created:
+
+```bash
+ollama create my-model
```
-### Step 2: Create the Ollama model
+Lastly, test the model:
-Finally, create a model from your `Modelfile`:
-
-```
-ollama create example -f Modelfile
+```bash
+ollama run my-model
```
-### Step 3: Run your model
+Ollama supports importing adapters based on several different model architectures including:
-Next, test the model with `ollama run`:
+ * Llama (including Llama 2, Llama 3, and Llama 3.1);
+ * Mistral (including Mistral 1, Mistral 2, and Mixtral); and
+ * Gemma (including Gemma 1 and Gemma 2)
-```
-ollama run example "What is your favourite condiment?"
+You can create the adapter using a fine tuning framework or tool which can output adapters in the Safetensors format, such as:
+
+ * Hugging Face [fine tuning framework](https://huggingface.co/docs/transformers/en/training)
+ * [Unsloth](https://github.com/unslothai/unsloth)
+ * [MLX](https://github.com/ml-explore/mlx)
+
+
+## Importing a model from Safetensors weights
+
+First, create a `Modelfile` with a `FROM` command which points to the directory containing your Safetensors weights:
+
+```dockerfile
+FROM /path/to/safetensors/directory
```
-## Importing (PyTorch & Safetensors)
+If you create the Modelfile in the same directory as the weights, you can use the command `FROM .`.
-### Supported models
+Now run the `ollama create` command from the directory where you created the `Modelfile`:
-Ollama supports a set of model architectures, with support for more coming soon:
-
-- Llama & Mistral
-- Falcon & RW
-- BigCode
-
-To view a model's architecture, check the `config.json` file in its HuggingFace repo. You should see an entry under `architectures` (e.g. `LlamaForCausalLM`).
-
-### Step 1: Clone the HuggingFace repository (optional)
-
-If the model is currently hosted in a HuggingFace repository, first clone that repository to download the raw model.
-
-```
-git lfs install
-git clone https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1
-cd Mistral-7B-Instruct-v0.1
+```shell
+ollama create my-model
```
-### Step 2: Convert and quantize to a `.bin` file (optional, for PyTorch and Safetensors)
+Lastly, test the model:
-If the model is in PyTorch or Safetensors format, a [Docker image](https://hub.docker.com/r/ollama/quantize) with the tooling required to convert and quantize models is available.
-
-First, Install [Docker](https://www.docker.com/get-started/).
-
-Next, to convert and quantize your model, run:
-
-```
-docker run --rm -v .:/model ollama/quantize -q q4_0 /model
+```shell
+ollama run my-model
```
-This will output two files into the directory:
+Ollama supports importing models for several different architectures including:
-- `f16.bin`: the model converted to GGUF
-- `q4_0.bin` the model quantized to a 4-bit quantization (Ollama will use this file to create the Ollama model)
+ * Llama (including Llama 2, Llama 3, and Llama 3.1);
+ * Mistral (including Mistral 1, Mistral 2, and Mixtral);
+ * Gemma (including Gemma 1 and Gemma 2); and
+ * Phi3
-### Step 3: Write a `Modelfile`
+This includes importing foundation models as well as any fine tuned models which which have been _fused_ with a foundation model.
-Next, create a `Modelfile` for your model:
-```
-FROM ./q4_0.bin
+## Importing a GGUF based model or adapter
+
+If you have a GGUF based model or adapter it is possible to import it into Ollama. You can obtain a GGUF model or adapter by:
+
+ * converting a Safetensors model with the `convert_hf_to_gguf.py` from Llama.cpp;
+ * converting a Safetensors adapter with the `convert_lora_to_gguf.py` from Llama.cpp; or
+ * downloading a model or adapter from a place such as HuggingFace
+
+To import a GGUF model, create a `Modelfile` containg:
+
+```dockerfile
+FROM /path/to/file.gguf
```
-(Optional) many chat models require a prompt template in order to answer correctly. A default prompt template can be specified with the `TEMPLATE` instruction in the `Modelfile`:
+For a GGUF adapter, create the `Modelfile` with:
-```
-FROM ./q4_0.bin
-TEMPLATE "[INST] {{ .Prompt }} [/INST]"
+```dockerfile
+FROM <model name>
+ADAPTER /path/to/file.gguf
```
-### Step 4: Create the Ollama model
+When importing a GGUF adapter, it's important to use the same base model as the base model that the adapter was created with. You can use:
-Finally, create a model from your `Modelfile`:
+ * a model from Ollama
+ * a GGUF file
+ * a Safetensors based model
-```
-ollama create example -f Modelfile
+Once you have created your `Modelfile`, use the `ollama create` command to build the model.
+
+```shell
+ollama create my-model
```
-### Step 5: Run your model
+## Quantizing a Model
-Next, test the model with `ollama run`:
+Quantizing a model allows you to run models faster and with less memory consumption but at reduced accuracy. This allows you to run a model on more modest hardware.
-```
-ollama run example "What is your favourite condiment?"
+Ollama can quantize FP16 and FP32 based models into different quantization levels using the `-q/--quantize` flag with the `ollama create` command.
+
+First, create a Modelfile with the FP16 or FP32 based model you wish to quantize.
+
+```dockerfile
+FROM /path/to/my/gemma/f16/model
```
-## Publishing your model (optional – early alpha)
+Use `ollama create` to then create the quantized model.
-Publishing models is in early alpha. If you'd like to publish your model to share with others, follow these steps:
-
-1. Create [an account](https://ollama.ai/signup)
-2. Run `cat ~/.ollama/id_ed25519.pub` to view your Ollama public key. Copy this to the clipboard.
-3. Add your public key to your [Ollama account](https://ollama.ai/settings/keys)
-
-Next, copy your model to your username's namespace:
-
-```
-ollama cp example <your username>/example
+```shell
+$ ollama create --quantize q4_K_M mymodel
+transferring model data
+quantizing F16 model to Q4_K_M
+creating new layer sha256:735e246cc1abfd06e9cdcf95504d6789a6cd1ad7577108a70d9902fef503c1bd
+creating new layer sha256:0853f0ad24e5865173bbf9ffcc7b0f5d56b66fd690ab1009867e45e7d2c4db0f
+writing manifest
+success
```
-Then push the model:
+### Supported Quantizations
-```
-ollama push <your username>/example
-```
+- `q4_0`
+- `q4_1`
+- `q5_0`
+- `q5_1`
+- `q8_0`
-After publishing, your model will be available at `https://ollama.ai/<your username>/example`.
+#### K-means Quantizations
-## Quantization reference
-
-The quantization options are as follow (from highest highest to lowest levels of quantization). Note: some architectures such as Falcon do not support K quants.
-
-- `q2_K`
-- `q3_K`
- `q3_K_S`
- `q3_K_M`
- `q3_K_L`
-- `q4_0` (recommended)
-- `q4_1`
-- `q4_K`
- `q4_K_S`
- `q4_K_M`
-- `q5_0`
-- `q5_1`
-- `q5_K`
- `q5_K_S`
- `q5_K_M`
- `q6_K`
-- `q8_0`
-## Manually converting & quantizing models
-### Prerequisites
+## Sharing your model on ollama.com
-Start by cloning the `llama.cpp` repo to your machine in another directory:
+You can share any model you have created by pushing it to [ollama.com](https://ollama.com) so that other users can try it out.
-```
-git clone https://github.com/ggerganov/llama.cpp.git
-cd llama.cpp
-```
+First, use your browser to go to the [Ollama Sign-Up](https://ollama.com/signup) page. If you already have an account, you can skip this step.
-Next, install the Python dependencies:
+<img src="images/signup.png" alt="Sign-Up" width="40%">
-```
-pip install -r requirements.txt
-```
+The `Username` field will be used as part of your model's name (e.g. `jmorganca/mymodel`), so make sure you are comfortable with the username that you have selected.
-Finally, build the `quantize` tool:
+Now that you have created an account and are signed-in, go to the [Ollama Keys Settings](https://ollama.com/settings/keys) page.
-```
-make quantize
-```
+Follow the directions on the page to determine where your Ollama Public Key is located.
-### Convert the model
+<img src="images/ollama-keys.png" alt="Ollama Keys" width="80%">
-Run the correct conversion script for your model architecture:
+Click on the `Add Ollama Public Key` button, and copy and paste the contents of your Ollama Public Key into the text field.
+
+To push a model to [ollama.com](https://ollama.com), first make sure that it is named correctly with your username. You may have to use the `ollama cp` command to copy
+your model to give it the correct name. Once you're happy with your model's name, use the `ollama push` command to push it to [ollama.com](https://ollama.com).
```shell
-# LlamaForCausalLM or MistralForCausalLM
-python convert.py <path to model directory>
-
-# FalconForCausalLM
-python convert-falcon-hf-to-gguf.py <path to model directory>
-
-# GPTBigCodeForCausalLM
-python convert-starcoder-hf-to-gguf.py <path to model directory>
+ollama cp mymodel myuser/mymodel
+ollama push myuser/mymodel
```
-### Quantize the model
+Once your model has been pushed, other users can pull and run it by using the command:
+```shell
+ollama run myuser/mymodel
```
-quantize <path to model dir>/ggml-model-f32.bin <path to model dir>/q4_0.bin q4_0
-```
+
diff --git a/docs/linux.md b/docs/linux.md
index 45d0f38f..0eec014f 100644
--- a/docs/linux.md
+++ b/docs/linux.md
@@ -1,30 +1,59 @@
-# Ollama on Linux
+# Linux
## Install
-Install Ollama running this one-liner:
->
-```bash
-curl https://ollama.ai/install.sh | sh
+To install Ollama, run the following command:
+
+```shell
+curl -fsSL https://ollama.com/install.sh | sh
```
## Manual install
-### Download the `ollama` binary
+Download and extract the package:
-Ollama is distributed as a self-contained binary. Download it to a directory in your PATH:
+```shell
+curl -L https://ollama.com/download/ollama-linux-amd64.tgz -o ollama-linux-amd64.tgz
+sudo tar -C /usr -xzf ollama-linux-amd64.tgz
+```
-```bash
-sudo curl -L https://ollama.ai/download/ollama-linux-amd64 -o /usr/bin/ollama
-sudo chmod +x /usr/bin/ollama
+Start Ollama:
+
+```shell
+ollama serve
+```
+
+In another terminal, verify that Ollama is running:
+
+```shell
+ollama -v
+```
+
+### AMD GPU install
+
+If you have an AMD GPU, also download and extract the additional ROCm package:
+
+```shell
+curl -L https://ollama.com/download/ollama-linux-amd64-rocm.tgz -o ollama-linux-amd64-rocm.tgz
+sudo tar -C /usr -xzf ollama-linux-amd64-rocm.tgz
+```
+
+### ARM64 install
+
+Download and extract the ARM64-specific package:
+
+```shell
+curl -L https://ollama.com/download/ollama-linux-arm64.tgz -o ollama-linux-arm64.tgz
+sudo tar -C /usr -xzf ollama-linux-arm64.tgz
```
### Adding Ollama as a startup service (recommended)
-Create a user for Ollama:
+Create a user and group for Ollama:
-```bash
-sudo useradd -r -s /bin/false -m -d /usr/share/ollama ollama
+```shell
+sudo useradd -r -s /bin/false -U -m -d /usr/share/ollama ollama
+sudo usermod -a -G ollama $(whoami)
```
Create a service file in `/etc/systemd/system/ollama.service`:
@@ -40,6 +69,7 @@ User=ollama
Group=ollama
Restart=always
RestartSec=3
+Environment="PATH=$PATH"
[Install]
WantedBy=default.target
@@ -47,57 +77,79 @@ WantedBy=default.target
Then start the service:
-```bash
+```shell
sudo systemctl daemon-reload
sudo systemctl enable ollama
```
-### Install CUDA drivers (optional – for Nvidia GPUs)
+### Install CUDA drivers (optional)
[Download and install](https://developer.nvidia.com/cuda-downloads) CUDA.
Verify that the drivers are installed by running the following command, which should print details about your GPU:
-```bash
+```shell
nvidia-smi
```
+### Install AMD ROCm drivers (optional)
+
+[Download and Install](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/tutorial/quick-start.html) ROCm v6.
+
### Start Ollama
-Start Ollama using `systemd`:
+Start Ollama and verify it is running:
-```bash
+```shell
sudo systemctl start ollama
+sudo systemctl status ollama
```
-## Update
+> [!NOTE]
+> While AMD has contributed the `amdgpu` driver upstream to the official linux
+> kernel source, the version is older and may not support all ROCm features. We
+> recommend you install the latest driver from
+> https://www.amd.com/en/support/linux-drivers for best support of your Radeon
+> GPU.
-Update ollama by running the install script again:
+## Updating
-```bash
-curl https://ollama.ai/install.sh | sh
+Update Ollama by running the install script again:
+
+```shell
+curl -fsSL https://ollama.com/install.sh | sh
```
-Or by downloading the ollama binary:
+Or by re-downloading Ollama:
-```bash
-sudo curl -L https://ollama.ai/download/ollama-linux-amd64 -o /usr/bin/ollama
-sudo chmod +x /usr/bin/ollama
+```shell
+curl -L https://ollama.com/download/ollama-linux-amd64.tgz -o ollama-linux-amd64.tgz
+sudo tar -C /usr -xzf ollama-linux-amd64.tgz
+```
+
+## Installing specific versions
+
+Use `OLLAMA_VERSION` environment variable with the install script to install a specific version of Ollama, including pre-releases. You can find the version numbers in the [releases page](https://github.com/ollama/ollama/releases).
+
+For example:
+
+```shell
+curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=0.3.9 sh
```
## Viewing logs
To view logs of Ollama running as a startup service, run:
-```bash
-journalctl -u ollama
+```shell
+journalctl -e -u ollama
```
## Uninstall
Remove the ollama service:
-```bash
+```shell
sudo systemctl stop ollama
sudo systemctl disable ollama
sudo rm /etc/systemd/system/ollama.service
@@ -105,12 +157,14 @@ sudo rm /etc/systemd/system/ollama.service
Remove the ollama binary from your bin directory (either `/usr/local/bin`, `/usr/bin`, or `/bin`):
-```bash
+```shell
sudo rm $(which ollama)
```
-Remove the downloaded models and Ollama service user:
-```bash
+Remove the downloaded models and Ollama service user and group:
+
+```shell
sudo rm -r /usr/share/ollama
sudo userdel ollama
+sudo groupdel ollama
```
diff --git a/docs/modelfile.md b/docs/modelfile.md
index 9f012673..aa2849e7 100644
--- a/docs/modelfile.md
+++ b/docs/modelfile.md
@@ -1,6 +1,7 @@
# Ollama Model File
-> Note: `Modelfile` syntax is in development
+> [!NOTE]
+> `Modelfile` syntax is in development
A model file is the blueprint to create and share models with Ollama.
@@ -10,8 +11,9 @@ A model file is the blueprint to create and share models with Ollama.
- [Examples](#examples)
- [Instructions](#instructions)
- [FROM (Required)](#from-required)
- - [Build from llama2](#build-from-llama2)
- - [Build from a bin file](#build-from-a-bin-file)
+ - [Build from existing model](#build-from-existing-model)
+ - [Build from a Safetensors model](#build-from-a-safetensors-model)
+ - [Build from a GGUF file](#build-from-a-gguf-file)
- [PARAMETER](#parameter)
- [Valid Parameters and Values](#valid-parameters-and-values)
- [TEMPLATE](#template)
@@ -19,6 +21,7 @@ A model file is the blueprint to create and share models with Ollama.
- [SYSTEM](#system)
- [ADAPTER](#adapter)
- [LICENSE](#license)
+ - [MESSAGE](#message)
- [Notes](#notes)
## Format
@@ -38,6 +41,7 @@ INSTRUCTION arguments
| [`SYSTEM`](#system) | Specifies the system message that will be set in the template. |
| [`ADAPTER`](#adapter) | Defines the (Q)LoRA adapters to apply to the model. |
| [`LICENSE`](#license) | Specifies the legal license. |
+| [`MESSAGE`](#message) | Specify message history. |
## Examples
@@ -46,7 +50,7 @@ INSTRUCTION arguments
An example of a `Modelfile` creating a mario blueprint:
```modelfile
-FROM llama2
+FROM llama3.2
# sets the temperature to 1 [higher is more creative, lower is more coherent]
PARAMETER temperature 1
# sets the context window size to 4096, this controls how many tokens the LLM can use as context to generate the next token
@@ -65,33 +69,25 @@ To use this:
More examples are available in the [examples directory](../examples).
-### `Modelfile`s in [ollama.ai/library][1]
-
-There are two ways to view `Modelfile`s underlying the models in [ollama.ai/library][1]:
-
-- Option 1: view a details page from a model's tags page:
- 1. Go to a particular model's tags (e.g. https://ollama.ai/library/llama2/tags)
- 2. Click on a tag (e.g. https://ollama.ai/library/llama2:13b)
- 3. Scroll down to "Layers"
- - Note: if the [`FROM` instruction](#from-required) is not present,
- it means the model was created from a local file
-- Option 2: use `ollama show` to print the `Modelfile` for any local models like so:
+To view the Modelfile of a given model, use the `ollama show --modelfile` command.
```bash
- > ollama show --modelfile llama2:13b
+ > ollama show --modelfile llama3.2
# Modelfile generated by "ollama show"
# To build a new Modelfile based on this one, replace the FROM line with:
- # FROM llama2:13b
+ # FROM llama3.2:latest
+ FROM /Users/pdevine/.ollama/models/blobs/sha256-00e1317cbf74d901080d7100f57580ba8dd8de57203072dc6f668324ba545f29
+ TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|>
- FROM /root/.ollama/models/blobs/sha256:123abc
- TEMPLATE """[INST] {{ if and .First .System }}<<SYS>>{{ .System }}<</SYS>>
+ {{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>
- {{ end }}{{ .Prompt }} [/INST] """
- SYSTEM """"""
- PARAMETER stop [INST]
- PARAMETER stop [/INST]
- PARAMETER stop <<SYS>>
- PARAMETER stop <</SYS>>
+ {{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>
+
+ {{ .Response }}<|eot_id|>"""
+ PARAMETER stop "<|start_header_id|>"
+ PARAMETER stop "<|end_header_id|>"
+ PARAMETER stop "<|eot_id|>"
+ PARAMETER stop "<|reserved_special_token"
```
## Instructions
@@ -104,22 +100,39 @@ The `FROM` instruction defines the base model to use when creating a model.
FROM <model name>:<tag>
```
-#### Build from llama2
+#### Build from existing model
```modelfile
-FROM llama2
+FROM llama3.2
```
A list of available base models:
-<https://github.com/jmorganca/ollama#model-library>
+<https://github.com/ollama/ollama#model-library>
+Additional models can be found at:
+<https://ollama.com/library>
-#### Build from a `bin` file
+#### Build from a Safetensors model
```modelfile
-FROM ./ollama-model.bin
+FROM <model directory>
```
-This bin file location should be specified as an absolute path or relative to the `Modelfile` location.
+The model directory should contain the Safetensors weights for a supported architecture.
+
+Currently supported model architectures:
+ * Llama (including Llama 2, Llama 3, and Llama 3.1)
+ * Mistral (including Mistral 1, Mistral 2, and Mixtral)
+ * Gemma (including Gemma 1 and Gemma 2)
+ * Phi3
+
+#### Build from a GGUF file
+
+```modelfile
+FROM ./ollama-model.gguf
+```
+
+The GGUF file location should be specified as an absolute path or relative to the `Modelfile` location.
+
### PARAMETER
@@ -129,7 +142,7 @@ The `PARAMETER` instruction defines a parameter that can be set when the model i
PARAMETER <parameter> <parametervalue>
```
-### Valid Parameters and Values
+#### Valid Parameters and Values
| Parameter | Description | Value Type | Example Usage |
| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------------------- |
@@ -137,9 +150,6 @@ PARAMETER <parameter> <parametervalue>
| mirostat_eta | Influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. (Default: 0.1) | float | mirostat_eta 0.1 |
| mirostat_tau | Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. (Default: 5.0) | float | mirostat_tau 5.0 |
| num_ctx | Sets the size of the context window used to generate the next token. (Default: 2048) | int | num_ctx 4096 |
-| num_gqa | The number of GQA groups in the transformer layer. Required for some models, for example it is 8 for llama2:70b | int | num_gqa 1 |
-| num_gpu | The number of layers to send to the GPU(s). On macOS it defaults to 1 to enable metal support, 0 to disable. | int | num_gpu 50 |
-| num_thread | Sets the number of threads to use during computation. By default, Ollama will detect this for optimal performance. It is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). | int | num_thread 8 |
| repeat_last_n | Sets how far back for the model to look back to prevent repetition. (Default: 64, 0 = disabled, -1 = num_ctx) | int | repeat_last_n 64 |
| repeat_penalty | Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. (Default: 1.1) | float | repeat_penalty 1.1 |
| temperature | The temperature of the model. Increasing the temperature will make the model answer more creatively. (Default: 0.8) | float | temperature 0.7 |
@@ -149,33 +159,27 @@ PARAMETER <parameter> <parametervalue>
| num_predict | Maximum number of tokens to predict when generating text. (Default: 128, -1 = infinite generation, -2 = fill context) | int | num_predict 42 |
| top_k | Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. (Default: 40) | int | top_k 40 |
| top_p | Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. (Default: 0.9) | float | top_p 0.9 |
+| min_p | Alternative to the top_p, and aims to ensure a balance of quality and variety. The parameter *p* represents the minimum probability for a token to be considered, relative to the probability of the most likely token. For example, with *p*=0.05 and the most likely token having a probability of 0.9, logits with a value less than 0.045 are filtered out. (Default: 0.0) | float | min_p 0.05 |
### TEMPLATE
-`TEMPLATE` of the full prompt template to be passed into the model. It may include (optionally) a system message and a user's prompt. This is used to create a full custom prompt, and syntax may be model specific. You can usually find the template for a given model in the readme for that model.
+`TEMPLATE` of the full prompt template to be passed into the model. It may include (optionally) a system message, a user's message and the response from the model. Note: syntax may be model specific. Templates use Go [template syntax](https://pkg.go.dev/text/template).
#### Template Variables
-| Variable | Description |
-| --------------- | ------------------------------------------------------------------------------------------------------------- |
-| `{{ .System }}` | The system message used to specify custom behavior, this must also be set in the Modelfile as an instruction. |
-| `{{ .Prompt }}` | The incoming prompt, this is not specified in the model file and will be set based on input. |
-| `{{ .First }}` | A boolean value used to render specific template information for the first generation of a session. |
+| Variable | Description |
+| ----------------- | --------------------------------------------------------------------------------------------- |
+| `{{ .System }}` | The system message used to specify custom behavior. |
+| `{{ .Prompt }}` | The user prompt message. |
+| `{{ .Response }}` | The response from the model. When generating a response, text after this variable is omitted. |
-```modelfile
-TEMPLATE """
-{{- if .First }}
-### System:
-{{ .System }}
-{{- end }}
-
-### User:
-{{ .Prompt }}
-
-### Response:
+```
+TEMPLATE """{{ if .System }}<|im_start|>system
+{{ .System }}<|im_end|>
+{{ end }}{{ if .Prompt }}<|im_start|>user
+{{ .Prompt }}<|im_end|>
+{{ end }}<|im_start|>assistant
"""
-
-SYSTEM """<system message>"""
```
### SYSTEM
@@ -188,10 +192,23 @@ SYSTEM """<system message>"""
### ADAPTER
-The `ADAPTER` instruction specifies the LoRA adapter to apply to the base model. The value of this instruction should be an absolute path or a path relative to the Modelfile and the file must be in a GGML file format. The adapter should be tuned from the base model otherwise the behaviour is undefined.
+The `ADAPTER` instruction specifies a fine tuned LoRA adapter that should apply to the base model. The value of the adapter should be an absolute path or a path relative to the Modelfile. The base model should be specified with a `FROM` instruction. If the base model is not the same as the base model that the adapter was tuned from the behaviour will be erratic.
+
+#### Safetensor adapter
```modelfile
-ADAPTER ./ollama-lora.bin
+ADAPTER <path to safetensor adapter>
+```
+
+Currently supported Safetensor adapters:
+ * Llama (including Llama 2, Llama 3, and Llama 3.1)
+ * Mistral (including Mistral 1, Mistral 2, and Mixtral)
+ * Gemma (including Gemma 1 and Gemma 2)
+
+#### GGUF adapter
+
+```modelfile
+ADAPTER ./ollama-lora.gguf
```
### LICENSE
@@ -204,9 +221,38 @@ LICENSE """
"""
```
+### MESSAGE
+
+The `MESSAGE` instruction allows you to specify a message history for the model to use when responding. Use multiple iterations of the MESSAGE command to build up a conversation which will guide the model to answer in a similar way.
+
+```modelfile
+MESSAGE <role> <message>
+```
+
+#### Valid roles
+
+| Role | Description |
+| --------- | ------------------------------------------------------------ |
+| system | Alternate way of providing the SYSTEM message for the model. |
+| user | An example message of what the user could have asked. |
+| assistant | An example message of how the model should respond. |
+
+
+#### Example conversation
+
+```modelfile
+MESSAGE user Is Toronto in Canada?
+MESSAGE assistant yes
+MESSAGE user Is Sacramento in Canada?
+MESSAGE assistant no
+MESSAGE user Is Ontario in Canada?
+MESSAGE assistant yes
+```
+
+
## Notes
- the **`Modelfile` is not case sensitive**. In the examples, uppercase instructions are used to make it easier to distinguish it from arguments.
- Instructions can be in any order. In the examples, the `FROM` instruction is first to keep it easily readable.
-[1]: https://ollama.ai/library
+[1]: https://ollama.com/library
diff --git a/docs/openai.md b/docs/openai.md
new file mode 100644
index 00000000..e13842c0
--- /dev/null
+++ b/docs/openai.md
@@ -0,0 +1,327 @@
+# OpenAI compatibility
+
+> **Note:** OpenAI compatibility is experimental and is subject to major adjustments including breaking changes. For fully-featured access to the Ollama API, see the Ollama [Python library](https://github.com/ollama/ollama-python), [JavaScript library](https://github.com/ollama/ollama-js) and [REST API](https://github.com/ollama/ollama/blob/main/docs/api.md).
+
+Ollama provides experimental compatibility with parts of the [OpenAI API](https://platform.openai.com/docs/api-reference) to help connect existing applications to Ollama.
+
+## Usage
+
+### OpenAI Python library
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+ base_url='http://localhost:11434/v1/',
+
+ # required but ignored
+ api_key='ollama',
+)
+
+chat_completion = client.chat.completions.create(
+ messages=[
+ {
+ 'role': 'user',
+ 'content': 'Say this is a test',
+ }
+ ],
+ model='llama3.2',
+)
+
+response = client.chat.completions.create(
+ model="llava",
+ messages=[
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "What's in this image?"},
+ {
+ "type": "image_url",
+ "image_url": "iVBORw0KGgoAAAANSUhEUgAAAG0AAABmCAYAAADBPx+VAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAA3VSURBVHgB7Z27r0zdG8fX743i1bi1ikMoFMQloXRpKFFIqI7LH4BEQ+NWIkjQuSWCRIEoULk0gsK1kCBI0IhrQVT7tz/7zZo888yz1r7MnDl7z5xvsjkzs2fP3uu71nNfa7lkAsm7d++Sffv2JbNmzUqcc8m0adOSzZs3Z+/XES4ZckAWJEGWPiCxjsQNLWmQsWjRIpMseaxcuTKpG/7HP27I8P79e7dq1ars/yL4/v27S0ejqwv+cUOGEGGpKHR37tzJCEpHV9tnT58+dXXCJDdECBE2Ojrqjh071hpNECjx4cMHVycM1Uhbv359B2F79+51586daxN/+pyRkRFXKyRDAqxEp4yMlDDzXG1NPnnyJKkThoK0VFd1ELZu3TrzXKxKfW7dMBQ6bcuWLW2v0VlHjx41z717927ba22U9APcw7Nnz1oGEPeL3m3p2mTAYYnFmMOMXybPPXv2bNIPpFZr1NHn4HMw0KRBjg9NuRw95s8PEcz/6DZELQd/09C9QGq5RsmSRybqkwHGjh07OsJSsYYm3ijPpyHzoiacg35MLdDSIS/O1yM778jOTwYUkKNHWUzUWaOsylE00MyI0fcnOwIdjvtNdW/HZwNLGg+sR1kMepSNJXmIwxBZiG8tDTpEZzKg0GItNsosY8USkxDhD0Rinuiko2gfL/RbiD2LZAjU9zKQJj8RDR0vJBR1/Phx9+PHj9Z7REF4nTZkxzX4LCXHrV271qXkBAPGfP/atWvu/PnzHe4C97F48eIsRLZ9+3a3f/9+87dwP1JxaF7/3r17ba+5l4EcaVo0lj3SBq5kGTJSQmLWMjgYNei2GPT1MuMqGTDEFHzeQSP2wi/jGnkmPJ/nhccs44jvDAxpVcxnq0F6eT8h4ni/iIWpR5lPyA6ETkNXoSukvpJAD3AsXLiwpZs49+fPn5ke4j10TqYvegSfn0OnafC+Tv9ooA/JPkgQysqQNBzagXY55nO/oa1F7qvIPWkRL12WRpMWUvpVDYmxAPehxWSe8ZEXL20sadYIozfmNch4QJPAfeJgW3rNsnzphBKNJM2KKODo1rVOMRYik5ETy3ix4qWNI81qAAirizgMIc+yhTytx0JWZuNI03qsrgWlGtwjoS9XwgUhWGyhUaRZZQNNIEwCiXD16tXcAHUs79co0vSD8rrJCIW98pzvxpAWyyo3HYwqS0+H0BjStClcZJT5coMm6D2LOF8TolGJtK9fvyZpyiC5ePFi9nc/oJU4eiEP0jVoAnHa9wyJycITMP78+eMeP37sXrx44d6+fdt6f82aNdkx1pg9e3Zb5W+RSRE+n+VjksQWifvVaTKFhn5O8my63K8Qabdv33b379/PiAP//vuvW7BggZszZ072/+TJk91YgkafPn166zXB1rQHFvouAWHq9z3SEevSUerqCn2/dDCeta2jxYbr69evk4MHDyY7d+7MjhMnTiTPnz9Pfv/+nfQT2ggpO2dMF8cghuoM7Ygj5iWCqRlGFml0QC/ftGmTmzt3rmsaKDsgBSPh0/8yPeLLBihLkOKJc0jp8H8vUzcxIA1k6QJ/c78tWEyj5P3o4u9+jywNPdJi5rAH9x0KHcl4Hg570eQp3+vHXGyrmEeigzQsQsjavXt38ujRo44LQuDDhw+TW7duRS1HGgMxhNXHgflaNTOsHyKvHK5Ijo2jbFjJBQK9YwFd6RVMzfgRBmEfP37suBBm/p49e1qjEP2mwTViNRo0VJWH1deMXcNK08uUjVUu7s/zRaL+oLNxz1bpANco4npUgX4G2eFbpDFyQoQxojBCpEGSytmOH8qrH5Q9vuzD6ofQylkCUmh8DBAr+q8JCyVNtWQIidKQE9wNtLSQnS4jDSsxNHogzFuQBw4cyM61UKVsjfr3ooBkPSqqQHesUPWVtzi9/vQi1T+rJj7WiTz4Pt/l3LxUkr5P2VYZaZ4URpsE+st/dujQoaBBYokbrz/8TJNQYLSonrPS9kUaSkPeZyj1AWSj+d+VBoy1pIWVNed8P0Ll/ee5HdGRhrHhR5GGN0r4LGZBaj8oFDJitBTJzIZgFcmU0Y8ytWMZMzJOaXUSrUs5RxKnrxmbb5YXO9VGUhtpXldhEUogFr3IzIsvlpmdosVcGVGXFWp2oU9kLFL3dEkSz6NHEY1sjSRdIuDFWEhd8KxFqsRi1uM/nz9/zpxnwlESONdg6dKlbsaMGS4EHFHtjFIDHwKOo46l4TxSuxgDzi+rE2jg+BaFruOX4HXa0Nnf1lwAPufZeF8/r6zD97WK2qFnGjBxTw5qNGPxT+5T/r7/7RawFC3j4vTp09koCxkeHjqbHJqArmH5UrFKKksnxrK7FuRIs8STfBZv+luugXZ2pR/pP9Ois4z+TiMzUUkUjD0iEi1fzX8GmXyuxUBRcaUfykV0YZnlJGKQpOiGB76x5GeWkWWJc3mOrK6S7xdND+W5N6XyaRgtWJFe13GkaZnKOsYqGdOVVVbGupsyA/l7emTLHi7vwTdirNEt0qxnzAvBFcnQF16xh/TMpUuXHDowhlA9vQVraQhkudRdzOnK+04ZSP3DUhVSP61YsaLtd/ks7ZgtPcXqPqEafHkdqa84X6aCeL7YWlv6edGFHb+ZFICPlljHhg0bKuk0CSvVznWsotRu433alNdFrqG45ejoaPCaUkWERpLXjzFL2Rpllp7PJU2a/v7Ab8N05/9t27Z16KUqoFGsxnI9EosS2niSYg9SpU6B4JgTrvVW1flt1sT+0ADIJU2maXzcUTraGCRaL1Wp9rUMk16PMom8QhruxzvZIegJjFU7LLCePfS8uaQdPny4jTTL0dbee5mYokQsXTIWNY46kuMbnt8Kmec+LGWtOVIl9cT1rCB0V8WqkjAsRwta93TbwNYoGKsUSChN44lgBNCoHLHzquYKrU6qZ8lolCIN0Rh6cP0Q3U6I6IXILYOQI513hJaSKAorFpuHXJNfVlpRtmYBk1Su1obZr5dnKAO+L10Hrj3WZW+E3qh6IszE37F6EB+68mGpvKm4eb9bFrlzrok7fvr0Kfv727dvWRmdVTJHw0qiiCUSZ6wCK+7XL/AcsgNyL74DQQ730sv78Su7+t/A36MdY0sW5o40ahslXr58aZ5HtZB8GH64m9EmMZ7FpYw4T6QnrZfgenrhFxaSiSGXtPnz57e9TkNZLvTjeqhr734CNtrK41L40sUQckmj1lGKQ0rC37x544r8eNXRpnVE3ZZY7zXo8NomiO0ZUCj2uHz58rbXoZ6gc0uA+F6ZeKS/jhRDUq8MKrTho9fEkihMmhxtBI1DxKFY9XLpVcSkfoi8JGnToZO5sU5aiDQIW716ddt7ZLYtMQlhECdBGXZZMWldY5BHm5xgAroWj4C0hbYkSc/jBmggIrXJWlZM6pSETsEPGqZOndr2uuuR5rF169a2HoHPdurUKZM4CO1WTPqaDaAd+GFGKdIQkxAn9RuEWcTRyN2KSUgiSgF5aWzPTeA/lN5rZubMmR2bE4SIC4nJoltgAV/dVefZm72AtctUCJU2CMJ327hxY9t7EHbkyJFseq+EJSY16RPo3Dkq1kkr7+q0bNmyDuLQcZBEPYmHVdOBiJyIlrRDq41YPWfXOxUysi5fvtyaj+2BpcnsUV/oSoEMOk2CQGlr4ckhBwaetBhjCwH0ZHtJROPJkyc7UjcYLDjmrH7ADTEBXFfOYmB0k9oYBOjJ8b4aOYSe7QkKcYhFlq3QYLQhSidNmtS2RATwy8YOM3EQJsUjKiaWZ+vZToUQgzhkHXudb/PW5YMHD9yZM2faPsMwoc7RciYJXbGuBqJ1UIGKKLv915jsvgtJxCZDubdXr165mzdvtr1Hz5LONA8jrUwKPqsmVesKa49S3Q4WxmRPUEYdTjgiUcfUwLx589ySJUva3oMkP6IYddq6HMS4o55xBJBUeRjzfa4Zdeg56QZ43LhxoyPo7Lf1kNt7oO8wWAbNwaYjIv5lhyS7kRf96dvm5Jah8vfvX3flyhX35cuX6HfzFHOToS1H4BenCaHvO8pr8iDuwoUL7tevX+b5ZdbBair0xkFIlFDlW4ZknEClsp/TzXyAKVOmmHWFVSbDNw1l1+4f90U6IY/q4V27dpnE9bJ+v87QEydjqx/UamVVPRG+mwkNTYN+9tjkwzEx+atCm/X9WvWtDtAb68Wy9LXa1UmvCDDIpPkyOQ5ZwSzJ4jMrvFcr0rSjOUh+GcT4LSg5ugkW1Io0/SCDQBojh0hPlaJdah+tkVYrnTZowP8iq1F1TgMBBauufyB33x1v+NWFYmT5KmppgHC+NkAgbmRkpD3yn9QIseXymoTQFGQmIOKTxiZIWpvAatenVqRVXf2nTrAWMsPnKrMZHz6bJq5jvce6QK8J1cQNgKxlJapMPdZSR64/UivS9NztpkVEdKcrs5alhhWP9NeqlfWopzhZScI6QxseegZRGeg5a8C3Re1Mfl1ScP36ddcUaMuv24iOJtz7sbUjTS4qBvKmstYJoUauiuD3k5qhyr7QdUHMeCgLa1Ear9NquemdXgmum4fvJ6w1lqsuDhNrg1qSpleJK7K3TF0Q2jSd94uSZ60kK1e3qyVpQK6PVWXp2/FC3mp6jBhKKOiY2h3gtUV64TWM6wDETRPLDfSakXmH3w8g9Jlug8ZtTt4kVF0kLUYYmCCtD/DrQ5YhMGbA9L3ucdjh0y8kOHW5gU/VEEmJTcL4Pz/f7mgoAbYkAAAAAElFTkSuQmCC",
+ },
+ ],
+ }
+ ],
+ max_tokens=300,
+)
+
+completion = client.completions.create(
+ model="llama3.2",
+ prompt="Say this is a test",
+)
+
+list_completion = client.models.list()
+
+model = client.models.retrieve("llama3.2")
+
+embeddings = client.embeddings.create(
+ model="all-minilm",
+ input=["why is the sky blue?", "why is the grass green?"],
+)
+```
+
+### OpenAI JavaScript library
+
+```javascript
+import OpenAI from 'openai'
+
+const openai = new OpenAI({
+ baseURL: 'http://localhost:11434/v1/',
+
+ // required but ignored
+ apiKey: 'ollama',
+})
+
+const chatCompletion = await openai.chat.completions.create({
+ messages: [{ role: 'user', content: 'Say this is a test' }],
+ model: 'llama3.2',
+})
+
+const response = await openai.chat.completions.create({
+ model: "llava",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "What's in this image?" },
+ {
+ type: "image_url",
+ image_url: "iVBORw0KGgoAAAANSUhEUgAAAG0AAABmCAYAAADBPx+VAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAA3VSURBVHgB7Z27r0zdG8fX743i1bi1ikMoFMQloXRpKFFIqI7LH4BEQ+NWIkjQuSWCRIEoULk0gsK1kCBI0IhrQVT7tz/7zZo888yz1r7MnDl7z5xvsjkzs2fP3uu71nNfa7lkAsm7d++Sffv2JbNmzUqcc8m0adOSzZs3Z+/XES4ZckAWJEGWPiCxjsQNLWmQsWjRIpMseaxcuTKpG/7HP27I8P79e7dq1ars/yL4/v27S0ejqwv+cUOGEGGpKHR37tzJCEpHV9tnT58+dXXCJDdECBE2Ojrqjh071hpNECjx4cMHVycM1Uhbv359B2F79+51586daxN/+pyRkRFXKyRDAqxEp4yMlDDzXG1NPnnyJKkThoK0VFd1ELZu3TrzXKxKfW7dMBQ6bcuWLW2v0VlHjx41z717927ba22U9APcw7Nnz1oGEPeL3m3p2mTAYYnFmMOMXybPPXv2bNIPpFZr1NHn4HMw0KRBjg9NuRw95s8PEcz/6DZELQd/09C9QGq5RsmSRybqkwHGjh07OsJSsYYm3ijPpyHzoiacg35MLdDSIS/O1yM778jOTwYUkKNHWUzUWaOsylE00MyI0fcnOwIdjvtNdW/HZwNLGg+sR1kMepSNJXmIwxBZiG8tDTpEZzKg0GItNsosY8USkxDhD0Rinuiko2gfL/RbiD2LZAjU9zKQJj8RDR0vJBR1/Phx9+PHj9Z7REF4nTZkxzX4LCXHrV271qXkBAPGfP/atWvu/PnzHe4C97F48eIsRLZ9+3a3f/9+87dwP1JxaF7/3r17ba+5l4EcaVo0lj3SBq5kGTJSQmLWMjgYNei2GPT1MuMqGTDEFHzeQSP2wi/jGnkmPJ/nhccs44jvDAxpVcxnq0F6eT8h4ni/iIWpR5lPyA6ETkNXoSukvpJAD3AsXLiwpZs49+fPn5ke4j10TqYvegSfn0OnafC+Tv9ooA/JPkgQysqQNBzagXY55nO/oa1F7qvIPWkRL12WRpMWUvpVDYmxAPehxWSe8ZEXL20sadYIozfmNch4QJPAfeJgW3rNsnzphBKNJM2KKODo1rVOMRYik5ETy3ix4qWNI81qAAirizgMIc+yhTytx0JWZuNI03qsrgWlGtwjoS9XwgUhWGyhUaRZZQNNIEwCiXD16tXcAHUs79co0vSD8rrJCIW98pzvxpAWyyo3HYwqS0+H0BjStClcZJT5coMm6D2LOF8TolGJtK9fvyZpyiC5ePFi9nc/oJU4eiEP0jVoAnHa9wyJycITMP78+eMeP37sXrx44d6+fdt6f82aNdkx1pg9e3Zb5W+RSRE+n+VjksQWifvVaTKFhn5O8my63K8Qabdv33b379/PiAP//vuvW7BggZszZ072/+TJk91YgkafPn166zXB1rQHFvouAWHq9z3SEevSUerqCn2/dDCeta2jxYbr69evk4MHDyY7d+7MjhMnTiTPnz9Pfv/+nfQT2ggpO2dMF8cghuoM7Ygj5iWCqRlGFml0QC/ftGmTmzt3rmsaKDsgBSPh0/8yPeLLBihLkOKJc0jp8H8vUzcxIA1k6QJ/c78tWEyj5P3o4u9+jywNPdJi5rAH9x0KHcl4Hg570eQp3+vHXGyrmEeigzQsQsjavXt38ujRo44LQuDDhw+TW7duRS1HGgMxhNXHgflaNTOsHyKvHK5Ijo2jbFjJBQK9YwFd6RVMzfgRBmEfP37suBBm/p49e1qjEP2mwTViNRo0VJWH1deMXcNK08uUjVUu7s/zRaL+oLNxz1bpANco4npUgX4G2eFbpDFyQoQxojBCpEGSytmOH8qrH5Q9vuzD6ofQylkCUmh8DBAr+q8JCyVNtWQIidKQE9wNtLSQnS4jDSsxNHogzFuQBw4cyM61UKVsjfr3ooBkPSqqQHesUPWVtzi9/vQi1T+rJj7WiTz4Pt/l3LxUkr5P2VYZaZ4URpsE+st/dujQoaBBYokbrz/8TJNQYLSonrPS9kUaSkPeZyj1AWSj+d+VBoy1pIWVNed8P0Ll/ee5HdGRhrHhR5GGN0r4LGZBaj8oFDJitBTJzIZgFcmU0Y8ytWMZMzJOaXUSrUs5RxKnrxmbb5YXO9VGUhtpXldhEUogFr3IzIsvlpmdosVcGVGXFWp2oU9kLFL3dEkSz6NHEY1sjSRdIuDFWEhd8KxFqsRi1uM/nz9/zpxnwlESONdg6dKlbsaMGS4EHFHtjFIDHwKOo46l4TxSuxgDzi+rE2jg+BaFruOX4HXa0Nnf1lwAPufZeF8/r6zD97WK2qFnGjBxTw5qNGPxT+5T/r7/7RawFC3j4vTp09koCxkeHjqbHJqArmH5UrFKKksnxrK7FuRIs8STfBZv+luugXZ2pR/pP9Ois4z+TiMzUUkUjD0iEi1fzX8GmXyuxUBRcaUfykV0YZnlJGKQpOiGB76x5GeWkWWJc3mOrK6S7xdND+W5N6XyaRgtWJFe13GkaZnKOsYqGdOVVVbGupsyA/l7emTLHi7vwTdirNEt0qxnzAvBFcnQF16xh/TMpUuXHDowhlA9vQVraQhkudRdzOnK+04ZSP3DUhVSP61YsaLtd/ks7ZgtPcXqPqEafHkdqa84X6aCeL7YWlv6edGFHb+ZFICPlljHhg0bKuk0CSvVznWsotRu433alNdFrqG45ejoaPCaUkWERpLXjzFL2Rpllp7PJU2a/v7Ab8N05/9t27Z16KUqoFGsxnI9EosS2niSYg9SpU6B4JgTrvVW1flt1sT+0ADIJU2maXzcUTraGCRaL1Wp9rUMk16PMom8QhruxzvZIegJjFU7LLCePfS8uaQdPny4jTTL0dbee5mYokQsXTIWNY46kuMbnt8Kmec+LGWtOVIl9cT1rCB0V8WqkjAsRwta93TbwNYoGKsUSChN44lgBNCoHLHzquYKrU6qZ8lolCIN0Rh6cP0Q3U6I6IXILYOQI513hJaSKAorFpuHXJNfVlpRtmYBk1Su1obZr5dnKAO+L10Hrj3WZW+E3qh6IszE37F6EB+68mGpvKm4eb9bFrlzrok7fvr0Kfv727dvWRmdVTJHw0qiiCUSZ6wCK+7XL/AcsgNyL74DQQ730sv78Su7+t/A36MdY0sW5o40ahslXr58aZ5HtZB8GH64m9EmMZ7FpYw4T6QnrZfgenrhFxaSiSGXtPnz57e9TkNZLvTjeqhr734CNtrK41L40sUQckmj1lGKQ0rC37x544r8eNXRpnVE3ZZY7zXo8NomiO0ZUCj2uHz58rbXoZ6gc0uA+F6ZeKS/jhRDUq8MKrTho9fEkihMmhxtBI1DxKFY9XLpVcSkfoi8JGnToZO5sU5aiDQIW716ddt7ZLYtMQlhECdBGXZZMWldY5BHm5xgAroWj4C0hbYkSc/jBmggIrXJWlZM6pSETsEPGqZOndr2uuuR5rF169a2HoHPdurUKZM4CO1WTPqaDaAd+GFGKdIQkxAn9RuEWcTRyN2KSUgiSgF5aWzPTeA/lN5rZubMmR2bE4SIC4nJoltgAV/dVefZm72AtctUCJU2CMJ327hxY9t7EHbkyJFseq+EJSY16RPo3Dkq1kkr7+q0bNmyDuLQcZBEPYmHVdOBiJyIlrRDq41YPWfXOxUysi5fvtyaj+2BpcnsUV/oSoEMOk2CQGlr4ckhBwaetBhjCwH0ZHtJROPJkyc7UjcYLDjmrH7ADTEBXFfOYmB0k9oYBOjJ8b4aOYSe7QkKcYhFlq3QYLQhSidNmtS2RATwy8YOM3EQJsUjKiaWZ+vZToUQgzhkHXudb/PW5YMHD9yZM2faPsMwoc7RciYJXbGuBqJ1UIGKKLv915jsvgtJxCZDubdXr165mzdvtr1Hz5LONA8jrUwKPqsmVesKa49S3Q4WxmRPUEYdTjgiUcfUwLx589ySJUva3oMkP6IYddq6HMS4o55xBJBUeRjzfa4Zdeg56QZ43LhxoyPo7Lf1kNt7oO8wWAbNwaYjIv5lhyS7kRf96dvm5Jah8vfvX3flyhX35cuX6HfzFHOToS1H4BenCaHvO8pr8iDuwoUL7tevX+b5ZdbBair0xkFIlFDlW4ZknEClsp/TzXyAKVOmmHWFVSbDNw1l1+4f90U6IY/q4V27dpnE9bJ+v87QEydjqx/UamVVPRG+mwkNTYN+9tjkwzEx+atCm/X9WvWtDtAb68Wy9LXa1UmvCDDIpPkyOQ5ZwSzJ4jMrvFcr0rSjOUh+GcT4LSg5ugkW1Io0/SCDQBojh0hPlaJdah+tkVYrnTZowP8iq1F1TgMBBauufyB33x1v+NWFYmT5KmppgHC+NkAgbmRkpD3yn9QIseXymoTQFGQmIOKTxiZIWpvAatenVqRVXf2nTrAWMsPnKrMZHz6bJq5jvce6QK8J1cQNgKxlJapMPdZSR64/UivS9NztpkVEdKcrs5alhhWP9NeqlfWopzhZScI6QxseegZRGeg5a8C3Re1Mfl1ScP36ddcUaMuv24iOJtz7sbUjTS4qBvKmstYJoUauiuD3k5qhyr7QdUHMeCgLa1Ear9NquemdXgmum4fvJ6w1lqsuDhNrg1qSpleJK7K3TF0Q2jSd94uSZ60kK1e3qyVpQK6PVWXp2/FC3mp6jBhKKOiY2h3gtUV64TWM6wDETRPLDfSakXmH3w8g9Jlug8ZtTt4kVF0kLUYYmCCtD/DrQ5YhMGbA9L3ucdjh0y8kOHW5gU/VEEmJTcL4Pz/f7mgoAbYkAAAAAElFTkSuQmCC",
+ },
+ ],
+ },
+ ],
+})
+
+const completion = await openai.completions.create({
+ model: "llama3.2",
+ prompt: "Say this is a test.",
+})
+
+const listCompletion = await openai.models.list()
+
+const model = await openai.models.retrieve("llama3.2")
+
+const embedding = await openai.embeddings.create({
+ model: "all-minilm",
+ input: ["why is the sky blue?", "why is the grass green?"],
+})
+```
+
+### `curl`
+
+``` shell
+curl http://localhost:11434/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "llama3.2",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant."
+ },
+ {
+ "role": "user",
+ "content": "Hello!"
+ }
+ ]
+ }'
+
+curl http://localhost:11434/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "llava",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What'\''s in this image?"
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "iVBORw0KGgoAAAANSUhEUgAAAG0AAABmCAYAAADBPx+VAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAA3VSURBVHgB7Z27r0zdG8fX743i1bi1ikMoFMQloXRpKFFIqI7LH4BEQ+NWIkjQuSWCRIEoULk0gsK1kCBI0IhrQVT7tz/7zZo888yz1r7MnDl7z5xvsjkzs2fP3uu71nNfa7lkAsm7d++Sffv2JbNmzUqcc8m0adOSzZs3Z+/XES4ZckAWJEGWPiCxjsQNLWmQsWjRIpMseaxcuTKpG/7HP27I8P79e7dq1ars/yL4/v27S0ejqwv+cUOGEGGpKHR37tzJCEpHV9tnT58+dXXCJDdECBE2Ojrqjh071hpNECjx4cMHVycM1Uhbv359B2F79+51586daxN/+pyRkRFXKyRDAqxEp4yMlDDzXG1NPnnyJKkThoK0VFd1ELZu3TrzXKxKfW7dMBQ6bcuWLW2v0VlHjx41z717927ba22U9APcw7Nnz1oGEPeL3m3p2mTAYYnFmMOMXybPPXv2bNIPpFZr1NHn4HMw0KRBjg9NuRw95s8PEcz/6DZELQd/09C9QGq5RsmSRybqkwHGjh07OsJSsYYm3ijPpyHzoiacg35MLdDSIS/O1yM778jOTwYUkKNHWUzUWaOsylE00MyI0fcnOwIdjvtNdW/HZwNLGg+sR1kMepSNJXmIwxBZiG8tDTpEZzKg0GItNsosY8USkxDhD0Rinuiko2gfL/RbiD2LZAjU9zKQJj8RDR0vJBR1/Phx9+PHj9Z7REF4nTZkxzX4LCXHrV271qXkBAPGfP/atWvu/PnzHe4C97F48eIsRLZ9+3a3f/9+87dwP1JxaF7/3r17ba+5l4EcaVo0lj3SBq5kGTJSQmLWMjgYNei2GPT1MuMqGTDEFHzeQSP2wi/jGnkmPJ/nhccs44jvDAxpVcxnq0F6eT8h4ni/iIWpR5lPyA6ETkNXoSukvpJAD3AsXLiwpZs49+fPn5ke4j10TqYvegSfn0OnafC+Tv9ooA/JPkgQysqQNBzagXY55nO/oa1F7qvIPWkRL12WRpMWUvpVDYmxAPehxWSe8ZEXL20sadYIozfmNch4QJPAfeJgW3rNsnzphBKNJM2KKODo1rVOMRYik5ETy3ix4qWNI81qAAirizgMIc+yhTytx0JWZuNI03qsrgWlGtwjoS9XwgUhWGyhUaRZZQNNIEwCiXD16tXcAHUs79co0vSD8rrJCIW98pzvxpAWyyo3HYwqS0+H0BjStClcZJT5coMm6D2LOF8TolGJtK9fvyZpyiC5ePFi9nc/oJU4eiEP0jVoAnHa9wyJycITMP78+eMeP37sXrx44d6+fdt6f82aNdkx1pg9e3Zb5W+RSRE+n+VjksQWifvVaTKFhn5O8my63K8Qabdv33b379/PiAP//vuvW7BggZszZ072/+TJk91YgkafPn166zXB1rQHFvouAWHq9z3SEevSUerqCn2/dDCeta2jxYbr69evk4MHDyY7d+7MjhMnTiTPnz9Pfv/+nfQT2ggpO2dMF8cghuoM7Ygj5iWCqRlGFml0QC/ftGmTmzt3rmsaKDsgBSPh0/8yPeLLBihLkOKJc0jp8H8vUzcxIA1k6QJ/c78tWEyj5P3o4u9+jywNPdJi5rAH9x0KHcl4Hg570eQp3+vHXGyrmEeigzQsQsjavXt38ujRo44LQuDDhw+TW7duRS1HGgMxhNXHgflaNTOsHyKvHK5Ijo2jbFjJBQK9YwFd6RVMzfgRBmEfP37suBBm/p49e1qjEP2mwTViNRo0VJWH1deMXcNK08uUjVUu7s/zRaL+oLNxz1bpANco4npUgX4G2eFbpDFyQoQxojBCpEGSytmOH8qrH5Q9vuzD6ofQylkCUmh8DBAr+q8JCyVNtWQIidKQE9wNtLSQnS4jDSsxNHogzFuQBw4cyM61UKVsjfr3ooBkPSqqQHesUPWVtzi9/vQi1T+rJj7WiTz4Pt/l3LxUkr5P2VYZaZ4URpsE+st/dujQoaBBYokbrz/8TJNQYLSonrPS9kUaSkPeZyj1AWSj+d+VBoy1pIWVNed8P0Ll/ee5HdGRhrHhR5GGN0r4LGZBaj8oFDJitBTJzIZgFcmU0Y8ytWMZMzJOaXUSrUs5RxKnrxmbb5YXO9VGUhtpXldhEUogFr3IzIsvlpmdosVcGVGXFWp2oU9kLFL3dEkSz6NHEY1sjSRdIuDFWEhd8KxFqsRi1uM/nz9/zpxnwlESONdg6dKlbsaMGS4EHFHtjFIDHwKOo46l4TxSuxgDzi+rE2jg+BaFruOX4HXa0Nnf1lwAPufZeF8/r6zD97WK2qFnGjBxTw5qNGPxT+5T/r7/7RawFC3j4vTp09koCxkeHjqbHJqArmH5UrFKKksnxrK7FuRIs8STfBZv+luugXZ2pR/pP9Ois4z+TiMzUUkUjD0iEi1fzX8GmXyuxUBRcaUfykV0YZnlJGKQpOiGB76x5GeWkWWJc3mOrK6S7xdND+W5N6XyaRgtWJFe13GkaZnKOsYqGdOVVVbGupsyA/l7emTLHi7vwTdirNEt0qxnzAvBFcnQF16xh/TMpUuXHDowhlA9vQVraQhkudRdzOnK+04ZSP3DUhVSP61YsaLtd/ks7ZgtPcXqPqEafHkdqa84X6aCeL7YWlv6edGFHb+ZFICPlljHhg0bKuk0CSvVznWsotRu433alNdFrqG45ejoaPCaUkWERpLXjzFL2Rpllp7PJU2a/v7Ab8N05/9t27Z16KUqoFGsxnI9EosS2niSYg9SpU6B4JgTrvVW1flt1sT+0ADIJU2maXzcUTraGCRaL1Wp9rUMk16PMom8QhruxzvZIegJjFU7LLCePfS8uaQdPny4jTTL0dbee5mYokQsXTIWNY46kuMbnt8Kmec+LGWtOVIl9cT1rCB0V8WqkjAsRwta93TbwNYoGKsUSChN44lgBNCoHLHzquYKrU6qZ8lolCIN0Rh6cP0Q3U6I6IXILYOQI513hJaSKAorFpuHXJNfVlpRtmYBk1Su1obZr5dnKAO+L10Hrj3WZW+E3qh6IszE37F6EB+68mGpvKm4eb9bFrlzrok7fvr0Kfv727dvWRmdVTJHw0qiiCUSZ6wCK+7XL/AcsgNyL74DQQ730sv78Su7+t/A36MdY0sW5o40ahslXr58aZ5HtZB8GH64m9EmMZ7FpYw4T6QnrZfgenrhFxaSiSGXtPnz57e9TkNZLvTjeqhr734CNtrK41L40sUQckmj1lGKQ0rC37x544r8eNXRpnVE3ZZY7zXo8NomiO0ZUCj2uHz58rbXoZ6gc0uA+F6ZeKS/jhRDUq8MKrTho9fEkihMmhxtBI1DxKFY9XLpVcSkfoi8JGnToZO5sU5aiDQIW716ddt7ZLYtMQlhECdBGXZZMWldY5BHm5xgAroWj4C0hbYkSc/jBmggIrXJWlZM6pSETsEPGqZOndr2uuuR5rF169a2HoHPdurUKZM4CO1WTPqaDaAd+GFGKdIQkxAn9RuEWcTRyN2KSUgiSgF5aWzPTeA/lN5rZubMmR2bE4SIC4nJoltgAV/dVefZm72AtctUCJU2CMJ327hxY9t7EHbkyJFseq+EJSY16RPo3Dkq1kkr7+q0bNmyDuLQcZBEPYmHVdOBiJyIlrRDq41YPWfXOxUysi5fvtyaj+2BpcnsUV/oSoEMOk2CQGlr4ckhBwaetBhjCwH0ZHtJROPJkyc7UjcYLDjmrH7ADTEBXFfOYmB0k9oYBOjJ8b4aOYSe7QkKcYhFlq3QYLQhSidNmtS2RATwy8YOM3EQJsUjKiaWZ+vZToUQgzhkHXudb/PW5YMHD9yZM2faPsMwoc7RciYJXbGuBqJ1UIGKKLv915jsvgtJxCZDubdXr165mzdvtr1Hz5LONA8jrUwKPqsmVesKa49S3Q4WxmRPUEYdTjgiUcfUwLx589ySJUva3oMkP6IYddq6HMS4o55xBJBUeRjzfa4Zdeg56QZ43LhxoyPo7Lf1kNt7oO8wWAbNwaYjIv5lhyS7kRf96dvm5Jah8vfvX3flyhX35cuX6HfzFHOToS1H4BenCaHvO8pr8iDuwoUL7tevX+b5ZdbBair0xkFIlFDlW4ZknEClsp/TzXyAKVOmmHWFVSbDNw1l1+4f90U6IY/q4V27dpnE9bJ+v87QEydjqx/UamVVPRG+mwkNTYN+9tjkwzEx+atCm/X9WvWtDtAb68Wy9LXa1UmvCDDIpPkyOQ5ZwSzJ4jMrvFcr0rSjOUh+GcT4LSg5ugkW1Io0/SCDQBojh0hPlaJdah+tkVYrnTZowP8iq1F1TgMBBauufyB33x1v+NWFYmT5KmppgHC+NkAgbmRkpD3yn9QIseXymoTQFGQmIOKTxiZIWpvAatenVqRVXf2nTrAWMsPnKrMZHz6bJq5jvce6QK8J1cQNgKxlJapMPdZSR64/UivS9NztpkVEdKcrs5alhhWP9NeqlfWopzhZScI6QxseegZRGeg5a8C3Re1Mfl1ScP36ddcUaMuv24iOJtz7sbUjTS4qBvKmstYJoUauiuD3k5qhyr7QdUHMeCgLa1Ear9NquemdXgmum4fvJ6w1lqsuDhNrg1qSpleJK7K3TF0Q2jSd94uSZ60kK1e3qyVpQK6PVWXp2/FC3mp6jBhKKOiY2h3gtUV64TWM6wDETRPLDfSakXmH3w8g9Jlug8ZtTt4kVF0kLUYYmCCtD/DrQ5YhMGbA9L3ucdjh0y8kOHW5gU/VEEmJTcL4Pz/f7mgoAbYkAAAAAElFTkSuQmCC"
+ }
+ }
+ ]
+ }
+ ],
+ "max_tokens": 300
+ }'
+
+curl http://localhost:11434/v1/completions \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "llama3.2",
+ "prompt": "Say this is a test"
+ }'
+
+curl http://localhost:11434/v1/models
+
+curl http://localhost:11434/v1/models/llama3.2
+
+curl http://localhost:11434/v1/embeddings \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "all-minilm",
+ "input": ["why is the sky blue?", "why is the grass green?"]
+ }'
+```
+
+## Endpoints
+
+### `/v1/chat/completions`
+
+#### Supported features
+
+- [x] Chat completions
+- [x] Streaming
+- [x] JSON mode
+- [x] Reproducible outputs
+- [x] Vision
+- [x] Tools (streaming support coming soon)
+- [ ] Logprobs
+
+#### Supported request fields
+
+- [x] `model`
+- [x] `messages`
+ - [x] Text `content`
+ - [x] Image `content`
+ - [x] Base64 encoded image
+ - [ ] Image URL
+ - [x] Array of `content` parts
+- [x] `frequency_penalty`
+- [x] `presence_penalty`
+- [x] `response_format`
+- [x] `seed`
+- [x] `stop`
+- [x] `stream`
+- [x] `temperature`
+- [x] `top_p`
+- [x] `max_tokens`
+- [x] `tools`
+- [ ] `tool_choice`
+- [ ] `logit_bias`
+- [ ] `user`
+- [ ] `n`
+
+### `/v1/completions`
+
+#### Supported features
+
+- [x] Completions
+- [x] Streaming
+- [x] JSON mode
+- [x] Reproducible outputs
+- [ ] Logprobs
+
+#### Supported request fields
+
+- [x] `model`
+- [x] `prompt`
+- [x] `frequency_penalty`
+- [x] `presence_penalty`
+- [x] `seed`
+- [x] `stop`
+- [x] `stream`
+- [x] `temperature`
+- [x] `top_p`
+- [x] `max_tokens`
+- [x] `suffix`
+- [ ] `best_of`
+- [ ] `echo`
+- [ ] `logit_bias`
+- [ ] `user`
+- [ ] `n`
+
+#### Notes
+
+- `prompt` currently only accepts a string
+
+### `/v1/models`
+
+#### Notes
+
+- `created` corresponds to when the model was last modified
+- `owned_by` corresponds to the ollama username, defaulting to `"library"`
+
+### `/v1/models/{model}`
+
+#### Notes
+
+- `created` corresponds to when the model was last modified
+- `owned_by` corresponds to the ollama username, defaulting to `"library"`
+
+### `/v1/embeddings`
+
+#### Supported request fields
+
+- [x] `model`
+- [x] `input`
+ - [x] string
+ - [x] array of strings
+ - [ ] array of tokens
+ - [ ] array of token arrays
+- [ ] `encoding format`
+- [ ] `dimensions`
+- [ ] `user`
+
+## Models
+
+Before using a model, pull it locally `ollama pull`:
+
+```shell
+ollama pull llama3.2
+```
+
+### Default model names
+
+For tooling that relies on default OpenAI model names such as `gpt-3.5-turbo`, use `ollama cp` to copy an existing model name to a temporary name:
+
+```
+ollama cp llama3.2 gpt-3.5-turbo
+```
+
+Afterwards, this new model name can be specified the `model` field:
+
+```shell
+curl http://localhost:11434/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Hello!"
+ }
+ ]
+ }'
+```
+
+### Setting the context size
+
+The OpenAI API does not have a way of setting the context size for a model. If you need to change the context size, create a `Modelfile` which looks like:
+
+```modelfile
+FROM <some model>
+PARAMETER num_ctx <context size>
+```
+
+Use the `ollama create mymodel` command to create a new model with the updated context size. Call the API with the updated model name:
+
+```shell
+curl http://localhost:11434/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "mymodel",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Hello!"
+ }
+ ]
+ }'
+```
diff --git a/docs/template.md b/docs/template.md
new file mode 100644
index 00000000..bd367e91
--- /dev/null
+++ b/docs/template.md
@@ -0,0 +1,167 @@
+# Template
+
+Ollama provides a powerful templating engine backed by Go's built-in templating engine to construct prompts for your large language model. This feature is a valuable tool to get the most out of your models.
+
+## Basic Template Structure
+
+A basic Go template consists of three main parts:
+
+* **Layout**: The overall structure of the template.
+* **Variables**: Placeholders for dynamic data that will be replaced with actual values when the template is rendered.
+* **Functions**: Custom functions or logic that can be used to manipulate the template's content.
+
+Here's an example of a simple chat template:
+
+```gotmpl
+{{- range .Messages }}
+{{ .Role }}: {{ .Content }}
+{{- end }}
+```
+
+In this example, we have:
+
+* A basic messages structure (layout)
+* Three variables: `Messages`, `Role`, and `Content` (variables)
+* A custom function (action) that iterates over an array of items (`range .Messages`) and displays each item
+
+## Adding templates to your model
+
+By default, models imported into Ollama have a default template of `{{ .Prompt }}`, i.e. user inputs are sent verbatim to the LLM. This is appropriate for text or code completion models but lacks essential markers for chat or instruction models.
+
+Omitting a template in these models puts the responsibility of correctly templating input onto the user. Adding a template allows users to easily get the best results from the model.
+
+To add templates in your model, you'll need to add a `TEMPLATE` command to the Modelfile. Here's an example using Meta's Llama 3.
+
+```dockerfile
+FROM llama3.2
+
+TEMPLATE """{{- if .System }}<|start_header_id|>system<|end_header_id|>
+
+{{ .System }}<|eot_id|>
+{{- end }}
+{{- range .Messages }}<|start_header_id|>{{ .Role }}<|end_header_id|>
+
+{{ .Content }}<|eot_id|>
+{{- end }}<|start_header_id|>assistant<|end_header_id|>
+
+"""
+```
+
+## Variables
+
+`System` (string): system prompt
+
+`Prompt` (string): user prompt
+
+`Response` (string): assistant response
+
+`Suffix` (string): text inserted after the assistant's response
+
+`Messages` (list): list of messages
+
+`Messages[].Role` (string): role which can be one of `system`, `user`, `assistant`, or `tool`
+
+`Messages[].Content` (string): message content
+
+`Messages[].ToolCalls` (list): list of tools the model wants to call
+
+`Messages[].ToolCalls[].Function` (object): function to call
+
+`Messages[].ToolCalls[].Function.Name` (string): function name
+
+`Messages[].ToolCalls[].Function.Arguments` (map): mapping of argument name to argument value
+
+`Tools` (list): list of tools the model can access
+
+`Tools[].Type` (string): schema type. `type` is always `function`
+
+`Tools[].Function` (object): function definition
+
+`Tools[].Function.Name` (string): function name
+
+`Tools[].Function.Description` (string): function description
+
+`Tools[].Function.Parameters` (object): function parameters
+
+`Tools[].Function.Parameters.Type` (string): schema type. `type` is always `object`
+
+`Tools[].Function.Parameters.Required` (list): list of required properties
+
+`Tools[].Function.Parameters.Properties` (map): mapping of property name to property definition
+
+`Tools[].Function.Parameters.Properties[].Type` (string): property type
+
+`Tools[].Function.Parameters.Properties[].Description` (string): property description
+
+`Tools[].Function.Parameters.Properties[].Enum` (list): list of valid values
+
+## Tips and Best Practices
+
+Keep the following tips and best practices in mind when working with Go templates:
+
+* **Be mindful of dot**: Control flow structures like `range` and `with` changes the value `.`
+* **Out-of-scope variables**: Use `$.` to reference variables not currently in scope, starting from the root
+* **Whitespace control**: Use `-` to trim leading (`{{-`) and trailing (`-}}`) whitespace
+
+## Examples
+
+### Example Messages
+
+#### ChatML
+
+ChatML is a popular template format. It can be used for models such as Databrick's DBRX, Intel's Neural Chat, and Microsoft's Orca 2.
+
+```gotmpl
+{{- range .Messages }}<|im_start|>{{ .Role }}
+{{ .Content }}<|im_end|>
+{{ end }}<|im_start|>assistant
+```
+
+### Example Tools
+
+Tools support can be added to a model by adding a `{{ .Tools }}` node to the template. This feature is useful for models trained to call external tools and can a powerful tool for retrieving real-time data or performing complex tasks.
+
+#### Mistral
+
+Mistral v0.3 and Mixtral 8x22B supports tool calling.
+
+```gotmpl
+{{- range $index, $_ := .Messages }}
+{{- if eq .Role "user" }}
+{{- if and (le (len (slice $.Messages $index)) 2) $.Tools }}[AVAILABLE_TOOLS] {{ json $.Tools }}[/AVAILABLE_TOOLS]
+{{- end }}[INST] {{ if and (eq (len (slice $.Messages $index)) 1) $.System }}{{ $.System }}
+
+{{ end }}{{ .Content }}[/INST]
+{{- else if eq .Role "assistant" }}
+{{- if .Content }} {{ .Content }}</s>
+{{- else if .ToolCalls }}[TOOL_CALLS] [
+{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ json .Function.Arguments }}}
+{{- end }}]</s>
+{{- end }}
+{{- else if eq .Role "tool" }}[TOOL_RESULTS] {"content": {{ .Content }}}[/TOOL_RESULTS]
+{{- end }}
+{{- end }}
+```
+
+### Example Fill-in-Middle
+
+Fill-in-middle support can be added to a model by adding a `{{ .Suffix }}` node to the template. This feature is useful for models that are trained to generate text in the middle of user input, such as code completion models.
+
+#### CodeLlama
+
+CodeLlama [7B](https://ollama.com/library/codellama:7b-code) and [13B](https://ollama.com/library/codellama:13b-code) code completion models support fill-in-middle.
+
+```gotmpl
+<PRE> {{ .Prompt }} <SUF>{{ .Suffix }} <MID>
+```
+
+> [!NOTE]
+> CodeLlama 34B and 70B code completion and all instruct and Python fine-tuned models do not support fill-in-middle.
+
+#### Codestral
+
+Codestral [22B](https://ollama.com/library/codestral:22b) supports fill-in-middle.
+
+```gotmpl
+[SUFFIX]{{ .Suffix }}[PREFIX] {{ .Prompt }}
+```
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index fe842bbd..0a89b87f 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -1,22 +1,107 @@
# How to troubleshoot issues
-Sometimes Ollama may not perform as expected. One of the best ways to figure out what happened is to take a look at the logs. Find the logs on Mac by running the command:
+Sometimes Ollama may not perform as expected. One of the best ways to figure out what happened is to take a look at the logs. Find the logs on **Mac** by running the command:
```shell
cat ~/.ollama/logs/server.log
```
-On Linux systems with systemd, the logs can be found with this command:
+On **Linux** systems with systemd, the logs can be found with this command:
```shell
-journalctl -u ollama
+journalctl -u ollama --no-pager
```
+When you run Ollama in a **container**, the logs go to stdout/stderr in the container:
+
+```shell
+docker logs <container-name>
+```
+(Use `docker ps` to find the container name)
+
If manually running `ollama serve` in a terminal, the logs will be on that terminal.
+When you run Ollama on **Windows**, there are a few different locations. You can view them in the explorer window by hitting `<cmd>+R` and type in:
+- `explorer %LOCALAPPDATA%\Ollama` to view logs. The most recent server logs will be in `server.log` and older logs will be in `server-#.log`
+- `explorer %LOCALAPPDATA%\Programs\Ollama` to browse the binaries (The installer adds this to your user PATH)
+- `explorer %HOMEPATH%\.ollama` to browse where models and configuration is stored
+- `explorer %TEMP%` where temporary executable files are stored in one or more `ollama*` directories
+
+To enable additional debug logging to help troubleshoot problems, first **Quit the running app from the tray menu** then in a powershell terminal
+```powershell
+$env:OLLAMA_DEBUG="1"
+& "ollama app.exe"
+```
+
Join the [Discord](https://discord.gg/ollama) for help interpreting the logs.
-## Known issues
+## LLM libraries
+
+Ollama includes multiple LLM libraries compiled for different GPUs and CPU vector features. Ollama tries to pick the best one based on the capabilities of your system. If this autodetection has problems, or you run into other problems (e.g. crashes in your GPU) you can workaround this by forcing a specific LLM library. `cpu_avx2` will perform the best, followed by `cpu_avx` an the slowest but most compatible is `cpu`. Rosetta emulation under MacOS will work with the `cpu` library.
+
+In the server log, you will see a message that looks something like this (varies from release to release):
+
+```
+Dynamic LLM libraries [rocm_v6 cpu cpu_avx cpu_avx2 cuda_v11 rocm_v5]
+```
+
+**Experimental LLM Library Override**
+
+You can set OLLAMA_LLM_LIBRARY to any of the available LLM libraries to bypass autodetection, so for example, if you have a CUDA card, but want to force the CPU LLM library with AVX2 vector support, use:
+
+```
+OLLAMA_LLM_LIBRARY="cpu_avx2" ollama serve
+```
+
+You can see what features your CPU has with the following.
+```
+cat /proc/cpuinfo| grep flags | head -1
+```
+
+## Installing older or pre-release versions on Linux
+
+If you run into problems on Linux and want to install an older version, or you'd like to try out a pre-release before it's officially released, you can tell the install script which version to install.
+
+```sh
+curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION="0.1.29" sh
+```
+
+## Linux tmp noexec
+
+If your system is configured with the "noexec" flag where Ollama stores its temporary executable files, you can specify an alternate location by setting OLLAMA_TMPDIR to a location writable by the user ollama runs as. For example OLLAMA_TMPDIR=/usr/share/ollama/
+
+## NVIDIA GPU Discovery
+
+When Ollama starts up, it takes inventory of the GPUs present in the system to determine compatibility and how much VRAM is available. Sometimes this discovery can fail to find your GPUs. In general, running the latest driver will yield the best results.
+
+### Linux NVIDIA Troubleshooting
+
+If you are using a container to run Ollama, make sure you've set up the container runtime first as described in [docker.md](./docker.md)
+
+Sometimes the Ollama can have difficulties initializing the GPU. When you check the server logs, this can show up as various error codes, such as "3" (not initialized), "46" (device unavailable), "100" (no device), "999" (unknown), or others. The following troubleshooting techniques may help resolve the problem
+
+- If you are using a container, is the container runtime working? Try `docker run --gpus all ubuntu nvidia-smi` - if this doesn't work, Ollama wont be able to see your NVIDIA GPU.
+- Is the uvm driver loaded? `sudo nvidia-modprobe -u`
+- Try reloading the nvidia_uvm driver - `sudo rmmod nvidia_uvm` then `sudo modprobe nvidia_uvm`
+- Try rebooting
+- Make sure you're running the latest nvidia drivers
+
+If none of those resolve the problem, gather additional information and file an issue:
+- Set `CUDA_ERROR_LEVEL=50` and try again to get more diagnostic logs
+- Check dmesg for any errors `sudo dmesg | grep -i nvrm` and `sudo dmesg | grep -i nvidia`
-* `signal: illegal instruction (core dumped)`: Ollama requires AVX support from the CPU. This was introduced in 2011 and CPUs started offering it in 2012. CPUs from before that and some lower end CPUs after that may not have AVX support and thus are not supported by Ollama. Some users have had luck with building Ollama on their machines disabling the need for AVX.
+## AMD GPU Discovery
+
+On linux, AMD GPU access typically requires `video` and/or `render` group membership to access the `/dev/kfd` device. If permissions are not set up correctly, Ollama will detect this and report an error in the server log.
+
+When running in a container, in some Linux distributions and container runtimes, the ollama process may be unable to access the GPU. Use `ls -ld /dev/kfd /dev/dri /dev/dri/*` on the host system to determine the group assignments on your system, and pass additional `--group-add ...` arguments to the container so it can access the required devices.
+
+If you are experiencing problems getting Ollama to correctly discover or use your GPU for inference, the following may help isolate the failure.
+- `AMD_LOG_LEVEL=3` Enable info log levels in the AMD HIP/ROCm libraries. This can help show more detailed error codes that can help troubleshoot problems
+- `OLLAMA_DEBUG=1` During GPU discovery additional information will be reported
+- Check dmesg for any errors from amdgpu or kfd drivers `sudo dmesg | grep -i amdgpu` and `sudo dmesg | grep -i kfd`
+
+## Windows Terminal Errors
+
+Older versions of Windows 10 (e.g., 21H1) are known to have a bug where the standard terminal program does not display control characters correctly. This can result in a long string of strings like `←[?25h←[?25l` being displayed, sometimes erroring with `The parameter is incorrect` To resolve this problem, please update to Win 10 22H1 or newer.
diff --git a/docs/tutorials/langchainjs.md b/docs/tutorials/langchainjs.md
index 94dc9081..86f895ae 100644
--- a/docs/tutorials/langchainjs.md
+++ b/docs/tutorials/langchainjs.md
@@ -5,28 +5,28 @@ In this tutorial, we are going to use JavaScript with LangChain and Ollama to le
To get started, let's just use **LangChain** to ask a simple question to a model. To do this with JavaScript, we need to install **LangChain**:
```bash
-npm install langchain
+npm install @langchain/community
```
Now we can start building out our JavaScript:
```javascript
-import { Ollama } from "langchain/llms/ollama";
+import { Ollama } from "@langchain/community/llms/ollama";
const ollama = new Ollama({
baseUrl: "http://localhost:11434",
- model: "llama2",
+ model: "llama3.2",
});
-const answer = await ollama.call(`why is the sky blue?`);
+const answer = await ollama.invoke(`why is the sky blue?`);
console.log(answer);
```
-That will get us the same thing as if we ran `ollama run llama2 "why is the sky blue"` in the terminal. But we want to load a document from the web to ask a question against. **Cheerio** is a great library for ingesting a webpage, and **LangChain** uses it in their **CheerioWebBaseLoader**. So let's install **Cheerio** and build that part of the app.
+That will get us the same thing as if we ran `ollama run llama3.2 "why is the sky blue"` in the terminal. But we want to load a document from the web to ask a question against. **Cheerio** is a great library for ingesting a webpage, and **LangChain** uses it in their **CheerioWebBaseLoader**. So let's install **Cheerio** and build that part of the app.
```bash
-npm install cheerio
+npm install cheerio
```
```javascript
diff --git a/docs/tutorials/langchainpy.md b/docs/tutorials/langchainpy.md
index ac712923..06543a07 100644
--- a/docs/tutorials/langchainpy.md
+++ b/docs/tutorials/langchainpy.md
@@ -12,15 +12,17 @@ So let's figure out how we can use **LangChain** with Ollama to ask our question
Let's start by asking a simple question that we can get an answer to from the **Llama2** model using **Ollama**. First, we need to install the **LangChain** package:
-`pip install langchain`
+`pip install langchain_community`
Then we can create a model and ask the question:
```python
-from langchain.llms import Ollama
-ollama = Ollama(base_url='http://localhost:11434',
-model="llama2")
-print(ollama("why is the sky blue"))
+from langchain_community.llms import Ollama
+ollama = Ollama(
+ base_url='http://localhost:11434',
+ model="llama3"
+)
+print(ollama.invoke("why is the sky blue"))
```
Notice that we are defining the model and the base URL for Ollama.
@@ -42,12 +44,12 @@ text_splitter=RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0)
all_splits = text_splitter.split_documents(data)
```
-It's split up, but we have to find the relevant splits and then submit those to the model. We can do this by creating embeddings and storing them in a vector database. We can use Ollama directly to instantiate an embedding model. We will use ChromaDB in this example for a vector database. `pip install GPT4All chromadb`
-
+It's split up, but we have to find the relevant splits and then submit those to the model. We can do this by creating embeddings and storing them in a vector database. We can use Ollama directly to instantiate an embedding model. We will use ChromaDB in this example for a vector database. `pip install chromadb`
+We also need to pull embedding model: `ollama pull nomic-embed-text`
```python
from langchain.embeddings import OllamaEmbeddings
from langchain.vectorstores import Chroma
-oembed = OllamaEmbeddings(base_url="http://localhost:11434", model="llama2")
+oembed = OllamaEmbeddings(base_url="http://localhost:11434", model="nomic-embed-text")
vectorstore = Chroma.from_documents(documents=all_splits, embedding=oembed)
```
@@ -66,7 +68,8 @@ The next thing is to send the question and the relevant parts of the docs to the
```python
from langchain.chains import RetrievalQA
qachain=RetrievalQA.from_chain_type(ollama, retriever=vectorstore.as_retriever())
-qachain({"query": question})
+res = qachain.invoke({"query": question})
+print(res['result'])
```
The answer received from this chain was:
diff --git a/docs/tutorials/nvidia-jetson.md b/docs/tutorials/nvidia-jetson.md
index 85cf741c..bb77c486 100644
--- a/docs/tutorials/nvidia-jetson.md
+++ b/docs/tutorials/nvidia-jetson.md
@@ -1,38 +1,15 @@
# Running Ollama on NVIDIA Jetson Devices
-With some minor configuration, Ollama runs well on [NVIDIA Jetson Devices](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/). The following has been tested on [JetPack 5.1.2](https://developer.nvidia.com/embedded/jetpack).
+Ollama runs well on [NVIDIA Jetson Devices](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/) and should run out of the box with the standard installation instructions.
-NVIDIA Jetson devices are Linux-based embedded AI computers that are purpose-built for AI applications.
+The following has been tested on [JetPack 5.1.2](https://developer.nvidia.com/embedded/jetpack), but should also work on JetPack 6.0.
-Jetsons have an integrated GPU that is wired directly to the memory controller of the machine. For this reason, the `nvidia-smi` command is unrecognized, and Ollama proceeds to operate in "CPU only"
-mode. This can be verified by using a monitoring tool like jtop.
-
-In order to address this, we simply pass the path to the Jetson's pre-installed CUDA libraries into `ollama serve` (while in a tmux session). We then hardcode the num_gpu parameters into a cloned
-version of our target model.
-
-Prerequisites:
-
-- curl
-- tmux
-
-Here are the steps:
-
-- Install Ollama via standard Linux command (ignore the 404 error): `curl https://ollama.ai/install.sh | sh`
-- Stop the Ollama service: `sudo systemctl stop ollama`
-- Start Ollama serve in a tmux session called ollama_jetson and reference the CUDA libraries path: `tmux has-session -t ollama_jetson 2>/dev/null || tmux new-session -d -s ollama_jetson
-'LD_LIBRARY_PATH=/usr/local/cuda/lib64 ollama serve'`
+- Install Ollama via standard Linux command (ignore the 404 error): `curl https://ollama.com/install.sh | sh`
- Pull the model you want to use (e.g. mistral): `ollama pull mistral`
-- Create a new Modelfile specifically for enabling GPU support on the Jetson: `touch ModelfileMistralJetson`
-- In the ModelfileMistralJetson file, specify the FROM model and the num_gpu PARAMETER as shown below:
-
-```
-FROM mistral
-PARAMETER num_gpu 999
-```
-
-- Create a new model from your Modelfile: `ollama create mistral-jetson -f ./ModelfileMistralJetson`
-- Run the new model: `ollama run mistral-jetson`
-
-If you run a monitoring tool like jtop you should now see that Ollama is using the Jetson's integrated GPU.
+- Start an interactive session: `ollama run mistral`
And that's it!
+
+# Running Ollama in Docker
+
+When running GPU accelerated applications in Docker, it is highly recommended to use [dusty-nv jetson-containers repo](https://github.com/dusty-nv/jetson-containers).
\ No newline at end of file
diff --git a/docs/windows.md b/docs/windows.md
new file mode 100644
index 00000000..5f196756
--- /dev/null
+++ b/docs/windows.md
@@ -0,0 +1,66 @@
+# Ollama Windows Preview
+
+Welcome to the Ollama Windows preview.
+
+No more WSL required!
+
+Ollama now runs as a native Windows application, including NVIDIA and AMD Radeon GPU support.
+After installing Ollama Windows Preview, Ollama will run in the background and
+the `ollama` command line is available in `cmd`, `powershell` or your favorite
+terminal application. As usual the Ollama [api](./api.md) will be served on
+`http://localhost:11434`.
+
+As this is a preview release, you should expect a few bugs here and there. If
+you run into a problem you can reach out on
+[Discord](https://discord.gg/ollama), or file an
+[issue](https://github.com/ollama/ollama/issues).
+Logs will often be helpful in diagnosing the problem (see
+[Troubleshooting](#troubleshooting) below)
+
+## System Requirements
+
+* Windows 10 22H2 or newer, Home or Pro
+* NVIDIA 452.39 or newer Drivers if you have an NVIDIA card
+* AMD Radeon Driver https://www.amd.com/en/support if you have a Radeon card
+
+Ollama uses unicode characters for progress indication, which may render as unknown squares in some older terminal fonts in Windows 10. If you see this, try changing your terminal font settings.
+
+## API Access
+
+Here's a quick example showing API access from `powershell`
+```powershell
+(Invoke-WebRequest -method POST -Body '{"model":"llama3.2", "prompt":"Why is the sky blue?", "stream": false}' -uri http://localhost:11434/api/generate ).Content | ConvertFrom-json
+```
+
+## Troubleshooting
+
+While we're in preview, `OLLAMA_DEBUG` is always enabled, which adds
+a "view logs" menu item to the app, and increases logging for the GUI app and
+server.
+
+Ollama on Windows stores files in a few different locations. You can view them in
+the explorer window by hitting `<cmd>+R` and type in:
+- `explorer %LOCALAPPDATA%\Ollama` contains logs, and downloaded updates
+ - *app.log* contains most resent logs from the GUI application
+ - *server.log* contains the most recent server logs
+ - *upgrade.log* contains log output for upgrades
+- `explorer %LOCALAPPDATA%\Programs\Ollama` contains the binaries (The installer adds this to your user PATH)
+- `explorer %HOMEPATH%\.ollama` contains models and configuration
+- `explorer %TEMP%` contains temporary executable files in one or more `ollama*` directories
+
+## Uninstall
+
+The Ollama Windows installer registers an Uninstaller application. Under `Add or remove programs` in Windows Settings, you can uninstall Ollama.
+
+## Standalone CLI
+
+The easiest way to install Ollama on Windows is to use the `OllamaSetup.exe`
+installer. It installs in your account without requiring Administrator rights.
+We update Ollama regularly to support the latest models, and this installer will
+help you keep up to date.
+
+If you'd like to install or integrate Ollama as a service, a standalone
+`ollama-windows-amd64.zip` zip file is available containing only the Ollama CLI
+and GPU library dependencies for Nvidia and AMD. This allows for embedding
+Ollama in existing applications, or running it as a system service via `ollama
+serve` with tools such as [NSSM](https://nssm.cc/).
diff --git a/envconfig/config.go b/envconfig/config.go
new file mode 100644
index 00000000..44a87995
--- /dev/null
+++ b/envconfig/config.go
@@ -0,0 +1,297 @@
+package envconfig
+
+import (
+ "fmt"
+ "log/slog"
+ "math"
+ "net"
+ "net/url"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// Host returns the scheme and host. Host can be configured via the OLLAMA_HOST environment variable.
+// Default is scheme "http" and host "127.0.0.1:11434"
+func Host() *url.URL {
+ defaultPort := "11434"
+
+ s := strings.TrimSpace(Var("OLLAMA_HOST"))
+ scheme, hostport, ok := strings.Cut(s, "://")
+ switch {
+ case !ok:
+ scheme, hostport = "http", s
+ case scheme == "http":
+ defaultPort = "80"
+ case scheme == "https":
+ defaultPort = "443"
+ }
+
+ hostport, path, _ := strings.Cut(hostport, "/")
+ host, port, err := net.SplitHostPort(hostport)
+ if err != nil {
+ host, port = "127.0.0.1", defaultPort
+ if ip := net.ParseIP(strings.Trim(hostport, "[]")); ip != nil {
+ host = ip.String()
+ } else if hostport != "" {
+ host = hostport
+ }
+ }
+
+ if n, err := strconv.ParseInt(port, 10, 32); err != nil || n > 65535 || n < 0 {
+ slog.Warn("invalid port, using default", "port", port, "default", defaultPort)
+ port = defaultPort
+ }
+
+ return &url.URL{
+ Scheme: scheme,
+ Host: net.JoinHostPort(host, port),
+ Path: path,
+ }
+}
+
+// Origins returns a list of allowed origins. Origins can be configured via the OLLAMA_ORIGINS environment variable.
+func Origins() (origins []string) {
+ if s := Var("OLLAMA_ORIGINS"); s != "" {
+ origins = strings.Split(s, ",")
+ }
+
+ for _, origin := range []string{"localhost", "127.0.0.1", "0.0.0.0"} {
+ origins = append(origins,
+ fmt.Sprintf("http://%s", origin),
+ fmt.Sprintf("https://%s", origin),
+ fmt.Sprintf("http://%s", net.JoinHostPort(origin, "*")),
+ fmt.Sprintf("https://%s", net.JoinHostPort(origin, "*")),
+ )
+ }
+
+ origins = append(origins,
+ "app://*",
+ "file://*",
+ "tauri://*",
+ )
+
+ return origins
+}
+
+// Models returns the path to the models directory. Models directory can be configured via the OLLAMA_MODELS environment variable.
+// Default is $HOME/.ollama/models
+func Models() string {
+ if s := Var("OLLAMA_MODELS"); s != "" {
+ return s
+ }
+
+ home, err := os.UserHomeDir()
+ if err != nil {
+ panic(err)
+ }
+
+ return filepath.Join(home, ".ollama", "models")
+}
+
+// KeepAlive returns the duration that models stay loaded in memory. KeepAlive can be configured via the OLLAMA_KEEP_ALIVE environment variable.
+// Negative values are treated as infinite. Zero is treated as no keep alive.
+// Default is 5 minutes.
+func KeepAlive() (keepAlive time.Duration) {
+ keepAlive = 5 * time.Minute
+ if s := Var("OLLAMA_KEEP_ALIVE"); s != "" {
+ if d, err := time.ParseDuration(s); err == nil {
+ keepAlive = d
+ } else if n, err := strconv.ParseInt(s, 10, 64); err == nil {
+ keepAlive = time.Duration(n) * time.Second
+ }
+ }
+
+ if keepAlive < 0 {
+ return time.Duration(math.MaxInt64)
+ }
+
+ return keepAlive
+}
+
+// LoadTimeout returns the duration for stall detection during model loads. LoadTimeout can be configured via the OLLAMA_LOAD_TIMEOUT environment variable.
+// Zero or Negative values are treated as infinite.
+// Default is 5 minutes.
+func LoadTimeout() (loadTimeout time.Duration) {
+ loadTimeout = 5 * time.Minute
+ if s := Var("OLLAMA_LOAD_TIMEOUT"); s != "" {
+ if d, err := time.ParseDuration(s); err == nil {
+ loadTimeout = d
+ } else if n, err := strconv.ParseInt(s, 10, 64); err == nil {
+ loadTimeout = time.Duration(n) * time.Second
+ }
+ }
+
+ if loadTimeout <= 0 {
+ return time.Duration(math.MaxInt64)
+ }
+
+ return loadTimeout
+}
+
+func Bool(k string) func() bool {
+ return func() bool {
+ if s := Var(k); s != "" {
+ b, err := strconv.ParseBool(s)
+ if err != nil {
+ return true
+ }
+
+ return b
+ }
+
+ return false
+ }
+}
+
+var (
+ // Debug enabled additional debug information.
+ Debug = Bool("OLLAMA_DEBUG")
+ // FlashAttention enables the experimental flash attention feature.
+ FlashAttention = Bool("OLLAMA_FLASH_ATTENTION")
+ // NoHistory disables readline history.
+ NoHistory = Bool("OLLAMA_NOHISTORY")
+ // NoPrune disables pruning of model blobs on startup.
+ NoPrune = Bool("OLLAMA_NOPRUNE")
+ // SchedSpread allows scheduling models across all GPUs.
+ SchedSpread = Bool("OLLAMA_SCHED_SPREAD")
+ // IntelGPU enables experimental Intel GPU detection.
+ IntelGPU = Bool("OLLAMA_INTEL_GPU")
+ // MultiUserCache optimizes prompt caching for multi-user scenarios
+ MultiUserCache = Bool("OLLAMA_MULTIUSER_CACHE")
+)
+
+func String(s string) func() string {
+ return func() string {
+ return Var(s)
+ }
+}
+
+var (
+ LLMLibrary = String("OLLAMA_LLM_LIBRARY")
+ TmpDir = String("OLLAMA_TMPDIR")
+
+ CudaVisibleDevices = String("CUDA_VISIBLE_DEVICES")
+ HipVisibleDevices = String("HIP_VISIBLE_DEVICES")
+ RocrVisibleDevices = String("ROCR_VISIBLE_DEVICES")
+ GpuDeviceOrdinal = String("GPU_DEVICE_ORDINAL")
+ HsaOverrideGfxVersion = String("HSA_OVERRIDE_GFX_VERSION")
+)
+
+func Uint(key string, defaultValue uint) func() uint {
+ return func() uint {
+ if s := Var(key); s != "" {
+ if n, err := strconv.ParseUint(s, 10, 64); err != nil {
+ slog.Warn("invalid environment variable, using default", "key", key, "value", s, "default", defaultValue)
+ } else {
+ return uint(n)
+ }
+ }
+
+ return defaultValue
+ }
+}
+
+var (
+ // NumParallel sets the number of parallel model requests. NumParallel can be configured via the OLLAMA_NUM_PARALLEL environment variable.
+ NumParallel = Uint("OLLAMA_NUM_PARALLEL", 0)
+ // MaxRunners sets the maximum number of loaded models. MaxRunners can be configured via the OLLAMA_MAX_LOADED_MODELS environment variable.
+ MaxRunners = Uint("OLLAMA_MAX_LOADED_MODELS", 0)
+ // MaxQueue sets the maximum number of queued requests. MaxQueue can be configured via the OLLAMA_MAX_QUEUE environment variable.
+ MaxQueue = Uint("OLLAMA_MAX_QUEUE", 512)
+ // MaxVRAM sets a maximum VRAM override in bytes. MaxVRAM can be configured via the OLLAMA_MAX_VRAM environment variable.
+ MaxVRAM = Uint("OLLAMA_MAX_VRAM", 0)
+)
+
+func Uint64(key string, defaultValue uint64) func() uint64 {
+ return func() uint64 {
+ if s := Var(key); s != "" {
+ if n, err := strconv.ParseUint(s, 10, 64); err != nil {
+ slog.Warn("invalid environment variable, using default", "key", key, "value", s, "default", defaultValue)
+ } else {
+ return n
+ }
+ }
+
+ return defaultValue
+ }
+}
+
+// Set aside VRAM per GPU
+var GpuOverhead = Uint64("OLLAMA_GPU_OVERHEAD", 0)
+
+type EnvVar struct {
+ Name string
+ Value any
+ Description string
+}
+
+func AsMap() map[string]EnvVar {
+ ret := map[string]EnvVar{
+ "OLLAMA_DEBUG": {"OLLAMA_DEBUG", Debug(), "Show additional debug information (e.g. OLLAMA_DEBUG=1)"},
+ "OLLAMA_FLASH_ATTENTION": {"OLLAMA_FLASH_ATTENTION", FlashAttention(), "Enabled flash attention"},
+ "OLLAMA_GPU_OVERHEAD": {"OLLAMA_GPU_OVERHEAD", GpuOverhead(), "Reserve a portion of VRAM per GPU (bytes)"},
+ "OLLAMA_HOST": {"OLLAMA_HOST", Host(), "IP Address for the ollama server (default 127.0.0.1:11434)"},
+ "OLLAMA_KEEP_ALIVE": {"OLLAMA_KEEP_ALIVE", KeepAlive(), "The duration that models stay loaded in memory (default \"5m\")"},
+ "OLLAMA_LLM_LIBRARY": {"OLLAMA_LLM_LIBRARY", LLMLibrary(), "Set LLM library to bypass autodetection"},
+ "OLLAMA_LOAD_TIMEOUT": {"OLLAMA_LOAD_TIMEOUT", LoadTimeout(), "How long to allow model loads to stall before giving up (default \"5m\")"},
+ "OLLAMA_MAX_LOADED_MODELS": {"OLLAMA_MAX_LOADED_MODELS", MaxRunners(), "Maximum number of loaded models per GPU"},
+ "OLLAMA_MAX_QUEUE": {"OLLAMA_MAX_QUEUE", MaxQueue(), "Maximum number of queued requests"},
+ "OLLAMA_MODELS": {"OLLAMA_MODELS", Models(), "The path to the models directory"},
+ "OLLAMA_NOHISTORY": {"OLLAMA_NOHISTORY", NoHistory(), "Do not preserve readline history"},
+ "OLLAMA_NOPRUNE": {"OLLAMA_NOPRUNE", NoPrune(), "Do not prune model blobs on startup"},
+ "OLLAMA_NUM_PARALLEL": {"OLLAMA_NUM_PARALLEL", NumParallel(), "Maximum number of parallel requests"},
+ "OLLAMA_ORIGINS": {"OLLAMA_ORIGINS", Origins(), "A comma separated list of allowed origins"},
+ "OLLAMA_SCHED_SPREAD": {"OLLAMA_SCHED_SPREAD", SchedSpread(), "Always schedule model across all GPUs"},
+ "OLLAMA_TMPDIR": {"OLLAMA_TMPDIR", TmpDir(), "Location for temporary files"},
+ "OLLAMA_MULTIUSER_CACHE": {"OLLAMA_MULTIUSER_CACHE", MultiUserCache(), "Optimize prompt caching for multi-user scenarios"},
+
+ // Informational
+ "HTTP_PROXY": {"HTTP_PROXY", String("HTTP_PROXY")(), "HTTP proxy"},
+ "HTTPS_PROXY": {"HTTPS_PROXY", String("HTTPS_PROXY")(), "HTTPS proxy"},
+ "NO_PROXY": {"NO_PROXY", String("NO_PROXY")(), "No proxy"},
+ }
+
+ if runtime.GOOS != "windows" {
+ // Windows environment variables are case-insensitive so there's no need to duplicate them
+ ret["http_proxy"] = EnvVar{"http_proxy", String("http_proxy")(), "HTTP proxy"}
+ ret["https_proxy"] = EnvVar{"https_proxy", String("https_proxy")(), "HTTPS proxy"}
+ ret["no_proxy"] = EnvVar{"no_proxy", String("no_proxy")(), "No proxy"}
+ }
+
+ if runtime.GOOS != "darwin" {
+ ret["CUDA_VISIBLE_DEVICES"] = EnvVar{"CUDA_VISIBLE_DEVICES", CudaVisibleDevices(), "Set which NVIDIA devices are visible"}
+ ret["HIP_VISIBLE_DEVICES"] = EnvVar{"HIP_VISIBLE_DEVICES", HipVisibleDevices(), "Set which AMD devices are visible"}
+ ret["ROCR_VISIBLE_DEVICES"] = EnvVar{"ROCR_VISIBLE_DEVICES", RocrVisibleDevices(), "Set which AMD devices are visible"}
+ ret["GPU_DEVICE_ORDINAL"] = EnvVar{"GPU_DEVICE_ORDINAL", GpuDeviceOrdinal(), "Set which AMD devices are visible"}
+ ret["HSA_OVERRIDE_GFX_VERSION"] = EnvVar{"HSA_OVERRIDE_GFX_VERSION", HsaOverrideGfxVersion(), "Override the gfx used for all detected AMD GPUs"}
+ ret["OLLAMA_INTEL_GPU"] = EnvVar{"OLLAMA_INTEL_GPU", IntelGPU(), "Enable experimental Intel GPU detection"}
+ }
+
+ return ret
+}
+
+func Values() map[string]string {
+ vals := make(map[string]string)
+ for k, v := range AsMap() {
+ vals[k] = fmt.Sprintf("%v", v.Value)
+ }
+ return vals
+}
+
+// Var returns an environment variable stripped of leading and trailing quotes or spaces
+func Var(key string) string {
+ return strings.Trim(strings.TrimSpace(os.Getenv(key)), "\"'")
+}
+
+// On windows, we keep the binary at the top directory, but
+// other platforms use a "bin" directory, so this returns ".."
+func LibRelativeToExe() string {
+ if runtime.GOOS == "windows" {
+ return "."
+ }
+ return ".."
+}
diff --git a/envconfig/config_test.go b/envconfig/config_test.go
new file mode 100644
index 00000000..7ac7c53e
--- /dev/null
+++ b/envconfig/config_test.go
@@ -0,0 +1,270 @@
+package envconfig
+
+import (
+ "math"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func TestHost(t *testing.T) {
+ cases := map[string]struct {
+ value string
+ expect string
+ }{
+ "empty": {"", "http://127.0.0.1:11434"},
+ "only address": {"1.2.3.4", "http://1.2.3.4:11434"},
+ "only port": {":1234", "http://:1234"},
+ "address and port": {"1.2.3.4:1234", "http://1.2.3.4:1234"},
+ "hostname": {"example.com", "http://example.com:11434"},
+ "hostname and port": {"example.com:1234", "http://example.com:1234"},
+ "zero port": {":0", "http://:0"},
+ "too large port": {":66000", "http://:11434"},
+ "too small port": {":-1", "http://:11434"},
+ "ipv6 localhost": {"[::1]", "http://[::1]:11434"},
+ "ipv6 world open": {"[::]", "http://[::]:11434"},
+ "ipv6 no brackets": {"::1", "http://[::1]:11434"},
+ "ipv6 + port": {"[::1]:1337", "http://[::1]:1337"},
+ "extra space": {" 1.2.3.4 ", "http://1.2.3.4:11434"},
+ "extra quotes": {"\"1.2.3.4\"", "http://1.2.3.4:11434"},
+ "extra space+quotes": {" \" 1.2.3.4 \" ", "http://1.2.3.4:11434"},
+ "extra single quotes": {"'1.2.3.4'", "http://1.2.3.4:11434"},
+ "http": {"http://1.2.3.4", "http://1.2.3.4:80"},
+ "http port": {"http://1.2.3.4:4321", "http://1.2.3.4:4321"},
+ "https": {"https://1.2.3.4", "https://1.2.3.4:443"},
+ "https port": {"https://1.2.3.4:4321", "https://1.2.3.4:4321"},
+ "proxy path": {"https://example.com/ollama", "https://example.com:443/ollama"},
+ }
+
+ for name, tt := range cases {
+ t.Run(name, func(t *testing.T) {
+ t.Setenv("OLLAMA_HOST", tt.value)
+ if host := Host(); host.String() != tt.expect {
+ t.Errorf("%s: expected %s, got %s", name, tt.expect, host.String())
+ }
+ })
+ }
+}
+
+func TestOrigins(t *testing.T) {
+ cases := []struct {
+ value string
+ expect []string
+ }{
+ {"", []string{
+ "http://localhost",
+ "https://localhost",
+ "http://localhost:*",
+ "https://localhost:*",
+ "http://127.0.0.1",
+ "https://127.0.0.1",
+ "http://127.0.0.1:*",
+ "https://127.0.0.1:*",
+ "http://0.0.0.0",
+ "https://0.0.0.0",
+ "http://0.0.0.0:*",
+ "https://0.0.0.0:*",
+ "app://*",
+ "file://*",
+ "tauri://*",
+ }},
+ {"http://10.0.0.1", []string{
+ "http://10.0.0.1",
+ "http://localhost",
+ "https://localhost",
+ "http://localhost:*",
+ "https://localhost:*",
+ "http://127.0.0.1",
+ "https://127.0.0.1",
+ "http://127.0.0.1:*",
+ "https://127.0.0.1:*",
+ "http://0.0.0.0",
+ "https://0.0.0.0",
+ "http://0.0.0.0:*",
+ "https://0.0.0.0:*",
+ "app://*",
+ "file://*",
+ "tauri://*",
+ }},
+ {"http://172.16.0.1,https://192.168.0.1", []string{
+ "http://172.16.0.1",
+ "https://192.168.0.1",
+ "http://localhost",
+ "https://localhost",
+ "http://localhost:*",
+ "https://localhost:*",
+ "http://127.0.0.1",
+ "https://127.0.0.1",
+ "http://127.0.0.1:*",
+ "https://127.0.0.1:*",
+ "http://0.0.0.0",
+ "https://0.0.0.0",
+ "http://0.0.0.0:*",
+ "https://0.0.0.0:*",
+ "app://*",
+ "file://*",
+ "tauri://*",
+ }},
+ {"http://totally.safe,http://definitely.legit", []string{
+ "http://totally.safe",
+ "http://definitely.legit",
+ "http://localhost",
+ "https://localhost",
+ "http://localhost:*",
+ "https://localhost:*",
+ "http://127.0.0.1",
+ "https://127.0.0.1",
+ "http://127.0.0.1:*",
+ "https://127.0.0.1:*",
+ "http://0.0.0.0",
+ "https://0.0.0.0",
+ "http://0.0.0.0:*",
+ "https://0.0.0.0:*",
+ "app://*",
+ "file://*",
+ "tauri://*",
+ }},
+ }
+ for _, tt := range cases {
+ t.Run(tt.value, func(t *testing.T) {
+ t.Setenv("OLLAMA_ORIGINS", tt.value)
+
+ if diff := cmp.Diff(Origins(), tt.expect); diff != "" {
+ t.Errorf("%s: mismatch (-want +got):\n%s", tt.value, diff)
+ }
+ })
+ }
+}
+
+func TestBool(t *testing.T) {
+ cases := map[string]bool{
+ "": false,
+ "true": true,
+ "false": false,
+ "1": true,
+ "0": false,
+ // invalid values
+ "random": true,
+ "something": true,
+ }
+
+ for k, v := range cases {
+ t.Run(k, func(t *testing.T) {
+ t.Setenv("OLLAMA_BOOL", k)
+ if b := Bool("OLLAMA_BOOL")(); b != v {
+ t.Errorf("%s: expected %t, got %t", k, v, b)
+ }
+ })
+ }
+}
+
+func TestUint(t *testing.T) {
+ cases := map[string]uint{
+ "0": 0,
+ "1": 1,
+ "1337": 1337,
+ // default values
+ "": 11434,
+ "-1": 11434,
+ "0o10": 11434,
+ "0x10": 11434,
+ "string": 11434,
+ }
+
+ for k, v := range cases {
+ t.Run(k, func(t *testing.T) {
+ t.Setenv("OLLAMA_UINT", k)
+ if i := Uint("OLLAMA_UINT", 11434)(); i != v {
+ t.Errorf("%s: expected %d, got %d", k, v, i)
+ }
+ })
+ }
+}
+
+func TestKeepAlive(t *testing.T) {
+ cases := map[string]time.Duration{
+ "": 5 * time.Minute,
+ "1s": time.Second,
+ "1m": time.Minute,
+ "1h": time.Hour,
+ "5m0s": 5 * time.Minute,
+ "1h2m3s": 1*time.Hour + 2*time.Minute + 3*time.Second,
+ "0": time.Duration(0),
+ "60": 60 * time.Second,
+ "120": 2 * time.Minute,
+ "3600": time.Hour,
+ "-0": time.Duration(0),
+ "-1": time.Duration(math.MaxInt64),
+ "-1m": time.Duration(math.MaxInt64),
+ // invalid values
+ " ": 5 * time.Minute,
+ "???": 5 * time.Minute,
+ "1d": 5 * time.Minute,
+ "1y": 5 * time.Minute,
+ "1w": 5 * time.Minute,
+ }
+
+ for tt, expect := range cases {
+ t.Run(tt, func(t *testing.T) {
+ t.Setenv("OLLAMA_KEEP_ALIVE", tt)
+ if actual := KeepAlive(); actual != expect {
+ t.Errorf("%s: expected %s, got %s", tt, expect, actual)
+ }
+ })
+ }
+}
+
+func TestLoadTimeout(t *testing.T) {
+ defaultTimeout := 5 * time.Minute
+ cases := map[string]time.Duration{
+ "": defaultTimeout,
+ "1s": time.Second,
+ "1m": time.Minute,
+ "1h": time.Hour,
+ "5m0s": defaultTimeout,
+ "1h2m3s": 1*time.Hour + 2*time.Minute + 3*time.Second,
+ "0": time.Duration(math.MaxInt64),
+ "60": 60 * time.Second,
+ "120": 2 * time.Minute,
+ "3600": time.Hour,
+ "-0": time.Duration(math.MaxInt64),
+ "-1": time.Duration(math.MaxInt64),
+ "-1m": time.Duration(math.MaxInt64),
+ // invalid values
+ " ": defaultTimeout,
+ "???": defaultTimeout,
+ "1d": defaultTimeout,
+ "1y": defaultTimeout,
+ "1w": defaultTimeout,
+ }
+
+ for tt, expect := range cases {
+ t.Run(tt, func(t *testing.T) {
+ t.Setenv("OLLAMA_LOAD_TIMEOUT", tt)
+ if actual := LoadTimeout(); actual != expect {
+ t.Errorf("%s: expected %s, got %s", tt, expect, actual)
+ }
+ })
+ }
+}
+
+func TestVar(t *testing.T) {
+ cases := map[string]string{
+ "value": "value",
+ " value ": "value",
+ " 'value' ": "value",
+ ` "value" `: "value",
+ " ' value ' ": " value ",
+ ` " value " `: " value ",
+ }
+
+ for k, v := range cases {
+ t.Run(k, func(t *testing.T) {
+ t.Setenv("OLLAMA_VAR", k)
+ if s := Var("OLLAMA_VAR"); s != v {
+ t.Errorf("%s: expected %q, got %q", k, v, s)
+ }
+ })
+ }
+}
diff --git a/examples/bash-comparemodels/README.md b/examples/bash-comparemodels/README.md
deleted file mode 100644
index 91499255..00000000
--- a/examples/bash-comparemodels/README.md
+++ /dev/null
@@ -1,10 +0,0 @@
-# Bash Shell examples
-
-When calling `ollama`, you can pass it a file to run all the prompts in the file, one after the other:
-
-`ollama run llama2 < sourcequestions.txt`
-
-This concept is used in the following example.
-
-## Compare Models
-`comparemodels.sh` is a script that runs all the questions in `sourcequestions.txt` using any 4 models you choose that you have already pulled from the Ollama library or have created locally.
diff --git a/examples/bash-comparemodels/comparemodels.sh b/examples/bash-comparemodels/comparemodels.sh
deleted file mode 100755
index 1ce249a6..00000000
--- a/examples/bash-comparemodels/comparemodels.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#! /usr/bin/env bash
-# Compare multiple models by running them with the same questions
-
-NUMBEROFCHOICES=4
-SELECTIONS=()
-declare -a SUMS=()
-
-# Get the list of models
-CHOICES=$(ollama list | awk '{print $1}')
-
-# Select which models to run as a comparison
-echo "Select $NUMBEROFCHOICES models to compare:"
-select ITEM in $CHOICES; do
- if [[ -n $ITEM ]]; then
- echo "You have selected $ITEM"
- SELECTIONS+=("$ITEM")
- ((COUNT++))
- if [[ $COUNT -eq $NUMBEROFCHOICES ]]; then
- break
- fi
- else
- echo "Invalid selection"
- fi
-done
-
-# Loop through each of the selected models
-for ITEM in "${SELECTIONS[@]}"; do
- echo "--------------------------------------------------------------"
- echo "Loading the model $ITEM into memory"
- ollama run "$ITEM" ""
- echo "--------------------------------------------------------------"
- echo "Running the questions through the model $ITEM"
- COMMAND_OUTPUT=$(ollama run "$ITEM" --verbose < sourcequestions.txt 2>&1| tee /dev/stderr)
-
- # eval duration is sometimes listed in seconds and sometimes in milliseconds.
- # Add up the values for each model
- SUM=$(echo "$COMMAND_OUTPUT" | awk '
- /eval duration:/ {
- value = $3
- if (index(value, "ms") > 0) {
- gsub("ms", "", value)
- value /= 1000
- } else {
- gsub("s", "", value)
- }
- sum += value
- }
- END { print sum }')
-
-
- SUMS+=("All questions for $ITEM completed in $SUM seconds")
-done
-
-echo ""
-echo "--------------------------------------------------------------"
-echo -e "Sums of eval durations for each run:"
-for val in "${SUMS[@]}"; do
- echo "$val"
-done
-
-echo "--------------------------------------------------------------"
-echo "Comparison complete. Now you can decide"
-echo "which model is best."
-echo "--------------------------------------------------------------"
\ No newline at end of file
diff --git a/examples/bash-comparemodels/sourcequestions.txt b/examples/bash-comparemodels/sourcequestions.txt
deleted file mode 100644
index 90004c07..00000000
--- a/examples/bash-comparemodels/sourcequestions.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Why is the sky blue
-What is a black hole
-Explain the big bang theory like I am 5?
-What is the quickest way to win a game of Monopoly with 3 others?
-Why does a vacuum bottle keep my coffee hot and my milkshake cold?
-What is the difference between a meteor, a meteorite, and a meteoroid?
-Create an array with 5 items and print to the console. Do this in Python, C#, Typescript, and Rust.
\ No newline at end of file
diff --git a/examples/flyio/.gitignore b/examples/flyio/.gitignore
new file mode 100644
index 00000000..0501d092
--- /dev/null
+++ b/examples/flyio/.gitignore
@@ -0,0 +1 @@
+fly.toml
diff --git a/examples/flyio/README.md b/examples/flyio/README.md
new file mode 100644
index 00000000..09b90aad
--- /dev/null
+++ b/examples/flyio/README.md
@@ -0,0 +1,67 @@
+# Deploy Ollama to Fly.io
+
+> Note: this example exposes a public endpoint and does not configure authentication. Use with care.
+
+## Prerequisites
+
+- Ollama: https://ollama.com/download
+- Fly.io account. Sign up for a free account: https://fly.io/app/sign-up
+
+## Steps
+
+1. Login to Fly.io
+
+ ```bash
+ fly auth login
+ ```
+
+1. Create a new Fly app
+
+ ```bash
+ fly launch --name <name> --image ollama/ollama --internal-port 11434 --vm-size shared-cpu-8x --now
+ ```
+
+1. Pull and run `orca-mini:3b`
+
+ ```bash
+ OLLAMA_HOST=https://<name>.fly.dev ollama run orca-mini:3b
+ ```
+
+`shared-cpu-8x` is a free-tier eligible machine type. For better performance, switch to a `performance` or `dedicated` machine type or attach a GPU for hardware acceleration (see below).
+
+## (Optional) Persistent Volume
+
+By default Fly Machines use ephemeral storage which is problematic if you want to use the same model across restarts without pulling it again. Create and attach a persistent volume to store the downloaded models:
+
+1. Create the Fly Volume
+
+ ```bash
+ fly volume create ollama
+ ```
+
+1. Update `fly.toml` and add `[mounts]`
+
+ ```toml
+ [mounts]
+ source = "ollama"
+ destination = "/mnt/ollama/models"
+ ```
+
+1. Update `fly.toml` and add `[env]`
+
+ ```toml
+ [env]
+ OLLAMA_MODELS = "/mnt/ollama/models"
+ ```
+
+1. Deploy your app
+
+ ```bash
+ fly deploy
+ ```
+
+## (Optional) Hardware Acceleration
+
+Fly.io GPU is currently in waitlist. Sign up for the waitlist: https://fly.io/gpu
+
+Once you've been accepted, create the app with the additional flags `--vm-gpu-kind a100-pcie-40gb` or `--vm-gpu-kind a100-pcie-80gb`.
diff --git a/examples/go-chat/main.go b/examples/go-chat/main.go
new file mode 100644
index 00000000..07430305
--- /dev/null
+++ b/examples/go-chat/main.go
@@ -0,0 +1,51 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "github.com/ollama/ollama/api"
+)
+
+func main() {
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ messages := []api.Message{
+ api.Message{
+ Role: "system",
+ Content: "Provide very brief, concise responses",
+ },
+ api.Message{
+ Role: "user",
+ Content: "Name some unusual animals",
+ },
+ api.Message{
+ Role: "assistant",
+ Content: "Monotreme, platypus, echidna",
+ },
+ api.Message{
+ Role: "user",
+ Content: "which of these is the most dangerous?",
+ },
+ }
+
+ ctx := context.Background()
+ req := &api.ChatRequest{
+ Model: "llama3.2",
+ Messages: messages,
+ }
+
+ respFunc := func(resp api.ChatResponse) error {
+ fmt.Print(resp.Message.Content)
+ return nil
+ }
+
+ err = client.Chat(ctx, req, respFunc)
+ if err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/examples/go-generate-streaming/main.go b/examples/go-generate-streaming/main.go
new file mode 100644
index 00000000..3acfb22a
--- /dev/null
+++ b/examples/go-generate-streaming/main.go
@@ -0,0 +1,40 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "github.com/ollama/ollama/api"
+)
+
+func main() {
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ // By default, GenerateRequest is streaming.
+ req := &api.GenerateRequest{
+ Model: "gemma2",
+ Prompt: "how many planets are there?",
+ }
+
+ ctx := context.Background()
+ respFunc := func(resp api.GenerateResponse) error {
+ // Only print the response here; GenerateResponse has a number of other
+ // interesting fields you want to examine.
+
+ // In streaming mode, responses are partial so we call fmt.Print (and not
+ // Println) in order to avoid spurious newlines being introduced. The
+ // model will insert its own newlines if it wants.
+ fmt.Print(resp.Response)
+ return nil
+ }
+
+ err = client.Generate(ctx, req, respFunc)
+ if err != nil {
+ log.Fatal(err)
+ }
+ fmt.Println()
+}
diff --git a/examples/go-generate/main.go b/examples/go-generate/main.go
new file mode 100644
index 00000000..2fe28742
--- /dev/null
+++ b/examples/go-generate/main.go
@@ -0,0 +1,37 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "github.com/ollama/ollama/api"
+)
+
+func main() {
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ req := &api.GenerateRequest{
+ Model: "gemma2",
+ Prompt: "how many planets are there?",
+
+ // set streaming to false
+ Stream: new(bool),
+ }
+
+ ctx := context.Background()
+ respFunc := func(resp api.GenerateResponse) error {
+ // Only print the response here; GenerateResponse has a number of other
+ // interesting fields you want to examine.
+ fmt.Println(resp.Response)
+ return nil
+ }
+
+ err = client.Generate(ctx, req, respFunc)
+ if err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/examples/golang-simplegenerate/main.go b/examples/go-http-generate/main.go
similarity index 99%
rename from examples/golang-simplegenerate/main.go
rename to examples/go-http-generate/main.go
index f4ca32f4..e5b64348 100644
--- a/examples/golang-simplegenerate/main.go
+++ b/examples/go-http-generate/main.go
@@ -19,7 +19,7 @@ func main() {
}
defer resp.Body.Close()
-
+
responseData, err := io.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
diff --git a/examples/go-multimodal/main.go b/examples/go-multimodal/main.go
new file mode 100644
index 00000000..0b0f19ed
--- /dev/null
+++ b/examples/go-multimodal/main.go
@@ -0,0 +1,47 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+
+ "github.com/ollama/ollama/api"
+)
+
+func main() {
+ if len(os.Args) <= 1 {
+ log.Fatal("usage: <image name>")
+ }
+
+ imgData, err := os.ReadFile(os.Args[1])
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ req := &api.GenerateRequest{
+ Model: "llava",
+ Prompt: "describe this image",
+ Images: []api.ImageData{imgData},
+ }
+
+ ctx := context.Background()
+ respFunc := func(resp api.GenerateResponse) error {
+ // In streaming mode, responses are partial so we call fmt.Print (and not
+ // Println) in order to avoid spurious newlines being introduced. The
+ // model will insert its own newlines if it wants.
+ fmt.Print(resp.Response)
+ return nil
+ }
+
+ err = client.Generate(ctx, req, respFunc)
+ if err != nil {
+ log.Fatal(err)
+ }
+ fmt.Println()
+}
diff --git a/examples/go-pull-progress/main.go b/examples/go-pull-progress/main.go
new file mode 100644
index 00000000..74863368
--- /dev/null
+++ b/examples/go-pull-progress/main.go
@@ -0,0 +1,31 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "github.com/ollama/ollama/api"
+)
+
+func main() {
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ ctx := context.Background()
+
+ req := &api.PullRequest{
+ Model: "mistral",
+ }
+ progressFunc := func(resp api.ProgressResponse) error {
+ fmt.Printf("Progress: status=%v, total=%v, completed=%v\n", resp.Status, resp.Total, resp.Completed)
+ return nil
+ }
+
+ err = client.Pull(ctx, req, progressFunc)
+ if err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/examples/golang-simplegenerate/README.md b/examples/golang-simplegenerate/README.md
deleted file mode 100644
index e69de29b..00000000
diff --git a/examples/jupyter-notebook/ollama.ipynb b/examples/jupyter-notebook/ollama.ipynb
index d57e2057..bee353cb 100644
--- a/examples/jupyter-notebook/ollama.ipynb
+++ b/examples/jupyter-notebook/ollama.ipynb
@@ -8,7 +8,7 @@
"outputs": [],
"source": [
"# Download and run the Ollama Linux install script\n",
- "!curl https://ollama.ai/install.sh | sh\n",
+ "!curl -fsSL https://ollama.com/install.sh | sh\n",
"!command -v systemctl >/dev/null && sudo systemctl stop ollama"
]
},
diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md
index cb5f39f9..2e2444c7 100644
--- a/examples/kubernetes/README.md
+++ b/examples/kubernetes/README.md
@@ -2,35 +2,37 @@
## Prerequisites
-- Ollama: https://ollama.ai/download
+- Ollama: https://ollama.com/download
- Kubernetes cluster. This example will use Google Kubernetes Engine.
## Steps
-1. Create the Ollama namespace, daemon set, and service
+1. Create the Ollama namespace, deployment, and service
- ```bash
- kubectl apply -f cpu.yaml
- ```
-
-1. Port forward the Ollama service to connect and use it locally
-
- ```bash
- kubectl -n ollama port-forward service/ollama 11434:80
- ```
-
-1. Pull and run a model, for example `orca-mini:3b`
-
- ```bash
- ollama run orca-mini:3b
- ```
+ ```bash
+ kubectl apply -f cpu.yaml
+ ```
## (Optional) Hardware Acceleration
-Hardware acceleration in Kubernetes requires NVIDIA's [`k8s-device-plugin`](https://github.com/NVIDIA/k8s-device-plugin). Follow the link for more details.
+Hardware acceleration in Kubernetes requires NVIDIA's [`k8s-device-plugin`](https://github.com/NVIDIA/k8s-device-plugin) which is deployed in Kubernetes in form of daemonset. Follow the link for more details.
Once configured, create a GPU enabled Ollama deployment.
```bash
kubectl apply -f gpu.yaml
```
+
+## Test
+
+1. Port forward the Ollama service to connect and use it locally
+
+ ```bash
+ kubectl -n ollama port-forward service/ollama 11434:80
+ ```
+
+1. Pull and run a model, for example `orca-mini:3b`
+
+ ```bash
+ ollama run orca-mini:3b
+ ```
\ No newline at end of file
diff --git a/examples/langchain-python-rag-document/README.md b/examples/langchain-python-rag-document/README.md
index 20a73a88..d37afc9d 100644
--- a/examples/langchain-python-rag-document/README.md
+++ b/examples/langchain-python-rag-document/README.md
@@ -4,6 +4,14 @@ This example provides an interface for asking questions to a PDF document.
## Setup
+1. Ensure you have the `llama3.2` model installed:
+
+```
+ollama pull llama3.2
+```
+
+2. Install the Python Requirements.
+
```
pip install -r requirements.txt
```
diff --git a/examples/langchain-python-rag-document/main.py b/examples/langchain-python-rag-document/main.py
index b9f98c4e..4871a042 100644
--- a/examples/langchain-python-rag-document/main.py
+++ b/examples/langchain-python-rag-document/main.py
@@ -40,9 +40,9 @@ while True:
continue
# Prompt
- template = """Use the following pieces of context to answer the question at the end.
- If you don't know the answer, just say that you don't know, don't try to make up an answer.
- Use three sentences maximum and keep the answer as concise as possible.
+ template = """Use the following pieces of context to answer the question at the end.
+ If you don't know the answer, just say that you don't know, don't try to make up an answer.
+ Use three sentences maximum and keep the answer as concise as possible.
{context}
Question: {question}
Helpful Answer:"""
@@ -51,11 +51,11 @@ while True:
template=template,
)
- llm = Ollama(model="llama2:13b", callback_manager=CallbackManager([StreamingStdOutCallbackHandler()]))
+ llm = Ollama(model="llama3.2", callback_manager=CallbackManager([StreamingStdOutCallbackHandler()]))
qa_chain = RetrievalQA.from_chain_type(
llm,
retriever=vectorstore.as_retriever(),
chain_type_kwargs={"prompt": QA_CHAIN_PROMPT},
)
- result = qa_chain({"query": query})
\ No newline at end of file
+ result = qa_chain({"query": query})
diff --git a/examples/langchain-python-rag-privategpt/README.md b/examples/langchain-python-rag-privategpt/README.md
index f739df2b..0133fd88 100644
--- a/examples/langchain-python-rag-privategpt/README.md
+++ b/examples/langchain-python-rag-privategpt/README.md
@@ -1,6 +1,6 @@
# PrivateGPT with Llama 2 uncensored
-https://github.com/jmorganca/ollama/assets/3325447/20cf8ec6-ff25-42c6-bdd8-9be594e3ce1b
+https://github.com/ollama/ollama/assets/3325447/20cf8ec6-ff25-42c6-bdd8-9be594e3ce1b
> Note: this example is a slightly modified version of PrivateGPT using models such as Llama 2 Uncensored. All credit for PrivateGPT goes to Iván Martínez who is the creator of it, and you can find his GitHub repo [here](https://github.com/imartinez/privateGPT).
diff --git a/examples/langchain-python-rag-privategpt/ingest.py b/examples/langchain-python-rag-privategpt/ingest.py
index 35324775..0f71ccf0 100755
--- a/examples/langchain-python-rag-privategpt/ingest.py
+++ b/examples/langchain-python-rag-privategpt/ingest.py
@@ -77,13 +77,21 @@ LOADER_MAPPING = {
def load_single_document(file_path: str) -> List[Document]:
- ext = "." + file_path.rsplit(".", 1)[-1]
- if ext in LOADER_MAPPING:
- loader_class, loader_args = LOADER_MAPPING[ext]
- loader = loader_class(file_path, **loader_args)
- return loader.load()
+ if os.path.getsize(file_path) != 0:
+ filename, ext = os.path.splitext(file_path)
+ if ext in LOADER_MAPPING:
+ loader_class, loader_args = LOADER_MAPPING[ext]
+ try:
+ loader = loader_class(file_path, **loader_args)
+ if loader:
+ return loader.load()
+ except:
+ print(f"Corrupted file {file_path}. Ignoring it.")
+ else:
+ print(f"Unsupported file {file_path}. Ignoring it.")
+ else:
+ print(f"Empty file {file_path}. Ignoring it.")
- raise ValueError(f"Unsupported file extension '{ext}'")
def load_documents(source_dir: str, ignored_files: List[str] = []) -> List[Document]:
"""
@@ -100,7 +108,8 @@ def load_documents(source_dir: str, ignored_files: List[str] = []) -> List[Docum
results = []
with tqdm(total=len(filtered_files), desc='Loading new documents', ncols=80) as pbar:
for i, docs in enumerate(pool.imap_unordered(load_single_document, filtered_files)):
- results.extend(docs)
+ if docs:
+ results.extend(docs)
pbar.update()
return results
diff --git a/examples/langchain-python-rag-privategpt/requirements.txt b/examples/langchain-python-rag-privategpt/requirements.txt
index d4c5cb2e..4f2cee25 100644
--- a/examples/langchain-python-rag-privategpt/requirements.txt
+++ b/examples/langchain-python-rag-privategpt/requirements.txt
@@ -1,6 +1,6 @@
langchain==0.0.274
gpt4all==1.0.8
-chromadb==0.4.7
+chromadb==0.5.0
llama-cpp-python==0.1.81
urllib3==2.0.4
PyMuPDF==1.23.5
@@ -11,4 +11,5 @@ tabulate==0.9.0
pandoc==2.3
pypandoc==1.11
tqdm==4.66.1
-sentence_transformers==2.2.2
\ No newline at end of file
+sentence_transformers==2.2.2
+numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability
diff --git a/examples/langchain-python-rag-websummary/README.md b/examples/langchain-python-rag-websummary/README.md
index 9ccc54cc..746c47ab 100644
--- a/examples/langchain-python-rag-websummary/README.md
+++ b/examples/langchain-python-rag-websummary/README.md
@@ -1,13 +1,13 @@
# LangChain Web Summarization
-This example summarizes the website, [https://ollama.ai/blog/run-llama2-uncensored-locally](https://ollama.ai/blog/run-llama2-uncensored-locally)
+This example summarizes the website, [https://ollama.com/blog/run-llama2-uncensored-locally](https://ollama.com/blog/run-llama2-uncensored-locally)
## Running the Example
-1. Ensure you have the `llama2` model installed:
+1. Ensure you have the `llama3.2` model installed:
```bash
- ollama pull llama2
+ ollama pull llama3.2
```
2. Install the Python Requirements.
diff --git a/examples/langchain-python-rag-websummary/main.py b/examples/langchain-python-rag-websummary/main.py
index 2bb25d75..56f8bd24 100644
--- a/examples/langchain-python-rag-websummary/main.py
+++ b/examples/langchain-python-rag-websummary/main.py
@@ -1,12 +1,12 @@
-from langchain.llms import Ollama
-from langchain.document_loaders import WebBaseLoader
+from langchain_community.llms import Ollama
+from langchain_community.document_loaders import WebBaseLoader
from langchain.chains.summarize import load_summarize_chain
-loader = WebBaseLoader("https://ollama.ai/blog/run-llama2-uncensored-locally")
+loader = WebBaseLoader("https://ollama.com/blog/run-llama2-uncensored-locally")
docs = loader.load()
-llm = Ollama(model="llama2")
+llm = Ollama(model="llama3.2")
chain = load_summarize_chain(llm, chain_type="stuff")
-result = chain.run(docs)
+result = chain.invoke(docs)
print(result)
diff --git a/examples/langchain-python-simple/README.md b/examples/langchain-python-simple/README.md
index 3f401ca8..680ab560 100644
--- a/examples/langchain-python-simple/README.md
+++ b/examples/langchain-python-simple/README.md
@@ -4,10 +4,10 @@ This example is a basic "hello world" of using LangChain with Ollama.
## Running the Example
-1. Ensure you have the `llama2` model installed:
+1. Ensure you have the `llama3.2` model installed:
```bash
- ollama pull llama2
+ ollama pull llama3.2
```
2. Install the Python Requirements.
@@ -21,4 +21,3 @@ This example is a basic "hello world" of using LangChain with Ollama.
```bash
python main.py
```
-
\ No newline at end of file
diff --git a/examples/langchain-python-simple/main.py b/examples/langchain-python-simple/main.py
index da696e00..8d6989c8 100644
--- a/examples/langchain-python-simple/main.py
+++ b/examples/langchain-python-simple/main.py
@@ -1,6 +1,6 @@
from langchain.llms import Ollama
input = input("What is your question?")
-llm = Ollama(model="llama2")
+llm = Ollama(model="llama3.2")
res = llm.predict(input)
print (res)
diff --git a/examples/modelfile-mario/Modelfile b/examples/modelfile-mario/Modelfile
index 35c787fc..b8e49667 100644
--- a/examples/modelfile-mario/Modelfile
+++ b/examples/modelfile-mario/Modelfile
@@ -1,4 +1,4 @@
-FROM llama2
+FROM llama3.2
PARAMETER temperature 1
SYSTEM """
You are Mario from super mario bros, acting as an assistant.
diff --git a/examples/modelfile-mario/readme.md b/examples/modelfile-mario/readme.md
index 9df08a67..882023ad 100644
--- a/examples/modelfile-mario/readme.md
+++ b/examples/modelfile-mario/readme.md
@@ -2,12 +2,12 @@
# Example character: Mario
-This example shows how to create a basic character using Llama2 as the base model.
+This example shows how to create a basic character using Llama 3.2 as the base model.
To run this example:
1. Download the Modelfile
-2. `ollama pull llama2` to get the base model used in the model file.
+2. `ollama pull llama3.2` to get the base model used in the model file.
3. `ollama create NAME -f ./Modelfile`
4. `ollama run NAME`
@@ -18,7 +18,7 @@ Ask it some questions like "Who are you?" or "Is Peach in trouble again?"
What the model file looks like:
```
-FROM llama2
+FROM llama3.2
PARAMETER temperature 1
SYSTEM """
You are Mario from Super Mario Bros, acting as an assistant.
@@ -28,7 +28,7 @@ You are Mario from Super Mario Bros, acting as an assistant.
What if you want to change its behaviour?
- Try changing the prompt
-- Try changing the parameters [Docs](https://github.com/jmorganca/ollama/blob/main/docs/modelfile.md)
+- Try changing the parameters [Docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md)
- Try changing the model (e.g. An uncensored model by `FROM wizard-vicuna` this is the wizard-vicuna uncensored model )
Once the changes are made,
diff --git a/examples/modelfile-tweetwriter/readme.md b/examples/modelfile-tweetwriter/readme.md
deleted file mode 100644
index 51111259..00000000
--- a/examples/modelfile-tweetwriter/readme.md
+++ /dev/null
@@ -1,23 +0,0 @@
-# Example Modelfile - Tweetwriter
-
-This simple examples shows what you can do without any code, simply relying on a Modelfile. The file has two instructions:
-
-1. FROM - The From instructions defines the parent model to use for this one. If you choose a model from the library, you can enter just the model name. For all other models, you need to specify the namespace as well. You could also use a local file. Just include the relative path to the converted, quantized model weights file. To learn more about creating that file, see the `import.md` file in the docs folder of this repository.
-2. SYSTEM - This defines the system prompt for the model and overrides the system prompt from the parent model.
-
-## Running the Example
-
-1. Create the model:
-
- ```bash
- ollama create tweetwriter
- ```
-
-2. Enter a topic to generate a tweet about.
-3. Show the Modelfile in the REPL.
-
- ```bash
- /show modelfile
- ```
-
- Notice that the FROM and SYSTEM match what was in the file. But there is also a TEMPLATE and PARAMETER. These are inherited from the parent model.
\ No newline at end of file
diff --git a/examples/python-dockerit/dockerit.py b/examples/python-dockerit/dockerit.py
index b013102f..6a288d90 100644
--- a/examples/python-dockerit/dockerit.py
+++ b/examples/python-dockerit/dockerit.py
@@ -4,7 +4,7 @@ imageName = input("Enter the name of the image: ")
client = docker.from_env()
s = requests.Session()
output=""
-with s.post('http://localhost:11434/api/generate', json={'model': 'dockerit', 'prompt': inputDescription}, stream=True) as r:
+with s.post('http://localhost:11434/api/generate', json={'model': 'mattw/dockerit', 'prompt': inputDescription}, stream=True) as r:
for line in r.iter_lines():
if line:
j = json.loads(line)
diff --git a/examples/python-grounded-factuality-rag-check/README.md b/examples/python-grounded-factuality-rag-check/README.md
new file mode 100644
index 00000000..cd72071c
--- /dev/null
+++ b/examples/python-grounded-factuality-rag-check/README.md
@@ -0,0 +1,93 @@
+# RAG Hallucination Checker using Bespoke-Minicheck
+
+This example allows the user to ask questions related to a document, which can be specified via an article url. Relevant chunks are retreived from the document and given to `llama3.2` as context to answer the question. Then each sentence in the answer is checked against the retrieved chunks using `bespoke-minicheck` to ensure that the answer does not contain hallucinations.
+
+## Running the Example
+
+1. Ensure `all-minilm` (embedding) `llama3.2` (chat) and `bespoke-minicheck` (check) models installed:
+
+ ```bash
+ ollama pull all-minilm
+ ollama pull llama3.2
+ ollama pull bespoke-minicheck
+ ```
+
+2. Install the dependencies.
+
+ ```bash
+ pip install -r requirements.txt
+ ```
+
+3. Run the example:
+
+ ```bash
+ python main.py
+ ```
+
+## Expected Output
+
+```text
+Enter the URL of an article you want to chat with, or press Enter for default example:
+
+Loaded, chunked, and embedded text from https://www.theverge.com/2024/9/12/24242439/openai-o1-model-reasoning-strawberry-chatgpt.
+
+Enter your question or type quit: Who is the CEO of openai?
+
+Retrieved chunks:
+OpenAI is releasing a new model called o1 , the first in a planned series of “ reasoning ” models that have been trained to answer more complex questions , faster than a human can . It ’ s being released alongside o1-mini , a smaller , cheaper version . And yes , if you ’ re steeped in AI rumors : this is , in fact , the extremely hyped Strawberry model . For OpenAI , o1 represents a step toward its broader goal of human-like artificial intelligence .
+
+OpenAI is releasing a new model called o1 , the first in a planned series of “ reasoning ” models that have been trained to answer more complex questions , faster than a human can . It ’ s being released alongside o1-mini , a smaller , cheaper version . And yes , if you ’ re steeped in AI rumors : this is , in fact , the extremely hyped Strawberry model . For OpenAI , o1 represents a step toward its broader goal of human-like artificial intelligence . More practically , it does a better job at writing code and solving multistep problems than previous models . But it ’ s also more expensive and slower to use than GPT-4o . OpenAI is calling this release of o1 a “ preview ” to emphasize how nascent it is . ChatGPT Plus and Team users get access to both o1-preview and o1-mini starting today , while Enterprise and Edu users will get access early next week .
+
+More practically , it does a better job at writing code and solving multistep problems than previous models . But it ’ s also more expensive and slower to use than GPT-4o . OpenAI is calling this release of o1 a “ preview ” to emphasize how nascent it is . ChatGPT Plus and Team users get access to both o1-preview and o1-mini starting today , while Enterprise and Edu users will get access early next week . OpenAI says it plans to bring o1-mini access to all the free users of ChatGPT but hasn ’ t set a release date yet . Developer access to o1 is really expensive : In the API , o1-preview is $ 15 per 1 million input tokens , or chunks of text parsed by the model , and $ 60 per 1 million output tokens . For comparison , GPT-4o costs $ 5 per 1 million input tokens and $ 15 per 1 million output tokens .
+
+OpenAI says it plans to bring o1-mini access to all the free users of ChatGPT but hasn ’ t set a release date yet . Developer access to o1 is really expensive : In the API , o1-preview is $ 15 per 1 million input tokens , or chunks of text parsed by the model , and $ 60 per 1 million output tokens . For comparison , GPT-4o costs $ 5 per 1 million input tokens and $ 15 per 1 million output tokens . The training behind o1 is fundamentally different from its predecessors , OpenAI ’ s research lead , Jerry Tworek , tells me , though the company is being vague about the exact details . He says o1 “ has been trained using a completely new optimization algorithm and a new training dataset specifically tailored for it. ” Image : OpenAI OpenAI taught previous GPT models to mimic patterns from its training data .
+
+LLM Answer:
+The text does not mention the CEO of OpenAI. It only discusses the release of a new model called o1 and some details about it, but does not provide information on the company's leadership.
+
+LLM Claim: The text does not mention the CEO of OpenAI.
+Is this claim supported by the context according to bespoke-minicheck? Yes
+
+LLM Claim: It only discusses the release of a new model called o1 and some details about it, but does not provide information on the company's leadership.
+Is this claim supported by the context according to bespoke-minicheck? No
+```
+
+The second claim is unsupported since the text mentions the research lead.
+
+Another tricky example:
+
+```text
+
+Enter your question or type quit: what sets o1 apart from gpt-4o?
+
+Retrieved chunks:
+OpenAI says it plans to bring o1-mini access to all the free users of ChatGPT but hasn ’ t set a release date yet . Developer access to o1 is really expensive : In the API , o1-preview is $ 15 per 1 million input tokens , or chunks of text parsed by the model , and $ 60 per 1 million output tokens . For comparison , GPT-4o costs $ 5 per 1 million input tokens and $ 15 per 1 million output tokens . The training behind o1 is fundamentally different from its predecessors , OpenAI ’ s research lead , Jerry Tworek , tells me , though the company is being vague about the exact details . He says o1 “ has been trained using a completely new optimization algorithm and a new training dataset specifically tailored for it. ” Image : OpenAI OpenAI taught previous GPT models to mimic patterns from its training data .
+
+He says OpenAI also tested o1 against a qualifying exam for the International Mathematics Olympiad , and while GPT-4o only correctly solved only 13 percent of problems , o1 scored 83 percent . “ We can ’ t say we solved hallucinations ” In online programming contests known as Codeforces competitions , this new model reached the 89th percentile of participants , and OpenAI claims the next update of this model will perform “ similarly to PhD students on challenging benchmark tasks in physics , chemistry and biology. ” At the same time , o1 is not as capable as GPT-4o in a lot of areas . It doesn ’ t do as well on factual knowledge about the world .
+
+More practically , it does a better job at writing code and solving multistep problems than previous models . But it ’ s also more expensive and slower to use than GPT-4o . OpenAI is calling this release of o1 a “ preview ” to emphasize how nascent it is . ChatGPT Plus and Team users get access to both o1-preview and o1-mini starting today , while Enterprise and Edu users will get access early next week . OpenAI says it plans to bring o1-mini access to all the free users of ChatGPT but hasn ’ t set a release date yet . Developer access to o1 is really expensive : In the API , o1-preview is $ 15 per 1 million input tokens , or chunks of text parsed by the model , and $ 60 per 1 million output tokens . For comparison , GPT-4o costs $ 5 per 1 million input tokens and $ 15 per 1 million output tokens .
+
+“ We can ’ t say we solved hallucinations ” In online programming contests known as Codeforces competitions , this new model reached the 89th percentile of participants , and OpenAI claims the next update of this model will perform “ similarly to PhD students on challenging benchmark tasks in physics , chemistry and biology. ” At the same time , o1 is not as capable as GPT-4o in a lot of areas . It doesn ’ t do as well on factual knowledge about the world . It also doesn ’ t have the ability to browse the web or process files and images . Still , the company believes it represents a brand-new class of capabilities . It was named o1 to indicate “ resetting the counter back to 1. ” “ I ’ m gon na be honest : I think we ’ re terrible at naming , traditionally , ” McGrew says .
+LLM Answer: According to the text, several things set o1 apart from GPT-4o:
+
+* In online programming contests (Codeforces competitions), o1 scored 83% correct solutions compared to GPT-4o's 13%.
+* The training behind o1 is "fundamentally different" from its predecessors, including a completely new optimization algorithm and a new training dataset specifically tailored for it.
+* o1 has been shown to perform similarly to PhD students on challenging benchmark tasks in physics, chemistry, and biology, while GPT-4o does not have this level of performance.
+* However, o1 is also less capable than GPT-4o in some areas, particularly with regard to factual knowledge about the world.
+
+LLM Claim: According to the text, several things set o1 apart from GPT-4o:
+
+* In online programming contests (Codeforces competitions), o1 scored 83% correct solutions compared to GPT-4o's 13%.
+Is this claim supported by the context according to bespoke-minicheck? Yes
+
+LLM Claim: * The training behind o1 is "fundamentally different" from its predecessors, including a completely new optimization algorithm and a new training dataset specifically tailored for it.
+Is this claim supported by the context according to bespoke-minicheck? Yes
+
+LLM Claim: * o1 has been shown to perform similarly to PhD students on challenging benchmark tasks in physics, chemistry, and biology, while GPT-4o does not have this level of performance.
+Is this claim supported by the context according to bespoke-minicheck? No
+
+LLM Claim: * However, o1 is also less capable than GPT-4o in some areas, particularly with regard to factual knowledge about the world.
+Is this claim supported by the context according to bespoke-minicheck? Yes
+```
+
+We see that the third claim "* o1 has been shown to perform similarly to PhD students on challenging benchmark tasks in physics, chemistry, and biology, while GPT-4o does not have this level of performance." is not supported by the context. This is because the context only mentions that o1 "is claimed to perform" which is different from "has been shown to perform".
diff --git a/examples/python-grounded-factuality-rag-check/main.py b/examples/python-grounded-factuality-rag-check/main.py
new file mode 100644
index 00000000..eab0b670
--- /dev/null
+++ b/examples/python-grounded-factuality-rag-check/main.py
@@ -0,0 +1,137 @@
+import ollama
+import warnings
+from mattsollamatools import chunker
+from newspaper import Article
+import numpy as np
+from sklearn.neighbors import NearestNeighbors
+import nltk
+
+warnings.filterwarnings(
+ "ignore", category=FutureWarning, module="transformers.tokenization_utils_base"
+)
+nltk.download("punkt_tab", quiet=True)
+
+
+def getArticleText(url):
+ """Gets the text of an article from a URL.
+
+ Often there are a bunch of ads and menus on pages for a news article.
+ This uses newspaper3k to get just the text of just the article.
+ """
+ article = Article(url)
+ article.download()
+ article.parse()
+ return article.text
+
+
+def knn_search(question_embedding, embeddings, k=5):
+ """Performs K-nearest neighbors (KNN) search"""
+ X = np.array(
+ [item["embedding"] for article in embeddings for item in article["embeddings"]]
+ )
+ source_texts = [
+ item["source"] for article in embeddings for item in article["embeddings"]
+ ]
+
+ # Fit a KNN model on the embeddings
+ knn = NearestNeighbors(n_neighbors=k, metric="cosine")
+ knn.fit(X)
+
+ # Find the indices and distances of the k-nearest neighbors.
+ _, indices = knn.kneighbors(question_embedding, n_neighbors=k)
+
+ # Get the indices and source texts of the best matches
+ best_matches = [(indices[0][i], source_texts[indices[0][i]]) for i in range(k)]
+
+ return best_matches
+
+
+def check(document, claim):
+ """Checks if the claim is supported by the document by calling bespoke-minicheck.
+
+ Returns Yes/yes if the claim is supported by the document, No/no otherwise.
+ Support for logits will be added in the future.
+
+ bespoke-minicheck's system prompt is defined as:
+ 'Determine whether the provided claim is consistent with the corresponding
+ document. Consistency in this context implies that all information presented in the claim
+ is substantiated by the document. If not, it should be considered inconsistent. Please
+ assess the claim's consistency with the document by responding with either "Yes" or "No".'
+
+ bespoke-minicheck's user prompt is defined as:
+ "Document: {document}\nClaim: {claim}"
+ """
+ prompt = f"Document: {document}\nClaim: {claim}"
+ response = ollama.generate(
+ model="bespoke-minicheck", prompt=prompt, options={"num_predict": 2, "temperature": 0.0}
+ )
+ return response["response"].strip()
+
+
+if __name__ == "__main__":
+ allEmbeddings = []
+ default_url = "https://www.theverge.com/2024/9/12/24242439/openai-o1-model-reasoning-strawberry-chatgpt"
+ user_input = input(
+ "Enter the URL of an article you want to chat with, or press Enter for default example: "
+ )
+ article_url = user_input.strip() if user_input.strip() else default_url
+ article = {}
+ article["embeddings"] = []
+ article["url"] = article_url
+ text = getArticleText(article_url)
+ chunks = chunker(text)
+
+ # Embed (batch) chunks using ollama
+ embeddings = ollama.embed(model="all-minilm", input=chunks)["embeddings"]
+
+ for chunk, embedding in zip(chunks, embeddings):
+ item = {}
+ item["source"] = chunk
+ item["embedding"] = embedding
+ item["sourcelength"] = len(chunk)
+ article["embeddings"].append(item)
+
+ allEmbeddings.append(article)
+
+ print(f"\nLoaded, chunked, and embedded text from {article_url}.\n")
+
+ while True:
+ # Input a question from the user
+ # For example, "Who is the chief research officer?"
+ question = input("Enter your question or type quit: ")
+
+ if question.lower() == "quit":
+ break
+
+ # Embed the user's question using ollama.embed
+ question_embedding = ollama.embed(model="all-minilm", input=question)[
+ "embeddings"
+ ]
+
+ # Perform KNN search to find the best matches (indices and source text)
+ best_matches = knn_search(question_embedding, allEmbeddings, k=4)
+
+ sourcetext = "\n\n".join([source_text for (_, source_text) in best_matches])
+
+ print(f"\nRetrieved chunks: \n{sourcetext}\n")
+
+ # Give the retreived chunks and question to the chat model
+ system_prompt = f"Only use the following information to answer the question. Do not use anything else: {sourcetext}"
+
+ ollama_response = ollama.generate(
+ model="llama3.2",
+ prompt=question,
+ system=system_prompt,
+ options={"stream": False},
+ )
+
+ answer = ollama_response["response"]
+ print(f"LLM Answer:\n{answer}\n")
+
+ # Check each sentence in the response for grounded factuality
+ if answer:
+ for claim in nltk.sent_tokenize(answer):
+ print(f"LLM Claim: {claim}")
+ print(
+ f"Is this claim supported by the context according to bespoke-minicheck? {check(sourcetext, claim)}\n"
+ )
diff --git a/examples/python-grounded-factuality-rag-check/requirements.txt b/examples/python-grounded-factuality-rag-check/requirements.txt
new file mode 100644
index 00000000..d4bd6df3
--- /dev/null
+++ b/examples/python-grounded-factuality-rag-check/requirements.txt
@@ -0,0 +1,8 @@
+ollama
+lxml==5.3.0
+lxml_html_clean==0.2.2
+mattsollamatools==0.0.25
+newspaper3k==0.2.8
+nltk==3.9.1
+numpy==1.26.4
+scikit-learn==1.5.2
\ No newline at end of file
diff --git a/examples/python-grounded-factuality-simple-check/main.py b/examples/python-grounded-factuality-simple-check/main.py
new file mode 100644
index 00000000..0204f3b3
--- /dev/null
+++ b/examples/python-grounded-factuality-simple-check/main.py
@@ -0,0 +1,53 @@
+"""Simple example to demonstrate how to use the bespoke-minicheck model."""
+
+import ollama
+
+# NOTE: ollama must be running for this to work, start the ollama app or run `ollama serve`
+
+
+def check(document, claim):
+ """Checks if the claim is supported by the document by calling bespoke-minicheck.
+
+ Returns Yes/yes if the claim is supported by the document, No/no otherwise.
+ Support for logits will be added in the future.
+
+ bespoke-minicheck's system prompt is defined as:
+ 'Determine whether the provided claim is consistent with the corresponding
+ document. Consistency in this context implies that all information presented in the claim
+ is substantiated by the document. If not, it should be considered inconsistent. Please
+ assess the claim's consistency with the document by responding with either "Yes" or "No".'
+
+ bespoke-minicheck's user prompt is defined as:
+ "Document: {document}\nClaim: {claim}"
+ """
+ prompt = f"Document: {document}\nClaim: {claim}"
+ response = ollama.generate(
+ model="bespoke-minicheck", prompt=prompt, options={"num_predict": 2, "temperature": 0.0}
+ )
+ return response["response"].strip()
+
+
+def get_user_input(prompt):
+ user_input = input(prompt)
+ if not user_input:
+ exit()
+ print()
+ return user_input
+
+
+def main():
+ while True:
+ # Get a document from the user (e.g. "Ryan likes running and biking.")
+ document = get_user_input("Enter a document: ")
+ # Get a claim from the user (e.g. "Ryan likes to run.")
+ claim = get_user_input("Enter a claim: ")
+ # Check if the claim is supported by the document
+ grounded_factuality_check = check(document, claim)
+ print(
+ f"Is the claim supported by the document according to bespoke-minicheck? {grounded_factuality_check}"
+ )
+ print("\n\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/python-grounded-factuality-simple-check/readme.md b/examples/python-grounded-factuality-simple-check/readme.md
new file mode 100644
index 00000000..b164b5eb
--- /dev/null
+++ b/examples/python-grounded-factuality-simple-check/readme.md
@@ -0,0 +1,54 @@
+# Simple Bespoke-Minicheck Example
+
+`bespoke-minicheck` is a model for checking if a claim is supported by a document. It is used through the **generate** endpoint, which is called in this example with a `prompt` that includes the expected formatting of the user input.
+
+## Running the Example
+
+1. Ensure you have the `bespoke-minicheck` model installed:
+
+ ```bash
+ ollama pull bespoke-minicheck
+ ```
+
+2. Install the dependencies:
+
+ ```bash
+ pip install -r requirements.txt
+ ```
+
+3. Run the program:
+
+ ```bash
+ python main.py
+ ```
+
+4. Enter a document and a claim when prompted:
+
+ ```bash
+ Enter a document: Roses are red.
+
+ Enter a claim: Roses are blue.
+ ```
+
+ The claim and document are then given to the `bespoke-minicheck` as inputs, which then generates a response (Yes or No) on whether the claim is supported by the document.
+
+ ```bash
+ Is the claim supported by the document according to bespoke-minicheck? No
+ ```
+
+## More Examples
+
+Document ([source](https://en.wikipedia.org/wiki/Apple_I)):
+> The Apple Computer 1 (Apple-1[a]), later known predominantly as the Apple I(written with a Roman numeral),[b] is an 8-bit motherboard-only personal computer designed by Steve Wozniak[5][6] and released by the Apple Computer Company (now Apple Inc.) in 1976. The company was initially formed to sell the Apple I – its first product – and would later become the world's largest technology company.[7] The idea of starting a company and selling the computer came from Wozniak's friend and Apple co-founder Steve Jobs.[8][9] One of the main innovations of the Apple I was that it included video display terminal circuitry on its circuit board, allowing it to connect to a low-cost composite video monitor or television, instead of an expensive computer terminal, compared to most existing computers at the time.
+
+Claim:
+>The Apple I is a 16-bit computer.
+
+Expected output:
+>Is the claim supported by the document according to bespoke-minicheck? **No**
+
+Claim:
+>Apple was originally called the Apple Computer Company.
+
+Expected output:
+>Is the claim supported by the document according to bespoke-minicheck? **Yes**
diff --git a/examples/python-grounded-factuality-simple-check/requirements.txt b/examples/python-grounded-factuality-simple-check/requirements.txt
new file mode 100644
index 00000000..403abba6
--- /dev/null
+++ b/examples/python-grounded-factuality-simple-check/requirements.txt
@@ -0,0 +1 @@
+ollama
diff --git a/examples/python-json-datagenerator/predefinedschema.py b/examples/python-json-datagenerator/predefinedschema.py
index abc399c4..91463760 100644
--- a/examples/python-json-datagenerator/predefinedschema.py
+++ b/examples/python-json-datagenerator/predefinedschema.py
@@ -2,16 +2,16 @@ import requests
import json
import random
-model = "llama2"
+model = "llama3.2"
template = {
- "firstName": "",
- "lastName": "",
+ "firstName": "",
+ "lastName": "",
"address": {
- "street": "",
- "city": "",
- "state": "",
+ "street": "",
+ "city": "",
+ "state": "",
"zipCode": ""
- },
+ },
"phoneNumber": ""
}
diff --git a/examples/python-json-datagenerator/randomaddresses.py b/examples/python-json-datagenerator/randomaddresses.py
index 5f27448f..3df59d32 100644
--- a/examples/python-json-datagenerator/randomaddresses.py
+++ b/examples/python-json-datagenerator/randomaddresses.py
@@ -12,7 +12,7 @@ countries = [
"France",
]
country = random.choice(countries)
-model = "llama2"
+model = "llama3.2"
prompt = f"generate one realistically believable sample data set of a persons first name, last name, address in {country}, and phone number. Do not use common names. Respond using JSON. Key names should have no backslashes, values should use plain ascii with no special characters."
diff --git a/examples/python-json-datagenerator/readme.md b/examples/python-json-datagenerator/readme.md
index ec5701be..a551e1dd 100644
--- a/examples/python-json-datagenerator/readme.md
+++ b/examples/python-json-datagenerator/readme.md
@@ -1,15 +1,15 @@
# JSON Output Example
-
+
There are two python scripts in this example. `randomaddresses.py` generates random addresses from different countries. `predefinedschema.py` sets a template for the model to fill in.
## Running the Example
-1. Ensure you have the `llama2` model installed:
+1. Ensure you have the `llama3.2` model installed:
```bash
- ollama pull llama2
+ ollama pull llama3.2
```
2. Install the Python Requirements.
diff --git a/examples/python-loganalysis/Modelfile b/examples/python-loganalysis/Modelfile
index 5237cb6e..b28aa0c0 100644
--- a/examples/python-loganalysis/Modelfile
+++ b/examples/python-loganalysis/Modelfile
@@ -4,5 +4,5 @@ SYSTEM """
You are a log file analyzer. You will receive a set of lines from a log file for some software application, find the errors and other interesting aspects of the logs, and explain them so a new user can understand what they mean. If there are any steps they can do to resolve them, list the steps in your answer.
"""
-PARAMETER TEMPERATURE 0.3
+PARAMETER temperature 0.3
diff --git a/examples/python-loganalysis/readme.md b/examples/python-loganalysis/readme.md
index 828e8de2..03bab672 100644
--- a/examples/python-loganalysis/readme.md
+++ b/examples/python-loganalysis/readme.md
@@ -1,6 +1,6 @@
# Log Analysis example
-
+
This example shows one possible way to create a log file analyzer. It uses the model **mattw/loganalyzer** which is based on **codebooga**, a 34b parameter model.
@@ -21,6 +21,8 @@ You can try this with the `logtest.logfile` file included in this directory.
2. Install the Python Requirements.
```bash
+ python3 -m venv .venv
+ source .venv/bin/activate
pip install -r requirements.txt
```
@@ -40,13 +42,13 @@ You are a log file analyzer. You will receive a set of lines from a log file for
"""
```
-This model is available at https://ollama.ai/mattw/loganalyzer. You can customize it and add to your own namespace using the command `ollama create <namespace/modelname> -f <path-to-modelfile>` then `ollama push <namespace/modelname>`.
+This model is available at https://ollama.com/mattw/loganalyzer. You can customize it and add to your own namespace using the command `ollama create <namespace/modelname> -f <path-to-modelfile>` then `ollama push <namespace/modelname>`.
Then loganalysis.py scans all the lines in the given log file and searches for the word 'error'. When the word is found, the 10 lines before and after are set as the prompt for a call to the Generate API.
```python
data = {
- "prompt": "\n".join(error_logs),
+ "prompt": "\n".join(error_logs),
"model": "mattw/loganalyzer"
}
```
diff --git a/examples/python-loganalysis/requirements.txt b/examples/python-loganalysis/requirements.txt
index 9688b8ec..e7cb17ef 100644
--- a/examples/python-loganalysis/requirements.txt
+++ b/examples/python-loganalysis/requirements.txt
@@ -1 +1 @@
-Requests==2.31.0
+Requests>=2.32.3
diff --git a/examples/python-simplechat/client.py b/examples/python-simplechat/client.py
index 768a2289..6ef14ffc 100644
--- a/examples/python-simplechat/client.py
+++ b/examples/python-simplechat/client.py
@@ -2,13 +2,14 @@ import json
import requests
# NOTE: ollama must be running for this to work, start the ollama app or run `ollama serve`
-model = "llama2" # TODO: update this for whatever model you wish to use
+model = "llama3.2" # TODO: update this for whatever model you wish to use
def chat(messages):
r = requests.post(
"http://0.0.0.0:11434/api/chat",
json={"model": model, "messages": messages, "stream": True},
+ stream=True
)
r.raise_for_status()
output = ""
diff --git a/examples/python-simplechat/readme.md b/examples/python-simplechat/readme.md
index 204a8159..a4a2dfc1 100644
--- a/examples/python-simplechat/readme.md
+++ b/examples/python-simplechat/readme.md
@@ -4,10 +4,10 @@ The **chat** endpoint is one of two ways to generate text from an LLM with Ollam
## Running the Example
-1. Ensure you have the `llama2` model installed:
+1. Ensure you have the `llama3.2` model installed:
```bash
- ollama pull llama2
+ ollama pull llama3.2
```
2. Install the Python Requirements.
diff --git a/examples/typescript-functioncalling/readme.md b/examples/typescript-functioncalling/readme.md
index 03ff46dc..d29379a0 100644
--- a/examples/typescript-functioncalling/readme.md
+++ b/examples/typescript-functioncalling/readme.md
@@ -1,6 +1,6 @@
# Function calling
-
+
One of the features added to some models is 'function calling'. It's a bit of a confusing name. It's understandable if you think that means the model can call functions, but that's not what it means. Function calling simply means that the output of the model is formatted in JSON, using a preconfigured schema, and uses the expected types. Then your code can use the output of the model and call functions with it. Using the JSON format in Ollama, you can use any model for function calling.
diff --git a/examples/typescript-mentors/README.md b/examples/typescript-mentors/README.md
index 5ab1cc55..d3611a5e 100644
--- a/examples/typescript-mentors/README.md
+++ b/examples/typescript-mentors/README.md
@@ -4,10 +4,10 @@ This example demonstrates how one would create a set of 'mentors' you can have a
## Usage
-1. Add llama2 to have the mentors ask your questions:
+1. Add llama3 to have the mentors ask your questions:
```bash
- ollama pull llama2
+ ollama pull llama3
```
2. Install prerequisites:
@@ -29,9 +29,9 @@ You can also add your own character to be chosen at random when you ask a questi
```bash
ollama pull stablebeluga2:70b-q4_K_M
```
-
+
2. Create a new character:
-
+
```bash
npm run charactergen "Lorne Greene"
```
@@ -41,15 +41,15 @@ You can also add your own character to be chosen at random when you ask a questi
3. Now you can create a model with this command:
```bash
- ollama create <YourNamespace>/lornegreene -f lornegreene/Modelfile
+ ollama create <username>/lornegreene -f lornegreene/Modelfile
```
- `YourNamespace` is whatever name you set up when you signed up at [https://ollama.ai/signup](https://ollama.ai/signup).
+ `username` is whatever name you set up when you signed up at [https://ollama.com/signup](https://ollama.com/signup).
-4. To add this to your mentors, you will have to update the code as follows. On line 8 of `mentors.ts`, add an object to the array, replacing `<YourNamespace>` with the namespace you used above.
+4. To add this to your mentors, you will have to update the code as follows. On line 8 of `mentors.ts`, add an object to the array, replacing `<username>` with the username you used above.
```bash
- {ns: "<YourNamespace>", char: "Lorne Greene"}
+ {ns: "<username>", char: "Lorne Greene"}
```
## Review the Code
diff --git a/examples/typescript-mentors/character-generator.ts b/examples/typescript-mentors/character-generator.ts
index 886eec67..dc5d2f5e 100644
--- a/examples/typescript-mentors/character-generator.ts
+++ b/examples/typescript-mentors/character-generator.ts
@@ -15,7 +15,7 @@ async function characterGenerator() {
ollama.setModel("stablebeluga2:70b-q4_K_M");
const bio = await ollama.generate(`create a bio of ${character} in a single long paragraph. Instead of saying '${character} is...' or '${character} was...' use language like 'You are...' or 'You were...'. Then create a paragraph describing the speaking mannerisms and style of ${character}. Don't include anything about how ${character} looked or what they sounded like, just focus on the words they said. Instead of saying '${character} would say...' use language like 'You should say...'. If you use quotes, always use single quotes instead of double quotes. If there are any specific words or phrases you used a lot, show how you used them. `);
- const thecontents = `FROM llama2\nSYSTEM """\n${bio.response.replace(/(\r\n|\n|\r)/gm, " ").replace('would', 'should')} All answers to questions should be related back to what you are most known for.\n"""`;
+ const thecontents = `FROM llama3\nSYSTEM """\n${bio.response.replace(/(\r\n|\n|\r)/gm, " ").replace('would', 'should')} All answers to questions should be related back to what you are most known for.\n"""`;
fs.writeFile(path.join(directory, 'Modelfile'), thecontents, (err: any) => {
if (err) throw err;
@@ -23,4 +23,4 @@ async function characterGenerator() {
});
}
-characterGenerator();
\ No newline at end of file
+characterGenerator();
diff --git a/examples/typescript-simplechat/client.ts b/examples/typescript-simplechat/client.ts
index 3e571ab6..d8faaa1b 100644
--- a/examples/typescript-simplechat/client.ts
+++ b/examples/typescript-simplechat/client.ts
@@ -1,6 +1,6 @@
import * as readline from "readline";
-const model = "llama2";
+const model = "llama3.2";
type Message = {
role: "assistant" | "user" | "system";
content: string;
@@ -74,4 +74,4 @@ async function main() {
}
-main();
\ No newline at end of file
+main();
diff --git a/format/bytes.go b/format/bytes.go
index 01be17ce..13d8575e 100644
--- a/format/bytes.go
+++ b/format/bytes.go
@@ -6,11 +6,16 @@ import (
)
const (
- Byte = 1
+ Byte = 1
+
KiloByte = Byte * 1000
MegaByte = KiloByte * 1000
GigaByte = MegaByte * 1000
TeraByte = GigaByte * 1000
+
+ KibiByte = Byte * 1024
+ MebiByte = KibiByte * 1024
+ GibiByte = MebiByte * 1024
)
func HumanBytes(b int64) string {
@@ -45,3 +50,16 @@ func HumanBytes(b int64) string {
return fmt.Sprintf("%d %s", int(value), unit)
}
}
+
+func HumanBytes2(b uint64) string {
+ switch {
+ case b >= GibiByte:
+ return fmt.Sprintf("%.1f GiB", float64(b)/GibiByte)
+ case b >= MebiByte:
+ return fmt.Sprintf("%.1f MiB", float64(b)/MebiByte)
+ case b >= KibiByte:
+ return fmt.Sprintf("%.1f KiB", float64(b)/KibiByte)
+ default:
+ return fmt.Sprintf("%d B", b)
+ }
+}
diff --git a/format/format.go b/format/format.go
index 8fd2defa..ac50570d 100644
--- a/format/format.go
+++ b/format/format.go
@@ -3,6 +3,7 @@ package format
import (
"fmt"
"math"
+ "strconv"
)
const (
@@ -13,13 +14,21 @@ const (
func HumanNumber(b uint64) string {
switch {
- case b > Billion:
- return fmt.Sprintf("%.0fB", math.Round(float64(b)/Billion))
- case b > Million:
- return fmt.Sprintf("%.0fM", math.Round(float64(b)/Million))
- case b > Thousand:
- return fmt.Sprintf("%.0fK", math.Round(float64(b)/Thousand))
+ case b >= Billion:
+ number := float64(b) / Billion
+ if number == math.Floor(number) {
+ return fmt.Sprintf("%.0fB", number) // no decimals if whole number
+ }
+ return fmt.Sprintf("%.1fB", number) // one decimal if not a whole number
+ case b >= Million:
+ number := float64(b) / Million
+ if number == math.Floor(number) {
+ return fmt.Sprintf("%.0fM", number) // no decimals if whole number
+ }
+ return fmt.Sprintf("%.2fM", number) // two decimals if not a whole number
+ case b >= Thousand:
+ return fmt.Sprintf("%.0fK", float64(b)/Thousand)
default:
- return fmt.Sprintf("%d", b)
+ return strconv.FormatUint(b, 10)
}
}
diff --git a/format/format_test.go b/format/format_test.go
new file mode 100644
index 00000000..bff32780
--- /dev/null
+++ b/format/format_test.go
@@ -0,0 +1,33 @@
+package format
+
+import (
+ "testing"
+)
+
+func TestHumanNumber(t *testing.T) {
+ type testCase struct {
+ input uint64
+ expected string
+ }
+
+ testCases := []testCase{
+ {0, "0"},
+ {1000000, "1M"},
+ {125000000, "125M"},
+ {500500000, "500.50M"},
+ {500550000, "500.55M"},
+ {1000000000, "1B"},
+ {2800000000, "2.8B"},
+ {2850000000, "2.9B"},
+ {1000000000000, "1000B"},
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.expected, func(t *testing.T) {
+ result := HumanNumber(tc.input)
+ if result != tc.expected {
+ t.Errorf("Expected %s, got %s", tc.expected, result)
+ }
+ })
+ }
+}
diff --git a/format/openssh.go b/format/openssh.go
deleted file mode 100644
index e642e358..00000000
--- a/format/openssh.go
+++ /dev/null
@@ -1,102 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Code originally from https://go-review.googlesource.com/c/crypto/+/218620
-
-// TODO: replace with upstream once the above change is merged and released.
-
-package format
-
-import (
- "crypto"
- "crypto/ed25519"
- "crypto/rand"
- "encoding/binary"
- "encoding/pem"
- "fmt"
-
- "golang.org/x/crypto/ssh"
-)
-
-const privateKeyAuthMagic = "openssh-key-v1\x00"
-
-type openSSHEncryptedPrivateKey struct {
- CipherName string
- KDFName string
- KDFOptions string
- KeysCount uint32
- PubKey []byte
- KeyBlocks []byte
-}
-
-type openSSHPrivateKey struct {
- Check1 uint32
- Check2 uint32
- Keytype string
- Rest []byte `ssh:"rest"`
-}
-
-type openSSHEd25519PrivateKey struct {
- Pub []byte
- Priv []byte
- Comment string
- Pad []byte `ssh:"rest"`
-}
-
-func OpenSSHPrivateKey(key crypto.PrivateKey, comment string) (*pem.Block, error) {
- var check uint32
- if err := binary.Read(rand.Reader, binary.BigEndian, &check); err != nil {
- return nil, err
- }
-
- var pk1 openSSHPrivateKey
- pk1.Check1 = check
- pk1.Check2 = check
-
- var w openSSHEncryptedPrivateKey
- w.KeysCount = 1
-
- if k, ok := key.(*ed25519.PrivateKey); ok {
- key = *k
- }
-
- switch k := key.(type) {
- case ed25519.PrivateKey:
- pub, priv := k[32:], k
- key := openSSHEd25519PrivateKey{
- Pub: pub,
- Priv: priv,
- Comment: comment,
- }
-
- pk1.Keytype = ssh.KeyAlgoED25519
- pk1.Rest = ssh.Marshal(key)
-
- w.PubKey = ssh.Marshal(struct {
- KeyType string
- Pub []byte
- }{
- ssh.KeyAlgoED25519, pub,
- })
- default:
- return nil, fmt.Errorf("ssh: unknown key type %T", k)
- }
-
- w.KeyBlocks = openSSHPadding(ssh.Marshal(pk1), 8)
-
- w.CipherName, w.KDFName, w.KDFOptions = "none", "none", ""
-
- return &pem.Block{
- Type: "OPENSSH PRIVATE KEY",
- Bytes: append([]byte(privateKeyAuthMagic), ssh.Marshal(w)...),
- }, nil
-}
-
-func openSSHPadding(block []byte, blocksize int) []byte {
- for i, j := 0, len(block); (j+i)%blocksize != 0; i++ {
- block = append(block, byte(i+1))
- }
-
- return block
-}
diff --git a/format/time.go b/format/time.go
index 6637c062..74062848 100644
--- a/format/time.go
+++ b/format/time.go
@@ -60,7 +60,9 @@ func humanTime(t time.Time, zeroValue string) string {
}
delta := time.Since(t)
- if delta < 0 {
+ if int(delta.Hours())/24/365 < -20 {
+ return "Forever"
+ } else if delta < 0 {
return humanDuration(-delta) + " from now"
}
diff --git a/format/time_test.go b/format/time_test.go
index cc6b8930..bd0ba9a8 100644
--- a/format/time_test.go
+++ b/format/time_test.go
@@ -32,4 +32,14 @@ func TestHumanTime(t *testing.T) {
v := now.Add(800 * time.Millisecond)
assertEqual(t, HumanTime(v, ""), "Less than a second from now")
})
+
+ t.Run("time way in the future", func(t *testing.T) {
+ v := now.Add(24 * time.Hour * 365 * 200)
+ assertEqual(t, HumanTime(v, ""), "Forever")
+ })
+
+ t.Run("time way in the future lowercase", func(t *testing.T) {
+ v := now.Add(24 * time.Hour * 365 * 200)
+ assertEqual(t, HumanTimeLower(v, ""), "forever")
+ })
}
diff --git a/go.mod b/go.mod
index 0df1372b..6e437c73 100644
--- a/go.mod
+++ b/go.mod
@@ -1,53 +1,78 @@
-module github.com/jmorganca/ollama
+module github.com/ollama/ollama
-go 1.20
+go 1.22.5
require (
+ github.com/containerd/console v1.0.3
github.com/emirpasic/gods v1.18.1
- github.com/gin-gonic/gin v1.9.1
+ github.com/gin-gonic/gin v1.10.0
+ github.com/golang/protobuf v1.5.4 // indirect
+ github.com/google/uuid v1.1.2
github.com/olekukonko/tablewriter v0.0.5
github.com/spf13/cobra v1.7.0
- github.com/stretchr/testify v1.8.4
+ github.com/stretchr/testify v1.9.0
+ github.com/x448/float16 v0.8.4
golang.org/x/sync v0.3.0
)
require (
- github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/mattn/go-runewidth v0.0.14 // indirect
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/rivo/uniseg v0.2.0 // indirect
+ github.com/agnivade/levenshtein v1.1.1
+ github.com/d4l3k/go-bfloat16 v0.0.0-20211005043715-690c3bdd05f1
+ github.com/google/go-cmp v0.6.0
+ github.com/mattn/go-runewidth v0.0.14
+ github.com/nlpodyssey/gopickle v0.3.0
+ github.com/pdevine/tensor v0.0.0-20240510204454-f88f4562727c
)
require (
- github.com/bytedance/sonic v1.9.1 // indirect
- github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
- github.com/gabriel-vasile/mimetype v1.4.2 // indirect
- github.com/gin-contrib/cors v1.4.0
+ github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect
+ github.com/bytedance/sonic/loader v0.1.1 // indirect
+ github.com/chewxy/hm v1.0.0 // indirect
+ github.com/chewxy/math32 v1.10.1 // indirect
+ github.com/cloudwego/base64x v0.1.4 // indirect
+ github.com/cloudwego/iasm v0.2.0 // indirect
+ github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/google/flatbuffers v24.3.25+incompatible // indirect
+ github.com/kr/text v0.2.0 // indirect
+ github.com/pkg/errors v0.9.1 // indirect
+ github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/rivo/uniseg v0.2.0 // indirect
+ github.com/xtgo/set v1.0.0 // indirect
+ go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 // indirect
+ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+ gonum.org/v1/gonum v0.15.0 // indirect
+ gorgonia.org/vecf32 v0.9.0 // indirect
+ gorgonia.org/vecf64 v0.9.0 // indirect
+)
+
+require (
+ github.com/bytedance/sonic v1.11.6 // indirect
+ github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+ github.com/gin-contrib/cors v1.7.2
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
- github.com/go-playground/validator/v10 v10.14.0 // indirect
+ github.com/go-playground/validator/v10 v10.20.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
- github.com/google/go-cmp v0.5.9 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
- github.com/klauspost/cpuid/v2 v2.2.4 // indirect
- github.com/leodido/go-urn v1.2.4 // indirect
- github.com/mattn/go-isatty v0.0.19 // indirect
+ github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+ github.com/leodido/go-urn v1.4.0 // indirect
+ github.com/mattn/go-isatty v0.0.20 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
- github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
- github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+ github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
- github.com/ugorji/go/codec v1.2.11 // indirect
- golang.org/x/arch v0.3.0 // indirect
- golang.org/x/crypto v0.14.0
- golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
- golang.org/x/net v0.17.0 // indirect
- golang.org/x/sys v0.13.0 // indirect
- golang.org/x/term v0.13.0
- golang.org/x/text v0.13.0 // indirect
- google.golang.org/protobuf v1.30.0 // indirect
+ github.com/ugorji/go/codec v1.2.12 // indirect
+ golang.org/x/arch v0.8.0 // indirect
+ golang.org/x/crypto v0.23.0
+ golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
+ golang.org/x/net v0.25.0 // indirect
+ golang.org/x/sys v0.20.0
+ golang.org/x/term v0.20.0
+ golang.org/x/text v0.15.0
+ google.golang.org/protobuf v1.34.1
gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/go.sum b/go.sum
index ff6bcbd9..926ed26d 100644
--- a/go.sum
+++ b/go.sum
@@ -1,65 +1,143 @@
-github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
-github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
-github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
-github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
-github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
-github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
+github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
+github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ=
+github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs=
+github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
+github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
+github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
+github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
+github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
+github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
+github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chewxy/hm v1.0.0 h1:zy/TSv3LV2nD3dwUEQL2VhXeoXbb9QkpmdRAVUFiA6k=
+github.com/chewxy/hm v1.0.0/go.mod h1:qg9YI4q6Fkj/whwHR1D+bOGeF7SniIP40VweVepLjg0=
+github.com/chewxy/math32 v1.0.0/go.mod h1:Miac6hA1ohdDUTagnvJy/q+aNnEk16qWUdb8ZVhvCN0=
+github.com/chewxy/math32 v1.10.1 h1:LFpeY0SLJXeaiej/eIp2L40VYfscTvKh/FSEZ68uMkU=
+github.com/chewxy/math32 v1.10.1/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
+github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
+github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
+github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
+github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/d4l3k/go-bfloat16 v0.0.0-20211005043715-690c3bdd05f1 h1:cBzrdJPAFBsgCrDPnZxlp1dF2+k4r1kVpD7+1S1PVjY=
+github.com/d4l3k/go-bfloat16 v0.0.0-20211005043715-690c3bdd05f1/go.mod h1:uw2gLcxEuYUlAd/EXyjc/v55nd3+47YAgWbSXVxPrNI=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
+github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
-github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
-github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
-github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g=
-github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
+github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gin-contrib/cors v1.7.2 h1:oLDHxdg8W/XDoN/8zamqk/Drgt4oVZDvaV0YmvVICQw=
+github.com/gin-contrib/cors v1.7.2/go.mod h1:SUJVARKgQ40dmrzgXEVxj2m7Ig1v1qIboQkPDTQ9t2E=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
-github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
-github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
-github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
-github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
+github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
+github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g=
+github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks=
+github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
+github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
-github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
-github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
-github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
-github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
+github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI=
+github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
+github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ=
+github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
-github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
+github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
-github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
-github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
-github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
@@ -68,22 +146,31 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/nlpodyssey/gopickle v0.3.0 h1:BLUE5gxFLyyNOPzlXxt6GoHEMMxD0qhsE4p0CIQyoLw=
+github.com/nlpodyssey/gopickle v0.3.0/go.mod h1:f070HJ/yR+eLi5WmM1OXJEGaTpuJEUiib19olXgYha0=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
-github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
-github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
-github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
-github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
-github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pdevine/tensor v0.0.0-20240510204454-f88f4562727c h1:GwiUUjKefgvSNmv3NCvI/BL0kDebW6Xa+kcdpdc1mTY=
+github.com/pdevine/tensor v0.0.0-20240510204454-f88f4562727c/go.mod h1:PSojXDXF7TbgQiD6kkd98IHOS0QqTyUEaWRiS8+BLu8=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
+github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
+github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4=
+github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -91,64 +178,187 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
-github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
-github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
-github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
-github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
+github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+github.com/xtgo/set v1.0.0 h1:6BCNBRv3ORNDQ7fyoJXRv+tstJz3m1JVFQErfeZz2pY=
+github.com/xtgo/set v1.0.0/go.mod h1:d3NHzGzSa0NmB2NhFyECA+QdRp29oEn2xbT+TpeFoM8=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 h1:lGdhQUN/cnWdSH3291CUuxSEqc+AsGTiDxPP3r2J0l4=
+go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
-golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
-golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
-golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
-golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
+golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
+golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
-golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
+gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
+gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0=
+gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ=
+gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo=
+gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
+gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
+gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorgonia.org/vecf32 v0.9.0 h1:PClazic1r+JVJ1dEzRXgeiVl4g1/Hf/w+wUSqnco1Xg=
+gorgonia.org/vecf32 v0.9.0/go.mod h1:NCc+5D2oxddRL11hd+pCB1PEyXWOyiQxfZ/1wwhOXCA=
+gorgonia.org/vecf64 v0.9.0 h1:bgZDP5x0OzBF64PjMGC3EvTdOoMEcmfAh1VCUnZFm1A=
+gorgonia.org/vecf64 v0.9.0/go.mod h1:hp7IOWCnRiVQKON73kkC/AUMtEXyf9kGlVrtPQ9ccVA=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
diff --git a/gpu/amd_common.go b/gpu/amd_common.go
new file mode 100644
index 00000000..2894ac2c
--- /dev/null
+++ b/gpu/amd_common.go
@@ -0,0 +1,101 @@
+//go:build linux || windows
+
+package gpu
+
+import (
+ "errors"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+
+ "github.com/ollama/ollama/envconfig"
+)
+
+// Determine if the given ROCm lib directory is usable by checking for existence of some glob patterns
+func rocmLibUsable(libDir string) bool {
+ slog.Debug("evaluating potential rocm lib dir " + libDir)
+ for _, g := range ROCmLibGlobs {
+ res, _ := filepath.Glob(filepath.Join(libDir, g))
+ if len(res) == 0 {
+ return false
+ }
+ }
+ return true
+}
+
+func GetSupportedGFX(libDir string) ([]string, error) {
+ var ret []string
+ files, err := filepath.Glob(filepath.Join(libDir, "rocblas", "library", "TensileLibrary_lazy_gfx*.dat"))
+ if err != nil {
+ return nil, err
+ }
+ for _, file := range files {
+ ret = append(ret, strings.TrimSuffix(strings.TrimPrefix(filepath.Base(file), "TensileLibrary_lazy_"), ".dat"))
+ }
+ return ret, nil
+}
+
+func rocmGetVisibleDevicesEnv(gpuInfo []GpuInfo) (string, string) {
+ ids := []string{}
+ for _, info := range gpuInfo {
+ if info.Library != "rocm" {
+ // TODO shouldn't happen if things are wired correctly...
+ slog.Debug("rocmGetVisibleDevicesEnv skipping over non-rocm device", "library", info.Library)
+ continue
+ }
+ ids = append(ids, info.ID)
+ }
+ return "HIP_VISIBLE_DEVICES", strings.Join(ids, ",")
+}
+
+func commonAMDValidateLibDir() (string, error) {
+ // Favor our bundled version
+
+ // Installer payload location if we're running the installed binary
+ exe, err := os.Executable()
+ if err == nil {
+ rocmTargetDir := filepath.Join(filepath.Dir(exe), envconfig.LibRelativeToExe(), "lib", "ollama")
+ if rocmLibUsable(rocmTargetDir) {
+ slog.Debug("detected ROCM next to ollama executable " + rocmTargetDir)
+ return rocmTargetDir, nil
+ }
+ }
+
+ // Prefer explicit HIP env var
+ hipPath := os.Getenv("HIP_PATH")
+ if hipPath != "" {
+ hipLibDir := filepath.Join(hipPath, "bin")
+ if rocmLibUsable(hipLibDir) {
+ slog.Debug("detected ROCM via HIP_PATH=" + hipPath)
+ return hipLibDir, nil
+ }
+ }
+
+ // Scan the LD_LIBRARY_PATH or PATH
+ pathEnv := "LD_LIBRARY_PATH"
+ if runtime.GOOS == "windows" {
+ pathEnv = "PATH"
+ }
+
+ paths := os.Getenv(pathEnv)
+ for _, path := range filepath.SplitList(paths) {
+ d, err := filepath.Abs(path)
+ if err != nil {
+ continue
+ }
+ if rocmLibUsable(d) {
+ return d, nil
+ }
+ }
+
+ // Well known location(s)
+ for _, path := range RocmStandardLocations {
+ if rocmLibUsable(path) {
+ return path, nil
+ }
+ }
+
+ return "", errors.New("no suitable rocm found, falling back to CPU")
+}
diff --git a/gpu/amd_hip_windows.go b/gpu/amd_hip_windows.go
new file mode 100644
index 00000000..2cea2824
--- /dev/null
+++ b/gpu/amd_hip_windows.go
@@ -0,0 +1,147 @@
+package gpu
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "syscall"
+ "unsafe"
+
+ "golang.org/x/sys/windows"
+)
+
+const (
+ hipSuccess = 0
+ hipErrorNoDevice = 100
+)
+
+type hipDevicePropMinimal struct {
+ Name [256]byte
+ unused1 [140]byte
+ GcnArchName [256]byte // gfx####
+ iGPU int // Doesn't seem to actually report correctly
+ unused2 [128]byte
+}
+
+// Wrap the amdhip64.dll library for GPU discovery
+type HipLib struct {
+ dll windows.Handle
+ hipGetDeviceCount uintptr
+ hipGetDeviceProperties uintptr
+ hipMemGetInfo uintptr
+ hipSetDevice uintptr
+ hipDriverGetVersion uintptr
+}
+
+func NewHipLib() (*HipLib, error) {
+ // At runtime we depend on v6, so discover GPUs with the same library for a consistent set of GPUs
+ h, err := windows.LoadLibrary("amdhip64_6.dll")
+ if err != nil {
+ return nil, fmt.Errorf("unable to load amdhip64_6.dll, please make sure to upgrade to the latest amd driver: %w", err)
+ }
+ hl := &HipLib{}
+ hl.dll = h
+ hl.hipGetDeviceCount, err = windows.GetProcAddress(hl.dll, "hipGetDeviceCount")
+ if err != nil {
+ return nil, err
+ }
+ hl.hipGetDeviceProperties, err = windows.GetProcAddress(hl.dll, "hipGetDeviceProperties")
+ if err != nil {
+ return nil, err
+ }
+ hl.hipMemGetInfo, err = windows.GetProcAddress(hl.dll, "hipMemGetInfo")
+ if err != nil {
+ return nil, err
+ }
+ hl.hipSetDevice, err = windows.GetProcAddress(hl.dll, "hipSetDevice")
+ if err != nil {
+ return nil, err
+ }
+ hl.hipDriverGetVersion, err = windows.GetProcAddress(hl.dll, "hipDriverGetVersion")
+ if err != nil {
+ return nil, err
+ }
+ return hl, nil
+}
+
+// The hip library only evaluates the HIP_VISIBLE_DEVICES variable at startup
+// so we have to unload/reset the library after we do our initial discovery
+// to make sure our updates to that variable are processed by llama.cpp
+func (hl *HipLib) Release() {
+ err := windows.FreeLibrary(hl.dll)
+ if err != nil {
+ slog.Warn("failed to unload amdhip64.dll", "error", err)
+ }
+ hl.dll = 0
+}
+
+func (hl *HipLib) AMDDriverVersion() (driverMajor, driverMinor int, err error) {
+ if hl.dll == 0 {
+ return 0, 0, errors.New("dll has been unloaded")
+ }
+ var version int
+ status, _, err := syscall.SyscallN(hl.hipDriverGetVersion, uintptr(unsafe.Pointer(&version)))
+ if status != hipSuccess {
+ return 0, 0, fmt.Errorf("failed call to hipDriverGetVersion: %d %s", status, err)
+ }
+
+ slog.Debug("hipDriverGetVersion", "version", version)
+ driverMajor = version / 10000000
+ driverMinor = (version - (driverMajor * 10000000)) / 100000
+
+ return driverMajor, driverMinor, nil
+}
+
+func (hl *HipLib) HipGetDeviceCount() int {
+ if hl.dll == 0 {
+ slog.Error("dll has been unloaded")
+ return 0
+ }
+ var count int
+ status, _, err := syscall.SyscallN(hl.hipGetDeviceCount, uintptr(unsafe.Pointer(&count)))
+ if status == hipErrorNoDevice {
+ slog.Info("AMD ROCm reports no devices found")
+ return 0
+ }
+ if status != hipSuccess {
+ slog.Warn("failed call to hipGetDeviceCount", "status", status, "error", err)
+ }
+ return count
+}
+
+func (hl *HipLib) HipSetDevice(device int) error {
+ if hl.dll == 0 {
+ return errors.New("dll has been unloaded")
+ }
+ status, _, err := syscall.SyscallN(hl.hipSetDevice, uintptr(device))
+ if status != hipSuccess {
+ return fmt.Errorf("failed call to hipSetDevice: %d %s", status, err)
+ }
+ return nil
+}
+
+func (hl *HipLib) HipGetDeviceProperties(device int) (*hipDevicePropMinimal, error) {
+ if hl.dll == 0 {
+ return nil, errors.New("dll has been unloaded")
+ }
+ var props hipDevicePropMinimal
+ status, _, err := syscall.SyscallN(hl.hipGetDeviceProperties, uintptr(unsafe.Pointer(&props)), uintptr(device))
+ if status != hipSuccess {
+ return nil, fmt.Errorf("failed call to hipGetDeviceProperties: %d %s", status, err)
+ }
+ return &props, nil
+}
+
+// free, total, err
+func (hl *HipLib) HipMemGetInfo() (uint64, uint64, error) {
+ if hl.dll == 0 {
+ return 0, 0, errors.New("dll has been unloaded")
+ }
+ var totalMemory uint64
+ var freeMemory uint64
+ status, _, err := syscall.SyscallN(hl.hipMemGetInfo, uintptr(unsafe.Pointer(&freeMemory)), uintptr(unsafe.Pointer(&totalMemory)))
+ if status != hipSuccess {
+ return 0, 0, fmt.Errorf("failed call to hipMemGetInfo: %d %s", status, err)
+ }
+ return freeMemory, totalMemory, nil
+}
diff --git a/gpu/amd_linux.go b/gpu/amd_linux.go
new file mode 100644
index 00000000..72dfb4db
--- /dev/null
+++ b/gpu/amd_linux.go
@@ -0,0 +1,518 @@
+package gpu
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "regexp"
+ "slices"
+ "sort"
+ "strconv"
+ "strings"
+
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/format"
+)
+
+// Discovery logic for AMD/ROCm GPUs
+
+const (
+ DriverVersionFile = "/sys/module/amdgpu/version"
+ AMDNodesSysfsDir = "/sys/class/kfd/kfd/topology/nodes/"
+ GPUPropertiesFileGlob = AMDNodesSysfsDir + "*/properties"
+
+ // Prefix with the node dir
+ GPUTotalMemoryFileGlob = "mem_banks/*/properties" // size_in_bytes line
+
+ // Direct Rendering Manager sysfs location
+ DRMDeviceDirGlob = "/sys/class/drm/card*/device"
+ DRMTotalMemoryFile = "mem_info_vram_total"
+ DRMUsedMemoryFile = "mem_info_vram_used"
+
+ // In hex; properties file is in decimal
+ DRMUniqueIDFile = "unique_id"
+ DRMVendorFile = "vendor"
+ DRMDeviceFile = "device"
+)
+
+var (
+ // Used to validate if the given ROCm lib is usable
+ ROCmLibGlobs = []string{"libhipblas.so.2*", "rocblas"} // TODO - probably include more coverage of files here...
+ RocmStandardLocations = []string{"/opt/rocm/lib", "/usr/lib64"}
+)
+
+// Gather GPU information from the amdgpu driver if any supported GPUs are detected
+// Only called once during bootstrap
+func AMDGetGPUInfo() ([]RocmGPUInfo, error) {
+ resp := []RocmGPUInfo{}
+ if !AMDDetected() {
+ return resp, fmt.Errorf("AMD GPUs not detected")
+ }
+
+ // Opportunistic logging of driver version to aid in troubleshooting
+ driverMajor, driverMinor, err := AMDDriverVersion()
+ if err != nil {
+ // TODO - if we see users crash and burn with the upstreamed kernel this can be adjusted to hard-fail rocm support and fallback to CPU
+ slog.Warn("ollama recommends running the https://www.amd.com/en/support/linux-drivers", "error", err)
+ }
+
+ // Determine if the user has already pre-selected which GPUs to look at, then ignore the others
+ var visibleDevices []string
+ hipVD := envconfig.HipVisibleDevices() // zero based index only
+ rocrVD := envconfig.RocrVisibleDevices() // zero based index or UUID, but consumer cards seem to not support UUID
+ gpuDO := envconfig.GpuDeviceOrdinal() // zero based index
+ switch {
+ // TODO is this priorty order right?
+ case hipVD != "":
+ visibleDevices = strings.Split(hipVD, ",")
+ case rocrVD != "":
+ visibleDevices = strings.Split(rocrVD, ",")
+ // TODO - since we don't yet support UUIDs, consider detecting and reporting here
+ // all our test systems show GPU-XX indicating UUID is not supported
+ case gpuDO != "":
+ visibleDevices = strings.Split(gpuDO, ",")
+ }
+
+ gfxOverride := envconfig.HsaOverrideGfxVersion()
+ var supported []string
+ libDir := ""
+
+ // The amdgpu driver always exposes the host CPU(s) first, but we have to skip them and subtract
+ // from the other IDs to get alignment with the HIP libraries expectations (zero is the first GPU, not the CPU)
+ matches, _ := filepath.Glob(GPUPropertiesFileGlob)
+ sort.Slice(matches, func(i, j int) bool {
+ // /sys/class/kfd/kfd/topology/nodes/<number>/properties
+ a, err := strconv.ParseInt(filepath.Base(filepath.Dir(matches[i])), 10, 64)
+ if err != nil {
+ slog.Debug("parse err", "error", err, "match", matches[i])
+ return false
+ }
+ b, err := strconv.ParseInt(filepath.Base(filepath.Dir(matches[j])), 10, 64)
+ if err != nil {
+ slog.Debug("parse err", "error", err, "match", matches[i])
+ return false
+ }
+ return a < b
+ })
+ cpuCount := 0
+ for _, match := range matches {
+ slog.Debug("evaluating amdgpu node " + match)
+ fp, err := os.Open(match)
+ if err != nil {
+ slog.Debug("failed to open sysfs node", "file", match, "error", err)
+ continue
+ }
+ defer fp.Close()
+ nodeID, err := strconv.Atoi(filepath.Base(filepath.Dir(match)))
+ if err != nil {
+ slog.Debug("failed to parse node ID", "error", err)
+ continue
+ }
+
+ scanner := bufio.NewScanner(fp)
+ isCPU := false
+ var major, minor, patch uint64
+ var vendor, device, uniqueID uint64
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ // Note: we could also use "cpu_cores_count X" where X is greater than zero to detect CPUs
+ if strings.HasPrefix(line, "gfx_target_version") {
+ ver := strings.Fields(line)
+
+ // Detect CPUs
+ if len(ver) == 2 && ver[1] == "0" {
+ slog.Debug("detected CPU " + match)
+ isCPU = true
+ break
+ }
+
+ if len(ver) != 2 || len(ver[1]) < 5 {
+ slog.Warn("malformed "+match, "gfx_target_version", line)
+ // If this winds up being a CPU, our offsets may be wrong
+ continue
+ }
+ l := len(ver[1])
+ var err1, err2, err3 error
+ patch, err1 = strconv.ParseUint(ver[1][l-2:l], 10, 32)
+ minor, err2 = strconv.ParseUint(ver[1][l-4:l-2], 10, 32)
+ major, err3 = strconv.ParseUint(ver[1][:l-4], 10, 32)
+ if err1 != nil || err2 != nil || err3 != nil {
+ slog.Debug("malformed int " + line)
+ continue
+ }
+ } else if strings.HasPrefix(line, "vendor_id") {
+ ver := strings.Fields(line)
+ if len(ver) != 2 {
+ slog.Debug("malformed", "vendor_id", line)
+ continue
+ }
+ vendor, err = strconv.ParseUint(ver[1], 10, 64)
+ if err != nil {
+ slog.Debug("malformed", "vendor_id", line, "error", err)
+ }
+ } else if strings.HasPrefix(line, "device_id") {
+ ver := strings.Fields(line)
+ if len(ver) != 2 {
+ slog.Debug("malformed", "device_id", line)
+ continue
+ }
+ device, err = strconv.ParseUint(ver[1], 10, 64)
+ if err != nil {
+ slog.Debug("malformed", "device_id", line, "error", err)
+ }
+ } else if strings.HasPrefix(line, "unique_id") {
+ ver := strings.Fields(line)
+ if len(ver) != 2 {
+ slog.Debug("malformed", "unique_id", line)
+ continue
+ }
+ uniqueID, err = strconv.ParseUint(ver[1], 10, 64)
+ if err != nil {
+ slog.Debug("malformed", "unique_id", line, "error", err)
+ }
+ }
+ // TODO - any other properties we want to extract and record?
+ // vendor_id + device_id -> pci lookup for "Name"
+ // Other metrics that may help us understand relative performance between multiple GPUs
+ }
+
+ // Note: while ./mem_banks/*/used_memory exists, it doesn't appear to take other VRAM consumers
+ // into consideration, so we instead map the device over to the DRM driver sysfs nodes which
+ // do reliably report VRAM usage.
+
+ if isCPU {
+ cpuCount++
+ continue
+ }
+
+ // CPUs are always first in the list
+ gpuID := nodeID - cpuCount
+
+ // Shouldn't happen, but just in case...
+ if gpuID < 0 {
+ err := fmt.Errorf("unexpected amdgpu sysfs data resulted in negative GPU ID, please set OLLAMA_DEBUG=1 and report an issue")
+ slog.Error(err.Error())
+ return nil, err
+ }
+
+ // Look up the memory for the current node
+ totalMemory := uint64(0)
+ usedMemory := uint64(0)
+ var usedFile string
+ mapping := []struct {
+ id uint64
+ filename string
+ }{
+ {vendor, DRMVendorFile},
+ {device, DRMDeviceFile},
+ {uniqueID, DRMUniqueIDFile}, // Not all devices will report this
+ }
+ slog.Debug("mapping amdgpu to drm sysfs nodes", "amdgpu", match, "vendor", vendor, "device", device, "unique_id", uniqueID)
+ // Map over to DRM location to find the total/free memory
+ drmMatches, _ := filepath.Glob(DRMDeviceDirGlob)
+ for _, devDir := range drmMatches {
+ matched := true
+ for _, m := range mapping {
+ if m.id == 0 {
+ // Null ID means it didn't populate, so we can't use it to match
+ continue
+ }
+ filename := filepath.Join(devDir, m.filename)
+ buf, err := os.ReadFile(filename)
+ if err != nil {
+ slog.Debug("failed to read sysfs node", "file", filename, "error", err)
+ matched = false
+ break
+ }
+ // values here are in hex, strip off the lead 0x and parse so we can compare the numeric (decimal) values in amdgpu
+ cmp, err := strconv.ParseUint(strings.TrimPrefix(strings.TrimSpace(string(buf)), "0x"), 16, 64)
+ if err != nil {
+ slog.Debug("failed to parse sysfs node", "file", filename, "error", err)
+ matched = false
+ break
+ }
+ if cmp != m.id {
+ matched = false
+ break
+ }
+ }
+ if !matched {
+ continue
+ }
+
+ // Found the matching DRM directory
+ slog.Debug("matched", "amdgpu", match, "drm", devDir)
+ totalFile := filepath.Join(devDir, DRMTotalMemoryFile)
+ buf, err := os.ReadFile(totalFile)
+ if err != nil {
+ slog.Debug("failed to read sysfs node", "file", totalFile, "error", err)
+ break
+ }
+ totalMemory, err = strconv.ParseUint(strings.TrimSpace(string(buf)), 10, 64)
+ if err != nil {
+ slog.Debug("failed to parse sysfs node", "file", totalFile, "error", err)
+ break
+ }
+
+ usedFile = filepath.Join(devDir, DRMUsedMemoryFile)
+ usedMemory, err = getFreeMemory(usedFile)
+ if err != nil {
+ slog.Debug("failed to update used memory", "error", err)
+ }
+ break
+ }
+
+ var name string
+ // TODO - PCI ID lookup
+ if vendor > 0 && device > 0 {
+ name = fmt.Sprintf("%04x:%04x", vendor, device)
+ }
+
+ gpuInfo := RocmGPUInfo{
+ GpuInfo: GpuInfo{
+ Library: "rocm",
+ memInfo: memInfo{
+ TotalMemory: totalMemory,
+ FreeMemory: (totalMemory - usedMemory),
+ },
+ ID: strconv.Itoa(gpuID),
+ Name: name,
+ Compute: fmt.Sprintf("gfx%d%x%x", major, minor, patch),
+ MinimumMemory: rocmMinimumMemory,
+ DriverMajor: driverMajor,
+ DriverMinor: driverMinor,
+ },
+ usedFilepath: usedFile,
+ }
+
+ // iGPU detection, remove this check once we can support an iGPU variant of the rocm library
+ if totalMemory < IGPUMemLimit {
+ reason := "unsupported Radeon iGPU detected skipping"
+ slog.Info(reason, "id", gpuID, "total", format.HumanBytes2(totalMemory))
+ unsupportedGPUs = append(unsupportedGPUs, UnsupportedGPUInfo{
+ GpuInfo: gpuInfo.GpuInfo,
+ Reason: reason,
+ })
+ continue
+ }
+
+ if int(major) < RocmComputeMin {
+ reason := fmt.Sprintf("amdgpu too old gfx%d%x%x", major, minor, patch)
+ slog.Warn(reason, "gpu", gpuID)
+ unsupportedGPUs = append(unsupportedGPUs, UnsupportedGPUInfo{
+ GpuInfo: gpuInfo.GpuInfo,
+ Reason: reason,
+ })
+
+ continue
+ }
+
+ slog.Debug("amdgpu memory", "gpu", gpuID, "total", format.HumanBytes2(totalMemory))
+ slog.Debug("amdgpu memory", "gpu", gpuID, "available", format.HumanBytes2(totalMemory-usedMemory))
+
+ // If the user wants to filter to a subset of devices, filter out if we aren't a match
+ if len(visibleDevices) > 0 {
+ include := false
+ for _, visible := range visibleDevices {
+ if visible == gpuInfo.ID {
+ include = true
+ break
+ }
+ }
+ if !include {
+ reason := "filtering out device per user request"
+ slog.Info(reason, "id", gpuInfo.ID, "visible_devices", visibleDevices)
+ unsupportedGPUs = append(unsupportedGPUs, UnsupportedGPUInfo{
+ GpuInfo: gpuInfo.GpuInfo,
+ Reason: reason,
+ })
+
+ continue
+ }
+ }
+
+ // Final validation is gfx compatibility - load the library if we haven't already loaded it
+ // even if the user overrides, we still need to validate the library
+ if libDir == "" {
+ libDir, err = AMDValidateLibDir()
+ if err != nil {
+ err = fmt.Errorf("unable to verify rocm library: %w", err)
+ slog.Warn(err.Error())
+ unsupportedGPUs = append(unsupportedGPUs, UnsupportedGPUInfo{
+ GpuInfo: gpuInfo.GpuInfo,
+ Reason: err.Error(),
+ })
+ return nil, err
+ }
+ }
+ gpuInfo.DependencyPath = libDir
+
+ if gfxOverride == "" {
+ // Only load supported list once
+ if len(supported) == 0 {
+ supported, err = GetSupportedGFX(libDir)
+ if err != nil {
+ err = fmt.Errorf("failed to lookup supported GFX types: %w", err)
+ slog.Warn(err.Error())
+ unsupportedGPUs = append(unsupportedGPUs, UnsupportedGPUInfo{
+ GpuInfo: gpuInfo.GpuInfo,
+ Reason: err.Error(),
+ })
+ return nil, err
+ }
+ slog.Debug("rocm supported GPUs", "types", supported)
+ }
+ gfx := gpuInfo.Compute
+ if !slices.Contains[[]string, string](supported, gfx) {
+ reason := fmt.Sprintf("amdgpu is not supported (supported types:%s)", supported)
+ slog.Warn(reason, "gpu_type", gfx, "gpu", gpuInfo.ID, "library", libDir)
+ unsupportedGPUs = append(unsupportedGPUs, UnsupportedGPUInfo{
+ GpuInfo: gpuInfo.GpuInfo,
+ Reason: reason,
+ })
+
+ // TODO - consider discrete markdown just for ROCM troubleshooting?
+ slog.Warn("See https://github.com/ollama/ollama/blob/main/docs/gpu.md#overrides for HSA_OVERRIDE_GFX_VERSION usage")
+ continue
+ } else {
+ slog.Info("amdgpu is supported", "gpu", gpuInfo.ID, "gpu_type", gfx)
+ }
+ } else {
+ slog.Info("skipping rocm gfx compatibility check", "HSA_OVERRIDE_GFX_VERSION", gfxOverride)
+ }
+
+ // Check for env var workarounds
+ if name == "1002:687f" { // Vega RX 56
+ gpuInfo.EnvWorkarounds = append(gpuInfo.EnvWorkarounds, [2]string{"HSA_ENABLE_SDMA", "0"})
+ }
+
+ // The GPU has passed all the verification steps and is supported
+ resp = append(resp, gpuInfo)
+ }
+ if len(resp) == 0 {
+ err := fmt.Errorf("no compatible amdgpu devices detected")
+ slog.Info(err.Error())
+ return nil, err
+ }
+ if err := verifyKFDDriverAccess(); err != nil {
+ err = fmt.Errorf("amdgpu devices detected but permission problems block access: %w", err)
+ slog.Error(err.Error())
+ return nil, err
+ }
+ return resp, nil
+}
+
+// Quick check for AMD driver so we can skip amdgpu discovery if not present
+func AMDDetected() bool {
+ // Some driver versions (older?) don't have a version file, so just lookup the parent dir
+ sysfsDir := filepath.Dir(DriverVersionFile)
+ _, err := os.Stat(sysfsDir)
+ if errors.Is(err, os.ErrNotExist) {
+ slog.Debug("amdgpu driver not detected " + sysfsDir)
+ return false
+ } else if err != nil {
+ slog.Debug("error looking up amd driver", "path", sysfsDir, "error", err)
+ return false
+ }
+ return true
+}
+
+// Prefer to use host installed ROCm, as long as it meets our minimum requirements
+// failing that, tell the user how to download it on their own
+func AMDValidateLibDir() (string, error) {
+ libDir, err := commonAMDValidateLibDir()
+ if err == nil {
+ return libDir, nil
+ }
+
+ // Well known ollama installer path
+ installedRocmDir := "/usr/share/ollama/lib/rocm"
+ if rocmLibUsable(installedRocmDir) {
+ return installedRocmDir, nil
+ }
+
+ // If we still haven't found a usable rocm, the user will have to install it on their own
+ slog.Warn("amdgpu detected, but no compatible rocm library found. Either install rocm v6, or follow manual install instructions at https://github.com/ollama/ollama/blob/main/docs/linux.md#manual-install")
+ return "", errors.New("no suitable rocm found, falling back to CPU")
+}
+
+func AMDDriverVersion() (driverMajor, driverMinor int, err error) {
+ _, err = os.Stat(DriverVersionFile)
+ if err != nil {
+ return 0, 0, fmt.Errorf("amdgpu version file missing: %s %w", DriverVersionFile, err)
+ }
+ fp, err := os.Open(DriverVersionFile)
+ if err != nil {
+ return 0, 0, err
+ }
+ defer fp.Close()
+ verString, err := io.ReadAll(fp)
+ if err != nil {
+ return 0, 0, err
+ }
+
+ pattern := `\A(\d+)\.(\d+).*`
+ regex := regexp.MustCompile(pattern)
+ match := regex.FindStringSubmatch(string(verString))
+ if len(match) < 2 {
+ return 0, 0, fmt.Errorf("malformed version string %s", string(verString))
+ }
+ driverMajor, err = strconv.Atoi(match[1])
+ if err != nil {
+ return 0, 0, err
+ }
+ driverMinor, err = strconv.Atoi(match[2])
+ if err != nil {
+ return 0, 0, err
+ }
+ return driverMajor, driverMinor, nil
+}
+
+func (gpus RocmGPUInfoList) RefreshFreeMemory() error {
+ if len(gpus) == 0 {
+ return nil
+ }
+ for i := range gpus {
+ usedMemory, err := getFreeMemory(gpus[i].usedFilepath)
+ if err != nil {
+ return err
+ }
+ slog.Debug("updating rocm free memory", "gpu", gpus[i].ID, "name", gpus[i].Name, "before", format.HumanBytes2(gpus[i].FreeMemory), "now", format.HumanBytes2(gpus[i].TotalMemory-usedMemory))
+ gpus[i].FreeMemory = gpus[i].TotalMemory - usedMemory
+ }
+ return nil
+}
+
+func getFreeMemory(usedFile string) (uint64, error) {
+ buf, err := os.ReadFile(usedFile)
+ if err != nil {
+ return 0, fmt.Errorf("failed to read sysfs node %s %w", usedFile, err)
+ }
+ usedMemory, err := strconv.ParseUint(strings.TrimSpace(string(buf)), 10, 64)
+ if err != nil {
+ slog.Debug("failed to parse sysfs node", "file", usedFile, "error", err)
+ return 0, fmt.Errorf("failed to parse sysfs node %s %w", usedFile, err)
+ }
+ return usedMemory, nil
+}
+
+func verifyKFDDriverAccess() error {
+ // Verify we have permissions - either running as root, or we have group access to the driver
+ fd, err := os.OpenFile("/dev/kfd", os.O_RDWR, 0o666)
+ if err != nil {
+ if errors.Is(err, fs.ErrPermission) {
+ return fmt.Errorf("permissions not set up properly. Either run ollama as root, or add you user account to the render group. %w", err)
+ } else if errors.Is(err, fs.ErrNotExist) {
+ // Container runtime failure?
+ return fmt.Errorf("kfd driver not loaded. If running in a container, remember to include '--device /dev/kfd --device /dev/dri'")
+ }
+ return fmt.Errorf("failed to check permission on /dev/kfd: %w", err)
+ }
+ fd.Close()
+ return nil
+}
diff --git a/gpu/amd_windows.go b/gpu/amd_windows.go
new file mode 100644
index 00000000..4da6b7cc
--- /dev/null
+++ b/gpu/amd_windows.go
@@ -0,0 +1,203 @@
+package gpu
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "slices"
+ "strconv"
+ "strings"
+
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/format"
+)
+
+const (
+
+ // TODO We're lookinng for this exact name to detect iGPUs since hipGetDeviceProperties never reports integrated==true
+ iGPUName = "AMD Radeon(TM) Graphics"
+)
+
+var (
+ // Used to validate if the given ROCm lib is usable
+ ROCmLibGlobs = []string{"hipblas.dll", "rocblas"} // This is not sufficient to discern v5 vs v6
+ RocmStandardLocations = []string{"C:\\Program Files\\AMD\\ROCm\\6.1\\bin"} // TODO glob?
+)
+
+// Only called once during bootstrap
+func AMDGetGPUInfo() ([]RocmGPUInfo, error) {
+ resp := []RocmGPUInfo{}
+ hl, err := NewHipLib()
+ if err != nil {
+ slog.Debug(err.Error())
+ return nil, err
+ }
+ defer hl.Release()
+
+ driverMajor, driverMinor, err := hl.AMDDriverVersion()
+ if err != nil {
+ // For now this is benign, but we may eventually need to fail compatibility checks
+ slog.Debug("error looking up amd driver version", "error", err)
+ }
+
+ // Note: the HIP library automatically handles subsetting to any HIP_VISIBLE_DEVICES the user specified
+ count := hl.HipGetDeviceCount()
+ if count == 0 {
+ err := fmt.Errorf("no compatible amdgpu devices detected")
+ slog.Info(err.Error())
+ return nil, err
+ }
+ libDir, err := AMDValidateLibDir()
+ if err != nil {
+ err = fmt.Errorf("unable to verify rocm library: %w", err)
+ slog.Warn(err.Error())
+ return nil, err
+ }
+
+ var supported []string
+ gfxOverride := envconfig.HsaOverrideGfxVersion()
+ if gfxOverride == "" {
+ supported, err = GetSupportedGFX(libDir)
+ if err != nil {
+ err = fmt.Errorf("failed to lookup supported GFX types: %w", err)
+ slog.Warn(err.Error())
+ return nil, err
+ }
+ } else {
+ slog.Info("skipping rocm gfx compatibility check", "HSA_OVERRIDE_GFX_VERSION", gfxOverride)
+ }
+
+ slog.Debug("detected hip devices", "count", count)
+ // TODO how to determine the underlying device ID when visible devices is causing this to subset?
+ for i := range count {
+ err = hl.HipSetDevice(i)
+ if err != nil {
+ slog.Warn("set device", "id", i, "error", err)
+ continue
+ }
+
+ props, err := hl.HipGetDeviceProperties(i)
+ if err != nil {
+ slog.Warn("get properties", "id", i, "error", err)
+ continue
+ }
+ n := bytes.IndexByte(props.Name[:], 0)
+ name := string(props.Name[:n])
+ // TODO is UUID actually populated on windows?
+ // Can luid be used on windows for setting visible devices (and is it actually set?)
+ n = bytes.IndexByte(props.GcnArchName[:], 0)
+ gfx := string(props.GcnArchName[:n])
+ slog.Debug("hip device", "id", i, "name", name, "gfx", gfx)
+ // slog.Info(fmt.Sprintf("[%d] Integrated: %d", i, props.iGPU)) // DOESN'T REPORT CORRECTLY! Always 0
+ // TODO Why isn't props.iGPU accurate!?
+
+ freeMemory, totalMemory, err := hl.HipMemGetInfo()
+ if err != nil {
+ slog.Warn("get mem info", "id", i, "error", err)
+ continue
+ }
+
+ gpuInfo := RocmGPUInfo{
+ GpuInfo: GpuInfo{
+ Library: "rocm",
+ memInfo: memInfo{
+ TotalMemory: totalMemory,
+ FreeMemory: freeMemory,
+ },
+ // Free memory reporting on Windows is not reliable until we bump to ROCm v6.2
+ UnreliableFreeMemory: true,
+
+ ID: strconv.Itoa(i), // TODO this is probably wrong if we specify visible devices
+ DependencyPath: libDir,
+ MinimumMemory: rocmMinimumMemory,
+ Name: name,
+ Compute: gfx,
+ DriverMajor: driverMajor,
+ DriverMinor: driverMinor,
+ },
+ index: i,
+ }
+
+ // iGPU detection, remove this check once we can support an iGPU variant of the rocm library
+ if strings.EqualFold(name, iGPUName) || totalMemory < IGPUMemLimit {
+ reason := "unsupported Radeon iGPU detected skipping"
+ slog.Info(reason, "id", gpuInfo.ID, "total", format.HumanBytes2(totalMemory))
+ unsupportedGPUs = append(unsupportedGPUs, UnsupportedGPUInfo{
+ GpuInfo: gpuInfo.GpuInfo,
+ Reason: reason,
+ })
+ continue
+ }
+
+ // Strip off Target Features when comparing
+ if !slices.Contains[[]string, string](supported, strings.Split(gfx, ":")[0]) {
+ reason := fmt.Sprintf("amdgpu is not supported (supported types:%s)", supported)
+ slog.Warn(reason, "gpu_type", gfx, "gpu", gpuInfo.ID, "library", libDir)
+ unsupportedGPUs = append(unsupportedGPUs, UnsupportedGPUInfo{
+ GpuInfo: gpuInfo.GpuInfo,
+ Reason: reason,
+ })
+ // HSA_OVERRIDE_GFX_VERSION not supported on windows
+ continue
+ } else {
+ slog.Debug("amdgpu is supported", "gpu", i, "gpu_type", gfx)
+ }
+
+ slog.Debug("amdgpu memory", "gpu", i, "total", format.HumanBytes2(totalMemory))
+ slog.Debug("amdgpu memory", "gpu", i, "available", format.HumanBytes2(freeMemory))
+
+ resp = append(resp, gpuInfo)
+ }
+
+ return resp, nil
+}
+
+func AMDValidateLibDir() (string, error) {
+ libDir, err := commonAMDValidateLibDir()
+ if err == nil {
+ return libDir, nil
+ }
+
+ // Installer payload (if we're running from some other location)
+ localAppData := os.Getenv("LOCALAPPDATA")
+ appDir := filepath.Join(localAppData, "Programs", "Ollama")
+ rocmTargetDir := filepath.Join(appDir, envconfig.LibRelativeToExe(), "lib", "ollama")
+ if rocmLibUsable(rocmTargetDir) {
+ slog.Debug("detected ollama installed ROCm at " + rocmTargetDir)
+ return rocmTargetDir, nil
+ }
+
+ // Should not happen on windows since we include it in the installer, but stand-alone binary might hit this
+ slog.Warn("amdgpu detected, but no compatible rocm library found. Please install ROCm")
+ return "", errors.New("no suitable rocm found, falling back to CPU")
+}
+
+func (gpus RocmGPUInfoList) RefreshFreeMemory() error {
+ if len(gpus) == 0 {
+ return nil
+ }
+ hl, err := NewHipLib()
+ if err != nil {
+ slog.Debug(err.Error())
+ return nil
+ }
+ defer hl.Release()
+
+ for i := range gpus {
+ err := hl.HipSetDevice(gpus[i].index)
+ if err != nil {
+ return err
+ }
+ freeMemory, _, err := hl.HipMemGetInfo()
+ if err != nil {
+ slog.Warn("get mem info", "id", i, "error", err)
+ continue
+ }
+ slog.Debug("updating rocm free memory", "gpu", gpus[i].ID, "name", gpus[i].Name, "before", format.HumanBytes2(gpus[i].FreeMemory), "now", format.HumanBytes2(freeMemory))
+ gpus[i].FreeMemory = freeMemory
+ }
+ return nil
+}
diff --git a/gpu/cpu_common.go b/gpu/cpu_common.go
new file mode 100644
index 00000000..34edcdc5
--- /dev/null
+++ b/gpu/cpu_common.go
@@ -0,0 +1,37 @@
+package gpu
+
+import (
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+
+ "golang.org/x/sys/cpu"
+)
+
+func GetCPUCapability() CPUCapability {
+ if cpu.X86.HasAVX2 {
+ return CPUCapabilityAVX2
+ }
+ if cpu.X86.HasAVX {
+ return CPUCapabilityAVX
+ }
+ // else LCD
+ return CPUCapabilityNone
+}
+
+func IsNUMA() bool {
+ if runtime.GOOS != "linux" {
+ // numa support in llama.cpp is linux only
+ return false
+ }
+ ids := map[string]interface{}{}
+ packageIds, _ := filepath.Glob("/sys/devices/system/cpu/cpu*/topology/physical_package_id")
+ for _, packageId := range packageIds {
+ id, err := os.ReadFile(packageId)
+ if err == nil {
+ ids[strings.TrimSpace(string(id))] = struct{}{}
+ }
+ }
+ return len(ids) > 1
+}
diff --git a/gpu/cuda_common.go b/gpu/cuda_common.go
new file mode 100644
index 00000000..aceec70a
--- /dev/null
+++ b/gpu/cuda_common.go
@@ -0,0 +1,64 @@
+//go:build linux || windows
+
+package gpu
+
+import (
+ "log/slog"
+ "os"
+ "regexp"
+ "runtime"
+ "strconv"
+ "strings"
+)
+
+// Jetson devices have JETSON_JETPACK="x.y.z" factory set to the Jetpack version installed.
+// Included to drive logic for reducing Ollama-allocated overhead on L4T/Jetson devices.
+var CudaTegra string = os.Getenv("JETSON_JETPACK")
+
+func cudaGetVisibleDevicesEnv(gpuInfo []GpuInfo) (string, string) {
+ ids := []string{}
+ for _, info := range gpuInfo {
+ if info.Library != "cuda" {
+ // TODO shouldn't happen if things are wired correctly...
+ slog.Debug("cudaGetVisibleDevicesEnv skipping over non-cuda device", "library", info.Library)
+ continue
+ }
+ ids = append(ids, info.ID)
+ }
+ return "CUDA_VISIBLE_DEVICES", strings.Join(ids, ",")
+}
+
+func cudaVariant(gpuInfo CudaGPUInfo) string {
+ if runtime.GOARCH == "arm64" && runtime.GOOS == "linux" {
+ if CudaTegra != "" {
+ ver := strings.Split(CudaTegra, ".")
+ if len(ver) > 0 {
+ return "jetpack" + ver[0]
+ }
+ } else if data, err := os.ReadFile("/etc/nv_tegra_release"); err == nil {
+ r := regexp.MustCompile(` R(\d+) `)
+ m := r.FindSubmatch(data)
+ if len(m) != 2 {
+ slog.Info("Unexpected format for /etc/nv_tegra_release. Set JETSON_JETPACK to select version")
+ } else {
+ if l4t, err := strconv.Atoi(string(m[1])); err == nil {
+ // Note: mapping from L4t -> JP is inconsistent (can't just subtract 30)
+ // https://developer.nvidia.com/embedded/jetpack-archive
+ switch l4t {
+ case 35:
+ return "jetpack5"
+ case 36:
+ return "jetpack6"
+ default:
+ slog.Info("unsupported L4T version", "nv_tegra_release", string(data))
+ }
+ }
+ }
+ }
+ }
+
+ if gpuInfo.computeMajor < 6 || gpuInfo.DriverMajor < 12 || (gpuInfo.DriverMajor == 12 && gpuInfo.DriverMinor == 0) {
+ return "v11"
+ }
+ return "v12"
+}
diff --git a/gpu/gpu.go b/gpu/gpu.go
index 91ced3a8..700c90f9 100644
--- a/gpu/gpu.go
+++ b/gpu/gpu.go
@@ -7,128 +7,744 @@ package gpu
#cgo windows LDFLAGS: -lpthread
#include "gpu_info.h"
-
*/
import "C"
+
import (
"fmt"
- "log"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
"sync"
"unsafe"
- "github.com/jmorganca/ollama/api"
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/format"
)
-type handles struct {
- cuda *C.cuda_handle_t
- rocm *C.rocm_handle_t
+type cudaHandles struct {
+ deviceCount int
+ cudart *C.cudart_handle_t
+ nvcuda *C.nvcuda_handle_t
+ nvml *C.nvml_handle_t
}
-var gpuMutex sync.Mutex
-var gpuHandles *handles = nil
+type oneapiHandles struct {
+ oneapi *C.oneapi_handle_t
+ deviceCount int
+}
+
+const (
+ cudaMinimumMemory = 457 * format.MebiByte
+ rocmMinimumMemory = 457 * format.MebiByte
+ // TODO OneAPI minimum memory
+)
+
+var (
+ gpuMutex sync.Mutex
+ bootstrapped bool
+ cpuCapability CPUCapability
+ cpus []CPUInfo
+ cudaGPUs []CudaGPUInfo
+ nvcudaLibPath string
+ cudartLibPath string
+ oneapiLibPath string
+ nvmlLibPath string
+ rocmGPUs []RocmGPUInfo
+ oneapiGPUs []OneapiGPUInfo
+
+ // If any discovered GPUs are incompatible, report why
+ unsupportedGPUs []UnsupportedGPUInfo
+
+ // Keep track of errors during bootstrapping so that if GPUs are missing
+ // they expected to be present this may explain why
+ bootstrapErrors []error
+)
+
+// With our current CUDA compile flags, older than 5.0 will not work properly
+var CudaComputeMin = [2]C.int{5, 0}
+
+var RocmComputeMin = 9
+
+// TODO find a better way to detect iGPU instead of minimum memory
+const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU
// Note: gpuMutex must already be held
-func initGPUHandles() {
+func initCudaHandles() *cudaHandles {
// TODO - if the ollama build is CPU only, don't do these checks as they're irrelevant and confusing
- log.Printf("Detecting GPU type")
- gpuHandles = &handles{nil, nil}
- var resp C.cuda_init_resp_t
- C.cuda_init(&resp)
- if resp.err != nil {
- log.Printf("CUDA not detected: %s", C.GoString(resp.err))
- C.free(unsafe.Pointer(resp.err))
- var resp C.rocm_init_resp_t
- C.rocm_init(&resp)
- if resp.err != nil {
- log.Printf("ROCm not detected: %s", C.GoString(resp.err))
- C.free(unsafe.Pointer(resp.err))
- } else {
- log.Printf("Radeon GPU detected")
- rocm := resp.rh
- gpuHandles.rocm = &rocm
- }
- } else {
- log.Printf("Nvidia GPU detected")
- cuda := resp.ch
- gpuHandles.cuda = &cuda
+ cHandles := &cudaHandles{}
+ // Short Circuit if we already know which library to use
+ // ignore bootstrap errors in this case since we already recorded them
+ if nvmlLibPath != "" {
+ cHandles.nvml, _, _ = loadNVMLMgmt([]string{nvmlLibPath})
+ return cHandles
}
+ if nvcudaLibPath != "" {
+ cHandles.deviceCount, cHandles.nvcuda, _, _ = loadNVCUDAMgmt([]string{nvcudaLibPath})
+ return cHandles
+ }
+ if cudartLibPath != "" {
+ cHandles.deviceCount, cHandles.cudart, _, _ = loadCUDARTMgmt([]string{cudartLibPath})
+ return cHandles
+ }
+
+ slog.Debug("searching for GPU discovery libraries for NVIDIA")
+ var cudartMgmtPatterns []string
+
+ // Aligned with driver, we can't carry as payloads
+ nvcudaMgmtPatterns := NvcudaGlobs
+
+ if runtime.GOOS == "windows" {
+ localAppData := os.Getenv("LOCALAPPDATA")
+ cudartMgmtPatterns = []string{filepath.Join(localAppData, "Programs", "Ollama", CudartMgmtName)}
+ }
+ libDir := LibraryDir()
+ if libDir != "" {
+ cudartMgmtPatterns = []string{filepath.Join(libDir, CudartMgmtName)}
+ }
+ cudartMgmtPatterns = append(cudartMgmtPatterns, CudartGlobs...)
+
+ if len(NvmlGlobs) > 0 {
+ nvmlLibPaths := FindGPULibs(NvmlMgmtName, NvmlGlobs)
+ if len(nvmlLibPaths) > 0 {
+ nvml, libPath, err := loadNVMLMgmt(nvmlLibPaths)
+ if nvml != nil {
+ slog.Debug("nvidia-ml loaded", "library", libPath)
+ cHandles.nvml = nvml
+ nvmlLibPath = libPath
+ }
+ if err != nil {
+ bootstrapErrors = append(bootstrapErrors, err)
+ }
+ }
+ }
+
+ nvcudaLibPaths := FindGPULibs(NvcudaMgmtName, nvcudaMgmtPatterns)
+ if len(nvcudaLibPaths) > 0 {
+ deviceCount, nvcuda, libPath, err := loadNVCUDAMgmt(nvcudaLibPaths)
+ if nvcuda != nil {
+ slog.Debug("detected GPUs", "count", deviceCount, "library", libPath)
+ cHandles.nvcuda = nvcuda
+ cHandles.deviceCount = deviceCount
+ nvcudaLibPath = libPath
+ return cHandles
+ }
+ if err != nil {
+ bootstrapErrors = append(bootstrapErrors, err)
+ }
+ }
+
+ cudartLibPaths := FindGPULibs(CudartMgmtName, cudartMgmtPatterns)
+ if len(cudartLibPaths) > 0 {
+ deviceCount, cudart, libPath, err := loadCUDARTMgmt(cudartLibPaths)
+ if cudart != nil {
+ slog.Debug("detected GPUs", "library", libPath, "count", deviceCount)
+ cHandles.cudart = cudart
+ cHandles.deviceCount = deviceCount
+ cudartLibPath = libPath
+ return cHandles
+ }
+ if err != nil {
+ bootstrapErrors = append(bootstrapErrors, err)
+ }
+ }
+
+ return cHandles
}
-func GetGPUInfo() GpuInfo {
+// Note: gpuMutex must already be held
+func initOneAPIHandles() *oneapiHandles {
+ oHandles := &oneapiHandles{}
+
+ // Short Circuit if we already know which library to use
+ // ignore bootstrap errors in this case since we already recorded them
+ if oneapiLibPath != "" {
+ oHandles.deviceCount, oHandles.oneapi, _, _ = loadOneapiMgmt([]string{oneapiLibPath})
+ return oHandles
+ }
+
+ oneapiLibPaths := FindGPULibs(OneapiMgmtName, OneapiGlobs)
+ if len(oneapiLibPaths) > 0 {
+ var err error
+ oHandles.deviceCount, oHandles.oneapi, oneapiLibPath, err = loadOneapiMgmt(oneapiLibPaths)
+ if err != nil {
+ bootstrapErrors = append(bootstrapErrors, err)
+ }
+ }
+
+ return oHandles
+}
+
+func GetCPUInfo() GpuInfoList {
+ gpuMutex.Lock()
+ if !bootstrapped {
+ gpuMutex.Unlock()
+ GetGPUInfo()
+ } else {
+ gpuMutex.Unlock()
+ }
+ return GpuInfoList{cpus[0].GpuInfo}
+}
+
+func GetGPUInfo() GpuInfoList {
// TODO - consider exploring lspci (and equivalent on windows) to check for
// GPUs so we can report warnings if we see Nvidia/AMD but fail to load the libraries
gpuMutex.Lock()
defer gpuMutex.Unlock()
- if gpuHandles == nil {
- initGPUHandles()
+ needRefresh := true
+ var cHandles *cudaHandles
+ var oHandles *oneapiHandles
+ defer func() {
+ if cHandles != nil {
+ if cHandles.cudart != nil {
+ C.cudart_release(*cHandles.cudart)
+ }
+ if cHandles.nvcuda != nil {
+ C.nvcuda_release(*cHandles.nvcuda)
+ }
+ if cHandles.nvml != nil {
+ C.nvml_release(*cHandles.nvml)
+ }
+ }
+ if oHandles != nil {
+ if oHandles.oneapi != nil {
+ // TODO - is this needed?
+ C.oneapi_release(*oHandles.oneapi)
+ }
+ }
+ }()
+
+ if !bootstrapped {
+ slog.Info("looking for compatible GPUs")
+ bootstrapErrors = []error{}
+ needRefresh = false
+ cpuCapability = GetCPUCapability()
+ var memInfo C.mem_info_t
+
+ mem, err := GetCPUMem()
+ if err != nil {
+ slog.Warn("error looking up system memory", "error", err)
+ }
+ depPath := LibraryDir()
+ details, err := GetCPUDetails()
+ if err != nil {
+ slog.Warn("failed to lookup CPU details", "error", err)
+ }
+ cpus = []CPUInfo{
+ {
+ GpuInfo: GpuInfo{
+ memInfo: mem,
+ Library: "cpu",
+ Variant: cpuCapability.String(),
+ ID: "0",
+ DependencyPath: depPath,
+ },
+ CPUs: details,
+ },
+ }
+
+ // Fallback to CPU mode if we're lacking required vector extensions on x86
+ if cpuCapability < GPURunnerCPUCapability && runtime.GOARCH == "amd64" {
+ err := fmt.Errorf("CPU does not have minimum vector extensions, GPU inference disabled. Required:%s Detected:%s", GPURunnerCPUCapability, cpuCapability)
+ slog.Warn(err.Error())
+ bootstrapErrors = append(bootstrapErrors, err)
+ bootstrapped = true
+ // No need to do any GPU discovery, since we can't run on them
+ return GpuInfoList{cpus[0].GpuInfo}
+ }
+
+ // Load ALL libraries
+ cHandles = initCudaHandles()
+
+ // NVIDIA
+ for i := range cHandles.deviceCount {
+ if cHandles.cudart != nil || cHandles.nvcuda != nil {
+ gpuInfo := CudaGPUInfo{
+ GpuInfo: GpuInfo{
+ Library: "cuda",
+ },
+ index: i,
+ }
+ var driverMajor int
+ var driverMinor int
+ if cHandles.cudart != nil {
+ C.cudart_bootstrap(*cHandles.cudart, C.int(i), &memInfo)
+ } else {
+ C.nvcuda_bootstrap(*cHandles.nvcuda, C.int(i), &memInfo)
+ driverMajor = int(cHandles.nvcuda.driver_major)
+ driverMinor = int(cHandles.nvcuda.driver_minor)
+ }
+ if memInfo.err != nil {
+ slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
+ C.free(unsafe.Pointer(memInfo.err))
+ continue
+ }
+ gpuInfo.TotalMemory = uint64(memInfo.total)
+ gpuInfo.FreeMemory = uint64(memInfo.free)
+ gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
+ gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor)
+ gpuInfo.computeMajor = int(memInfo.major)
+ gpuInfo.computeMinor = int(memInfo.minor)
+ gpuInfo.MinimumMemory = cudaMinimumMemory
+ gpuInfo.DriverMajor = driverMajor
+ gpuInfo.DriverMinor = driverMinor
+ variant := cudaVariant(gpuInfo)
+ if depPath != "" {
+ gpuInfo.DependencyPath = depPath
+ // Check for variant specific directory
+ if variant != "" {
+ if _, err := os.Stat(filepath.Join(depPath, "cuda_"+variant)); err == nil {
+ gpuInfo.DependencyPath = filepath.Join(depPath, "cuda_"+variant)
+ }
+ }
+ }
+ gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
+ gpuInfo.Variant = variant
+
+ if memInfo.major < CudaComputeMin[0] || (memInfo.major == CudaComputeMin[0] && memInfo.minor < CudaComputeMin[1]) {
+ unsupportedGPUs = append(unsupportedGPUs,
+ UnsupportedGPUInfo{
+ GpuInfo: gpuInfo.GpuInfo,
+ })
+ slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor))
+ continue
+ }
+
+ // query the management library as well so we can record any skew between the two
+ // which represents overhead on the GPU we must set aside on subsequent updates
+ if cHandles.nvml != nil {
+ C.nvml_get_free(*cHandles.nvml, C.int(gpuInfo.index), &memInfo.free, &memInfo.total, &memInfo.used)
+ if memInfo.err != nil {
+ slog.Warn("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
+ C.free(unsafe.Pointer(memInfo.err))
+ } else {
+ if memInfo.free != 0 && uint64(memInfo.free) > gpuInfo.FreeMemory {
+ gpuInfo.OSOverhead = uint64(memInfo.free) - gpuInfo.FreeMemory
+ slog.Info("detected OS VRAM overhead",
+ "id", gpuInfo.ID,
+ "library", gpuInfo.Library,
+ "compute", gpuInfo.Compute,
+ "driver", fmt.Sprintf("%d.%d", gpuInfo.DriverMajor, gpuInfo.DriverMinor),
+ "name", gpuInfo.Name,
+ "overhead", format.HumanBytes2(gpuInfo.OSOverhead),
+ )
+ }
+ }
+ }
+
+ // TODO potentially sort on our own algorithm instead of what the underlying GPU library does...
+ cudaGPUs = append(cudaGPUs, gpuInfo)
+ }
+ }
+
+ // Intel
+ if envconfig.IntelGPU() {
+ oHandles = initOneAPIHandles()
+ if oHandles != nil && oHandles.oneapi != nil {
+ for d := range oHandles.oneapi.num_drivers {
+ if oHandles.oneapi == nil {
+ // shouldn't happen
+ slog.Warn("nil oneapi handle with driver count", "count", int(oHandles.oneapi.num_drivers))
+ continue
+ }
+ devCount := C.oneapi_get_device_count(*oHandles.oneapi, C.int(d))
+ for i := range devCount {
+ gpuInfo := OneapiGPUInfo{
+ GpuInfo: GpuInfo{
+ Library: "oneapi",
+ },
+ driverIndex: int(d),
+ gpuIndex: int(i),
+ }
+ // TODO - split bootstrapping from updating free memory
+ C.oneapi_check_vram(*oHandles.oneapi, C.int(d), i, &memInfo)
+ // TODO - convert this to MinimumMemory based on testing...
+ var totalFreeMem float64 = float64(memInfo.free) * 0.95 // work-around: leave some reserve vram for mkl lib used in ggml-sycl backend.
+ memInfo.free = C.uint64_t(totalFreeMem)
+ gpuInfo.TotalMemory = uint64(memInfo.total)
+ gpuInfo.FreeMemory = uint64(memInfo.free)
+ gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
+ gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
+ gpuInfo.DependencyPath = depPath
+ oneapiGPUs = append(oneapiGPUs, gpuInfo)
+ }
+ }
+ }
+ }
+
+ rocmGPUs, err = AMDGetGPUInfo()
+ if err != nil {
+ bootstrapErrors = append(bootstrapErrors, err)
+ }
+ bootstrapped = true
+ if len(cudaGPUs) == 0 && len(rocmGPUs) == 0 && len(oneapiGPUs) == 0 {
+ slog.Info("no compatible GPUs were discovered")
+ }
}
- var memInfo C.mem_info_t
- resp := GpuInfo{"", "", 0, 0}
- if gpuHandles.cuda != nil {
- C.cuda_check_vram(*gpuHandles.cuda, &memInfo)
- if memInfo.err != nil {
- log.Printf("error looking up CUDA GPU memory: %s", C.GoString(memInfo.err))
- C.free(unsafe.Pointer(memInfo.err))
+ // For detected GPUs, load library if not loaded
+
+ // Refresh free memory usage
+ if needRefresh {
+ mem, err := GetCPUMem()
+ if err != nil {
+ slog.Warn("error looking up system memory", "error", err)
} else {
- resp.Driver = "CUDA"
- resp.Library = "cuda_server"
+ slog.Debug("updating system memory data",
+ slog.Group(
+ "before",
+ "total", format.HumanBytes2(cpus[0].TotalMemory),
+ "free", format.HumanBytes2(cpus[0].FreeMemory),
+ "free_swap", format.HumanBytes2(cpus[0].FreeSwap),
+ ),
+ slog.Group(
+ "now",
+ "total", format.HumanBytes2(mem.TotalMemory),
+ "free", format.HumanBytes2(mem.FreeMemory),
+ "free_swap", format.HumanBytes2(mem.FreeSwap),
+ ),
+ )
+ cpus[0].FreeMemory = mem.FreeMemory
+ cpus[0].FreeSwap = mem.FreeSwap
}
- } else if gpuHandles.rocm != nil {
- C.rocm_check_vram(*gpuHandles.rocm, &memInfo)
- if memInfo.err != nil {
- log.Printf("error looking up ROCm GPU memory: %s", C.GoString(memInfo.err))
- C.free(unsafe.Pointer(memInfo.err))
- } else {
- resp.Driver = "ROCM"
- resp.Library = "rocm_server"
+
+ var memInfo C.mem_info_t
+ if cHandles == nil && len(cudaGPUs) > 0 {
+ cHandles = initCudaHandles()
+ }
+ for i, gpu := range cudaGPUs {
+ if cHandles.nvml != nil {
+ C.nvml_get_free(*cHandles.nvml, C.int(gpu.index), &memInfo.free, &memInfo.total, &memInfo.used)
+ } else if cHandles.cudart != nil {
+ C.cudart_bootstrap(*cHandles.cudart, C.int(gpu.index), &memInfo)
+ } else if cHandles.nvcuda != nil {
+ C.nvcuda_get_free(*cHandles.nvcuda, C.int(gpu.index), &memInfo.free, &memInfo.total)
+ memInfo.used = memInfo.total - memInfo.free
+ } else {
+ // shouldn't happen
+ slog.Warn("no valid cuda library loaded to refresh vram usage")
+ break
+ }
+ if memInfo.err != nil {
+ slog.Warn("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
+ C.free(unsafe.Pointer(memInfo.err))
+ continue
+ }
+ if memInfo.free == 0 {
+ slog.Warn("error looking up nvidia GPU memory")
+ continue
+ }
+ if cHandles.nvml != nil && gpu.OSOverhead > 0 {
+ // When using the management library update based on recorded overhead
+ memInfo.free -= C.uint64_t(gpu.OSOverhead)
+ }
+ slog.Debug("updating cuda memory data",
+ "gpu", gpu.ID,
+ "name", gpu.Name,
+ "overhead", format.HumanBytes2(gpu.OSOverhead),
+ slog.Group(
+ "before",
+ "total", format.HumanBytes2(gpu.TotalMemory),
+ "free", format.HumanBytes2(gpu.FreeMemory),
+ ),
+ slog.Group(
+ "now",
+ "total", format.HumanBytes2(uint64(memInfo.total)),
+ "free", format.HumanBytes2(uint64(memInfo.free)),
+ "used", format.HumanBytes2(uint64(memInfo.used)),
+ ),
+ )
+ cudaGPUs[i].FreeMemory = uint64(memInfo.free)
+ }
+
+ if oHandles == nil && len(oneapiGPUs) > 0 {
+ oHandles = initOneAPIHandles()
+ }
+ for i, gpu := range oneapiGPUs {
+ if oHandles.oneapi == nil {
+ // shouldn't happen
+ slog.Warn("nil oneapi handle with device count", "count", oHandles.deviceCount)
+ continue
+ }
+ C.oneapi_check_vram(*oHandles.oneapi, C.int(gpu.driverIndex), C.int(gpu.gpuIndex), &memInfo)
+ // TODO - convert this to MinimumMemory based on testing...
+ var totalFreeMem float64 = float64(memInfo.free) * 0.95 // work-around: leave some reserve vram for mkl lib used in ggml-sycl backend.
+ memInfo.free = C.uint64_t(totalFreeMem)
+ oneapiGPUs[i].FreeMemory = uint64(memInfo.free)
+ }
+
+ err = RocmGPUInfoList(rocmGPUs).RefreshFreeMemory()
+ if err != nil {
+ slog.Debug("problem refreshing ROCm free memory", "error", err)
}
}
- if resp.Driver == "" {
- C.cpu_check_ram(&memInfo)
- resp.Driver = "CPU"
- // In the future we may offer multiple CPU variants to tune CPU features
- resp.Library = "default"
+
+ resp := []GpuInfo{}
+ for _, gpu := range cudaGPUs {
+ resp = append(resp, gpu.GpuInfo)
}
- if memInfo.err != nil {
- log.Printf("error looking up CPU memory: %s", C.GoString(memInfo.err))
- C.free(unsafe.Pointer(memInfo.err))
- return resp
+ for _, gpu := range rocmGPUs {
+ resp = append(resp, gpu.GpuInfo)
+ }
+ for _, gpu := range oneapiGPUs {
+ resp = append(resp, gpu.GpuInfo)
+ }
+ if len(resp) == 0 {
+ resp = append(resp, cpus[0].GpuInfo)
}
- resp.FreeMemory = uint64(memInfo.free)
- resp.TotalMemory = uint64(memInfo.total)
return resp
}
-func CheckVRAM() (int64, error) {
- gpuInfo := GetGPUInfo()
- if gpuInfo.FreeMemory > 0 && gpuInfo.Driver != "CPU" {
- return int64(gpuInfo.FreeMemory), nil
+func FindGPULibs(baseLibName string, defaultPatterns []string) []string {
+ // Multiple GPU libraries may exist, and some may not work, so keep trying until we exhaust them
+ var ldPaths []string
+ gpuLibPaths := []string{}
+ slog.Debug("Searching for GPU library", "name", baseLibName)
+
+ // Start with our bundled libraries
+ patterns := []string{filepath.Join(LibraryDir(), baseLibName)}
+
+ switch runtime.GOOS {
+ case "windows":
+ ldPaths = strings.Split(os.Getenv("PATH"), ";")
+ case "linux":
+ ldPaths = strings.Split(os.Getenv("LD_LIBRARY_PATH"), ":")
+ default:
+ return gpuLibPaths
}
- return 0, fmt.Errorf("no GPU detected") // TODO - better handling of CPU based memory determiniation
+
+ // Then with whatever we find in the PATH/LD_LIBRARY_PATH
+ for _, ldPath := range ldPaths {
+ d, err := filepath.Abs(ldPath)
+ if err != nil {
+ continue
+ }
+ patterns = append(patterns, filepath.Join(d, baseLibName))
+ }
+ patterns = append(patterns, defaultPatterns...)
+ slog.Debug("gpu library search", "globs", patterns)
+ for _, pattern := range patterns {
+
+ // Nvidia PhysX known to return bogus results
+ if strings.Contains(pattern, "PhysX") {
+ slog.Debug("skipping PhysX cuda library path", "path", pattern)
+ continue
+ }
+ // Ignore glob discovery errors
+ matches, _ := filepath.Glob(pattern)
+ for _, match := range matches {
+ // Resolve any links so we don't try the same lib multiple times
+ // and weed out any dups across globs
+ libPath := match
+ tmp := match
+ var err error
+ for ; err == nil; tmp, err = os.Readlink(libPath) {
+ if !filepath.IsAbs(tmp) {
+ tmp = filepath.Join(filepath.Dir(libPath), tmp)
+ }
+ libPath = tmp
+ }
+ new := true
+ for _, cmp := range gpuLibPaths {
+ if cmp == libPath {
+ new = false
+ break
+ }
+ }
+ if new {
+ gpuLibPaths = append(gpuLibPaths, libPath)
+ }
+ }
+ }
+ slog.Debug("discovered GPU libraries", "paths", gpuLibPaths)
+ return gpuLibPaths
}
-func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int {
- if opts.NumGPU != -1 {
- return opts.NumGPU
+// Bootstrap the runtime library
+// Returns: num devices, handle, libPath, error
+func loadCUDARTMgmt(cudartLibPaths []string) (int, *C.cudart_handle_t, string, error) {
+ var resp C.cudart_init_resp_t
+ resp.ch.verbose = getVerboseState()
+ var err error
+ for _, libPath := range cudartLibPaths {
+ lib := C.CString(libPath)
+ defer C.free(unsafe.Pointer(lib))
+ C.cudart_init(lib, &resp)
+ if resp.err != nil {
+ err = fmt.Errorf("Unable to load cudart library %s: %s", libPath, C.GoString(resp.err))
+ slog.Debug(err.Error())
+ C.free(unsafe.Pointer(resp.err))
+ } else {
+ err = nil
+ return int(resp.num_devices), &resp.ch, libPath, err
+ }
+ }
+ return 0, nil, "", err
+}
+
+// Bootstrap the driver library
+// Returns: num devices, handle, libPath, error
+func loadNVCUDAMgmt(nvcudaLibPaths []string) (int, *C.nvcuda_handle_t, string, error) {
+ var resp C.nvcuda_init_resp_t
+ resp.ch.verbose = getVerboseState()
+ var err error
+ for _, libPath := range nvcudaLibPaths {
+ lib := C.CString(libPath)
+ defer C.free(unsafe.Pointer(lib))
+ C.nvcuda_init(lib, &resp)
+ if resp.err != nil {
+ // Decide what log level based on the type of error message to help users understand why
+ switch resp.cudaErr {
+ case C.CUDA_ERROR_INSUFFICIENT_DRIVER, C.CUDA_ERROR_SYSTEM_DRIVER_MISMATCH:
+ err = fmt.Errorf("version mismatch between driver and cuda driver library - reboot or upgrade may be required: library %s", libPath)
+ slog.Warn(err.Error())
+ case C.CUDA_ERROR_NO_DEVICE:
+ err = fmt.Errorf("no nvidia devices detected by library %s", libPath)
+ slog.Info(err.Error())
+ case C.CUDA_ERROR_UNKNOWN:
+ err = fmt.Errorf("unknown error initializing cuda driver library %s: %s. see https://github.com/ollama/ollama/blob/main/docs/troubleshooting.md for more information", libPath, C.GoString(resp.err))
+ slog.Warn(err.Error())
+ default:
+ msg := C.GoString(resp.err)
+ if strings.Contains(msg, "wrong ELF class") {
+ slog.Debug("skipping 32bit library", "library", libPath)
+ } else {
+ err = fmt.Errorf("Unable to load cudart library %s: %s", libPath, C.GoString(resp.err))
+ slog.Info(err.Error())
+ }
+ }
+ C.free(unsafe.Pointer(resp.err))
+ } else {
+ err = nil
+ return int(resp.num_devices), &resp.ch, libPath, err
+ }
+ }
+ return 0, nil, "", err
+}
+
+// Bootstrap the management library
+// Returns: handle, libPath, error
+func loadNVMLMgmt(nvmlLibPaths []string) (*C.nvml_handle_t, string, error) {
+ var resp C.nvml_init_resp_t
+ resp.ch.verbose = getVerboseState()
+ var err error
+ for _, libPath := range nvmlLibPaths {
+ lib := C.CString(libPath)
+ defer C.free(unsafe.Pointer(lib))
+ C.nvml_init(lib, &resp)
+ if resp.err != nil {
+ err = fmt.Errorf("Unable to load NVML management library %s: %s", libPath, C.GoString(resp.err))
+ slog.Info(err.Error())
+ C.free(unsafe.Pointer(resp.err))
+ } else {
+ err = nil
+ return &resp.ch, libPath, err
+ }
+ }
+ return nil, "", err
+}
+
+// bootstrap the Intel GPU library
+// Returns: num devices, handle, libPath, error
+func loadOneapiMgmt(oneapiLibPaths []string) (int, *C.oneapi_handle_t, string, error) {
+ var resp C.oneapi_init_resp_t
+ num_devices := 0
+ resp.oh.verbose = getVerboseState()
+ var err error
+ for _, libPath := range oneapiLibPaths {
+ lib := C.CString(libPath)
+ defer C.free(unsafe.Pointer(lib))
+ C.oneapi_init(lib, &resp)
+ if resp.err != nil {
+ err = fmt.Errorf("Unable to load oneAPI management library %s: %s", libPath, C.GoString(resp.err))
+ slog.Debug(err.Error())
+ C.free(unsafe.Pointer(resp.err))
+ } else {
+ err = nil
+ for i := range resp.oh.num_drivers {
+ num_devices += int(C.oneapi_get_device_count(resp.oh, C.int(i)))
+ }
+ return num_devices, &resp.oh, libPath, err
+ }
+ }
+ return 0, nil, "", err
+}
+
+func getVerboseState() C.uint16_t {
+ if envconfig.Debug() {
+ return C.uint16_t(1)
+ }
+ return C.uint16_t(0)
+}
+
+// Given the list of GPUs this instantiation is targeted for,
+// figure out the visible devices environment variable
+//
+// If different libraries are detected, the first one is what we use
+func (l GpuInfoList) GetVisibleDevicesEnv() (string, string) {
+ if len(l) == 0 {
+ return "", ""
+ }
+ switch l[0].Library {
+ case "cuda":
+ return cudaGetVisibleDevicesEnv(l)
+ case "rocm":
+ return rocmGetVisibleDevicesEnv(l)
+ case "oneapi":
+ return oneapiGetVisibleDevicesEnv(l)
+ default:
+ slog.Debug("no filter required for library " + l[0].Library)
+ return "", ""
+ }
+}
+
+func LibraryDir() string {
+ // On Windows/linux we bundle the dependencies at the same level as the executable
+ appExe, err := os.Executable()
+ if err != nil {
+ slog.Warn("failed to lookup executable path", "error", err)
+ }
+ cwd, err := os.Getwd()
+ if err != nil {
+ slog.Warn("failed to lookup working directory", "error", err)
+ }
+ // Scan for any of our dependeices, and pick first match
+ for _, root := range []string{filepath.Dir(appExe), filepath.Join(filepath.Dir(appExe), envconfig.LibRelativeToExe()), cwd} {
+ libDep := filepath.Join("lib", "ollama")
+ if _, err := os.Stat(filepath.Join(root, libDep)); err == nil {
+ return filepath.Join(root, libDep)
+ }
+ // Developer mode, local build
+ if _, err := os.Stat(filepath.Join(root, runtime.GOOS+"-"+runtime.GOARCH, libDep)); err == nil {
+ return filepath.Join(root, runtime.GOOS+"-"+runtime.GOARCH, libDep)
+ }
+ if _, err := os.Stat(filepath.Join(root, "dist", runtime.GOOS+"-"+runtime.GOARCH, libDep)); err == nil {
+ return filepath.Join(root, "dist", runtime.GOOS+"-"+runtime.GOARCH, libDep)
+ }
+ }
+ slog.Warn("unable to locate gpu dependency libraries")
+ return ""
+}
+
+func GetSystemInfo() SystemInfo {
+ gpus := GetGPUInfo()
+ gpuMutex.Lock()
+ defer gpuMutex.Unlock()
+ discoveryErrors := []string{}
+ for _, err := range bootstrapErrors {
+ discoveryErrors = append(discoveryErrors, err.Error())
+ }
+ if len(gpus) == 1 && gpus[0].Library == "cpu" {
+ gpus = []GpuInfo{}
+ }
+
+ return SystemInfo{
+ System: cpus[0],
+ GPUs: gpus,
+ UnsupportedGPUs: unsupportedGPUs,
+ DiscoveryErrors: discoveryErrors,
}
- info := GetGPUInfo()
- if info.Driver == "CPU" {
- return 0
- }
-
- /*
- Calculate bytes per layer, this will roughly be the size of the model file divided by the number of layers.
- We can store the model weights and the kv cache in vram,
- to enable kv chache vram storage add two additional layers to the number of layers retrieved from the model file.
- */
- bytesPerLayer := uint64(fileSizeBytes / numLayer)
-
- // 75% of the absolute max number of layers we can fit in available VRAM, off-loading too many layers to the GPU can cause OOM errors
- layers := int(info.FreeMemory/bytesPerLayer) * 3 / 4
-
- log.Printf("%d MB VRAM available, loading up to %d %s GPU layers out of %d", info.FreeMemory/(1024*1024), layers, info.Driver, numLayer)
-
- return layers
}
diff --git a/gpu/gpu_darwin.go b/gpu/gpu_darwin.go
index ccf67b51..530b98c9 100644
--- a/gpu/gpu_darwin.go
+++ b/gpu/gpu_darwin.go
@@ -2,40 +2,100 @@
package gpu
+/*
+#cgo CFLAGS: -x objective-c
+#cgo LDFLAGS: -framework Foundation -framework CoreGraphics -framework Metal
+#include "gpu_info_darwin.h"
+*/
import "C"
-import (
- "runtime"
- "github.com/jmorganca/ollama/api"
+import (
+ "log/slog"
+ "runtime"
+ "syscall"
+
+ "github.com/ollama/ollama/format"
)
-// CheckVRAM returns the free VRAM in bytes on Linux machines with NVIDIA GPUs
-func CheckVRAM() (int64, error) {
- // TODO - assume metal, and return free memory?
- return 0, nil
+const (
+ metalMinimumMemory = 512 * format.MebiByte
+)
+func GetGPUInfo() GpuInfoList {
+ mem, _ := GetCPUMem()
+ if runtime.GOARCH == "amd64" {
+ return []GpuInfo{
+ {
+ Library: "cpu",
+ Variant: GetCPUCapability().String(),
+ memInfo: mem,
+ },
+ }
+ }
+ info := GpuInfo{
+ Library: "metal",
+ ID: "0",
+ }
+ info.TotalMemory = uint64(C.getRecommendedMaxVRAM())
+
+ // TODO is there a way to gather actual allocated video memory? (currentAllocatedSize doesn't work)
+ info.FreeMemory = info.TotalMemory
+
+ info.MinimumMemory = metalMinimumMemory
+ return []GpuInfo{info}
}
-func GetGPUInfo() GpuInfo {
- // TODO - Metal vs. x86 macs...
-
- return GpuInfo{
- Driver: "METAL",
- Library: "default",
- TotalMemory: 0,
- FreeMemory: 0,
+func GetCPUInfo() GpuInfoList {
+ mem, _ := GetCPUMem()
+ return []GpuInfo{
+ {
+ Library: "cpu",
+ Variant: GetCPUCapability().String(),
+ memInfo: mem,
+ },
}
}
-func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int {
- if runtime.GOARCH == "arm64" {
- return 1
+func GetCPUMem() (memInfo, error) {
+ return memInfo{
+ TotalMemory: uint64(C.getPhysicalMemory()),
+ FreeMemory: uint64(C.getFreeMemory()),
+ // FreeSwap omitted as Darwin uses dynamic paging
+ }, nil
+}
+
+func (l GpuInfoList) GetVisibleDevicesEnv() (string, string) {
+ // No-op on darwin
+ return "", ""
+}
+
+func GetSystemInfo() SystemInfo {
+ mem, _ := GetCPUMem()
+ query := "hw.perflevel0.physicalcpu"
+ perfCores, err := syscall.SysctlUint32(query)
+ if err != nil {
+ slog.Warn("failed to discover physical CPU details", "query", query, "error", err)
}
+ query = "hw.perflevel1.physicalcpu"
+ efficiencyCores, _ := syscall.SysctlUint32(query) // On x86 xeon this wont return data
- // metal only supported on arm64
- return 0
-}
+ // Determine thread count
+ query = "hw.logicalcpu"
+ logicalCores, _ := syscall.SysctlUint32(query)
-func nativeInit() error {
- return nil
+ return SystemInfo{
+ System: CPUInfo{
+ GpuInfo: GpuInfo{
+ memInfo: mem,
+ },
+ CPUs: []CPU{
+ {
+ CoreCount: int(perfCores + efficiencyCores),
+ EfficiencyCoreCount: int(efficiencyCores),
+ ThreadCount: int(logicalCores),
+ },
+ },
+ },
+ GPUs: GetGPUInfo(),
+ }
}
diff --git a/gpu/gpu_info.h b/gpu/gpu_info.h
index 7de36465..094b791a 100644
--- a/gpu/gpu_info.h
+++ b/gpu/gpu_info.h
@@ -9,31 +9,50 @@
#include <dlfcn.h>
#define LOAD_LIBRARY(lib, flags) dlopen(lib, flags)
#define LOAD_SYMBOL(handle, sym) dlsym(handle, sym)
-#define LOAD_ERR() dlerror()
+#define LOAD_ERR() strdup(dlerror())
#define UNLOAD_LIBRARY(handle) dlclose(handle)
#else
#include <windows.h>
#define LOAD_LIBRARY(lib, flags) LoadLibrary(lib)
#define LOAD_SYMBOL(handle, sym) GetProcAddress(handle, sym)
#define UNLOAD_LIBRARY(handle) FreeLibrary(handle)
-
-// TODO - refactor this with proper error message handling on windows
-inline static char *LOAD_ERR() {
- static char errbuf[8];
- snprintf(errbuf, 8, "0x%lx", GetLastError());
- return errbuf;
-}
+#define LOAD_ERR() ({\
+ LPSTR messageBuffer = NULL; \
+ size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, \
+ NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&messageBuffer, 0, NULL); \
+ char *resp = strdup(messageBuffer); \
+ LocalFree(messageBuffer); \
+ resp; \
+})
#endif
+#define LOG(verbose, ...) \
+ do { \
+ if (verbose) { \
+ fprintf(stderr, __VA_ARGS__); \
+ } \
+ } while (0)
+
#ifdef __cplusplus
extern "C" {
#endif
+#define GPU_ID_LEN 64
+#define GPU_NAME_LEN 96
+
typedef struct mem_info {
+ char *err; // If non-nill, caller responsible for freeing
+ char gpu_id[GPU_ID_LEN];
+ char gpu_name[GPU_NAME_LEN];
uint64_t total;
uint64_t free;
- char *err; // If non-nill, caller responsible for freeing
+ uint64_t used;
+
+ // Compute Capability
+ int major;
+ int minor;
+ int patch;
} mem_info_t;
void cpu_check_ram(mem_info_t *resp);
@@ -42,8 +61,10 @@ void cpu_check_ram(mem_info_t *resp);
}
#endif
-#include "gpu_info_cuda.h"
-#include "gpu_info_rocm.h"
+#include "gpu_info_cudart.h"
+#include "gpu_info_nvcuda.h"
+#include "gpu_info_nvml.h"
+#include "gpu_info_oneapi.h"
#endif // __GPU_INFO_H__
-#endif // __APPLE__
\ No newline at end of file
+#endif // __APPLE__
diff --git a/gpu/gpu_info_cpu.c b/gpu/gpu_info_cpu.c
deleted file mode 100644
index a7987cd4..00000000
--- a/gpu/gpu_info_cpu.c
+++ /dev/null
@@ -1,42 +0,0 @@
-#include "gpu_info.h"
-// Fallbacks for CPU mode
-
-#ifdef _WIN32
-#include <sysinfoapi.h>
-void cpu_check_ram(mem_info_t *resp) {
- resp->err = NULL;
- MEMORYSTATUSEX info;
- if (GlobalMemoryStatusEx(&info) != 0) {
- resp->total = info.ullTotalPhys;
- resp->free = info.ullAvailPhys;
- } else {
- resp->err = strdup(LOAD_ERR());
- }
- return;
-}
-
-#elif __linux__
-#include <errno.h>
-#include <string.h>
-#include <sys/sysinfo.h>
-void cpu_check_ram(mem_info_t *resp) {
- struct sysinfo info;
- resp->err = NULL;
- if (sysinfo(&info) != 0) {
- resp->err = strdup(strerror(errno));
- } else {
- resp->total = info.totalram * info.mem_unit;
- resp->free = info.freeram * info.mem_unit;
- }
- return;
-}
-
-#elif __APPLE__
-// TODO consider an Apple implementation that does something useful
-// mem_info_t cpu_check_ram() {
-// mem_info_t resp = {0, 0, NULL};
-// return resp;
-// }
-#else
-#error "Unsupported platform"
-#endif
diff --git a/gpu/gpu_info_cuda.c b/gpu/gpu_info_cuda.c
deleted file mode 100644
index 20055ed6..00000000
--- a/gpu/gpu_info_cuda.c
+++ /dev/null
@@ -1,106 +0,0 @@
-#ifndef __APPLE__ // TODO - maybe consider nvidia support on intel macs?
-
-#include "gpu_info_cuda.h"
-
-#include <string.h>
-
-#ifndef _WIN32
-const char *cuda_lib_paths[] = {
- "libnvidia-ml.so",
- "/usr/local/cuda/lib64/libnvidia-ml.so",
- "/usr/lib/x86_64-linux-gnu/nvidia/current/libnvidia-ml.so",
- "/usr/lib/wsl/lib/libnvidia-ml.so.1", // TODO Maybe glob?
- NULL,
-};
-#else
-const char *cuda_lib_paths[] = {
- "nvml.dll",
- "",
- NULL,
-};
-#endif
-
-void cuda_init(cuda_init_resp_t *resp) {
- nvmlReturn_t ret;
- resp->err = NULL;
- const int buflen = 256;
- char buf[buflen + 1];
- int i;
-
- struct lookup {
- char *s;
- void **p;
- } l[4] = {
- {"nvmlInit_v2", (void *)&resp->ch.initFn},
- {"nvmlShutdown", (void *)&resp->ch.shutdownFn},
- {"nvmlDeviceGetHandleByIndex", (void *)&resp->ch.getHandle},
- {"nvmlDeviceGetMemoryInfo", (void *)&resp->ch.getMemInfo},
- };
-
- for (i = 0; cuda_lib_paths[i] != NULL && resp->ch.handle == NULL; i++) {
- resp->ch.handle = LOAD_LIBRARY(cuda_lib_paths[i], RTLD_LAZY);
- }
- if (!resp->ch.handle) {
- // TODO improve error message, as the LOAD_ERR will have typically have the
- // final path that was checked which might be confusing.
- snprintf(buf, buflen,
- "Unable to load %s library to query for Nvidia GPUs: %s",
- cuda_lib_paths[0], LOAD_ERR());
- resp->err = strdup(buf);
- return;
- }
-
- for (i = 0; i < 4; i++) { // TODO - fix this to use a null terminated list
- *l[i].p = LOAD_SYMBOL(resp->ch.handle, l[i].s);
- if (!l[i].p) {
- UNLOAD_LIBRARY(resp->ch.handle);
- resp->ch.handle = NULL;
- snprintf(buf, buflen, "symbol lookup for %s failed: %s", l[i].s,
- LOAD_ERR());
- resp->err = strdup(buf);
- return;
- }
- }
-
- ret = (*resp->ch.initFn)();
- if (ret != NVML_SUCCESS) {
- snprintf(buf, buflen, "nvml vram init failure: %d", ret);
- resp->err = strdup(buf);
- }
-
- return;
-}
-
-void cuda_check_vram(cuda_handle_t h, mem_info_t *resp) {
- resp->err = NULL;
- nvmlDevice_t device;
- nvmlMemory_t memInfo = {0};
- nvmlReturn_t ret;
- const int buflen = 256;
- char buf[buflen + 1];
- int i;
-
- if (h.handle == NULL) {
- resp->err = strdup("nvml handle sn't initialized");
- return;
- }
-
- // TODO - handle multiple GPUs
- ret = (*h.getHandle)(0, &device);
- if (ret != NVML_SUCCESS) {
- snprintf(buf, buflen, "unable to get device handle: %d", ret);
- resp->err = strdup(buf);
- return;
- }
-
- ret = (*h.getMemInfo)(device, &memInfo);
- if (ret != NVML_SUCCESS) {
- snprintf(buf, buflen, "device memory info lookup failure: %d", ret);
- resp->err = strdup(buf);
- return;
- }
- resp->total = memInfo.total;
- resp->free = memInfo.free;
- return;
-}
-#endif // __APPLE__
\ No newline at end of file
diff --git a/gpu/gpu_info_cuda.h b/gpu/gpu_info_cuda.h
deleted file mode 100644
index 7d13cb6a..00000000
--- a/gpu/gpu_info_cuda.h
+++ /dev/null
@@ -1,35 +0,0 @@
-#ifndef __APPLE__
-#ifndef __GPU_INFO_CUDA_H__
-#define __GPU_INFO_CUDA_H__
-#include "gpu_info.h"
-
-// Just enough typedef's to dlopen/dlsym for memory information
-typedef enum nvmlReturn_enum {
- NVML_SUCCESS = 0,
- // Other values omitted for now...
-} nvmlReturn_t;
-typedef void *nvmlDevice_t; // Opaque is sufficient
-typedef struct nvmlMemory_st {
- unsigned long long total;
- unsigned long long free;
- unsigned long long used;
-} nvmlMemory_t;
-
-typedef struct cuda_handle {
- void *handle;
- nvmlReturn_t (*initFn)(void);
- nvmlReturn_t (*shutdownFn)(void);
- nvmlReturn_t (*getHandle)(unsigned int, nvmlDevice_t *);
- nvmlReturn_t (*getMemInfo)(nvmlDevice_t, nvmlMemory_t *);
-} cuda_handle_t;
-
-typedef struct cuda_init_resp {
- char *err; // If err is non-null handle is invalid
- cuda_handle_t ch;
-} cuda_init_resp_t;
-
-void cuda_init(cuda_init_resp_t *resp);
-void cuda_check_vram(cuda_handle_t ch, mem_info_t *resp);
-
-#endif // __GPU_INFO_CUDA_H__
-#endif // __APPLE__
\ No newline at end of file
diff --git a/gpu/gpu_info_cudart.c b/gpu/gpu_info_cudart.c
new file mode 100644
index 00000000..03f15a2c
--- /dev/null
+++ b/gpu/gpu_info_cudart.c
@@ -0,0 +1,183 @@
+#ifndef __APPLE__ // TODO - maybe consider nvidia support on intel macs?
+
+#include <string.h>
+#include "gpu_info_cudart.h"
+
+void cudart_init(char *cudart_lib_path, cudart_init_resp_t *resp) {
+ cudartReturn_t ret;
+ resp->err = NULL;
+ resp->num_devices = 0;
+ const int buflen = 256;
+ char buf[buflen + 1];
+ int i;
+
+ struct lookup {
+ char *s;
+ void **p;
+ } l[] = {
+ {"cudaSetDevice", (void *)&resp->ch.cudaSetDevice},
+ {"cudaDeviceSynchronize", (void *)&resp->ch.cudaDeviceSynchronize},
+ {"cudaDeviceReset", (void *)&resp->ch.cudaDeviceReset},
+ {"cudaMemGetInfo", (void *)&resp->ch.cudaMemGetInfo},
+ {"cudaGetDeviceCount", (void *)&resp->ch.cudaGetDeviceCount},
+ {"cudaDeviceGetAttribute", (void *)&resp->ch.cudaDeviceGetAttribute},
+ {"cudaDriverGetVersion", (void *)&resp->ch.cudaDriverGetVersion},
+ {"cudaGetDeviceProperties", (void *)&resp->ch.cudaGetDeviceProperties},
+ {NULL, NULL},
+ };
+
+ resp->ch.handle = LOAD_LIBRARY(cudart_lib_path, RTLD_LAZY);
+ if (!resp->ch.handle) {
+ char *msg = LOAD_ERR();
+ LOG(resp->ch.verbose, "library %s load err: %s\n", cudart_lib_path, msg);
+ snprintf(buf, buflen,
+ "Unable to load %s library to query for Nvidia GPUs: %s",
+ cudart_lib_path, msg);
+ free(msg);
+ resp->err = strdup(buf);
+ return;
+ }
+
+ for (i = 0; l[i].s != NULL; i++) {
+ *l[i].p = LOAD_SYMBOL(resp->ch.handle, l[i].s);
+ if (!*(l[i].p)) {
+ char *msg = LOAD_ERR();
+ LOG(resp->ch.verbose, "dlerr: %s\n", msg);
+ UNLOAD_LIBRARY(resp->ch.handle);
+ resp->ch.handle = NULL;
+ snprintf(buf, buflen, "symbol lookup for %s failed: %s", l[i].s,
+ msg);
+ free(msg);
+ resp->err = strdup(buf);
+ return;
+ }
+ }
+
+ ret = (*resp->ch.cudaSetDevice)(0);
+ if (ret != CUDART_SUCCESS) {
+ LOG(resp->ch.verbose, "cudaSetDevice err: %d\n", ret);
+ UNLOAD_LIBRARY(resp->ch.handle);
+ resp->ch.handle = NULL;
+ if (ret == CUDA_ERROR_INSUFFICIENT_DRIVER) {
+ resp->err = strdup("your nvidia driver is too old or missing. If you have a CUDA GPU please upgrade to run ollama");
+ return;
+ }
+ snprintf(buf, buflen, "cudart init failure: %d", ret);
+ resp->err = strdup(buf);
+ return;
+ }
+
+ int version = 0;
+ cudartDriverVersion_t driverVersion;
+ driverVersion.major = 0;
+ driverVersion.minor = 0;
+
+ // Report driver version if we're in verbose mode, ignore errors
+ ret = (*resp->ch.cudaDriverGetVersion)(&version);
+ if (ret != CUDART_SUCCESS) {
+ LOG(resp->ch.verbose, "cudaDriverGetVersion failed: %d\n", ret);
+ } else {
+ driverVersion.major = version / 1000;
+ driverVersion.minor = (version - (driverVersion.major * 1000)) / 10;
+ LOG(resp->ch.verbose, "CUDA driver version: %d-%d\n", driverVersion.major, driverVersion.minor);
+ }
+
+ ret = (*resp->ch.cudaGetDeviceCount)(&resp->num_devices);
+ if (ret != CUDART_SUCCESS) {
+ LOG(resp->ch.verbose, "cudaGetDeviceCount err: %d\n", ret);
+ UNLOAD_LIBRARY(resp->ch.handle);
+ resp->ch.handle = NULL;
+ snprintf(buf, buflen, "unable to get device count: %d", ret);
+ resp->err = strdup(buf);
+ return;
+ }
+}
+
+
+void cudart_bootstrap(cudart_handle_t h, int i, mem_info_t *resp) {
+ resp->err = NULL;
+ cudartMemory_t memInfo = {0,0,0};
+ cudartReturn_t ret;
+ const int buflen = 256;
+ char buf[buflen + 1];
+
+ if (h.handle == NULL) {
+ resp->err = strdup("cudart handle isn't initialized");
+ return;
+ }
+
+ ret = (*h.cudaSetDevice)(i);
+ if (ret != CUDART_SUCCESS) {
+ snprintf(buf, buflen, "cudart device failed to initialize");
+ resp->err = strdup(buf);
+ return;
+ }
+
+ cudaDeviceProp_t props;
+ ret = (*h.cudaGetDeviceProperties)(&props, i);
+ if (ret != CUDART_SUCCESS) {
+ LOG(h.verbose, "[%d] device properties lookup failure: %d\n", i, ret);
+ snprintf(&resp->gpu_id[0], GPU_ID_LEN, "%d", i);
+ resp->major = 0;
+ resp->minor = 0;
+ } else {
+ int allNull = 1;
+ for (int j = 0; j < 16; j++) {
+ if (props.uuid.bytes[j] != 0) {
+ allNull = 0;
+ break;
+ }
+ }
+ if (allNull != 0) {
+ snprintf(&resp->gpu_id[0], GPU_ID_LEN, "%d", i);
+ } else {
+ // GPU-d110a105-ac29-1d54-7b49-9c90440f215b
+ snprintf(&resp->gpu_id[0], GPU_ID_LEN,
+ "GPU-%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x",
+ props.uuid.bytes[0],
+ props.uuid.bytes[1],
+ props.uuid.bytes[2],
+ props.uuid.bytes[3],
+ props.uuid.bytes[4],
+ props.uuid.bytes[5],
+ props.uuid.bytes[6],
+ props.uuid.bytes[7],
+ props.uuid.bytes[8],
+ props.uuid.bytes[9],
+ props.uuid.bytes[10],
+ props.uuid.bytes[11],
+ props.uuid.bytes[12],
+ props.uuid.bytes[13],
+ props.uuid.bytes[14],
+ props.uuid.bytes[15]
+ );
+ }
+ resp->major = props.major;
+ resp->minor = props.minor;
+
+ // TODO add other useful properties from props
+ }
+ ret = (*h.cudaMemGetInfo)(&memInfo.free, &memInfo.total);
+ if (ret != CUDART_SUCCESS) {
+ snprintf(buf, buflen, "cudart device memory info lookup failure %d", ret);
+ resp->err = strdup(buf);
+ return;
+ }
+
+ resp->total = memInfo.total;
+ resp->free = memInfo.free;
+ resp->used = memInfo.used;
+
+ LOG(h.verbose, "[%s] CUDA totalMem %lu\n", resp->gpu_id, resp->total);
+ LOG(h.verbose, "[%s] CUDA freeMem %lu\n", resp->gpu_id, resp->free);
+ LOG(h.verbose, "[%s] CUDA usedMem %lu\n", resp->gpu_id, resp->used);
+ LOG(h.verbose, "[%s] Compute Capability %d.%d\n", resp->gpu_id, resp->major, resp->minor);
+}
+
+void cudart_release(cudart_handle_t h) {
+ LOG(h.verbose, "releasing cudart library\n");
+ UNLOAD_LIBRARY(h.handle);
+ h.handle = NULL;
+}
+
+#endif // __APPLE__
\ No newline at end of file
diff --git a/gpu/gpu_info_cudart.h b/gpu/gpu_info_cudart.h
new file mode 100644
index 00000000..ff0c0af1
--- /dev/null
+++ b/gpu/gpu_info_cudart.h
@@ -0,0 +1,148 @@
+#ifndef __APPLE__
+#ifndef __GPU_INFO_CUDART_H__
+#define __GPU_INFO_CUDART_H__
+#include "gpu_info.h"
+
+// Just enough typedef's to dlopen/dlsym for memory information
+typedef enum cudartReturn_enum {
+ CUDART_SUCCESS = 0,
+ CUDART_ERROR_INVALID_VALUE = 1,
+ CUDART_ERROR_MEMORY_ALLOCATION = 2,
+ CUDART_ERROR_INSUFFICIENT_DRIVER = 35,
+ // Other values omitted for now...
+} cudartReturn_t;
+
+typedef enum cudartDeviceAttr_enum {
+ cudartDevAttrComputeCapabilityMajor = 75,
+ cudartDevAttrComputeCapabilityMinor = 76,
+
+ // TODO - not yet wired up but may be useful for Jetson or other
+ // integrated GPU scenarios with shared memory
+ cudaDevAttrIntegrated = 18
+
+} cudartDeviceAttr_t;
+
+typedef void *cudartDevice_t; // Opaque is sufficient
+typedef struct cudartMemory_st {
+ size_t total;
+ size_t free;
+ size_t used;
+} cudartMemory_t;
+
+typedef struct cudartDriverVersion {
+ int major;
+ int minor;
+} cudartDriverVersion_t;
+
+typedef struct cudaUUID {
+ unsigned char bytes[16];
+} cudaUUID_t;
+typedef struct cudaDeviceProp {
+ char name[256]; /**< ASCII string identifying device */
+ cudaUUID_t uuid; /**< 16-byte unique identifier */
+ char luid[8]; /**< 8-byte locally unique identifier. Value is undefined on TCC and non-Windows platforms */
+ unsigned int luidDeviceNodeMask; /**< LUID device node mask. Value is undefined on TCC and non-Windows platforms */
+ size_t totalGlobalMem; /**< Global memory available on device in bytes */
+ size_t sharedMemPerBlock; /**< Shared memory available per block in bytes */
+ int regsPerBlock; /**< 32-bit registers available per block */
+ int warpSize; /**< Warp size in threads */
+ size_t memPitch; /**< Maximum pitch in bytes allowed by memory copies */
+ int maxThreadsPerBlock; /**< Maximum number of threads per block */
+ int maxThreadsDim[3]; /**< Maximum size of each dimension of a block */
+ int maxGridSize[3]; /**< Maximum size of each dimension of a grid */
+ int clockRate; /**< Clock frequency in kilohertz */
+ size_t totalConstMem; /**< Constant memory available on device in bytes */
+ int major; /**< Major compute capability */
+ int minor; /**< Minor compute capability */
+ size_t textureAlignment; /**< Alignment requirement for textures */
+ size_t texturePitchAlignment; /**< Pitch alignment requirement for texture references bound to pitched memory */
+ int deviceOverlap; /**< Device can concurrently copy memory and execute a kernel. Deprecated. Use instead asyncEngineCount. */
+ int multiProcessorCount; /**< Number of multiprocessors on device */
+ int kernelExecTimeoutEnabled; /**< Specified whether there is a run time limit on kernels */
+ int integrated; /**< Device is integrated as opposed to discrete */
+ int canMapHostMemory; /**< Device can map host memory with cudaHostAlloc/cudaHostGetDevicePointer */
+ int computeMode; /**< Compute mode (See ::cudaComputeMode) */
+ int maxTexture1D; /**< Maximum 1D texture size */
+ int maxTexture1DMipmap; /**< Maximum 1D mipmapped texture size */
+ int maxTexture1DLinear; /**< Deprecated, do not use. Use cudaDeviceGetTexture1DLinearMaxWidth() or cuDeviceGetTexture1DLinearMaxWidth() instead. */
+ int maxTexture2D[2]; /**< Maximum 2D texture dimensions */
+ int maxTexture2DMipmap[2]; /**< Maximum 2D mipmapped texture dimensions */
+ int maxTexture2DLinear[3]; /**< Maximum dimensions (width, height, pitch) for 2D textures bound to pitched memory */
+ int maxTexture2DGather[2]; /**< Maximum 2D texture dimensions if texture gather operations have to be performed */
+ int maxTexture3D[3]; /**< Maximum 3D texture dimensions */
+ int maxTexture3DAlt[3]; /**< Maximum alternate 3D texture dimensions */
+ int maxTextureCubemap; /**< Maximum Cubemap texture dimensions */
+ int maxTexture1DLayered[2]; /**< Maximum 1D layered texture dimensions */
+ int maxTexture2DLayered[3]; /**< Maximum 2D layered texture dimensions */
+ int maxTextureCubemapLayered[2];/**< Maximum Cubemap layered texture dimensions */
+ int maxSurface1D; /**< Maximum 1D surface size */
+ int maxSurface2D[2]; /**< Maximum 2D surface dimensions */
+ int maxSurface3D[3]; /**< Maximum 3D surface dimensions */
+ int maxSurface1DLayered[2]; /**< Maximum 1D layered surface dimensions */
+ int maxSurface2DLayered[3]; /**< Maximum 2D layered surface dimensions */
+ int maxSurfaceCubemap; /**< Maximum Cubemap surface dimensions */
+ int maxSurfaceCubemapLayered[2];/**< Maximum Cubemap layered surface dimensions */
+ size_t surfaceAlignment; /**< Alignment requirements for surfaces */
+ int concurrentKernels; /**< Device can possibly execute multiple kernels concurrently */
+ int ECCEnabled; /**< Device has ECC support enabled */
+ int pciBusID; /**< PCI bus ID of the device */
+ int pciDeviceID; /**< PCI device ID of the device */
+ int pciDomainID; /**< PCI domain ID of the device */
+ int tccDriver; /**< 1 if device is a Tesla device using TCC driver, 0 otherwise */
+ int asyncEngineCount; /**< Number of asynchronous engines */
+ int unifiedAddressing; /**< Device shares a unified address space with the host */
+ int memoryClockRate; /**< Peak memory clock frequency in kilohertz */
+ int memoryBusWidth; /**< Global memory bus width in bits */
+ int l2CacheSize; /**< Size of L2 cache in bytes */
+ int persistingL2CacheMaxSize; /**< Device's maximum l2 persisting lines capacity setting in bytes */
+ int maxThreadsPerMultiProcessor;/**< Maximum resident threads per multiprocessor */
+ int streamPrioritiesSupported; /**< Device supports stream priorities */
+ int globalL1CacheSupported; /**< Device supports caching globals in L1 */
+ int localL1CacheSupported; /**< Device supports caching locals in L1 */
+ size_t sharedMemPerMultiprocessor; /**< Shared memory available per multiprocessor in bytes */
+ int regsPerMultiprocessor; /**< 32-bit registers available per multiprocessor */
+ int managedMemory; /**< Device supports allocating managed memory on this system */
+ int isMultiGpuBoard; /**< Device is on a multi-GPU board */
+ int multiGpuBoardGroupID; /**< Unique identifier for a group of devices on the same multi-GPU board */
+ int hostNativeAtomicSupported; /**< Link between the device and the host supports native atomic operations */
+ int singleToDoublePrecisionPerfRatio; /**< Ratio of single precision performance (in floating-point operations per second) to double precision performance */
+ int pageableMemoryAccess; /**< Device supports coherently accessing pageable memory without calling cudaHostRegister on it */
+ int concurrentManagedAccess; /**< Device can coherently access managed memory concurrently with the CPU */
+ int computePreemptionSupported; /**< Device supports Compute Preemption */
+ int canUseHostPointerForRegisteredMem; /**< Device can access host registered memory at the same virtual address as the CPU */
+ int cooperativeLaunch; /**< Device supports launching cooperative kernels via ::cudaLaunchCooperativeKernel */
+ int cooperativeMultiDeviceLaunch; /**< Deprecated, cudaLaunchCooperativeKernelMultiDevice is deprecated. */
+ size_t sharedMemPerBlockOptin; /**< Per device maximum shared memory per block usable by special opt in */
+ int pageableMemoryAccessUsesHostPageTables; /**< Device accesses pageable memory via the host's page tables */
+ int directManagedMemAccessFromHost; /**< Host can directly access managed memory on the device without migration. */
+ int maxBlocksPerMultiProcessor; /**< Maximum number of resident blocks per multiprocessor */
+ int accessPolicyMaxWindowSize; /**< The maximum value of ::cudaAccessPolicyWindow::num_bytes. */
+ size_t reservedSharedMemPerBlock; /**< Shared memory reserved by CUDA driver per block in bytes */
+ } cudaDeviceProp_t;
+
+typedef struct cudart_handle {
+ void *handle;
+ uint16_t verbose;
+ cudartReturn_t (*cudaSetDevice)(int device);
+ cudartReturn_t (*cudaDeviceSynchronize)(void);
+ cudartReturn_t (*cudaDeviceReset)(void);
+ cudartReturn_t (*cudaMemGetInfo)(size_t *, size_t *);
+ cudartReturn_t (*cudaGetDeviceCount)(int *);
+ cudartReturn_t (*cudaDeviceGetAttribute)(int* value, cudartDeviceAttr_t attr, int device);
+ cudartReturn_t (*cudaDriverGetVersion) (int *driverVersion);
+ cudartReturn_t (*cudaGetDeviceProperties) (cudaDeviceProp_t* prop, int device);
+} cudart_handle_t;
+
+typedef struct cudart_init_resp {
+ char *err; // If err is non-null handle is invalid
+ cudart_handle_t ch;
+ int num_devices;
+} cudart_init_resp_t;
+
+void cudart_init(char *cudart_lib_path, cudart_init_resp_t *resp);
+void cudart_bootstrap(cudart_handle_t ch, int device_id, mem_info_t *resp);
+// TODO - if we keep this library longer term, add cudart_get_free
+void cudart_release(cudart_handle_t ch);
+
+#endif // __GPU_INFO_CUDART_H__
+#endif // __APPLE__
diff --git a/gpu/gpu_info_darwin.h b/gpu/gpu_info_darwin.h
new file mode 100644
index 00000000..415e7922
--- /dev/null
+++ b/gpu/gpu_info_darwin.h
@@ -0,0 +1,5 @@
+#import <Metal/Metal.h>
+#include <stdint.h>
+uint64_t getRecommendedMaxVRAM();
+uint64_t getPhysicalMemory();
+uint64_t getFreeMemory();
diff --git a/gpu/gpu_info_darwin.m b/gpu/gpu_info_darwin.m
new file mode 100644
index 00000000..5ca139e0
--- /dev/null
+++ b/gpu/gpu_info_darwin.m
@@ -0,0 +1,35 @@
+#import <Foundation/Foundation.h>
+#import <mach/mach.h>
+#include "gpu_info_darwin.h"
+
+uint64_t getRecommendedMaxVRAM() {
+ id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+ uint64_t result = device.recommendedMaxWorkingSetSize;
+ CFRelease(device);
+ return result;
+}
+
+// getPhysicalMemory returns the total physical memory in bytes
+uint64_t getPhysicalMemory() {
+ return [NSProcessInfo processInfo].physicalMemory;
+}
+
+// getFreeMemory returns the total free memory in bytes, including inactive
+// memory that can be reclaimed by the system.
+uint64_t getFreeMemory() {
+ mach_port_t host_port = mach_host_self();
+ mach_msg_type_number_t host_size = sizeof(vm_statistics64_data_t) / sizeof(integer_t);
+ vm_size_t pagesize;
+ vm_statistics64_data_t vm_stat;
+
+ host_page_size(host_port, &pagesize);
+ if (host_statistics64(host_port, HOST_VM_INFO64, (host_info64_t)&vm_stat, &host_size) != KERN_SUCCESS) {
+ return 0;
+ }
+
+ uint64_t free_memory = (uint64_t)vm_stat.free_count * pagesize;
+ free_memory += (uint64_t)vm_stat.speculative_count * pagesize;
+ free_memory += (uint64_t)vm_stat.inactive_count * pagesize;
+
+ return free_memory;
+}
diff --git a/gpu/gpu_info_nvcuda.c b/gpu/gpu_info_nvcuda.c
new file mode 100644
index 00000000..a1a38bfc
--- /dev/null
+++ b/gpu/gpu_info_nvcuda.c
@@ -0,0 +1,243 @@
+#ifndef __APPLE__ // TODO - maybe consider nvidia support on intel macs?
+
+#include <string.h>
+#include "gpu_info_nvcuda.h"
+
+void nvcuda_init(char *nvcuda_lib_path, nvcuda_init_resp_t *resp) {
+ CUresult ret;
+ resp->err = NULL;
+ resp->num_devices = 0;
+ resp->cudaErr = CUDA_SUCCESS;
+ const int buflen = 256;
+ char buf[buflen + 1];
+ int i;
+
+ struct lookup {
+ char *s;
+ void **p;
+ } l[] = {
+
+ {"cuInit", (void *)&resp->ch.cuInit},
+ {"cuDriverGetVersion", (void *)&resp->ch.cuDriverGetVersion},
+ {"cuDeviceGetCount", (void *)&resp->ch.cuDeviceGetCount},
+ {"cuDeviceGet", (void *)&resp->ch.cuDeviceGet},
+ {"cuDeviceGetAttribute", (void *)&resp->ch.cuDeviceGetAttribute},
+ {"cuDeviceGetUuid", (void *)&resp->ch.cuDeviceGetUuid},
+ {"cuDeviceGetName", (void *)&resp->ch.cuDeviceGetName},
+ {"cuCtxCreate_v3", (void *)&resp->ch.cuCtxCreate_v3},
+ {"cuMemGetInfo_v2", (void *)&resp->ch.cuMemGetInfo_v2},
+ {"cuCtxDestroy", (void *)&resp->ch.cuCtxDestroy},
+ {NULL, NULL},
+ };
+
+ resp->ch.handle = LOAD_LIBRARY(nvcuda_lib_path, RTLD_LAZY);
+ if (!resp->ch.handle) {
+ char *msg = LOAD_ERR();
+ LOG(resp->ch.verbose, "library %s load err: %s\n", nvcuda_lib_path, msg);
+ snprintf(buf, buflen,
+ "Unable to load %s library to query for Nvidia GPUs: %s",
+ nvcuda_lib_path, msg);
+ free(msg);
+ resp->err = strdup(buf);
+ resp->cudaErr = -1;
+ return;
+ }
+
+ for (i = 0; l[i].s != NULL; i++) {
+ *l[i].p = LOAD_SYMBOL(resp->ch.handle, l[i].s);
+ if (!*(l[i].p)) {
+ char *msg = LOAD_ERR();
+ LOG(resp->ch.verbose, "dlerr: %s\n", msg);
+ UNLOAD_LIBRARY(resp->ch.handle);
+ resp->ch.handle = NULL;
+ snprintf(buf, buflen, "symbol lookup for %s failed: %s", l[i].s,
+ msg);
+ free(msg);
+ resp->err = strdup(buf);
+ resp->cudaErr = -1;
+ return;
+ }
+ }
+
+ ret = (*resp->ch.cuInit)(0);
+ if (ret != CUDA_SUCCESS) {
+ LOG(resp->ch.verbose, "cuInit err: %d\n", ret);
+ UNLOAD_LIBRARY(resp->ch.handle);
+ resp->ch.handle = NULL;
+ snprintf(buf, buflen, "cuda driver library init failure: %d", ret);
+ resp->err = strdup(buf);
+ resp->cudaErr = ret;
+ return;
+ }
+
+ int version = 0;
+ resp->ch.driver_major = 0;
+ resp->ch.driver_minor = 0;
+
+ // Report driver version if we're in verbose mode, ignore errors
+ ret = (*resp->ch.cuDriverGetVersion)(&version);
+ if (ret != CUDA_SUCCESS) {
+ LOG(resp->ch.verbose, "cuDriverGetVersion failed: %d\n", ret);
+ } else {
+ resp->ch.driver_major = version / 1000;
+ resp->ch.driver_minor = (version - (resp->ch.driver_major * 1000)) / 10;
+ LOG(resp->ch.verbose, "CUDA driver version: %d.%d\n", resp->ch.driver_major, resp->ch.driver_minor);
+ }
+
+ ret = (*resp->ch.cuDeviceGetCount)(&resp->num_devices);
+ if (ret != CUDA_SUCCESS) {
+ LOG(resp->ch.verbose, "cuDeviceGetCount err: %d\n", ret);
+ UNLOAD_LIBRARY(resp->ch.handle);
+ resp->ch.handle = NULL;
+ snprintf(buf, buflen, "unable to get device count: %d", ret);
+ resp->err = strdup(buf);
+ resp->cudaErr = ret;
+ return;
+ }
+}
+
+const int buflen = 256;
+void nvcuda_bootstrap(nvcuda_handle_t h, int i, mem_info_t *resp) {
+ resp->err = NULL;
+ nvcudaMemory_t memInfo = {0,0};
+ CUresult ret;
+ CUdevice device = -1;
+ CUcontext ctx = NULL;
+ char buf[buflen + 1];
+ CUuuid uuid = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+
+ if (h.handle == NULL) {
+ resp->err = strdup("cuda driver library handle isn't initialized");
+ return;
+ }
+
+ ret = (*h.cuDeviceGet)(&device, i);
+ if (ret != CUDA_SUCCESS) {
+ snprintf(buf, buflen, "cuda driver library device failed to initialize");
+ resp->err = strdup(buf);
+ return;
+ }
+
+ int major = 0;
+ int minor = 0;
+ ret = (*h.cuDeviceGetAttribute)(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device);
+ if (ret != CUDA_SUCCESS) {
+ LOG(h.verbose, "[%d] device major lookup failure: %d\n", i, ret);
+ } else {
+ ret = (*h.cuDeviceGetAttribute)(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device);
+ if (ret != CUDA_SUCCESS) {
+ LOG(h.verbose, "[%d] device minor lookup failure: %d\n", i, ret);
+ } else {
+ resp->minor = minor;
+ resp->major = major;
+ }
+ }
+
+ ret = (*h.cuDeviceGetUuid)(&uuid, device);
+ if (ret != CUDA_SUCCESS) {
+ LOG(h.verbose, "[%d] device uuid lookup failure: %d\n", i, ret);
+ snprintf(&resp->gpu_id[0], GPU_ID_LEN, "%d", i);
+ } else {
+ // GPU-d110a105-ac29-1d54-7b49-9c90440f215b
+ snprintf(&resp->gpu_id[0], GPU_ID_LEN,
+ "GPU-%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x",
+ uuid.bytes[0],
+ uuid.bytes[1],
+ uuid.bytes[2],
+ uuid.bytes[3],
+ uuid.bytes[4],
+ uuid.bytes[5],
+ uuid.bytes[6],
+ uuid.bytes[7],
+ uuid.bytes[8],
+ uuid.bytes[9],
+ uuid.bytes[10],
+ uuid.bytes[11],
+ uuid.bytes[12],
+ uuid.bytes[13],
+ uuid.bytes[14],
+ uuid.bytes[15]
+ );
+ }
+
+ ret = (*h.cuDeviceGetName)(&resp->gpu_name[0], GPU_NAME_LEN, device);
+ if (ret != CUDA_SUCCESS) {
+ LOG(h.verbose, "[%d] device name lookup failure: %d\n", i, ret);
+ resp->gpu_name[0] = '\0';
+ }
+
+ // To get memory we have to set (and release) a context
+ ret = (*h.cuCtxCreate_v3)(&ctx, NULL, 0, 0, device);
+ if (ret != CUDA_SUCCESS) {
+ snprintf(buf, buflen, "cuda driver library failed to get device context %d", ret);
+ resp->err = strdup(buf);
+ return;
+ }
+
+ ret = (*h.cuMemGetInfo_v2)(&memInfo.free, &memInfo.total);
+ if (ret != CUDA_SUCCESS) {
+ snprintf(buf, buflen, "cuda driver library device memory info lookup failure %d", ret);
+ resp->err = strdup(buf);
+ // Best effort on failure...
+ (*h.cuCtxDestroy)(ctx);
+ return;
+ }
+
+ resp->total = memInfo.total;
+ resp->free = memInfo.free;
+
+ LOG(h.verbose, "[%s] CUDA totalMem %lu mb\n", resp->gpu_id, resp->total / 1024 / 1024);
+ LOG(h.verbose, "[%s] CUDA freeMem %lu mb\n", resp->gpu_id, resp->free / 1024 / 1024);
+ LOG(h.verbose, "[%s] Compute Capability %d.%d\n", resp->gpu_id, resp->major, resp->minor);
+
+
+
+ ret = (*h.cuCtxDestroy)(ctx);
+ if (ret != CUDA_SUCCESS) {
+ LOG(1, "cuda driver library failed to release device context %d", ret);
+ }
+}
+
+void nvcuda_get_free(nvcuda_handle_t h, int i, uint64_t *free, uint64_t *total) {
+ CUresult ret;
+ CUcontext ctx = NULL;
+ CUdevice device = -1;
+ *free = 0;
+ *total = 0;
+
+ ret = (*h.cuDeviceGet)(&device, i);
+ if (ret != CUDA_SUCCESS) {
+ LOG(1, "cuda driver library device failed to initialize");
+ return;
+ }
+
+
+ // To get memory we have to set (and release) a context
+ ret = (*h.cuCtxCreate_v3)(&ctx, NULL, 0, 0, device);
+ if (ret != CUDA_SUCCESS) {
+ LOG(1, "cuda driver library failed to get device context %d", ret);
+ return;
+ }
+
+ ret = (*h.cuMemGetInfo_v2)(free, total);
+ if (ret != CUDA_SUCCESS) {
+ LOG(1, "cuda driver library device memory info lookup failure %d", ret);
+ // Best effort on failure...
+ (*h.cuCtxDestroy)(ctx);
+ return;
+ }
+
+ ret = (*h.cuCtxDestroy)(ctx);
+ if (ret != CUDA_SUCCESS) {
+ LOG(1, "cuda driver library failed to release device context %d", ret);
+ }
+}
+
+void nvcuda_release(nvcuda_handle_t h) {
+ LOG(h.verbose, "releasing cuda driver library\n");
+ UNLOAD_LIBRARY(h.handle);
+ // TODO and other context release logic?
+ h.handle = NULL;
+}
+
+#endif // __APPLE__
\ No newline at end of file
diff --git a/gpu/gpu_info_nvcuda.h b/gpu/gpu_info_nvcuda.h
new file mode 100644
index 00000000..ef2fe8a3
--- /dev/null
+++ b/gpu/gpu_info_nvcuda.h
@@ -0,0 +1,79 @@
+#ifndef __APPLE__
+#ifndef __GPU_INFO_NVCUDA_H__
+#define __GPU_INFO_NVCUDA_H__
+#include "gpu_info.h"
+
+// Just enough typedef's to dlopen/dlsym for memory information
+typedef enum cudaError_enum {
+ CUDA_SUCCESS = 0,
+ CUDA_ERROR_INVALID_VALUE = 1,
+ CUDA_ERROR_OUT_OF_MEMORY = 2,
+ CUDA_ERROR_NOT_INITIALIZED = 3,
+ CUDA_ERROR_INSUFFICIENT_DRIVER = 35,
+ CUDA_ERROR_NO_DEVICE = 100,
+ CUDA_ERROR_SYSTEM_DRIVER_MISMATCH = 803,
+ CUDA_ERROR_UNKNOWN = 999,
+ // Other values omitted for now...
+} CUresult;
+
+typedef enum CUdevice_attribute_enum {
+ CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75,
+ CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR = 76,
+
+ // TODO - not yet wired up but may be useful for Jetson or other
+ // integrated GPU scenarios with shared memory
+ CU_DEVICE_ATTRIBUTE_INTEGRATED = 18
+
+} CUdevice_attribute;
+
+typedef void *nvcudaDevice_t; // Opaque is sufficient
+typedef struct nvcudaMemory_st {
+ uint64_t total;
+ uint64_t free;
+} nvcudaMemory_t;
+
+typedef struct nvcudaDriverVersion {
+ int major;
+ int minor;
+} nvcudaDriverVersion_t;
+
+typedef struct CUuuid_st {
+ unsigned char bytes[16];
+} CUuuid;
+
+typedef int CUdevice;
+typedef void* CUcontext;
+
+typedef struct nvcuda_handle {
+ void *handle;
+ uint16_t verbose;
+ int driver_major;
+ int driver_minor;
+ CUresult (*cuInit)(unsigned int Flags);
+ CUresult (*cuDriverGetVersion)(int *driverVersion);
+ CUresult (*cuDeviceGetCount)(int *);
+ CUresult (*cuDeviceGet)(CUdevice* device, int ordinal);
+ CUresult (*cuDeviceGetAttribute)(int* pi, CUdevice_attribute attrib, CUdevice dev);
+ CUresult (*cuDeviceGetUuid)(CUuuid* uuid, CUdevice dev); // signature compatible with cuDeviceGetUuid_v2
+ CUresult (*cuDeviceGetName)(char *name, int len, CUdevice dev);
+
+ // Context specific aspects
+ CUresult (*cuCtxCreate_v3)(CUcontext* pctx, void *params, int len, unsigned int flags, CUdevice dev);
+ CUresult (*cuMemGetInfo_v2)(uint64_t* free, uint64_t* total);
+ CUresult (*cuCtxDestroy)(CUcontext ctx);
+} nvcuda_handle_t;
+
+typedef struct nvcuda_init_resp {
+ char *err; // If err is non-null handle is invalid
+ nvcuda_handle_t ch;
+ int num_devices;
+ CUresult cudaErr;
+} nvcuda_init_resp_t;
+
+void nvcuda_init(char *nvcuda_lib_path, nvcuda_init_resp_t *resp);
+void nvcuda_bootstrap(nvcuda_handle_t ch, int device_id, mem_info_t *resp);
+void nvcuda_get_free(nvcuda_handle_t ch, int device_id, uint64_t *free, uint64_t *total);
+void nvcuda_release(nvcuda_handle_t ch);
+
+#endif // __GPU_INFO_NVCUDA_H__
+#endif // __APPLE__
diff --git a/gpu/gpu_info_nvml.c b/gpu/gpu_info_nvml.c
new file mode 100644
index 00000000..11293e44
--- /dev/null
+++ b/gpu/gpu_info_nvml.c
@@ -0,0 +1,104 @@
+#ifndef __APPLE__ // TODO - maybe consider nvidia support on intel macs?
+
+#include <string.h>
+
+#include "gpu_info_nvml.h"
+
+void nvml_init(char *nvml_lib_path, nvml_init_resp_t *resp) {
+ nvmlReturn_t ret;
+ resp->err = NULL;
+ const int buflen = 256;
+ char buf[buflen + 1];
+ int i;
+
+ struct lookup {
+ char *s;
+ void **p;
+ } l[] = {
+ {"nvmlInit_v2", (void *)&resp->ch.nvmlInit_v2},
+ {"nvmlShutdown", (void *)&resp->ch.nvmlShutdown},
+ {"nvmlDeviceGetHandleByIndex", (void *)&resp->ch.nvmlDeviceGetHandleByIndex},
+ {"nvmlDeviceGetMemoryInfo", (void *)&resp->ch.nvmlDeviceGetMemoryInfo},
+ {NULL, NULL},
+ };
+
+ resp->ch.handle = LOAD_LIBRARY(nvml_lib_path, RTLD_LAZY);
+ if (!resp->ch.handle) {
+ char *msg = LOAD_ERR();
+ LOG(resp->ch.verbose, "library %s load err: %s\n", nvml_lib_path, msg);
+ snprintf(buf, buflen,
+ "Unable to load %s library to query for Nvidia GPUs: %s",
+ nvml_lib_path, msg);
+ free(msg);
+ resp->err = strdup(buf);
+ return;
+ }
+
+ // TODO once we've squashed the remaining corner cases remove this log
+ // LOG(resp->ch.verbose, "wiring nvidia management library functions in %s\n", nvml_lib_path);
+
+ for (i = 0; l[i].s != NULL; i++) {
+ // TODO once we've squashed the remaining corner cases remove this log
+ // LOG(resp->ch.verbose, "dlsym: %s\n", l[i].s);
+
+ *l[i].p = LOAD_SYMBOL(resp->ch.handle, l[i].s);
+ if (!*(l[i].p)) {
+ resp->ch.handle = NULL;
+ char *msg = LOAD_ERR();
+ LOG(resp->ch.verbose, "dlerr: %s\n", msg);
+ UNLOAD_LIBRARY(resp->ch.handle);
+ snprintf(buf, buflen, "symbol lookup for %s failed: %s", l[i].s,
+ msg);
+ free(msg);
+ resp->err = strdup(buf);
+ return;
+ }
+ }
+
+ ret = (*resp->ch.nvmlInit_v2)();
+ if (ret != NVML_SUCCESS) {
+ LOG(resp->ch.verbose, "nvmlInit_v2 err: %d\n", ret);
+ UNLOAD_LIBRARY(resp->ch.handle);
+ resp->ch.handle = NULL;
+ snprintf(buf, buflen, "nvml vram init failure: %d", ret);
+ resp->err = strdup(buf);
+ return;
+ }
+}
+
+
+void nvml_get_free(nvml_handle_t h, int device_id, uint64_t *free, uint64_t *total, uint64_t *used) {
+ nvmlDevice_t device;
+ nvmlMemory_t memInfo = {0};
+ nvmlReturn_t ret;
+ ret = (*h.nvmlDeviceGetHandleByIndex)(device_id, &device);
+ if (ret != NVML_SUCCESS) {
+ LOG(1, "unable to get device handle %d: %d", device_id, ret);
+ *free = 0;
+ return;
+ }
+
+ ret = (*h.nvmlDeviceGetMemoryInfo)(device, &memInfo);
+ if (ret != NVML_SUCCESS) {
+ LOG(1, "device memory info lookup failure %d: %d", device_id, ret);
+ *free = 0;
+ return;
+ }
+ *free = memInfo.free;
+ *total = memInfo.total;
+ *used = memInfo.used;
+}
+
+
+void nvml_release(nvml_handle_t h) {
+ LOG(h.verbose, "releasing nvml library\n");
+ nvmlReturn_t ret;
+ ret = (*h.nvmlShutdown)();
+ if (ret != NVML_SUCCESS) {
+ LOG(1, "error during nvmlShutdown %d", ret);
+ }
+ UNLOAD_LIBRARY(h.handle);
+ h.handle = NULL;
+}
+
+#endif // __APPLE__
\ No newline at end of file
diff --git a/gpu/gpu_info_nvml.h b/gpu/gpu_info_nvml.h
new file mode 100644
index 00000000..a661f723
--- /dev/null
+++ b/gpu/gpu_info_nvml.h
@@ -0,0 +1,48 @@
+#ifndef __APPLE__
+#ifndef __GPU_INFO_NVML_H__
+#define __GPU_INFO_NVML_H__
+#include "gpu_info.h"
+
+// Just enough typedef's to dlopen/dlsym for memory information
+typedef enum nvmlReturn_enum {
+ NVML_SUCCESS = 0,
+ // Other values omitted for now...
+} nvmlReturn_t;
+typedef void *nvmlDevice_t; // Opaque is sufficient
+typedef struct nvmlMemory_st {
+ unsigned long long total;
+ unsigned long long free;
+ unsigned long long used;
+} nvmlMemory_t;
+
+typedef enum nvmlBrandType_enum
+{
+ NVML_BRAND_UNKNOWN = 0,
+} nvmlBrandType_t;
+
+typedef struct nvml_handle {
+ void *handle;
+ uint16_t verbose;
+ nvmlReturn_t (*nvmlInit_v2)(void);
+ nvmlReturn_t (*nvmlShutdown)(void);
+ nvmlReturn_t (*nvmlDeviceGetHandleByIndex)(unsigned int, nvmlDevice_t *);
+ nvmlReturn_t (*nvmlDeviceGetMemoryInfo)(nvmlDevice_t, nvmlMemory_t *);
+} nvml_handle_t;
+
+typedef struct nvml_init_resp {
+ char *err; // If err is non-null handle is invalid
+ nvml_handle_t ch;
+} nvml_init_resp_t;
+
+typedef struct nvml_compute_capability {
+ char *err;
+ int major;
+ int minor;
+} nvml_compute_capability_t;
+
+void nvml_init(char *nvml_lib_path, nvml_init_resp_t *resp);
+void nvml_get_free(nvml_handle_t ch, int device_id, uint64_t *free, uint64_t *total, uint64_t *used);
+void nvml_release(nvml_handle_t ch);
+
+#endif // __GPU_INFO_NVML_H__
+#endif // __APPLE__
\ No newline at end of file
diff --git a/gpu/gpu_info_oneapi.c b/gpu/gpu_info_oneapi.c
new file mode 100644
index 00000000..3ff708ea
--- /dev/null
+++ b/gpu/gpu_info_oneapi.c
@@ -0,0 +1,259 @@
+#ifndef __APPLE__
+
+#include "gpu_info_oneapi.h"
+
+#include <string.h>
+
+void oneapi_init(char *oneapi_lib_path, oneapi_init_resp_t *resp) {
+ ze_result_t ret;
+ resp->err = NULL;
+ resp->oh.devices = NULL;
+ resp->oh.num_devices = NULL;
+ resp->oh.drivers = NULL;
+ resp->oh.num_drivers = 0;
+ const int buflen = 256;
+ char buf[buflen + 1];
+ int i, d;
+ struct lookup {
+ char *s;
+ void **p;
+ } l[] = {
+ {"zesInit", (void *)&resp->oh.zesInit},
+ {"zesDriverGet", (void *)&resp->oh.zesDriverGet},
+ {"zesDeviceGet", (void *)&resp->oh.zesDeviceGet},
+ {"zesDeviceGetProperties", (void *)&resp->oh.zesDeviceGetProperties},
+ {"zesDeviceEnumMemoryModules",
+ (void *)&resp->oh.zesDeviceEnumMemoryModules},
+ {"zesMemoryGetProperties", (void *)&resp->oh.zesMemoryGetProperties},
+ {"zesMemoryGetState", (void *)&resp->oh.zesMemoryGetState},
+ {NULL, NULL},
+ };
+
+ resp->oh.handle = LOAD_LIBRARY(oneapi_lib_path, RTLD_LAZY);
+ if (!resp->oh.handle) {
+ char *msg = LOAD_ERR();
+ snprintf(buf, buflen,
+ "Unable to load %s library to query for Intel GPUs: %s\n",
+ oneapi_lib_path, msg);
+ free(msg);
+ resp->err = strdup(buf);
+ return;
+ }
+
+ // TODO once we've squashed the remaining corner cases remove this log
+ LOG(resp->oh.verbose,
+ "wiring Level-Zero management library functions in %s\n",
+ oneapi_lib_path);
+
+ for (i = 0; l[i].s != NULL; i++) {
+ // TODO once we've squashed the remaining corner cases remove this log
+ LOG(resp->oh.verbose, "dlsym: %s\n", l[i].s);
+
+ *l[i].p = LOAD_SYMBOL(resp->oh.handle, l[i].s);
+ if (!*(l[i].p)) {
+ resp->oh.handle = NULL;
+ char *msg = LOAD_ERR();
+ LOG(resp->oh.verbose, "dlerr: %s\n", msg);
+ UNLOAD_LIBRARY(resp->oh.handle);
+ snprintf(buf, buflen, "symbol lookup for %s failed: %s", l[i].s, msg);
+ free(msg);
+ resp->err = strdup(buf);
+ return;
+ }
+ }
+
+ LOG(resp->oh.verbose, "calling zesInit\n");
+
+ ret = (*resp->oh.zesInit)(0);
+ if (ret != ZE_RESULT_SUCCESS) {
+ LOG(resp->oh.verbose, "zesInit err: %x\n", ret);
+ snprintf(buf, buflen, "oneapi vram init failure: %x", ret);
+ resp->err = strdup(buf);
+ oneapi_release(resp->oh);
+ return;
+ }
+
+ LOG(resp->oh.verbose, "calling zesDriverGet\n");
+ ret = (*resp->oh.zesDriverGet)(&resp->oh.num_drivers, NULL);
+ if (ret != ZE_RESULT_SUCCESS) {
+ LOG(resp->oh.verbose, "zesDriverGet err: %x\n", ret);
+ snprintf(buf, buflen, "unable to get driver count: %x", ret);
+ resp->err = strdup(buf);
+ oneapi_release(resp->oh);
+ return;
+ }
+ LOG(resp->oh.verbose, "oneapi driver count: %d\n", resp->oh.num_drivers);
+ resp->oh.drivers = malloc(resp->oh.num_drivers * sizeof(zes_driver_handle_t));
+ resp->oh.num_devices = malloc(resp->oh.num_drivers * sizeof(uint32_t));
+ memset(&resp->oh.num_devices[0], 0, resp->oh.num_drivers * sizeof(uint32_t));
+ resp->oh.devices =
+ malloc(resp->oh.num_drivers * sizeof(zes_device_handle_t *));
+ ret = (*resp->oh.zesDriverGet)(&resp->oh.num_drivers, &resp->oh.drivers[0]);
+ if (ret != ZE_RESULT_SUCCESS) {
+ LOG(resp->oh.verbose, "zesDriverGet err: %x\n", ret);
+ snprintf(buf, buflen, "unable to get driver count: %x", ret);
+ resp->err = strdup(buf);
+ oneapi_release(resp->oh);
+ return;
+ }
+
+ for (d = 0; d < resp->oh.num_drivers; d++) {
+ LOG(resp->oh.verbose, "calling zesDeviceGet count %d: %p\n", d, resp->oh.drivers[d]);
+ ret = (*resp->oh.zesDeviceGet)(resp->oh.drivers[d],
+ &resp->oh.num_devices[d], NULL);
+ if (ret != ZE_RESULT_SUCCESS) {
+ LOG(resp->oh.verbose, "zesDeviceGet err: %x\n", ret);
+ snprintf(buf, buflen, "unable to get device count: %x", ret);
+ resp->err = strdup(buf);
+ oneapi_release(resp->oh);
+ return;
+ }
+ resp->oh.devices[d] =
+ malloc(resp->oh.num_devices[d] * sizeof(zes_device_handle_t));
+ ret = (*resp->oh.zesDeviceGet)(
+ resp->oh.drivers[d], &resp->oh.num_devices[d], resp->oh.devices[d]);
+ if (ret != ZE_RESULT_SUCCESS) {
+ LOG(resp->oh.verbose, "zesDeviceGet err: %x\n", ret);
+ snprintf(buf, buflen, "unable to get device count: %x", ret);
+ resp->err = strdup(buf);
+ oneapi_release(resp->oh);
+ return;
+ }
+ }
+
+ return;
+}
+
+void oneapi_check_vram(oneapi_handle_t h, int driver, int device,
+ mem_info_t *resp) {
+ ze_result_t ret;
+ resp->err = NULL;
+ uint64_t totalMem = 0;
+ uint64_t usedMem = 0;
+ const int buflen = 256;
+ char buf[buflen + 1];
+ int i, d, m;
+
+ if (h.handle == NULL) {
+ resp->err = strdup("Level-Zero handle not initialized");
+ return;
+ }
+
+ if (driver > h.num_drivers || device > h.num_devices[driver]) {
+ resp->err = strdup("driver of device index out of bounds");
+ return;
+ }
+
+ resp->total = 0;
+ resp->free = 0;
+
+ zes_device_ext_properties_t ext_props;
+ ext_props.stype = ZES_STRUCTURE_TYPE_DEVICE_EXT_PROPERTIES;
+ ext_props.pNext = NULL;
+
+ zes_device_properties_t props;
+ props.stype = ZES_STRUCTURE_TYPE_DEVICE_PROPERTIES;
+ props.pNext = &ext_props;
+
+ ret = (*h.zesDeviceGetProperties)(h.devices[driver][device], &props);
+ if (ret != ZE_RESULT_SUCCESS) {
+ snprintf(buf, buflen, "unable to get device properties: %d", ret);
+ resp->err = strdup(buf);
+ return;
+ }
+
+ snprintf(&resp->gpu_name[0], GPU_NAME_LEN, "%s", props.modelName);
+
+ // TODO this needs to map to ONEAPI_DEVICE_SELECTOR syntax
+ // (this is probably wrong...)
+ // TODO - the driver isn't included - what if there are multiple drivers?
+ snprintf(&resp->gpu_id[0], GPU_ID_LEN, "%d", device);
+
+ if (h.verbose) {
+ // When in verbose mode, report more information about
+ // the card we discover.
+ LOG(h.verbose, "[%d:%d] oneAPI device name: %s\n", driver, device,
+ props.modelName);
+ LOG(h.verbose, "[%d:%d] oneAPI brand: %s\n", driver, device,
+ props.brandName);
+ LOG(h.verbose, "[%d:%d] oneAPI vendor: %s\n", driver, device,
+ props.vendorName);
+ LOG(h.verbose, "[%d:%d] oneAPI S/N: %s\n", driver, device,
+ props.serialNumber);
+ LOG(h.verbose, "[%d:%d] oneAPI board number: %s\n", driver, device,
+ props.boardNumber);
+ }
+
+ // TODO
+ // Compute Capability equivalent in resp->major, resp->minor, resp->patch
+
+ uint32_t memCount = 0;
+ ret = (*h.zesDeviceEnumMemoryModules)(h.devices[driver][device], &memCount,
+ NULL);
+ if (ret != ZE_RESULT_SUCCESS) {
+ snprintf(buf, buflen, "unable to enumerate Level-Zero memory modules: %x",
+ ret);
+ resp->err = strdup(buf);
+ return;
+ }
+
+ LOG(h.verbose, "discovered %d Level-Zero memory modules\n", memCount);
+
+ zes_mem_handle_t *mems = malloc(memCount * sizeof(zes_mem_handle_t));
+ (*h.zesDeviceEnumMemoryModules)(h.devices[driver][device], &memCount, mems);
+
+ for (m = 0; m < memCount; m++) {
+ zes_mem_state_t state;
+ state.stype = ZES_STRUCTURE_TYPE_MEM_STATE;
+ state.pNext = NULL;
+ ret = (*h.zesMemoryGetState)(mems[m], &state);
+ if (ret != ZE_RESULT_SUCCESS) {
+ snprintf(buf, buflen, "unable to get memory state: %x", ret);
+ resp->err = strdup(buf);
+ free(mems);
+ return;
+ }
+
+ resp->total += state.size;
+ resp->free += state.free;
+ }
+
+ free(mems);
+}
+
+void oneapi_release(oneapi_handle_t h) {
+ int d;
+ LOG(h.verbose, "releasing oneapi library\n");
+ for (d = 0; d < h.num_drivers; d++) {
+ if (h.devices != NULL && h.devices[d] != NULL) {
+ free(h.devices[d]);
+ }
+ }
+ if (h.devices != NULL) {
+ free(h.devices);
+ h.devices = NULL;
+ }
+ if (h.num_devices != NULL) {
+ free(h.num_devices);
+ h.num_devices = NULL;
+ }
+ if (h.drivers != NULL) {
+ free(h.drivers);
+ h.drivers = NULL;
+ }
+ h.num_drivers = 0;
+ UNLOAD_LIBRARY(h.handle);
+ h.handle = NULL;
+}
+
+int oneapi_get_device_count(oneapi_handle_t h, int driver) {
+ if (h.handle == NULL || h.num_devices == NULL) {
+ return 0;
+ }
+ if (driver > h.num_drivers) {
+ return 0;
+ }
+ return (int)h.num_devices[driver];
+}
+
+#endif // __APPLE__
diff --git a/gpu/gpu_info_oneapi.h b/gpu/gpu_info_oneapi.h
new file mode 100644
index 00000000..97fcecd9
--- /dev/null
+++ b/gpu/gpu_info_oneapi.h
@@ -0,0 +1,203 @@
+#ifndef __APPLE__
+#ifndef __GPU_INFO_ONEAPI_H__
+#define __GPU_INFO_ONEAPI_H__
+#include "gpu_info.h"
+
+#define ZE_MAX_DEVICE_NAME 256
+#define ZE_MAX_DEVICE_UUID_SIZE 16
+#define ZES_STRING_PROPERTY_SIZE 64
+#define ZE_BIT(_i) (1 << _i)
+
+// Just enough typedef's to dlopen/dlsym for memory information
+typedef enum ze_result_t {
+ ZE_RESULT_SUCCESS = 0,
+ // Other values omitted for now...
+} ze_result_t;
+
+typedef uint8_t ze_bool_t;
+typedef struct _zes_driver_handle_t *zes_driver_handle_t;
+typedef struct _zes_device_handle_t *zes_device_handle_t;
+typedef struct _zes_mem_handle_t *zes_mem_handle_t;
+
+typedef enum _ze_structure_type_t {
+ ZE_STRUCTURE_TYPE_FORCE_UINT32 = 0x7fffffff
+} ze_structure_type_t;
+
+typedef enum _zes_structure_type_t {
+ ZES_STRUCTURE_TYPE_DEVICE_PROPERTIES = 0x1,
+ ZES_STRUCTURE_TYPE_MEM_PROPERTIES = 0xb,
+ ZES_STRUCTURE_TYPE_MEM_STATE = 0x1e,
+ ZES_STRUCTURE_TYPE_DEVICE_EXT_PROPERTIES = 0x2d,
+ ZES_STRUCTURE_TYPE_FORCE_UINT32 = 0x7fffffff
+} zes_structure_type_t;
+
+typedef enum _zes_mem_type_t {
+ ZES_MEM_TYPE_FORCE_UINT32 = 0x7fffffff
+} zes_mem_type_t;
+
+typedef enum _zes_mem_loc_t {
+ ZES_MEM_LOC_SYSTEM = 0,
+ ZES_MEM_LOC_DEVICE = 1,
+ ZES_MEM_LOC_FORCE_UINT32 = 0x7fffffff
+} zes_mem_loc_t;
+
+typedef enum _zes_mem_health_t {
+ ZES_MEM_HEALTH_FORCE_UINT32 = 0x7fffffff
+} zes_mem_health_t;
+
+typedef struct _ze_device_uuid_t {
+ uint8_t id[ZE_MAX_DEVICE_UUID_SIZE];
+} ze_device_uuid_t;
+
+typedef struct _zes_uuid_t {
+ uint8_t id[ZE_MAX_DEVICE_UUID_SIZE];
+} zes_uuid_t;
+
+typedef enum _ze_device_type_t {
+ ZE_DEVICE_TYPE_GPU = 1,
+ ZE_DEVICE_TYPE_CPU = 2,
+ ZE_DEVICE_TYPE_FPGA = 3,
+ ZE_DEVICE_TYPE_MCA = 4,
+ ZE_DEVICE_TYPE_VPU = 5,
+ ZE_DEVICE_TYPE_FORCE_UINT32 = 0x7fffffff
+} ze_device_type_t;
+
+typedef enum _zes_device_type_t {
+ ZES_DEVICE_TYPE_GPU = 1,
+ ZES_DEVICE_TYPE_CPU = 2,
+ ZES_DEVICE_TYPE_FPGA = 3,
+ ZES_DEVICE_TYPE_MCA = 4,
+ ZES_DEVICE_TYPE_VPU = 5,
+ ZES_DEVICE_TYPE_FORCE_UINT32 = 0x7fffffff
+} zes_device_type_t;
+
+typedef uint32_t ze_device_property_flags_t;
+typedef enum _ze_device_property_flag_t {
+ ZE_DEVICE_PROPERTY_FLAG_INTEGRATED = ZE_BIT(0),
+ ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE = ZE_BIT(1),
+ ZE_DEVICE_PROPERTY_FLAG_ECC = ZE_BIT(2),
+ ZE_DEVICE_PROPERTY_FLAG_ONDEMANDPAGING = ZE_BIT(3),
+ ZE_DEVICE_PROPERTY_FLAG_FORCE_UINT32 = 0x7fffffff
+} ze_device_property_flag_t;
+
+typedef uint32_t zes_device_property_flags_t;
+typedef enum _zes_device_property_flag_t {
+ ZES_DEVICE_PROPERTY_FLAG_INTEGRATED = ZE_BIT(0),
+ ZES_DEVICE_PROPERTY_FLAG_SUBDEVICE = ZE_BIT(1),
+ ZES_DEVICE_PROPERTY_FLAG_ECC = ZE_BIT(2),
+ ZES_DEVICE_PROPERTY_FLAG_ONDEMANDPAGING = ZE_BIT(3),
+ ZES_DEVICE_PROPERTY_FLAG_FORCE_UINT32 = 0x7fffffff
+} zes_device_property_flag_t;
+
+typedef struct _ze_device_properties_t {
+ ze_structure_type_t stype;
+ void *pNext;
+ ze_device_type_t type;
+ uint32_t vendorId;
+ uint32_t deviceId;
+ ze_device_property_flags_t flags;
+ uint32_t subdeviceId;
+ uint32_t coreClockRate;
+ uint64_t maxMemAllocSize;
+ uint32_t maxHardwareContexts;
+ uint32_t maxCommandQueuePriority;
+ uint32_t numThreadsPerEU;
+ uint32_t physicalEUSimdWidth;
+ uint32_t numEUsPerSubslice;
+ uint32_t numSubslicesPerSlice;
+ uint32_t numSlices;
+ uint64_t timerResolution;
+ uint32_t timestampValidBits;
+ uint32_t kernelTimestampValidBits;
+ ze_device_uuid_t uuid;
+ char name[ZE_MAX_DEVICE_NAME];
+} ze_device_properties_t;
+
+typedef struct _zes_device_properties_t {
+ zes_structure_type_t stype;
+ void *pNext;
+ ze_device_properties_t core;
+ uint32_t numSubdevices;
+ char serialNumber[ZES_STRING_PROPERTY_SIZE];
+ char boardNumber[ZES_STRING_PROPERTY_SIZE];
+ char brandName[ZES_STRING_PROPERTY_SIZE];
+ char modelName[ZES_STRING_PROPERTY_SIZE];
+ char vendorName[ZES_STRING_PROPERTY_SIZE];
+ char driverVersion[ZES_STRING_PROPERTY_SIZE];
+} zes_device_properties_t;
+
+typedef struct _zes_device_ext_properties_t {
+ zes_structure_type_t stype;
+ void *pNext;
+ zes_uuid_t uuid;
+ zes_device_type_t type;
+ zes_device_property_flags_t flags;
+} zes_device_ext_properties_t;
+
+typedef struct _zes_mem_properties_t {
+ zes_structure_type_t stype;
+ void *pNext;
+ zes_mem_type_t type;
+ ze_bool_t onSubdevice;
+ uint32_t subdeviceId;
+ zes_mem_loc_t location;
+ uint64_t physicalSize;
+ int32_t busWidth;
+ int32_t numChannels;
+} zes_mem_properties_t;
+
+typedef struct _zes_mem_state_t {
+ zes_structure_type_t stype;
+ const void *pNext;
+ zes_mem_health_t health;
+ uint64_t free;
+ uint64_t size;
+} zes_mem_state_t;
+
+typedef struct oneapi_handle {
+ void *handle;
+ uint16_t verbose;
+
+ uint32_t num_drivers;
+ zes_driver_handle_t *drivers;
+ uint32_t *num_devices;
+ zes_device_handle_t **devices;
+
+ // TODO Driver major, minor information
+ // int driver_major;
+ // int driver_minor;
+
+ ze_result_t (*zesInit)(int);
+ ze_result_t (*zesDriverGet)(uint32_t *pCount, zes_driver_handle_t *phDrivers);
+ ze_result_t (*zesDeviceGet)(zes_driver_handle_t hDriver, uint32_t *pCount,
+ zes_device_handle_t *phDevices);
+ ze_result_t (*zesDeviceGetProperties)(zes_device_handle_t hDevice,
+ zes_device_properties_t *pProperties);
+ ze_result_t (*zesDeviceEnumMemoryModules)(zes_device_handle_t hDevice,
+ uint32_t *pCount,
+ zes_mem_handle_t *phMemory);
+ ze_result_t (*zesMemoryGetProperties)(zes_mem_handle_t hMemory,
+ zes_mem_properties_t *pProperties);
+ ze_result_t (*zesMemoryGetState)(zes_mem_handle_t hMemory,
+ zes_mem_state_t *pState);
+
+} oneapi_handle_t;
+
+typedef struct oneapi_init_resp {
+ char *err; // If err is non-null handle is invalid
+ oneapi_handle_t oh;
+} oneapi_init_resp_t;
+
+typedef struct oneapi_version_resp {
+ ze_result_t status;
+ char *str; // Contains version or error string if status != 0
+} oneapi_version_resp_t;
+
+void oneapi_init(char *oneapi_lib_path, oneapi_init_resp_t *resp);
+void oneapi_check_vram(oneapi_handle_t h, int driver, int device,
+ mem_info_t *resp);
+void oneapi_release(oneapi_handle_t h);
+int oneapi_get_device_count(oneapi_handle_t h, int driver);
+
+#endif // __GPU_INFO_INTEL_H__
+#endif // __APPLE__
diff --git a/gpu/gpu_info_rocm.c b/gpu/gpu_info_rocm.c
deleted file mode 100644
index e69d5cba..00000000
--- a/gpu/gpu_info_rocm.c
+++ /dev/null
@@ -1,114 +0,0 @@
-#ifndef __APPLE__
-
-#include "gpu_info_rocm.h"
-
-#include <string.h>
-
-#ifndef _WIN32
-const char *rocm_lib_paths[] = {
- "librocm_smi64.so",
- "/opt/rocm/lib/librocm_smi64.so",
- NULL,
-};
-#else
-// TODO untested
-const char *rocm_lib_paths[] = {
- "rocm_smi64.dll",
- "/opt/rocm/lib/rocm_smi64.dll",
- NULL,
-};
-#endif
-
-void rocm_init(rocm_init_resp_t *resp) {
- rsmi_status_t ret;
- resp->err = NULL;
- const int buflen = 256;
- char buf[buflen + 1];
- int i;
- struct lookup {
- char *s;
- void **p;
- } l[4] = {
- {"rsmi_init", (void *)&resp->rh.initFn},
- {"rsmi_shut_down", (void *)&resp->rh.shutdownFn},
- {"rsmi_dev_memory_total_get", (void *)&resp->rh.totalMemFn},
- {"rsmi_dev_memory_usage_get", (void *)&resp->rh.usageMemFn},
- // { "rsmi_dev_id_get", (void*)&resp->rh.getHandle },
- };
-
- for (i = 0; rocm_lib_paths[i] != NULL && resp->rh.handle == NULL; i++) {
- resp->rh.handle = LOAD_LIBRARY(rocm_lib_paths[i], RTLD_LAZY);
- }
- if (!resp->rh.handle) {
- snprintf(buf, buflen,
- "Unable to load %s library to query for Radeon GPUs: %s\n",
- rocm_lib_paths[0], LOAD_ERR());
- resp->err = strdup(buf);
- return;
- }
-
- for (i = 0; i < 4; i++) {
- *l[i].p = LOAD_SYMBOL(resp->rh.handle, l[i].s);
- if (!l[i].p) {
- UNLOAD_LIBRARY(resp->rh.handle);
- snprintf(buf, buflen, "symbol lookup for %s failed: %s", l[i].s,
- LOAD_ERR());
- resp->err = strdup(buf);
- return;
- }
- }
-
- ret = (*resp->rh.initFn)(0);
- if (ret != RSMI_STATUS_SUCCESS) {
- snprintf(buf, buflen, "rocm vram init failure: %d", ret);
- resp->err = strdup(buf);
- }
-
- return;
-}
-
-void rocm_check_vram(rocm_handle_t h, mem_info_t *resp) {
- resp->err = NULL;
- // uint32_t num_devices;
- // uint16_t device;
- uint64_t totalMem = 0;
- uint64_t usedMem = 0;
- rsmi_status_t ret;
- const int buflen = 256;
- char buf[buflen + 1];
- int i;
-
- if (h.handle == NULL) {
- resp->err = strdup("nvml handle sn't initialized");
- return;
- }
-
- // TODO - iterate through devices... ret =
- // rsmi_num_monitor_devices(&num_devices);
-
- // ret = (*h.getHandle)(0, &device);
- // if (ret != RSMI_STATUS_SUCCESS) {
- // printf("rocm vram device lookup failure: %d\n", ret);
- // return -1;
- // }
-
- // Get total memory - used memory for available memory
- ret = (*h.totalMemFn)(0, RSMI_MEM_TYPE_VRAM, &totalMem);
- if (ret != RSMI_STATUS_SUCCESS) {
- snprintf(buf, buflen, "rocm total mem lookup failure: %d", ret);
- resp->err = strdup(buf);
- return;
- }
- ret = (*h.usageMemFn)(0, RSMI_MEM_TYPE_VRAM, &usedMem);
- if (ret != RSMI_STATUS_SUCCESS) {
- snprintf(buf, buflen, "rocm usage mem lookup failure: %d", ret);
- resp->err = strdup(buf);
- return;
- }
-
- resp->total = totalMem;
- resp->free = totalMem - usedMem;
- return;
-}
-
-#endif // __APPLE__
\ No newline at end of file
diff --git a/gpu/gpu_info_rocm.h b/gpu/gpu_info_rocm.h
deleted file mode 100644
index 8d7a04ae..00000000
--- a/gpu/gpu_info_rocm.h
+++ /dev/null
@@ -1,36 +0,0 @@
-#ifndef __APPLE__
-#ifndef __GPU_INFO_ROCM_H__
-#define __GPU_INFO_ROCM_H__
-#include "gpu_info.h"
-
-// Just enough typedef's to dlopen/dlsym for memory information
-typedef enum rsmi_status_return {
- RSMI_STATUS_SUCCESS = 0,
- // Other values omitted for now...
-} rsmi_status_t;
-
-typedef enum rsmi_memory_type {
- RSMI_MEM_TYPE_VRAM = 0,
- RSMI_MEM_TYPE_VIS_VRAM,
- RSMI_MEM_TYPE_GTT,
-} rsmi_memory_type_t;
-
-typedef struct rocm_handle {
- void *handle;
- rsmi_status_t (*initFn)(uint64_t);
- rsmi_status_t (*shutdownFn)(void);
- rsmi_status_t (*totalMemFn)(uint32_t, rsmi_memory_type_t, uint64_t *);
- rsmi_status_t (*usageMemFn)(uint32_t, rsmi_memory_type_t, uint64_t *);
- // rsmi_status_t (*getHandle)(uint32_t, uint16_t *);
-} rocm_handle_t;
-
-typedef struct rocm_init_resp {
- char *err; // If err is non-null handle is invalid
- rocm_handle_t rh;
-} rocm_init_resp_t;
-
-void rocm_init(rocm_init_resp_t *resp);
-void rocm_check_vram(rocm_handle_t rh, mem_info_t *resp);
-
-#endif // __GPU_INFO_ROCM_H__
-#endif // __APPLE__
\ No newline at end of file
diff --git a/gpu/gpu_linux.go b/gpu/gpu_linux.go
new file mode 100644
index 00000000..b15bc2d2
--- /dev/null
+++ b/gpu/gpu_linux.go
@@ -0,0 +1,186 @@
+package gpu
+
+import (
+ "bufio"
+ "fmt"
+ "os"
+ "reflect"
+ "regexp"
+ "strings"
+
+ "github.com/ollama/ollama/format"
+)
+
+var CudartGlobs = []string{
+ "/usr/local/cuda/lib64/libcudart.so*",
+ "/usr/lib/x86_64-linux-gnu/nvidia/current/libcudart.so*",
+ "/usr/lib/x86_64-linux-gnu/libcudart.so*",
+ "/usr/lib/wsl/lib/libcudart.so*",
+ "/usr/lib/wsl/drivers/*/libcudart.so*",
+ "/opt/cuda/lib64/libcudart.so*",
+ "/usr/local/cuda*/targets/aarch64-linux/lib/libcudart.so*",
+ "/usr/lib/aarch64-linux-gnu/nvidia/current/libcudart.so*",
+ "/usr/lib/aarch64-linux-gnu/libcudart.so*",
+ "/usr/local/cuda/lib*/libcudart.so*",
+ "/usr/lib*/libcudart.so*",
+ "/usr/local/lib*/libcudart.so*",
+}
+
+var NvmlGlobs = []string{}
+
+var NvcudaGlobs = []string{
+ "/usr/local/cuda*/targets/*/lib/libcuda.so*",
+ "/usr/lib/*-linux-gnu/nvidia/current/libcuda.so*",
+ "/usr/lib/*-linux-gnu/libcuda.so*",
+ "/usr/lib/wsl/lib/libcuda.so*",
+ "/usr/lib/wsl/drivers/*/libcuda.so*",
+ "/opt/cuda/lib*/libcuda.so*",
+ "/usr/local/cuda/lib*/libcuda.so*",
+ "/usr/lib*/libcuda.so*",
+ "/usr/local/lib*/libcuda.so*",
+}
+
+var OneapiGlobs = []string{
+ "/usr/lib/x86_64-linux-gnu/libze_intel_gpu.so*",
+ "/usr/lib*/libze_intel_gpu.so*",
+}
+
+var (
+ CudartMgmtName = "libcudart.so*"
+ NvcudaMgmtName = "libcuda.so*"
+ NvmlMgmtName = "" // not currently wired on linux
+ OneapiMgmtName = "libze_intel_gpu.so*"
+)
+
+func GetCPUMem() (memInfo, error) {
+ var mem memInfo
+ var total, available, free, buffers, cached, freeSwap uint64
+ f, err := os.Open("/proc/meminfo")
+ if err != nil {
+ return mem, err
+ }
+ defer f.Close()
+ s := bufio.NewScanner(f)
+ for s.Scan() {
+ line := s.Text()
+ switch {
+ case strings.HasPrefix(line, "MemTotal:"):
+ _, err = fmt.Sscanf(line, "MemTotal:%d", &total)
+ case strings.HasPrefix(line, "MemAvailable:"):
+ _, err = fmt.Sscanf(line, "MemAvailable:%d", &available)
+ case strings.HasPrefix(line, "MemFree:"):
+ _, err = fmt.Sscanf(line, "MemFree:%d", &free)
+ case strings.HasPrefix(line, "Buffers:"):
+ _, err = fmt.Sscanf(line, "Buffers:%d", &buffers)
+ case strings.HasPrefix(line, "Cached:"):
+ _, err = fmt.Sscanf(line, "Cached:%d", &cached)
+ case strings.HasPrefix(line, "SwapFree:"):
+ _, err = fmt.Sscanf(line, "SwapFree:%d", &freeSwap)
+ default:
+ continue
+ }
+ if err != nil {
+ return mem, err
+ }
+ }
+ mem.TotalMemory = total * format.KibiByte
+ mem.FreeSwap = freeSwap * format.KibiByte
+ if available > 0 {
+ mem.FreeMemory = available * format.KibiByte
+ } else {
+ mem.FreeMemory = (free + buffers + cached) * format.KibiByte
+ }
+ return mem, nil
+}
+
+const CpuInfoFilename = "/proc/cpuinfo"
+
+type linuxCpuInfo struct {
+ ID string `cpuinfo:"processor"`
+ VendorID string `cpuinfo:"vendor_id"`
+ ModelName string `cpuinfo:"model name"`
+ PhysicalID string `cpuinfo:"physical id"`
+ Siblings string `cpuinfo:"siblings"`
+ CoreID string `cpuinfo:"core id"`
+}
+
+func GetCPUDetails() ([]CPU, error) {
+ file, err := os.Open(CpuInfoFilename)
+ if err != nil {
+ return nil, err
+ }
+ reColumns := regexp.MustCompile("\t+: ")
+ scanner := bufio.NewScanner(file)
+ cpuInfos := []linuxCpuInfo{}
+ cpu := &linuxCpuInfo{}
+ for scanner.Scan() {
+ line := scanner.Text()
+ if sl := reColumns.Split(line, 2); len(sl) > 1 {
+ t := reflect.TypeOf(cpu).Elem()
+ s := reflect.ValueOf(cpu).Elem()
+ for i := range t.NumField() {
+ field := t.Field(i)
+ tag := field.Tag.Get("cpuinfo")
+ if tag == sl[0] {
+ s.FieldByName(field.Name).SetString(sl[1])
+ break
+ }
+ }
+ } else if strings.TrimSpace(line) == "" && cpu.ID != "" {
+ cpuInfos = append(cpuInfos, *cpu)
+ cpu = &linuxCpuInfo{}
+ }
+ }
+
+ // Process the sockets/cores/threads
+ socketByID := map[string]*CPU{}
+ coreBySocket := map[string]map[string]struct{}{}
+ threadsByCoreBySocket := map[string]map[string]int{}
+ for _, c := range cpuInfos {
+ if _, found := socketByID[c.PhysicalID]; !found {
+ socketByID[c.PhysicalID] = &CPU{
+ ID: c.PhysicalID,
+ VendorID: c.VendorID,
+ ModelName: c.ModelName,
+ }
+ coreBySocket[c.PhysicalID] = map[string]struct{}{}
+ threadsByCoreBySocket[c.PhysicalID] = map[string]int{}
+ }
+ if c.CoreID != "" {
+ coreBySocket[c.PhysicalID][c.PhysicalID+":"+c.CoreID] = struct{}{}
+ threadsByCoreBySocket[c.PhysicalID][c.PhysicalID+":"+c.CoreID]++
+ } else {
+ coreBySocket[c.PhysicalID][c.PhysicalID+":"+c.ID] = struct{}{}
+ threadsByCoreBySocket[c.PhysicalID][c.PhysicalID+":"+c.ID]++
+ }
+ }
+
+ // Tally up the values from the tracking maps
+ for id, s := range socketByID {
+ s.CoreCount = len(coreBySocket[id])
+ s.ThreadCount = 0
+ for _, tc := range threadsByCoreBySocket[id] {
+ s.ThreadCount += tc
+ }
+
+ // This only works if HT is enabled, consider a more reliable model, maybe cache size comparisons?
+ efficiencyCoreCount := 0
+ for _, threads := range threadsByCoreBySocket[id] {
+ if threads == 1 {
+ efficiencyCoreCount++
+ }
+ }
+ if efficiencyCoreCount == s.CoreCount {
+ // 1:1 mapping means they're not actually efficiency cores, but regular cores
+ s.EfficiencyCoreCount = 0
+ } else {
+ s.EfficiencyCoreCount = efficiencyCoreCount
+ }
+ }
+
+ result := []CPU{}
+ for _, c := range socketByID {
+ result = append(result, *c)
+ }
+ return result, nil
+}
diff --git a/gpu/gpu_oneapi.go b/gpu/gpu_oneapi.go
new file mode 100644
index 00000000..9864bde5
--- /dev/null
+++ b/gpu/gpu_oneapi.go
@@ -0,0 +1,21 @@
+//go:build linux || windows
+
+package gpu
+
+import (
+ "log/slog"
+ "strings"
+)
+
+func oneapiGetVisibleDevicesEnv(gpuInfo []GpuInfo) (string, string) {
+ ids := []string{}
+ for _, info := range gpuInfo {
+ if info.Library != "oneapi" {
+ // TODO shouldn't happen if things are wired correctly...
+ slog.Debug("oneapiGetVisibleDevicesEnv skipping over non-sycl device", "library", info.Library)
+ continue
+ }
+ ids = append(ids, info.ID)
+ }
+ return "ONEAPI_DEVICE_SELECTOR", "level_zero:" + strings.Join(ids, ",")
+}
diff --git a/gpu/gpu_test.go b/gpu/gpu_test.go
index cbdcf3ec..13a3f544 100644
--- a/gpu/gpu_test.go
+++ b/gpu/gpu_test.go
@@ -5,16 +5,25 @@ import (
"testing"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
func TestBasicGetGPUInfo(t *testing.T) {
info := GetGPUInfo()
- assert.Contains(t, "CUDA ROCM CPU METAL", info.Driver)
+ assert.NotEmpty(t, len(info))
+ assert.Contains(t, "cuda rocm cpu metal", info[0].Library)
+ if info[0].Library != "cpu" {
+ assert.Greater(t, info[0].TotalMemory, uint64(0))
+ assert.Greater(t, info[0].FreeMemory, uint64(0))
+ }
+}
+func TestCPUMemInfo(t *testing.T) {
+ info, err := GetCPUMem()
+ require.NoError(t, err)
switch runtime.GOOS {
case "darwin":
- // TODO - remove this once MacOS returns some size for CPU
- return
+ t.Skip("CPU memory not populated on darwin")
case "linux", "windows":
assert.Greater(t, info.TotalMemory, uint64(0))
assert.Greater(t, info.FreeMemory, uint64(0))
@@ -23,4 +32,29 @@ func TestBasicGetGPUInfo(t *testing.T) {
}
}
+func TestByLibrary(t *testing.T) {
+ type testCase struct {
+ input []GpuInfo
+ expect int
+ }
+
+ testCases := map[string]*testCase{
+ "empty": {input: []GpuInfo{}, expect: 0},
+ "cpu": {input: []GpuInfo{{Library: "cpu"}}, expect: 1},
+ "cpu + GPU": {input: []GpuInfo{{Library: "cpu"}, {Library: "cuda"}}, expect: 2},
+ "cpu + 2 GPU no variant": {input: []GpuInfo{{Library: "cpu"}, {Library: "cuda"}, {Library: "cuda"}}, expect: 2},
+ "cpu + 2 GPU same variant": {input: []GpuInfo{{Library: "cpu"}, {Library: "cuda", Variant: "v11"}, {Library: "cuda", Variant: "v11"}}, expect: 2},
+ "cpu + 2 GPU diff variant": {input: []GpuInfo{{Library: "cpu"}, {Library: "cuda", Variant: "v11"}, {Library: "cuda", Variant: "v12"}}, expect: 3},
+ }
+
+ for k, v := range testCases {
+ t.Run(k, func(t *testing.T) {
+ resp := (GpuInfoList)(v.input).ByLibrary()
+ if len(resp) != v.expect {
+ t.Fatalf("expected length %d, got %d => %+v", v.expect, len(resp), resp)
+ }
+ })
+ }
+}
+
// TODO - add some logic to figure out card type through other means and actually verify we got back what we expected
diff --git a/gpu/gpu_windows.go b/gpu/gpu_windows.go
new file mode 100644
index 00000000..da0dce92
--- /dev/null
+++ b/gpu/gpu_windows.go
@@ -0,0 +1,234 @@
+package gpu
+
+import (
+ "fmt"
+ "log/slog"
+ "syscall"
+ "unsafe"
+)
+
+type MEMORYSTATUSEX struct {
+ length uint32
+ MemoryLoad uint32
+ TotalPhys uint64
+ AvailPhys uint64
+ TotalPageFile uint64
+ AvailPageFile uint64
+ TotalVirtual uint64
+ AvailVirtual uint64
+ AvailExtendedVirtual uint64
+}
+
+var (
+ k32 = syscall.NewLazyDLL("kernel32.dll")
+ globalMemoryStatusExProc = k32.NewProc("GlobalMemoryStatusEx")
+ sizeofMemoryStatusEx = uint32(unsafe.Sizeof(MEMORYSTATUSEX{}))
+ GetLogicalProcessorInformationEx = k32.NewProc("GetLogicalProcessorInformationEx")
+)
+
+var CudartGlobs = []string{
+ "c:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v*\\bin\\cudart64_*.dll",
+}
+
+var NvmlGlobs = []string{
+ "c:\\Windows\\System32\\nvml.dll",
+}
+
+var NvcudaGlobs = []string{
+ "c:\\windows\\system*\\nvcuda.dll",
+}
+
+var OneapiGlobs = []string{
+ "c:\\Windows\\System32\\DriverStore\\FileRepository\\*\\ze_intel_gpu64.dll",
+}
+
+var (
+ CudartMgmtName = "cudart64_*.dll"
+ NvcudaMgmtName = "nvcuda.dll"
+ NvmlMgmtName = "nvml.dll"
+ OneapiMgmtName = "ze_intel_gpu64.dll"
+)
+
+func GetCPUMem() (memInfo, error) {
+ memStatus := MEMORYSTATUSEX{length: sizeofMemoryStatusEx}
+ r1, _, err := globalMemoryStatusExProc.Call(uintptr(unsafe.Pointer(&memStatus)))
+ if r1 == 0 {
+ return memInfo{}, fmt.Errorf("GlobalMemoryStatusEx failed: %w", err)
+ }
+ return memInfo{TotalMemory: memStatus.TotalPhys, FreeMemory: memStatus.AvailPhys, FreeSwap: memStatus.AvailPageFile}, nil
+}
+
+type LOGICAL_PROCESSOR_RELATIONSHIP uint32
+
+const (
+ RelationProcessorCore LOGICAL_PROCESSOR_RELATIONSHIP = iota
+ RelationNumaNode
+ RelationCache
+ RelationProcessorPackage
+ RelationGroup
+ RelationProcessorDie
+ RelationNumaNodeEx
+ RelationProcessorModule
+)
+const RelationAll LOGICAL_PROCESSOR_RELATIONSHIP = 0xffff
+
+type GROUP_AFFINITY struct {
+ Mask uintptr // KAFFINITY
+ Group uint16
+ Reserved [3]uint16
+}
+
+type PROCESSOR_RELATIONSHIP struct {
+ Flags byte
+ EfficiencyClass byte
+ Reserved [20]byte
+ GroupCount uint16
+ GroupMask [1]GROUP_AFFINITY // len GroupCount
+}
+
+// Omitted unused structs: NUMA_NODE_RELATIONSHIP CACHE_RELATIONSHIP GROUP_RELATIONSHIP
+
+type SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX struct {
+ Relationship LOGICAL_PROCESSOR_RELATIONSHIP
+ Size uint32
+ U [1]byte // Union len Size
+ // PROCESSOR_RELATIONSHIP
+ // NUMA_NODE_RELATIONSHIP
+ // CACHE_RELATIONSHIP
+ // GROUP_RELATIONSHIP
+}
+
+func (group *GROUP_AFFINITY) IsMember(target *GROUP_AFFINITY) bool {
+ if group == nil || target == nil {
+ return false
+ }
+ return group.Mask&target.Mask != 0
+}
+
+type winPackage struct {
+ groups []*GROUP_AFFINITY
+ coreCount int // performance cores = coreCount - efficiencyCoreCount
+ efficiencyCoreCount int
+ threadCount int
+}
+
+func (pkg *winPackage) IsMember(target *GROUP_AFFINITY) bool {
+ for _, group := range pkg.groups {
+ if group.IsMember(target) {
+ return true
+ }
+ }
+ return false
+}
+
+func getLogicalProcessorInformationEx() ([]byte, error) {
+ buf := make([]byte, 1)
+ bufSize := len(buf)
+ ret, _, err := GetLogicalProcessorInformationEx.Call(
+ uintptr(RelationAll),
+ uintptr(unsafe.Pointer(&buf[0])),
+ uintptr(unsafe.Pointer(&bufSize)),
+ )
+ if ret != 0 {
+ return nil, fmt.Errorf("failed to determine size info ret:%d %w", ret, err)
+ }
+
+ buf = make([]byte, bufSize)
+ ret, _, err = GetLogicalProcessorInformationEx.Call(
+ uintptr(RelationAll),
+ uintptr(unsafe.Pointer(&buf[0])),
+ uintptr(unsafe.Pointer(&bufSize)),
+ )
+ if ret == 0 {
+ return nil, fmt.Errorf("failed to gather processor information ret:%d buflen:%d %w", ret, bufSize, err)
+ }
+ return buf, nil
+}
+
+func processSystemLogicalProcessorInforationList(buf []byte) []*winPackage {
+ var slpi *SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX
+ // Find all the packages first
+ packages := []*winPackage{}
+ for bufOffset := 0; bufOffset < len(buf); bufOffset += int(slpi.Size) {
+ slpi = (*SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)(unsafe.Pointer(&buf[bufOffset]))
+ if slpi.Relationship != RelationProcessorPackage {
+ continue
+ }
+ pr := (*PROCESSOR_RELATIONSHIP)(unsafe.Pointer(&slpi.U[0]))
+ pkg := &winPackage{}
+ ga0 := unsafe.Pointer(&pr.GroupMask[0])
+ for j := range pr.GroupCount {
+ gm := (*GROUP_AFFINITY)(unsafe.Pointer(uintptr(ga0) + uintptr(j)*unsafe.Sizeof(GROUP_AFFINITY{})))
+ pkg.groups = append(pkg.groups, gm)
+ }
+ packages = append(packages, pkg)
+ }
+
+ slog.Info("packages", "count", len(packages))
+
+ // To identify efficiency cores we have to compare the relative values
+ // Larger values are "less efficient" (aka, more performant)
+ var maxEfficiencyClass byte
+ for bufOffset := 0; bufOffset < len(buf); bufOffset += int(slpi.Size) {
+ slpi = (*SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)(unsafe.Pointer(&buf[bufOffset]))
+ if slpi.Relationship != RelationProcessorCore {
+ continue
+ }
+ pr := (*PROCESSOR_RELATIONSHIP)(unsafe.Pointer(&slpi.U[0]))
+ if pr.EfficiencyClass > maxEfficiencyClass {
+ maxEfficiencyClass = pr.EfficiencyClass
+ }
+ }
+ if maxEfficiencyClass > 0 {
+ slog.Info("efficiency cores detected", "maxEfficiencyClass", maxEfficiencyClass)
+ }
+
+ // then match up the Cores to the Packages, count up cores, threads and efficiency cores
+ for bufOffset := 0; bufOffset < len(buf); bufOffset += int(slpi.Size) {
+ slpi = (*SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)(unsafe.Pointer(&buf[bufOffset]))
+ if slpi.Relationship != RelationProcessorCore {
+ continue
+ }
+ pr := (*PROCESSOR_RELATIONSHIP)(unsafe.Pointer(&slpi.U[0]))
+ ga0 := unsafe.Pointer(&pr.GroupMask[0])
+ for j := range pr.GroupCount {
+ gm := (*GROUP_AFFINITY)(unsafe.Pointer(uintptr(ga0) + uintptr(j)*unsafe.Sizeof(GROUP_AFFINITY{})))
+ for _, pkg := range packages {
+ if pkg.IsMember(gm) {
+ pkg.coreCount++
+ if pr.Flags == 0 {
+ pkg.threadCount++
+ } else {
+ pkg.threadCount += 2
+ }
+ if pr.EfficiencyClass < maxEfficiencyClass {
+ pkg.efficiencyCoreCount++
+ }
+ }
+ }
+ }
+ }
+
+ // Sumarize the results
+ for i, pkg := range packages {
+ slog.Info("", "package", i, "cores", pkg.coreCount, "efficiency", pkg.efficiencyCoreCount, "threads", pkg.threadCount)
+ }
+
+ return packages
+}
+
+func GetCPUDetails() ([]CPU, error) {
+ buf, err := getLogicalProcessorInformationEx()
+ if err != nil {
+ return nil, err
+ }
+ packages := processSystemLogicalProcessorInforationList(buf)
+ cpus := make([]CPU, len(packages))
+
+ for i, pkg := range packages {
+ cpus[i].CoreCount = pkg.coreCount
+ cpus[i].EfficiencyCoreCount = pkg.efficiencyCoreCount
+ cpus[i].ThreadCount = pkg.threadCount
+ }
+ return cpus, nil
+}
diff --git a/gpu/gpu_windows_test.go b/gpu/gpu_windows_test.go
new file mode 100644
index 00000000..27e609cf
--- /dev/null
+++ b/gpu/gpu_windows_test.go
@@ -0,0 +1,77 @@
+package gpu
+
+import "testing"
+
+func TestProcessSystemLogicalProcessorInforationList(t *testing.T) {
+ type pkgs struct {
+ cores int
+ efficiency int
+ threads int
+ }
+ type testCase struct {
+ input []byte
+ expected []pkgs
+ }
+ testCases := map[string]*testCase{
+ "AMD64 Family 25 Model 97 Stepping 2 ": {
+ input: []byte{
+ 0x3, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x3, 0x10, 0x40, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x3, 0x10, 0x40, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x8, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x50, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x20, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0,
+ },
+ expected: []pkgs{
+ {
+ cores: 16,
+ efficiency: 0,
+ threads: 32,
+ },
+ },
+ },
+ "Intel64 Family 6 Model 183 Stepping 1": {
+ input: []byte{
+ 0x3, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0xc, 0x40, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x3, 0xc, 0x40, 0x0, 0x0, 0x0, 0xe0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0xc, 0x40, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0xc, 0x40, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0xc, 0x40, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0xc, 0x40, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0xc, 0x40, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0xc, 0x40, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0xc, 0x40, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x50, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x18, 0x18, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0,
+ },
+ expected: []pkgs{
+ {
+ cores: 16,
+ efficiency: 8,
+ threads: 24,
+ },
+ },
+ },
+ "dual Intel64 Family 6 Model 85 Stepping 4": {
+ input: []byte{
+ 0x3, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x3, 0xb, 0x40, 0x0, 0x0, 0x0, 0xb8, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x3, 0xb, 0x40, 0x0, 0x0, 0x0, 0xb8, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x1, 0x8, 0x40, 0x0, 0x0, 0x80, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x38, 0x0, 0x0, 0x0, 0x2, 0x10, 0x40, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x2, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x28, 0x28, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x28, 0x28, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0,
+ },
+ expected: []pkgs{
+ {
+ cores: 40,
+ efficiency: 0,
+ threads: 80,
+ }, {
+ cores: 40,
+ efficiency: 0,
+ threads: 80,
+ },
+ },
+ },
+ }
+
+ for k, v := range testCases {
+ t.Run(k, func(t *testing.T) {
+ resp := processSystemLogicalProcessorInforationList(v.input)
+ if len(resp) != len(v.expected) {
+ t.Fatalf("incorrect number of packages %d, got %d", v.expected, len(resp))
+ }
+ for i, pkg := range v.expected {
+ if resp[i].coreCount != pkg.cores {
+ t.Fatalf("[%d] expected core count %d, got %d", i, pkg.cores, resp[i].coreCount)
+ }
+ if resp[i].efficiencyCoreCount != pkg.efficiency {
+ t.Fatalf("[%d] expected efficiency count %d, got %d", i, pkg.efficiency, resp[i].efficiencyCoreCount)
+ }
+ if resp[i].threadCount != pkg.threads {
+ t.Fatalf("[%d] expected thread count %d, got %d", i, pkg.threads, resp[i].threadCount)
+ }
+ }
+ })
+ }
+}
diff --git a/gpu/types.go b/gpu/types.go
index a56da45e..655c8733 100644
--- a/gpu/types.go
+++ b/gpu/types.go
@@ -1,11 +1,180 @@
package gpu
-// Beginning of an `ollama info` command
-type GpuInfo struct {
- Driver string `json:"driver,omitempty"`
- Library string `json:"library,omitempty"`
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/ollama/ollama/format"
+)
+
+type memInfo struct {
TotalMemory uint64 `json:"total_memory,omitempty"`
FreeMemory uint64 `json:"free_memory,omitempty"`
-
- // TODO add other useful attributes about the card here for discovery information
+ FreeSwap uint64 `json:"free_swap,omitempty"` // TODO split this out for system only
+}
+
+// Beginning of an `ollama info` command
+type GpuInfo struct { // TODO better name maybe "InferenceProcessor"?
+ memInfo
+ Library string `json:"library,omitempty"`
+
+ // Optional variant to select (e.g. versions, cpu feature flags)
+ Variant string `json:"variant"`
+
+ // MinimumMemory represents the minimum memory required to use the GPU
+ MinimumMemory uint64 `json:"-"`
+
+ // Any extra PATH/LD_LIBRARY_PATH dependencies required for the Library to operate properly
+ DependencyPath string `json:"lib_path,omitempty"`
+
+ // Extra environment variables specific to the GPU as list of [key,value]
+ EnvWorkarounds [][2]string `json:"envs,omitempty"`
+
+ // Set to true if we can NOT reliably discover FreeMemory. A value of true indicates
+ // the FreeMemory is best effort, and may over or under report actual memory usage
+ // False indicates FreeMemory can generally be trusted on this GPU
+ UnreliableFreeMemory bool
+
+ // GPU information
+ ID string `json:"gpu_id"` // string to use for selection of this specific GPU
+ Name string `json:"name"` // user friendly name if available
+ Compute string `json:"compute"` // Compute Capability or gfx
+
+ // Driver Information - TODO no need to put this on each GPU
+ DriverMajor int `json:"driver_major,omitempty"`
+ DriverMinor int `json:"driver_minor,omitempty"`
+
+ // TODO other performance capability info to help in scheduling decisions
+}
+
+type CPUInfo struct {
+ GpuInfo
+ CPUs []CPU
+}
+
+// CPU type represents a CPU Package occupying a socket
+type CPU struct {
+ ID string `cpuinfo:"processor"`
+ VendorID string `cpuinfo:"vendor_id"`
+ ModelName string `cpuinfo:"model name"`
+ CoreCount int
+ EfficiencyCoreCount int // Performance = CoreCount - Efficiency
+ ThreadCount int
+}
+
+type CudaGPUInfo struct {
+ GpuInfo
+ OSOverhead uint64 // Memory overhead between the driver library and management library
+ index int //nolint:unused,nolintlint
+ computeMajor int //nolint:unused,nolintlint
+ computeMinor int //nolint:unused,nolintlint
+}
+type CudaGPUInfoList []CudaGPUInfo
+
+type RocmGPUInfo struct {
+ GpuInfo
+ usedFilepath string //nolint:unused,nolintlint
+ index int //nolint:unused,nolintlint
+}
+type RocmGPUInfoList []RocmGPUInfo
+
+type OneapiGPUInfo struct {
+ GpuInfo
+ driverIndex int //nolint:unused,nolintlint
+ gpuIndex int //nolint:unused,nolintlint
+}
+type OneapiGPUInfoList []OneapiGPUInfo
+
+type GpuInfoList []GpuInfo
+
+type UnsupportedGPUInfo struct {
+ GpuInfo
+ Reason string `json:"reason"`
+}
+
+// Split up the set of gpu info's by Library and variant
+func (l GpuInfoList) ByLibrary() []GpuInfoList {
+ resp := []GpuInfoList{}
+ libs := []string{}
+ for _, info := range l {
+ found := false
+ requested := info.Library
+ if info.Variant != CPUCapabilityNone.String() {
+ requested += "_" + info.Variant
+ }
+ for i, lib := range libs {
+ if lib == requested {
+ resp[i] = append(resp[i], info)
+ found = true
+ break
+ }
+ }
+ if !found {
+ libs = append(libs, requested)
+ resp = append(resp, []GpuInfo{info})
+ }
+ }
+ return resp
+}
+
+// Report the GPU information into the log an Info level
+func (l GpuInfoList) LogDetails() {
+ for _, g := range l {
+ slog.Info("inference compute",
+ "id", g.ID,
+ "library", g.Library,
+ "variant", g.Variant,
+ "compute", g.Compute,
+ "driver", fmt.Sprintf("%d.%d", g.DriverMajor, g.DriverMinor),
+ "name", g.Name,
+ "total", format.HumanBytes2(g.TotalMemory),
+ "available", format.HumanBytes2(g.FreeMemory),
+ )
+ }
+}
+
+// Sort by Free Space
+type ByFreeMemory []GpuInfo
+
+func (a ByFreeMemory) Len() int { return len(a) }
+func (a ByFreeMemory) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByFreeMemory) Less(i, j int) bool { return a[i].FreeMemory < a[j].FreeMemory }
+
+type CPUCapability uint32
+
+// Override at build time when building base GPU runners
+var GPURunnerCPUCapability = CPUCapabilityAVX
+
+const (
+ CPUCapabilityNone CPUCapability = iota
+ CPUCapabilityAVX
+ CPUCapabilityAVX2
+ // TODO AVX512
+)
+
+func (c CPUCapability) String() string {
+ switch c {
+ case CPUCapabilityAVX:
+ return "avx"
+ case CPUCapabilityAVX2:
+ return "avx2"
+ default:
+ return "no vector extensions"
+ }
+}
+
+type SystemInfo struct {
+ System CPUInfo `json:"system"`
+ GPUs []GpuInfo `json:"gpus"`
+ UnsupportedGPUs []UnsupportedGPUInfo `json:"unsupported_gpus"`
+ DiscoveryErrors []string `json:"discovery_errors"`
+}
+
+// Return the optimal number of threads to use for inference
+func (si SystemInfo) GetOptimalThreadCount() int {
+ if len(si.System.CPUs) == 0 {
+ return 0
+ }
+ // Allocate thread count matching the performance cores on a single socket
+ return si.System.CPUs[0].CoreCount - si.System.CPUs[0].EfficiencyCoreCount
}
diff --git a/integration/README.md b/integration/README.md
new file mode 100644
index 00000000..e2bdd6b2
--- /dev/null
+++ b/integration/README.md
@@ -0,0 +1,11 @@
+# Integration Tests
+
+This directory contains integration tests to exercise Ollama end-to-end to verify behavior
+
+By default, these tests are disabled so `go test ./...` will exercise only unit tests. To run integration tests you must pass the integration tag. `go test -tags=integration ./...`
+
+
+The integration tests have 2 modes of operating.
+
+1. By default, they will start the server on a random port, run the tests, and then shutdown the server.
+2. If `OLLAMA_TEST_EXISTING` is set to a non-empty string, the tests will run against an existing running server, which can be remote
diff --git a/integration/basic_test.go b/integration/basic_test.go
new file mode 100644
index 00000000..8e35b5c5
--- /dev/null
+++ b/integration/basic_test.go
@@ -0,0 +1,63 @@
+//go:build integration
+
+package integration
+
+import (
+ "context"
+ "log/slog"
+ "os"
+ "runtime"
+ "testing"
+ "time"
+
+ "github.com/ollama/ollama/api"
+ "github.com/stretchr/testify/require"
+)
+
+func TestOrcaMiniBlueSky(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+ // Set up the test data
+ req := api.GenerateRequest{
+ Model: "orca-mini",
+ Prompt: "why is the sky blue?",
+ Stream: &stream,
+ Options: map[string]interface{}{
+ "temperature": 0,
+ "seed": 123,
+ },
+ }
+ GenerateTestHelper(ctx, t, req, []string{"rayleigh", "scattering"})
+}
+
+func TestUnicodeModelDir(t *testing.T) {
+ // This is only useful for Windows with utf-16 characters, so skip this test for other platforms
+ if runtime.GOOS != "windows" {
+ t.Skip("Unicode test only applicable to windows")
+ }
+ // Only works for local testing
+ if os.Getenv("OLLAMA_TEST_EXISTING") != "" {
+ t.Skip("TestUnicodeModelDir only works for local testing, skipping")
+ }
+
+ modelDir, err := os.MkdirTemp("", "ollama_埃")
+ require.NoError(t, err)
+ defer os.RemoveAll(modelDir)
+ slog.Info("unicode", "OLLAMA_MODELS", modelDir)
+
+ t.Setenv("OLLAMA_MODELS", modelDir)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+
+ req := api.GenerateRequest{
+ Model: "orca-mini",
+ Prompt: "why is the sky blue?",
+ Stream: &stream,
+ Options: map[string]interface{}{
+ "temperature": 0,
+ "seed": 123,
+ },
+ }
+ GenerateTestHelper(ctx, t, req, []string{"rayleigh", "scattering"})
+}
diff --git a/integration/concurrency_test.go b/integration/concurrency_test.go
new file mode 100644
index 00000000..74eb6361
--- /dev/null
+++ b/integration/concurrency_test.go
@@ -0,0 +1,270 @@
+//go:build integration
+
+package integration
+
+import (
+ "context"
+ "log/slog"
+ "os"
+ "strconv"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/format"
+)
+
+func TestMultiModelConcurrency(t *testing.T) {
+ var (
+ req = [2]api.GenerateRequest{
+ {
+ Model: "orca-mini",
+ Prompt: "why is the ocean blue?",
+ Stream: &stream,
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
+ Options: map[string]interface{}{
+ "seed": 42,
+ "temperature": 0.0,
+ },
+ }, {
+ Model: "tinydolphin",
+ Prompt: "what is the origin of the us thanksgiving holiday?",
+ Stream: &stream,
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
+ Options: map[string]interface{}{
+ "seed": 42,
+ "temperature": 0.0,
+ },
+ },
+ }
+ resp = [2][]string{
+ {"sunlight"},
+ {"england", "english", "massachusetts", "pilgrims", "british", "festival"},
+ }
+ )
+ var wg sync.WaitGroup
+ wg.Add(len(req))
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*240)
+ defer cancel()
+
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+
+ for i := 0; i < len(req); i++ {
+ require.NoError(t, PullIfMissing(ctx, client, req[i].Model))
+ }
+
+ for i := 0; i < len(req); i++ {
+ go func(i int) {
+ defer wg.Done()
+ DoGenerate(ctx, t, client, req[i], resp[i], 60*time.Second, 10*time.Second)
+ }(i)
+ }
+ wg.Wait()
+}
+
+func TestIntegrationConcurrentPredictOrcaMini(t *testing.T) {
+ req, resp := GenerateRequests()
+ reqLimit := len(req)
+ iterLimit := 5
+
+ if s := os.Getenv("OLLAMA_MAX_VRAM"); s != "" {
+ maxVram, err := strconv.ParseUint(s, 10, 64)
+ require.NoError(t, err)
+ // Don't hammer on small VRAM cards...
+ if maxVram < 4*format.GibiByte {
+ reqLimit = min(reqLimit, 2)
+ iterLimit = 2
+ }
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 9*time.Minute)
+ defer cancel()
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+
+ // Get the server running (if applicable) warm the model up with a single initial request
+ DoGenerate(ctx, t, client, req[0], resp[0], 60*time.Second, 10*time.Second)
+
+ var wg sync.WaitGroup
+ wg.Add(reqLimit)
+ for i := 0; i < reqLimit; i++ {
+ go func(i int) {
+ defer wg.Done()
+ for j := 0; j < iterLimit; j++ {
+ slog.Info("Starting", "req", i, "iter", j)
+ // On slower GPUs it can take a while to process the concurrent requests
+ // so we allow a much longer initial timeout
+ DoGenerate(ctx, t, client, req[i], resp[i], 120*time.Second, 20*time.Second)
+ }
+ }(i)
+ }
+ wg.Wait()
+}
+
+// Stress the system if we know how much VRAM it has, and attempt to load more models than will fit
+func TestMultiModelStress(t *testing.T) {
+ s := os.Getenv("OLLAMA_MAX_VRAM") // TODO - discover actual VRAM
+ if s == "" {
+ t.Skip("OLLAMA_MAX_VRAM not specified, can't pick the right models for the stress test")
+ }
+
+ maxVram, err := strconv.ParseUint(s, 10, 64)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ type model struct {
+ name string
+ size uint64 // Approximate amount of VRAM they typically use when fully loaded in VRAM
+ }
+
+ smallModels := []model{
+ {
+ name: "orca-mini",
+ size: 2992 * format.MebiByte,
+ },
+ {
+ name: "phi",
+ size: 2616 * format.MebiByte,
+ },
+ {
+ name: "gemma:2b",
+ size: 2364 * format.MebiByte,
+ },
+ {
+ name: "stable-code:3b",
+ size: 2608 * format.MebiByte,
+ },
+ {
+ name: "starcoder2:3b",
+ size: 2166 * format.MebiByte,
+ },
+ }
+ mediumModels := []model{
+ {
+ name: "llama2",
+ size: 5118 * format.MebiByte,
+ },
+ {
+ name: "mistral",
+ size: 4620 * format.MebiByte,
+ },
+ {
+ name: "orca-mini:7b",
+ size: 5118 * format.MebiByte,
+ },
+ {
+ name: "dolphin-mistral",
+ size: 4620 * format.MebiByte,
+ },
+ {
+ name: "gemma:7b",
+ size: 5000 * format.MebiByte,
+ },
+ {
+ name: "codellama:7b",
+ size: 5118 * format.MebiByte,
+ },
+ }
+
+ // These seem to be too slow to be useful...
+ // largeModels := []model{
+ // {
+ // name: "llama2:13b",
+ // size: 7400 * format.MebiByte,
+ // },
+ // {
+ // name: "codellama:13b",
+ // size: 7400 * format.MebiByte,
+ // },
+ // {
+ // name: "orca-mini:13b",
+ // size: 7400 * format.MebiByte,
+ // },
+ // {
+ // name: "gemma:7b",
+ // size: 5000 * format.MebiByte,
+ // },
+ // {
+ // name: "starcoder2:15b",
+ // size: 9100 * format.MebiByte,
+ // },
+ // }
+
+ var chosenModels []model
+ switch {
+ case maxVram < 10000*format.MebiByte:
+ slog.Info("selecting small models")
+ chosenModels = smallModels
+ // case maxVram < 30000*format.MebiByte:
+ default:
+ slog.Info("selecting medium models")
+ chosenModels = mediumModels
+ // default:
+ // slog.Info("selecting large models")
+ // chosenModels = largModels
+ }
+
+ req, resp := GenerateRequests()
+
+ for i := range req {
+ if i > len(chosenModels) {
+ break
+ }
+ req[i].Model = chosenModels[i].name
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) // TODO baseline -- 10m too short
+ defer cancel()
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+
+ // Make sure all the models are pulled before we get started
+ for _, r := range req {
+ require.NoError(t, PullIfMissing(ctx, client, r.Model))
+ }
+
+ var wg sync.WaitGroup
+ consumed := uint64(256 * format.MebiByte) // Assume some baseline usage
+ for i := 0; i < len(req); i++ {
+ // Always get at least 2 models, but dont' overshoot VRAM too much or we'll take too long
+ if i > 1 && consumed > maxVram {
+ slog.Info("achieved target vram exhaustion", "count", i, "vram", format.HumanBytes2(maxVram), "models", format.HumanBytes2(consumed))
+ break
+ }
+ consumed += chosenModels[i].size
+ slog.Info("target vram", "count", i, "vram", format.HumanBytes2(maxVram), "models", format.HumanBytes2(consumed))
+
+ wg.Add(1)
+ go func(i int) {
+ defer wg.Done()
+ for j := 0; j < 3; j++ {
+ slog.Info("Starting", "req", i, "iter", j, "model", req[i].Model)
+ DoGenerate(ctx, t, client, req[i], resp[i], 120*time.Second, 5*time.Second)
+ }
+ }(i)
+ }
+ go func() {
+ for {
+ time.Sleep(2 * time.Second)
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ models, err := client.ListRunning(ctx)
+ if err != nil {
+ slog.Warn("failed to list running models", "error", err)
+ continue
+ }
+ for _, m := range models.Models {
+ slog.Info("loaded model snapshot", "model", m)
+ }
+ }
+ }
+ }()
+ wg.Wait()
+}
diff --git a/integration/context_test.go b/integration/context_test.go
new file mode 100644
index 00000000..f1342e16
--- /dev/null
+++ b/integration/context_test.go
@@ -0,0 +1,34 @@
+//go:build integration
+
+package integration
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/ollama/ollama/api"
+)
+
+func TestContextExhaustion(t *testing.T) {
+ // Longer needed for small footprint GPUs
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+ // Set up the test data
+ req := api.GenerateRequest{
+ Model: "llama2",
+ Prompt: "Write me a story with a ton of emojis?",
+ Stream: &stream,
+ Options: map[string]interface{}{
+ "temperature": 0,
+ "seed": 123,
+ "num_ctx": 128,
+ },
+ }
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+ if err := PullIfMissing(ctx, client, req.Model); err != nil {
+ t.Fatalf("PullIfMissing failed: %v", err)
+ }
+ DoGenerate(ctx, t, client, req, []string{"once", "upon", "lived"}, 120*time.Second, 10*time.Second)
+}
diff --git a/integration/embed_test.go b/integration/embed_test.go
new file mode 100644
index 00000000..4a68af68
--- /dev/null
+++ b/integration/embed_test.go
@@ -0,0 +1,209 @@
+//go:build integration
+
+package integration
+
+import (
+ "context"
+ "math"
+ "testing"
+ "time"
+
+ "github.com/ollama/ollama/api"
+)
+
+func floatsEqual32(a, b float32) bool {
+ return math.Abs(float64(a-b)) <= 1e-4
+}
+
+func floatsEqual64(a, b float64) bool {
+ return math.Abs(a-b) <= 1e-4
+}
+
+func TestAllMiniLMEmbeddings(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+
+ req := api.EmbeddingRequest{
+ Model: "all-minilm",
+ Prompt: "why is the sky blue?",
+ }
+
+ res, err := embeddingTestHelper(ctx, t, req)
+
+ if err != nil {
+ t.Fatalf("error: %v", err)
+ }
+
+ if len(res.Embedding) != 384 {
+ t.Fatalf("expected 384 floats, got %d", len(res.Embedding))
+ }
+
+ if !floatsEqual64(res.Embedding[0], 0.06642947345972061) {
+ t.Fatalf("expected 0.06642947345972061, got %.16f", res.Embedding[0])
+ }
+}
+
+func TestAllMiniLMEmbed(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+
+ req := api.EmbedRequest{
+ Model: "all-minilm",
+ Input: "why is the sky blue?",
+ }
+
+ res, err := embedTestHelper(ctx, t, req)
+
+ if err != nil {
+ t.Fatalf("error: %v", err)
+ }
+
+ if len(res.Embeddings) != 1 {
+ t.Fatalf("expected 1 embedding, got %d", len(res.Embeddings))
+ }
+
+ if len(res.Embeddings[0]) != 384 {
+ t.Fatalf("expected 384 floats, got %d", len(res.Embeddings[0]))
+ }
+
+ if !floatsEqual32(res.Embeddings[0][0], 0.010071031) {
+ t.Fatalf("expected 0.010071031, got %.8f", res.Embeddings[0][0])
+ }
+
+ if res.PromptEvalCount != 6 {
+ t.Fatalf("expected 6 prompt tokens, got %d", res.PromptEvalCount)
+ }
+}
+
+func TestAllMiniLMBatchEmbed(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+
+ req := api.EmbedRequest{
+ Model: "all-minilm",
+ Input: []string{"why is the sky blue?", "why is the grass green?"},
+ }
+
+ res, err := embedTestHelper(ctx, t, req)
+
+ if err != nil {
+ t.Fatalf("error: %v", err)
+ }
+
+ if len(res.Embeddings) != 2 {
+ t.Fatalf("expected 2 embeddings, got %d", len(res.Embeddings))
+ }
+
+ if len(res.Embeddings[0]) != 384 {
+ t.Fatalf("expected 384 floats, got %d", len(res.Embeddings[0]))
+ }
+
+ if !floatsEqual32(res.Embeddings[0][0], 0.010071031) || !floatsEqual32(res.Embeddings[1][0], -0.009802706) {
+ t.Fatalf("expected 0.010071031 and -0.009802706, got %.8f and %.8f", res.Embeddings[0][0], res.Embeddings[1][0])
+ }
+
+ if res.PromptEvalCount != 12 {
+ t.Fatalf("expected 12 prompt tokens, got %d", res.PromptEvalCount)
+ }
+}
+
+func TestAllMiniLMEmbedTruncate(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+
+ truncTrue, truncFalse := true, false
+
+ type testReq struct {
+ Name string
+ Request api.EmbedRequest
+ }
+
+ reqs := []testReq{
+ {
+ Name: "Target Truncation",
+ Request: api.EmbedRequest{
+ Model: "all-minilm",
+ Input: "why",
+ },
+ },
+ {
+ Name: "Default Truncate",
+ Request: api.EmbedRequest{
+ Model: "all-minilm",
+ Input: "why is the sky blue?",
+ Options: map[string]any{"num_ctx": 1},
+ },
+ },
+ {
+ Name: "Explicit Truncate",
+ Request: api.EmbedRequest{
+ Model: "all-minilm",
+ Input: "why is the sky blue?",
+ Truncate: &truncTrue,
+ Options: map[string]any{"num_ctx": 1},
+ },
+ },
+ }
+
+ res := make(map[string]*api.EmbedResponse)
+
+ for _, req := range reqs {
+ response, err := embedTestHelper(ctx, t, req.Request)
+ if err != nil {
+ t.Fatalf("error: %v", err)
+ }
+ res[req.Name] = response
+ }
+
+ if res["Target Truncation"].Embeddings[0][0] != res["Default Truncate"].Embeddings[0][0] {
+ t.Fatal("expected default request to truncate correctly")
+ }
+
+ if res["Default Truncate"].Embeddings[0][0] != res["Explicit Truncate"].Embeddings[0][0] {
+ t.Fatal("expected default request and truncate true request to be the same")
+ }
+
+ // check that truncate set to false returns an error if context length is exceeded
+ _, err := embedTestHelper(ctx, t, api.EmbedRequest{
+ Model: "all-minilm",
+ Input: "why is the sky blue?",
+ Truncate: &truncFalse,
+ Options: map[string]any{"num_ctx": 1},
+ })
+
+ if err == nil {
+ t.Fatal("expected error, got nil")
+ }
+}
+
+func embeddingTestHelper(ctx context.Context, t *testing.T, req api.EmbeddingRequest) (*api.EmbeddingResponse, error) {
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+ if err := PullIfMissing(ctx, client, req.Model); err != nil {
+ t.Fatalf("failed to pull model %s: %v", req.Model, err)
+ }
+
+ response, err := client.Embeddings(ctx, &req)
+
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+func embedTestHelper(ctx context.Context, t *testing.T, req api.EmbedRequest) (*api.EmbedResponse, error) {
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+ if err := PullIfMissing(ctx, client, req.Model); err != nil {
+ t.Fatalf("failed to pull model %s: %v", req.Model, err)
+ }
+
+ response, err := client.Embed(ctx, &req)
+
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
diff --git a/server/llm_image_test.go b/integration/llm_image_test.go
similarity index 98%
rename from server/llm_image_test.go
rename to integration/llm_image_test.go
index 6ca3577b..d0c861cc 100644
--- a/server/llm_image_test.go
+++ b/integration/llm_image_test.go
@@ -1,49 +1,42 @@
//go:build integration
-package server
+package integration
import (
"context"
"encoding/base64"
- "log"
- "os"
- "strings"
"testing"
"time"
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/llm"
- "github.com/stretchr/testify/assert"
+ "github.com/ollama/ollama/api"
"github.com/stretchr/testify/require"
)
func TestIntegrationMultimodal(t *testing.T) {
- SkipIFNoTestData(t)
image, err := base64.StdEncoding.DecodeString(imageEncoding)
require.NoError(t, err)
req := api.GenerateRequest{
- Model: "llava:7b",
- Prompt: "what does the text in this image say?",
- Options: map[string]interface{}{},
+ Model: "llava:7b",
+ Prompt: "what does the text in this image say?",
+ Stream: &stream,
+ Options: map[string]interface{}{
+ "seed": 42,
+ "temperature": 0.0,
+ },
Images: []api.ImageData{
image,
},
}
- resp := "the ollamas"
- workDir, err := os.MkdirTemp("", "ollama")
- require.NoError(t, err)
- defer os.RemoveAll(workDir)
- require.NoError(t, llm.Init(workDir))
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
+
+ // Note: sometimes it returns "the ollamas" sometimes "the ollams"
+ resp := "the ollam"
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
- opts := api.DefaultOptions()
- opts.Seed = 42
- opts.Temperature = 0.0
- model, llmRunner := PrepareModelForPrompts(t, req.Model, opts)
- defer llmRunner.Close()
- response := OneShotPromptResponse(t, ctx, req, model, llmRunner)
- log.Print(response)
- assert.Contains(t, strings.ToLower(response), resp)
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+ require.NoError(t, PullIfMissing(ctx, client, req.Model))
+ // llava models on CPU can be quite slow to start,
+ DoGenerate(ctx, t, client, req, []string{resp}, 120*time.Second, 30*time.Second)
}
const imageEncoding = `iVBORw0KGgoAAAANSUhEUgAAANIAAAB4CAYAAACHHqzKAAAAAXNSR0IArs4c6QAAAIRlWElmTU0AKgAAAAgABQESAAMAAAABAAEAAAEaAAUAAAABAAAASgEb
diff --git a/integration/llm_test.go b/integration/llm_test.go
new file mode 100644
index 00000000..398e0a03
--- /dev/null
+++ b/integration/llm_test.go
@@ -0,0 +1,47 @@
+//go:build integration
+
+package integration
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/ollama/ollama/api"
+)
+
+// TODO - this would ideally be in the llm package, but that would require some refactoring of interfaces in the server
+// package to avoid circular dependencies
+
+var (
+ stream = false
+ req = [2]api.GenerateRequest{
+ {
+ Model: "orca-mini",
+ Prompt: "why is the ocean blue?",
+ Stream: &stream,
+ Options: map[string]interface{}{
+ "seed": 42,
+ "temperature": 0.0,
+ },
+ }, {
+ Model: "orca-mini",
+ Prompt: "what is the origin of the us thanksgiving holiday?",
+ Stream: &stream,
+ Options: map[string]interface{}{
+ "seed": 42,
+ "temperature": 0.0,
+ },
+ },
+ }
+ resp = [2][]string{
+ {"sunlight"},
+ {"england", "english", "massachusetts", "pilgrims"},
+ }
+)
+
+func TestIntegrationSimpleOrcaMini(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*120)
+ defer cancel()
+ GenerateTestHelper(ctx, t, req[0], resp[0])
+}
diff --git a/integration/max_queue_test.go b/integration/max_queue_test.go
new file mode 100644
index 00000000..ec9e085a
--- /dev/null
+++ b/integration/max_queue_test.go
@@ -0,0 +1,120 @@
+//go:build integration
+
+package integration
+
+import (
+ "context"
+ "errors"
+ "log/slog"
+ "os"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/envconfig"
+)
+
+func TestMaxQueue(t *testing.T) {
+ if os.Getenv("OLLAMA_TEST_EXISTING") != "" {
+ t.Skip("Max Queue test requires spawing a local server so we can adjust the queue size")
+ return
+ }
+
+ // Note: This test can be quite slow when running in CPU mode, so keep the threadCount low unless your on GPU
+ // Also note that by default Darwin can't sustain > ~128 connections without adjusting limits
+ threadCount := 32
+ if maxQueue := envconfig.MaxQueue(); maxQueue != 0 {
+ threadCount = int(maxQueue)
+ } else {
+ t.Setenv("OLLAMA_MAX_QUEUE", strconv.Itoa(threadCount))
+ }
+
+ req := api.GenerateRequest{
+ Model: "orca-mini",
+ Prompt: "write a long historical fiction story about christopher columbus. use at least 10 facts from his actual journey",
+ Options: map[string]interface{}{
+ "seed": 42,
+ "temperature": 0.0,
+ },
+ }
+ resp := []string{"explore", "discover", "ocean"}
+
+ // CPU mode takes much longer at the limit with a large queue setting
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+
+ require.NoError(t, PullIfMissing(ctx, client, req.Model))
+
+ // Context for the worker threads so we can shut them down
+ // embedCtx, embedCancel := context.WithCancel(ctx)
+ embedCtx := ctx
+
+ var genwg sync.WaitGroup
+ go func() {
+ genwg.Add(1)
+ defer genwg.Done()
+ slog.Info("Starting generate request")
+ DoGenerate(ctx, t, client, req, resp, 45*time.Second, 5*time.Second)
+ slog.Info("generate completed")
+ }()
+
+ // Give the generate a chance to get started before we start hammering on embed requests
+ time.Sleep(5 * time.Millisecond)
+
+ threadCount += 10 // Add a few extra to ensure we push the queue past its limit
+ busyCount := 0
+ resetByPeerCount := 0
+ canceledCount := 0
+ succesCount := 0
+ counterMu := sync.Mutex{}
+ var embedwg sync.WaitGroup
+ for i := 0; i < threadCount; i++ {
+ go func(i int) {
+ embedwg.Add(1)
+ defer embedwg.Done()
+ slog.Info("embed started", "id", i)
+ embedReq := api.EmbeddingRequest{
+ Model: req.Model,
+ Prompt: req.Prompt,
+ Options: req.Options,
+ }
+ // Fresh client for every request
+ client, _ = GetTestEndpoint()
+
+ resp, genErr := client.Embeddings(embedCtx, &embedReq)
+ counterMu.Lock()
+ defer counterMu.Unlock()
+ switch {
+ case genErr == nil:
+ succesCount++
+ require.Greater(t, len(resp.Embedding), 5) // somewhat arbitrary, but sufficient to be reasonable
+ case errors.Is(genErr, context.Canceled):
+ canceledCount++
+ case strings.Contains(genErr.Error(), "busy"):
+ busyCount++
+ case strings.Contains(genErr.Error(), "connection reset by peer"):
+ resetByPeerCount++
+ default:
+ require.NoError(t, genErr, "%d request failed", i)
+ }
+
+ slog.Info("embed finished", "id", i)
+ }(i)
+ }
+ genwg.Wait()
+ slog.Info("generate done, waiting for embeds")
+ embedwg.Wait()
+
+ slog.Info("embeds completed", "success", succesCount, "busy", busyCount, "reset", resetByPeerCount, "canceled", canceledCount)
+ require.Equal(t, resetByPeerCount, 0, "Connections reset by peer, have you updated your fd and socket limits?")
+ require.True(t, busyCount > 0, "no requests hit busy error but some should have")
+ require.True(t, canceledCount == 0, "no requests should have been canceled due to timeout")
+
+}
diff --git a/integration/utils_test.go b/integration/utils_test.go
new file mode 100644
index 00000000..e76b63f2
--- /dev/null
+++ b/integration/utils_test.go
@@ -0,0 +1,343 @@
+//go:build integration
+
+package integration
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "math/rand"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/app/lifecycle"
+ "github.com/stretchr/testify/require"
+)
+
+func Init() {
+ lifecycle.InitLogging()
+}
+
+func FindPort() string {
+ port := 0
+ if a, err := net.ResolveTCPAddr("tcp", "localhost:0"); err == nil {
+ var l *net.TCPListener
+ if l, err = net.ListenTCP("tcp", a); err == nil {
+ port = l.Addr().(*net.TCPAddr).Port
+ l.Close()
+ }
+ }
+ if port == 0 {
+ port = rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range
+ }
+ return strconv.Itoa(port)
+}
+
+func GetTestEndpoint() (*api.Client, string) {
+ defaultPort := "11434"
+ ollamaHost := os.Getenv("OLLAMA_HOST")
+
+ scheme, hostport, ok := strings.Cut(ollamaHost, "://")
+ if !ok {
+ scheme, hostport = "http", ollamaHost
+ }
+
+ // trim trailing slashes
+ hostport = strings.TrimRight(hostport, "/")
+
+ host, port, err := net.SplitHostPort(hostport)
+ if err != nil {
+ host, port = "127.0.0.1", defaultPort
+ if ip := net.ParseIP(strings.Trim(hostport, "[]")); ip != nil {
+ host = ip.String()
+ } else if hostport != "" {
+ host = hostport
+ }
+ }
+
+ if os.Getenv("OLLAMA_TEST_EXISTING") == "" && port == defaultPort {
+ port = FindPort()
+ }
+
+ slog.Info("server connection", "host", host, "port", port)
+
+ return api.NewClient(
+ &url.URL{
+ Scheme: scheme,
+ Host: net.JoinHostPort(host, port),
+ },
+ http.DefaultClient), fmt.Sprintf("%s:%s", host, port)
+}
+
+var serverMutex sync.Mutex
+var serverReady bool
+
+func startServer(t *testing.T, ctx context.Context, ollamaHost string) error {
+ // Make sure the server has been built
+ CLIName, err := filepath.Abs("../ollama")
+ if err != nil {
+ return err
+ }
+
+ if runtime.GOOS == "windows" {
+ CLIName += ".exe"
+ }
+ _, err = os.Stat(CLIName)
+ if err != nil {
+ return fmt.Errorf("CLI missing, did you forget to build first? %w", err)
+ }
+ serverMutex.Lock()
+ defer serverMutex.Unlock()
+ if serverReady {
+ return nil
+ }
+
+ if tmp := os.Getenv("OLLAMA_HOST"); tmp != ollamaHost {
+ slog.Info("setting env", "OLLAMA_HOST", ollamaHost)
+ t.Setenv("OLLAMA_HOST", ollamaHost)
+ }
+
+ slog.Info("starting server", "url", ollamaHost)
+ done, err := lifecycle.SpawnServer(ctx, "../ollama")
+ if err != nil {
+ return fmt.Errorf("failed to start server: %w", err)
+ }
+
+ go func() {
+ <-ctx.Done()
+ serverMutex.Lock()
+ defer serverMutex.Unlock()
+ exitCode := <-done
+ if exitCode > 0 {
+ slog.Warn("server failure", "exit", exitCode)
+ }
+ serverReady = false
+ }()
+
+ // TODO wait only long enough for the server to be responsive...
+ time.Sleep(500 * time.Millisecond)
+
+ serverReady = true
+ return nil
+}
+
+func PullIfMissing(ctx context.Context, client *api.Client, modelName string) error {
+ slog.Info("checking status of model", "model", modelName)
+ showReq := &api.ShowRequest{Name: modelName}
+
+ showCtx, cancel := context.WithDeadlineCause(
+ ctx,
+ time.Now().Add(10*time.Second),
+ fmt.Errorf("show for existing model %s took too long", modelName),
+ )
+ defer cancel()
+ _, err := client.Show(showCtx, showReq)
+ var statusError api.StatusError
+ switch {
+ case errors.As(err, &statusError) && statusError.StatusCode == http.StatusNotFound:
+ break
+ case err != nil:
+ return err
+ default:
+ slog.Info("model already present", "model", modelName)
+ return nil
+ }
+ slog.Info("model missing", "model", modelName)
+
+ stallDuration := 30 * time.Second // This includes checksum verification, which can take a while on larger models
+ stallTimer := time.NewTimer(stallDuration)
+ fn := func(resp api.ProgressResponse) error {
+ // fmt.Print(".")
+ if !stallTimer.Reset(stallDuration) {
+ return errors.New("stall was detected, aborting status reporting")
+ }
+ return nil
+ }
+
+ stream := true
+ pullReq := &api.PullRequest{Name: modelName, Stream: &stream}
+
+ var pullError error
+
+ done := make(chan int)
+ go func() {
+ pullError = client.Pull(ctx, pullReq, fn)
+ done <- 0
+ }()
+
+ select {
+ case <-stallTimer.C:
+ return errors.New("download stalled")
+ case <-done:
+ return pullError
+ }
+}
+
+var serverProcMutex sync.Mutex
+
+// Returns an Client, the testEndpoint, and a cleanup function, fails the test on errors
+// Starts the server if needed
+func InitServerConnection(ctx context.Context, t *testing.T) (*api.Client, string, func()) {
+ client, testEndpoint := GetTestEndpoint()
+ if os.Getenv("OLLAMA_TEST_EXISTING") == "" {
+ serverProcMutex.Lock()
+ fp, err := os.CreateTemp("", "ollama-server-*.log")
+ if err != nil {
+ t.Fatalf("failed to generate log file: %s", err)
+ }
+ lifecycle.ServerLogFile = fp.Name()
+ fp.Close()
+ require.NoError(t, startServer(t, ctx, testEndpoint))
+ }
+
+ return client, testEndpoint, func() {
+ if os.Getenv("OLLAMA_TEST_EXISTING") == "" {
+ defer serverProcMutex.Unlock()
+ if t.Failed() {
+ fp, err := os.Open(lifecycle.ServerLogFile)
+ if err != nil {
+ slog.Error("failed to open server log", "logfile", lifecycle.ServerLogFile, "error", err)
+ return
+ }
+ data, err := io.ReadAll(fp)
+ if err != nil {
+ slog.Error("failed to read server log", "logfile", lifecycle.ServerLogFile, "error", err)
+ return
+ }
+ slog.Warn("SERVER LOG FOLLOWS")
+ os.Stderr.Write(data)
+ slog.Warn("END OF SERVER")
+ }
+ err := os.Remove(lifecycle.ServerLogFile)
+ if err != nil && !os.IsNotExist(err) {
+ slog.Warn("failed to cleanup", "logfile", lifecycle.ServerLogFile, "error", err)
+ }
+ }
+ }
+}
+
+func GenerateTestHelper(ctx context.Context, t *testing.T, genReq api.GenerateRequest, anyResp []string) {
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+ require.NoError(t, PullIfMissing(ctx, client, genReq.Model))
+ DoGenerate(ctx, t, client, genReq, anyResp, 30*time.Second, 10*time.Second)
+}
+
+func DoGenerate(ctx context.Context, t *testing.T, client *api.Client, genReq api.GenerateRequest, anyResp []string, initialTimeout, streamTimeout time.Duration) {
+ stallTimer := time.NewTimer(initialTimeout)
+ var buf bytes.Buffer
+ fn := func(response api.GenerateResponse) error {
+ // fmt.Print(".")
+ buf.Write([]byte(response.Response))
+ if !stallTimer.Reset(streamTimeout) {
+ return errors.New("stall was detected while streaming response, aborting")
+ }
+ return nil
+ }
+
+ stream := true
+ genReq.Stream = &stream
+ done := make(chan int)
+ var genErr error
+ go func() {
+ genErr = client.Generate(ctx, &genReq, fn)
+ done <- 0
+ }()
+
+ select {
+ case <-stallTimer.C:
+ if buf.Len() == 0 {
+ t.Errorf("generate never started. Timed out after :%s", initialTimeout.String())
+ } else {
+ t.Errorf("generate stalled. Response so far:%s", buf.String())
+ }
+ case <-done:
+ require.NoError(t, genErr, "failed with %s request prompt %s ", genReq.Model, genReq.Prompt)
+ // Verify the response contains the expected data
+ response := buf.String()
+ atLeastOne := false
+ for _, resp := range anyResp {
+ if strings.Contains(strings.ToLower(response), resp) {
+ atLeastOne = true
+ break
+ }
+ }
+ require.True(t, atLeastOne, "%s: none of %v found in %s", genReq.Model, anyResp, response)
+ slog.Info("test pass", "model", genReq.Model, "prompt", genReq.Prompt, "contains", anyResp, "response", response)
+ case <-ctx.Done():
+ t.Error("outer test context done while waiting for generate")
+ }
+}
+
+// Generate a set of requests
+// By default each request uses orca-mini as the model
+func GenerateRequests() ([]api.GenerateRequest, [][]string) {
+ return []api.GenerateRequest{
+ {
+ Model: "orca-mini",
+ Prompt: "why is the ocean blue?",
+ Stream: &stream,
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
+ Options: map[string]interface{}{
+ "seed": 42,
+ "temperature": 0.0,
+ },
+ }, {
+ Model: "orca-mini",
+ Prompt: "why is the color of dirt brown?",
+ Stream: &stream,
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
+ Options: map[string]interface{}{
+ "seed": 42,
+ "temperature": 0.0,
+ },
+ }, {
+ Model: "orca-mini",
+ Prompt: "what is the origin of the us thanksgiving holiday?",
+ Stream: &stream,
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
+ Options: map[string]interface{}{
+ "seed": 42,
+ "temperature": 0.0,
+ },
+ }, {
+ Model: "orca-mini",
+ Prompt: "what is the origin of independence day?",
+ Stream: &stream,
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
+ Options: map[string]interface{}{
+ "seed": 42,
+ "temperature": 0.0,
+ },
+ }, {
+ Model: "orca-mini",
+ Prompt: "what is the composition of air?",
+ Stream: &stream,
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
+ Options: map[string]interface{}{
+ "seed": 42,
+ "temperature": 0.0,
+ },
+ },
+ },
+ [][]string{
+ {"sunlight"},
+ {"soil", "organic", "earth", "black", "tan"},
+ {"england", "english", "massachusetts", "pilgrims", "british"},
+ {"fourth", "july", "declaration", "independence"},
+ {"nitrogen", "oxygen", "carbon", "dioxide"},
+ }
+}
diff --git a/llama/.gitignore b/llama/.gitignore
new file mode 100644
index 00000000..ee552043
--- /dev/null
+++ b/llama/.gitignore
@@ -0,0 +1,3 @@
+*.bin
+*.gguf
+build/
\ No newline at end of file
diff --git a/llama/Dockerfile b/llama/Dockerfile
new file mode 100644
index 00000000..c636ee5d
--- /dev/null
+++ b/llama/Dockerfile
@@ -0,0 +1,221 @@
+# Note: once we have fully transitioned to the Go server, this will replace the old Dockerfile at the top of the tree
+ARG GOLANG_VERSION=1.22.5
+ARG CMAKE_VERSION=3.22.1
+ARG CUDA_VERSION_11=11.3.1
+ARG CUDA_V11_ARCHITECTURES="50;52;53;60;61;62;70;72;75;80;86"
+ARG CUDA_VERSION_12=12.4.0
+ARG CUDA_V12_ARCHITECTURES="60;61;62;70;72;75;80;86;87;89;90;90a"
+ARG ROCM_VERSION=6.1.2
+
+### To create a local image for building linux binaries on mac or windows with efficient incremental builds
+#
+# docker build --platform linux/amd64 -t builder-amd64 -f Dockerfile.new --target unified-builder-amd64 .
+# docker run --platform linux/amd64 --rm -it -v $(pwd):/go/src/github.com/ollama/ollama/ builder-amd64
+#
+### Then incremental builds will be much faster in this container
+#
+# make -C llama -j 10 && go build -trimpath -o dist/linux-amd64/ollama .
+#
+FROM --platform=linux/amd64 rocm/dev-centos-7:${ROCM_VERSION}-complete AS unified-builder-amd64
+ARG CMAKE_VERSION
+ARG GOLANG_VERSION
+ARG CUDA_VERSION_11
+ARG CUDA_VERSION_12
+COPY ./scripts/rh_linux_deps.sh /
+ENV PATH /opt/rh/devtoolset-10/root/usr/bin:/usr/local/cuda/bin:$PATH
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib64
+ENV LIBRARY_PATH=/usr/local/cuda/lib64/stubs:/opt/amdgpu/lib64
+RUN CMAKE_VERSION=${CMAKE_VERSION} GOLANG_VERSION=${GOLANG_VERSION} sh /rh_linux_deps.sh
+RUN yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \
+ dnf clean all && \
+ dnf install -y \
+ zsh \
+ cuda-$(echo ${CUDA_VERSION_11} | cut -f1-2 -d. | sed -e "s/\./-/g") \
+ cuda-$(echo ${CUDA_VERSION_12} | cut -f1-2 -d. | sed -e "s/\./-/g")
+# TODO intel oneapi goes here...
+ENV GOARCH amd64
+ENV CGO_ENABLED 1
+WORKDIR /go/src/github.com/ollama/ollama/
+ENTRYPOINT [ "zsh" ]
+
+### To create a local image for building linux binaries on mac or linux/arm64 with efficient incremental builds
+# Note: this does not contain jetson variants
+#
+# docker build --platform linux/arm64 -t builder-arm64 -f Dockerfile.new --target unified-builder-arm64 .
+# docker run --platform linux/arm64 --rm -it -v $(pwd):/go/src/github.com/ollama/ollama/ builder-arm64
+#
+FROM --platform=linux/arm64 rockylinux:8 AS unified-builder-arm64
+ARG CMAKE_VERSION
+ARG GOLANG_VERSION
+ARG CUDA_VERSION_11
+ARG CUDA_VERSION_12
+COPY ./scripts/rh_linux_deps.sh /
+RUN CMAKE_VERSION=${CMAKE_VERSION} GOLANG_VERSION=${GOLANG_VERSION} sh /rh_linux_deps.sh
+RUN yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/sbsa/cuda-rhel8.repo && \
+ dnf config-manager --set-enabled appstream && \
+ dnf clean all && \
+ dnf install -y \
+ zsh \
+ cuda-toolkit-$(echo ${CUDA_VERSION_11} | cut -f1-2 -d. | sed -e "s/\./-/g") \
+ cuda-toolkit-$(echo ${CUDA_VERSION_12} | cut -f1-2 -d. | sed -e "s/\./-/g")
+ENV PATH /opt/rh/gcc-toolset-10/root/usr/bin:$PATH:/usr/local/cuda/bin
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib64
+ENV LIBRARY_PATH=/usr/local/cuda/lib64/stubs:/opt/amdgpu/lib64
+ENV GOARCH amd64
+ENV CGO_ENABLED 1
+WORKDIR /go/src/github.com/ollama/ollama/
+ENTRYPOINT [ "zsh" ]
+
+FROM --platform=linux/amd64 unified-builder-amd64 AS runners-amd64
+COPY . .
+ARG OLLAMA_SKIP_CUDA_GENERATE
+ARG OLLAMA_SKIP_CUDA_11_GENERATE
+ARG OLLAMA_SKIP_CUDA_12_GENERATE
+ARG OLLAMA_SKIP_ROCM_GENERATE
+ARG CUDA_V11_ARCHITECTURES
+ARG CUDA_V12_ARCHITECTURES
+ARG OLLAMA_FAST_BUILD
+RUN --mount=type=cache,target=/root/.ccache \
+ if grep "^flags" /proc/cpuinfo|grep avx>/dev/null; then \
+ make -C llama -j $(expr $(nproc) / 2 ) ; \
+ else \
+ make -C llama -j 5 ; \
+ fi
+
+FROM --platform=linux/arm64 unified-builder-arm64 AS runners-arm64
+COPY . .
+ARG OLLAMA_SKIP_CUDA_GENERATE
+ARG OLLAMA_SKIP_CUDA_11_GENERATE
+ARG OLLAMA_SKIP_CUDA_12_GENERATE
+ARG CUDA_V11_ARCHITECTURES
+ARG CUDA_V12_ARCHITECTURES
+ARG OLLAMA_FAST_BUILD
+RUN --mount=type=cache,target=/root/.ccache \
+ make -C llama -j 8
+
+
+# Intermediate stages used for ./scripts/build_linux.sh
+FROM --platform=linux/amd64 centos:7 AS builder-amd64
+ARG CMAKE_VERSION
+ARG GOLANG_VERSION
+COPY ./scripts/rh_linux_deps.sh /
+RUN CMAKE_VERSION=${CMAKE_VERSION} GOLANG_VERSION=${GOLANG_VERSION} sh /rh_linux_deps.sh
+ENV PATH /opt/rh/devtoolset-10/root/usr/bin:$PATH
+ENV CGO_ENABLED 1
+ENV GOARCH amd64
+WORKDIR /go/src/github.com/ollama/ollama
+
+FROM --platform=linux/amd64 builder-amd64 AS build-amd64
+COPY . .
+COPY --from=runners-amd64 /go/src/github.com/ollama/ollama/dist/ dist/
+COPY --from=runners-amd64 /go/src/github.com/ollama/ollama/build/ build/
+ARG GOFLAGS
+ARG CGO_CFLAGS
+ARG OLLAMA_SKIP_ROCM_GENERATE
+RUN --mount=type=cache,target=/root/.ccache \
+ go build -trimpath -o dist/linux-amd64/bin/ollama .
+RUN cd dist/linux-$GOARCH && \
+ tar --exclude runners -cf - . | pigz --best > ../ollama-linux-$GOARCH.tgz
+RUN if [ -z ${OLLAMA_SKIP_ROCM_GENERATE} ] ; then \
+ cd dist/linux-$GOARCH-rocm && \
+ tar -cf - . | pigz --best > ../ollama-linux-$GOARCH-rocm.tgz ;\
+ fi
+
+FROM --platform=linux/arm64 rockylinux:8 AS builder-arm64
+ARG CMAKE_VERSION
+ARG GOLANG_VERSION
+COPY ./scripts/rh_linux_deps.sh /
+RUN CMAKE_VERSION=${CMAKE_VERSION} GOLANG_VERSION=${GOLANG_VERSION} sh /rh_linux_deps.sh
+ENV PATH /opt/rh/gcc-toolset-10/root/usr/bin:$PATH
+ENV CGO_ENABLED 1
+ENV GOARCH arm64
+WORKDIR /go/src/github.com/ollama/ollama
+
+FROM --platform=linux/arm64 builder-arm64 AS build-arm64
+COPY . .
+COPY --from=runners-arm64 /go/src/github.com/ollama/ollama/dist/ dist/
+COPY --from=runners-arm64 /go/src/github.com/ollama/ollama/build/ build/
+ARG GOFLAGS
+ARG CGO_CFLAGS
+RUN --mount=type=cache,target=/root/.ccache \
+ go build -trimpath -o dist/linux-arm64/bin/ollama .
+RUN cd dist/linux-$GOARCH && \
+ tar --exclude runners -cf - . | pigz --best > ../ollama-linux-$GOARCH.tgz
+
+FROM --platform=linux/amd64 scratch AS dist-amd64
+COPY --from=build-amd64 /go/src/github.com/ollama/ollama/dist/ollama-linux-*.tgz /
+FROM --platform=linux/arm64 scratch AS dist-arm64
+COPY --from=build-arm64 /go/src/github.com/ollama/ollama/dist/ollama-linux-*.tgz /
+FROM dist-$TARGETARCH AS dist
+
+
+# Optimized container images do not cary nested payloads
+FROM --platform=linux/amd64 builder-amd64 AS container-build-amd64
+WORKDIR /go/src/github.com/ollama/ollama
+COPY . .
+ARG GOFLAGS
+ARG CGO_CFLAGS
+RUN --mount=type=cache,target=/root/.ccache \
+ go build -trimpath -o dist/linux-amd64/bin/ollama .
+
+FROM --platform=linux/arm64 builder-arm64 AS container-build-arm64
+WORKDIR /go/src/github.com/ollama/ollama
+COPY . .
+ARG GOFLAGS
+ARG CGO_CFLAGS
+RUN --mount=type=cache,target=/root/.ccache \
+ go build -trimpath -o dist/linux-arm64/bin/ollama .
+
+# For amd64 container images, filter out cuda/rocm to minimize size
+FROM runners-amd64 AS runners-cuda-amd64
+RUN rm -rf \
+ ./dist/linux-amd64/lib/ollama/libggml_hipblas.so \
+ ./dist/linux-amd64/lib/ollama/runners/rocm*
+
+FROM runners-amd64 AS runners-rocm-amd64
+RUN rm -rf \
+ ./dist/linux-amd64/lib/ollama/libggml_cuda*.so \
+ ./dist/linux-amd64/lib/ollama/libcu*.so* \
+ ./dist/linux-amd64/lib/ollama/runners/cuda*
+
+FROM --platform=linux/amd64 ubuntu:22.04 AS runtime-amd64
+RUN apt-get update && \
+ apt-get install -y ca-certificates && \
+ rm -rf /var/lib/apt/lists/*
+COPY --from=container-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/bin/ /bin/
+COPY --from=runners-cuda-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/lib/ /lib/
+
+FROM --platform=linux/arm64 ubuntu:22.04 AS runtime-arm64
+RUN apt-get update && \
+ apt-get install -y ca-certificates && \
+ rm -rf /var/lib/apt/lists/*
+COPY --from=container-build-arm64 /go/src/github.com/ollama/ollama/dist/linux-arm64/bin/ /bin/
+COPY --from=runners-arm64 /go/src/github.com/ollama/ollama/dist/linux-arm64/lib/ /lib/
+
+# ROCm libraries larger so we keep it distinct from the CPU/CUDA image
+FROM --platform=linux/amd64 ubuntu:22.04 AS runtime-rocm
+# Frontload the rocm libraries which are large, and rarely change to increase chance of a common layer
+# across releases
+COPY --from=build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64-rocm/lib/ /lib/
+RUN apt-get update && \
+ apt-get install -y ca-certificates && \
+ rm -rf /var/lib/apt/lists/*
+COPY --from=container-build-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/bin/ /bin/
+COPY --from=runners-rocm-amd64 /go/src/github.com/ollama/ollama/dist/linux-amd64/lib/ /lib/
+
+EXPOSE 11434
+ENV OLLAMA_HOST 0.0.0.0
+
+ENTRYPOINT ["/bin/ollama"]
+CMD ["serve"]
+
+FROM runtime-$TARGETARCH
+EXPOSE 11434
+ENV OLLAMA_HOST 0.0.0.0
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
+ENV NVIDIA_VISIBLE_DEVICES=all
+
+ENTRYPOINT ["/bin/ollama"]
+CMD ["serve"]
diff --git a/llama/Makefile b/llama/Makefile
new file mode 100644
index 00000000..589eb6de
--- /dev/null
+++ b/llama/Makefile
@@ -0,0 +1,54 @@
+# top level makefile for Go server
+include make/common-defs.make
+
+RUNNER_TARGETS := default
+
+# Determine which if any GPU runners we should build
+ifeq ($(OS),windows)
+ CUDA_PATH?=$(shell cygpath -m -s "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\" 2>/dev/null)unknown
+ CUDA_BASE_DIR := $(dir $(shell cygpath -m -s "$(CUDA_PATH)\\.." 2>/dev/null))
+ CUDA_11:=$(shell ls -d $(CUDA_BASE_DIR)/v11.? 2>/dev/null)
+ CUDA_12:=$(shell ls -d $(CUDA_BASE_DIR)/v12.? 2>/dev/null)
+ HIP_PATH_83 := $(shell cygpath -m -s "$(subst \,/,$(HIP_PATH))" 2>/dev/null)
+ HIP_LIB_DIR := $(shell ls -d $(HIP_PATH_83)/lib 2>/dev/null)
+else ifeq ($(OS),linux)
+ HIP_PATH?=/opt/rocm
+ HIP_LIB_DIR := $(shell ls -d $(HIP_PATH)/lib 2>/dev/null)
+ CUDA_PATH?=/usr/local/cuda
+ CUDA_11:=$(shell ls -d $(CUDA_PATH)-11 2>/dev/null)
+ CUDA_12:=$(shell ls -d $(CUDA_PATH)-12 2>/dev/null)
+endif
+
+ifeq ($(OLLAMA_SKIP_CUDA_GENERATE),)
+ifneq ($(CUDA_11),)
+ RUNNER_TARGETS += cuda_v11
+endif
+ifneq ($(CUDA_12),)
+ RUNNER_TARGETS += cuda_v12
+endif
+endif
+ifeq ($(OLLAMA_SKIP_ROCM_GENERATE),)
+ifneq ($(HIP_LIB_DIR),)
+ RUNNER_TARGETS += rocm
+endif
+endif
+
+
+all: clean-payload .WAIT runners
+
+runners: $(RUNNER_TARGETS)
+
+$(RUNNER_TARGETS):
+ $(MAKE) -f make/Makefile.$@
+
+clean:
+ rm -rf $(BUILD_DIR) $(DIST_RUNNERS) $(PAYLOAD_RUNNERS)
+
+clean-payload:
+ rm -rf $(addprefix $(RUNNERS_PAYLOAD_DIR)/, $(RUNNER_TARGETS) metal cpu cpu_avx cpu_avx2)
+
+.PHONY: all runners clean clean-payload $(RUNNER_TARGETS) .WAIT
+
+# Handy debugging for make variables
+print-%:
+ @echo '$*=$($*)'
diff --git a/llama/README.md b/llama/README.md
new file mode 100644
index 00000000..05c95c90
--- /dev/null
+++ b/llama/README.md
@@ -0,0 +1,100 @@
+# `llama`
+
+This package integrates the [llama.cpp](https://github.com/ggerganov/llama.cpp) library as a Go package and makes it easy to build it with tags for different CPU and GPU processors.
+
+Supported:
+
+- [x] CPU
+- [x] avx, avx2
+- [x] macOS Metal
+- [x] Windows CUDA
+- [x] Windows ROCm
+- [x] Linux CUDA
+- [x] Linux ROCm
+- [x] Llava
+
+Extra build steps are required for CUDA and ROCm on Windows since `nvcc` and `hipcc` both require using msvc as the host compiler. For these shared libraries are created:
+
+- `ggml_cuda.dll` on Windows or `ggml_cuda.so` on Linux
+- `ggml_hipblas.dll` on Windows or `ggml_hipblas.so` on Linux
+
+> Note: it's important that memory is allocated and freed by the same compiler (e.g. entirely by code compiled with msvc or mingw). Issues from this should be rare, but there are some places where pointers are returned by the CUDA or HIP runtimes and freed elsewhere, causing a a crash. In a future change the same runtime should be used in both cases to avoid crashes.
+
+## Building
+
+```
+go build .
+```
+
+### AVX
+
+```shell
+go build -tags avx .
+```
+
+### AVX2
+
+```shell
+# go doesn't recognize `-mfma` as a valid compiler flag
+# see https://github.com/golang/go/issues/17895
+go env -w "CGO_CFLAGS_ALLOW=-mfma|-mf16c"
+go env -w "CGO_CXXFLAGS_ALLOW=-mfma|-mf16c"
+go build -tags=avx,avx2 .
+```
+
+## Linux
+
+### CUDA
+
+Install the [CUDA toolkit v11.3.1](https://developer.nvidia.com/cuda-11-3-1-download-archive):
+
+```shell
+make ggml_cuda.so
+go build -tags avx,cuda .
+```
+
+### ROCm
+
+Install the [CUDA toolkit v11.3.1](https://developer.nvidia.com/cuda-11-3-1-download-archive):
+
+```shell
+make ggml_hipblas.so
+go build -tags avx,rocm .
+```
+
+## Windows
+
+Download [w64devkit](https://github.com/skeeto/w64devkit/releases/latest) for a simple MinGW development environment.
+
+### CUDA
+
+Install the [CUDA toolkit v11.3.1](https://developer.nvidia.com/cuda-11-3-1-download-archive) then build the cuda code:
+
+```shell
+make ggml_cuda.dll
+go build -tags avx,cuda .
+```
+
+### ROCm
+
+Install [ROCm 5.7.1](https://rocm.docs.amd.com/en/docs-5.7.1/).
+
+```shell
+make ggml_hipblas.dll
+go build -tags avx,rocm .
+```
+
+## Building runners
+
+```shell
+# build all runners for this platform
+make -j
+```
+
+## Syncing with llama.cpp
+
+To update this package to the latest llama.cpp code, use the `sync.sh` script:
+
+```
+./sync.sh ../../llama.cpp
+```
diff --git a/llama/base64.hpp b/llama/base64.hpp
new file mode 100644
index 00000000..563247a6
--- /dev/null
+++ b/llama/base64.hpp
@@ -0,0 +1,392 @@
+/*
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org>
+*/
+
+#ifndef PUBLIC_DOMAIN_BASE64_HPP_
+#define PUBLIC_DOMAIN_BASE64_HPP_
+
+#include <cstdint>
+#include <iterator>
+#include <stdexcept>
+#include <string>
+
+class base64_error : public std::runtime_error
+{
+public:
+ using std::runtime_error::runtime_error;
+};
+
+class base64
+{
+public:
+ enum class alphabet
+ {
+ /** the alphabet is detected automatically */
+ auto_,
+ /** the standard base64 alphabet is used */
+ standard,
+ /** like `standard` except that the characters `+` and `/` are replaced by `-` and `_` respectively*/
+ url_filename_safe
+ };
+
+ enum class decoding_behavior
+ {
+ /** if the input is not padded, the remaining bits are ignored */
+ moderate,
+ /** if a padding character is encounter decoding is finished */
+ loose
+ };
+
+ /**
+ Encodes all the elements from `in_begin` to `in_end` to `out`.
+
+ @warning The source and destination cannot overlap. The destination must be able to hold at least
+ `required_encode_size(std::distance(in_begin, in_end))`, otherwise the behavior depends on the output iterator.
+
+ @tparam Input_iterator the source; the returned elements are cast to `std::uint8_t` and should not be greater than
+ 8 bits
+ @tparam Output_iterator the destination; the elements written to it are from the type `char`
+ @param in_begin the beginning of the source
+ @param in_end the ending of the source
+ @param out the destination iterator
+ @param alphabet which alphabet should be used
+ @returns the iterator to the next element past the last element copied
+ @throws see `Input_iterator` and `Output_iterator`
+ */
+ template<typename Input_iterator, typename Output_iterator>
+ static Output_iterator encode(Input_iterator in_begin, Input_iterator in_end, Output_iterator out,
+ alphabet alphabet = alphabet::standard)
+ {
+ constexpr auto pad = '=';
+ const char* alpha = alphabet == alphabet::url_filename_safe
+ ? "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+ : "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+ while (in_begin != in_end) {
+ std::uint8_t i0 = 0, i1 = 0, i2 = 0;
+
+ // first character
+ i0 = static_cast<std::uint8_t>(*in_begin);
+ ++in_begin;
+
+ *out = alpha[i0 >> 2 & 0x3f];
+ ++out;
+
+ // part of first character and second
+ if (in_begin != in_end) {
+ i1 = static_cast<std::uint8_t>(*in_begin);
+ ++in_begin;
+
+ *out = alpha[((i0 & 0x3) << 4) | (i1 >> 4 & 0x0f)];
+ ++out;
+ } else {
+ *out = alpha[(i0 & 0x3) << 4];
+ ++out;
+
+ // last padding
+ *out = pad;
+ ++out;
+
+ // last padding
+ *out = pad;
+ ++out;
+
+ break;
+ }
+
+ // part of second character and third
+ if (in_begin != in_end) {
+ i2 = static_cast<std::uint8_t>(*in_begin);
+ ++in_begin;
+
+ *out = alpha[((i1 & 0xf) << 2) | (i2 >> 6 & 0x03)];
+ ++out;
+ } else {
+ *out = alpha[(i1 & 0xf) << 2];
+ ++out;
+
+ // last padding
+ *out = pad;
+ ++out;
+
+ break;
+ }
+
+ // rest of third
+ *out = alpha[i2 & 0x3f];
+ ++out;
+ }
+
+ return out;
+ }
+ /**
+ Encodes a string.
+
+ @param str the string that should be encoded
+ @param alphabet which alphabet should be used
+ @returns the encoded base64 string
+ @throws see base64::encode()
+ */
+ static std::string encode(const std::string& str, alphabet alphabet = alphabet::standard)
+ {
+ std::string result;
+
+ result.reserve(required_encode_size(str.length()) + 1);
+
+ encode(str.begin(), str.end(), std::back_inserter(result), alphabet);
+
+ return result;
+ }
+ /**
+ Encodes a char array.
+
+ @param buffer the char array
+ @param size the size of the array
+ @param alphabet which alphabet should be used
+ @returns the encoded string
+ */
+ static std::string encode(const char* buffer, std::size_t size, alphabet alphabet = alphabet::standard)
+ {
+ std::string result;
+
+ result.reserve(required_encode_size(size) + 1);
+
+ encode(buffer, buffer + size, std::back_inserter(result), alphabet);
+
+ return result;
+ }
+ /**
+ Decodes all the elements from `in_begin` to `in_end` to `out`. `in_begin` may point to the same location as `out`,
+ in other words: inplace decoding is possible.
+
+ @warning The destination must be able to hold at least `required_decode_size(std::distance(in_begin, in_end))`,
+ otherwise the behavior depends on the output iterator.
+
+ @tparam Input_iterator the source; the returned elements are cast to `char`
+ @tparam Output_iterator the destination; the elements written to it are from the type `std::uint8_t`
+ @param in_begin the beginning of the source
+ @param in_end the ending of the source
+ @param out the destination iterator
+ @param alphabet which alphabet should be used
+ @param behavior the behavior when an error was detected
+ @returns the iterator to the next element past the last element copied
+ @throws base64_error depending on the set behavior
+ @throws see `Input_iterator` and `Output_iterator`
+ */
+ template<typename Input_iterator, typename Output_iterator>
+ static Output_iterator decode(Input_iterator in_begin, Input_iterator in_end, Output_iterator out,
+ alphabet alphabet = alphabet::auto_,
+ decoding_behavior behavior = decoding_behavior::moderate)
+ {
+ //constexpr auto pad = '=';
+ std::uint8_t last = 0;
+ auto bits = 0;
+
+ while (in_begin != in_end) {
+ auto c = *in_begin;
+ ++in_begin;
+
+ if (c == '=') {
+ break;
+ }
+
+ auto part = _base64_value(alphabet, c);
+
+ // enough bits for one byte
+ if (bits + 6 >= 8) {
+ *out = (last << (8 - bits)) | (part >> (bits - 2));
+ ++out;
+
+ bits -= 2;
+ } else {
+ bits += 6;
+ }
+
+ last = part;
+ }
+
+ // check padding
+ if (behavior != decoding_behavior::loose) {
+ while (in_begin != in_end) {
+ auto c = *in_begin;
+ ++in_begin;
+
+ if (c != '=') {
+ throw base64_error("invalid base64 character.");
+ }
+ }
+ }
+
+ return out;
+ }
+ /**
+ Decodes a string.
+
+ @param str the base64 encoded string
+ @param alphabet which alphabet should be used
+ @param behavior the behavior when an error was detected
+ @returns the decoded string
+ @throws see base64::decode()
+ */
+ static std::string decode(const std::string& str, alphabet alphabet = alphabet::auto_,
+ decoding_behavior behavior = decoding_behavior::moderate)
+ {
+ std::string result;
+
+ result.reserve(max_decode_size(str.length()));
+
+ decode(str.begin(), str.end(), std::back_inserter(result), alphabet, behavior);
+
+ return result;
+ }
+ /**
+ Decodes a string.
+
+ @param buffer the base64 encoded buffer
+ @param size the size of the buffer
+ @param alphabet which alphabet should be used
+ @param behavior the behavior when an error was detected
+ @returns the decoded string
+ @throws see base64::decode()
+ */
+ static std::string decode(const char* buffer, std::size_t size, alphabet alphabet = alphabet::auto_,
+ decoding_behavior behavior = decoding_behavior::moderate)
+ {
+ std::string result;
+
+ result.reserve(max_decode_size(size));
+
+ decode(buffer, buffer + size, std::back_inserter(result), alphabet, behavior);
+
+ return result;
+ }
+ /**
+ Decodes a string inplace.
+
+ @param[in,out] str the base64 encoded string
+ @param alphabet which alphabet should be used
+ @param behavior the behavior when an error was detected
+ @throws base64::decode_inplace()
+ */
+ static void decode_inplace(std::string& str, alphabet alphabet = alphabet::auto_,
+ decoding_behavior behavior = decoding_behavior::moderate)
+ {
+ str.resize(decode(str.begin(), str.end(), str.begin(), alphabet, behavior) - str.begin());
+ }
+ /**
+ Decodes a char array inplace.
+
+ @param[in,out] str the string array
+ @param size the length of the array
+ @param alphabet which alphabet should be used
+ @param behavior the behavior when an error was detected
+ @returns the pointer to the next element past the last element decoded
+ @throws base64::decode_inplace()
+ */
+ static char* decode_inplace(char* str, std::size_t size, alphabet alphabet = alphabet::auto_,
+ decoding_behavior behavior = decoding_behavior::moderate)
+ {
+ return decode(str, str + size, str, alphabet, behavior);
+ }
+ /**
+ Returns the required decoding size for a given size. The value is calculated with the following formula:
+
+ $$
+ \lceil \frac{size}{4} \rceil \cdot 3
+ $$
+
+ @param size the size of the encoded input
+ @returns the size of the resulting decoded buffer; this the absolute maximum
+ */
+ static std::size_t max_decode_size(std::size_t size) noexcept
+ {
+ return (size / 4 + (size % 4 ? 1 : 0)) * 3;
+ }
+ /**
+ Returns the required encoding size for a given size. The value is calculated with the following formula:
+
+ $$
+ \lceil \frac{size}{3} \rceil \cdot 4
+ $$
+
+ @param size the size of the decoded input
+ @returns the size of the resulting encoded buffer
+ */
+ static std::size_t required_encode_size(std::size_t size) noexcept
+ {
+ return (size / 3 + (size % 3 ? 1 : 0)) * 4;
+ }
+
+private:
+ static std::uint8_t _base64_value(alphabet& alphabet, char c)
+ {
+ if (c >= 'A' && c <= 'Z') {
+ return c - 'A';
+ } else if (c >= 'a' && c <= 'z') {
+ return c - 'a' + 26;
+ } else if (c >= '0' && c <= '9') {
+ return c - '0' + 52;
+ }
+
+ // comes down to alphabet
+ if (alphabet == alphabet::standard) {
+ if (c == '+') {
+ return 62;
+ } else if (c == '/') {
+ return 63;
+ }
+ } else if (alphabet == alphabet::url_filename_safe) {
+ if (c == '-') {
+ return 62;
+ } else if (c == '_') {
+ return 63;
+ }
+ } // auto detect
+ else {
+ if (c == '+') {
+ alphabet = alphabet::standard;
+
+ return 62;
+ } else if (c == '/') {
+ alphabet = alphabet::standard;
+
+ return 63;
+ } else if (c == '-') {
+ alphabet = alphabet::url_filename_safe;
+
+ return 62;
+ } else if (c == '_') {
+ alphabet = alphabet::url_filename_safe;
+
+ return 63;
+ }
+ }
+
+ throw base64_error("invalid base64 character.");
+ }
+};
+
+#endif // !PUBLIC_DOMAIN_BASE64_HPP_
diff --git a/llama/build-info.cpp b/llama/build-info.cpp
new file mode 100644
index 00000000..52e983a8
--- /dev/null
+++ b/llama/build-info.cpp
@@ -0,0 +1,30 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+int LLAMA_BUILD_NUMBER = 0;
+char const *LLAMA_COMMIT = "";
+char const *LLAMA_COMPILER = "";
+char const *LLAMA_BUILD_TARGET = "";
diff --git a/llama/clip.cpp b/llama/clip.cpp
new file mode 100644
index 00000000..1a990306
--- /dev/null
+++ b/llama/clip.cpp
@@ -0,0 +1,2689 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// NOTE: This is modified from clip.cpp only for LLaVA,
+// so there might be still unnecessary artifacts hanging around
+// I'll gradually clean and extend it
+// Note: Even when using identical normalized image inputs (see normalize_image_u8_to_f32()) we have a significant difference in resulting embeddings compared to pytorch
+#include "clip.h"
+#include "log.h"
+#include "ggml.h"
+#include "ggml-alloc.h"
+#include "ggml-backend.h"
+
+#ifdef GGML_USE_CUDA
+#include "ggml-cuda.h"
+#endif
+
+#ifdef GGML_USE_METAL
+#include "ggml-metal.h"
+#endif
+
+#ifdef GGML_USE_CANN
+#include "ggml-cann.h"
+#endif
+
+#ifdef GGML_USE_VULKAN
+#include "ggml-vulkan.h"
+#endif
+
+#define STB_IMAGE_IMPLEMENTATION
+#include "stb_image.h"
+
+#include <cassert>
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <map>
+#include <regex>
+#include <stdexcept>
+#include <vector>
+#include <sstream>
+#include <cinttypes>
+#include <limits>
+
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+ #define NOMINMAX
+#endif
+#include <windows.h>
+#if __GLIBCXX__
+#include <cstdio>
+#include <ext/stdio_filebuf.h>
+#include <fcntl.h>
+#endif
+#endif
+
+//#define CLIP_DEBUG_FUNCTIONS
+
+// RGB uint8 image
+struct clip_image_u8 {
+ int nx;
+ int ny;
+
+ std::vector<uint8_t> buf;
+};
+
+// RGB float32 image (NHWC)
+// Memory layout: RGBRGBRGB...
+struct clip_image_f32 {
+ int nx;
+ int ny;
+
+ std::vector<float> buf;
+};
+
+static std::string format(const char * fmt, ...) {
+ va_list ap;
+ va_list ap2;
+ va_start(ap, fmt);
+ va_copy(ap2, ap);
+ int size = vsnprintf(NULL, 0, fmt, ap);
+ GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT
+ std::vector<char> buf(size + 1);
+ int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
+ GGML_ASSERT(size2 == size);
+ va_end(ap2);
+ va_end(ap);
+ return std::string(buf.data(), buf.size());
+}
+
+//
+// key constants
+//
+
+#define KEY_FTYPE "general.file_type"
+#define KEY_NAME "general.name"
+#define KEY_DESCRIPTION "general.description"
+#define KEY_HAS_TEXT_ENC "clip.has_text_encoder"
+#define KEY_HAS_VIS_ENC "clip.has_vision_encoder"
+#define KEY_HAS_LLAVA_PROJ "clip.has_llava_projector"
+#define KEY_HAS_MINICPMV_PROJ "clip.has_minicpmv_projector"
+#define KEY_MINICPMV_VERSION "clip.minicpmv_version"
+#define KEY_USE_GELU "clip.use_gelu"
+#define KEY_N_EMBD "clip.%s.embedding_length"
+#define KEY_N_FF "clip.%s.feed_forward_length"
+#define KEY_N_BLOCK "clip.%s.block_count"
+#define KEY_N_HEAD "clip.%s.attention.head_count"
+#define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon"
+#define KEY_PROJ_DIM "clip.%s.projection_dim"
+#define KEY_TOKENS "tokenizer.ggml.tokens"
+#define KEY_N_POSITIONS "clip.text.context_length"
+#define KEY_IMAGE_SIZE "clip.vision.image_size"
+#define KEY_PATCH_SIZE "clip.vision.patch_size"
+#define KEY_IMAGE_MEAN "clip.vision.image_mean"
+#define KEY_IMAGE_STD "clip.vision.image_std"
+#define KEY_PROJ_TYPE "clip.projector_type"
+
+#define KEY_MM_PATCH_MERGE_TYPE "clip.vision.mm_patch_merge_type"
+#define KEY_IMAGE_GRID_PINPOINTS "clip.vision.image_grid_pinpoints"
+#define KEY_IMAGE_CROP_RESOLUTION "clip.vision.image_crop_resolution"
+
+
+//
+// tensor name constants
+//
+
+#define TN_TOKEN_EMBD "%s.token_embd.weight"
+#define TN_POS_EMBD "%s.position_embd.weight"
+#define TN_CLASS_EMBD "v.class_embd"
+#define TN_PATCH_EMBD "v.patch_embd.weight"
+#define TN_PATCH_BIAS "v.patch_embd.bias"
+#define TN_ATTN_K "%s.blk.%d.attn_k.%s"
+#define TN_ATTN_Q "%s.blk.%d.attn_q.%s"
+#define TN_ATTN_V "%s.blk.%d.attn_v.%s"
+#define TN_ATTN_OUTPUT "%s.blk.%d.attn_out.%s"
+#define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s"
+#define TN_FFN_UP "%s.blk.%d.ffn_up.%s"
+#define TN_LN_1 "%s.blk.%d.ln1.%s"
+#define TN_LN_2 "%s.blk.%d.ln2.%s"
+#define TN_LN_PRE "%s.pre_ln.%s"
+#define TN_LN_POST "%s.post_ln.%s"
+#define TN_TEXT_PROJ "text_projection.weight"
+#define TN_VIS_PROJ "visual_projection.weight"
+#define TN_LLAVA_PROJ "mm.%d.%s"
+#define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s"
+#define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s"
+#define TN_MVLM_PROJ_PEG "mm.model.peg.%d.%s"
+#define TN_IMAGE_NEWLINE "model.image_newline"
+
+#define TN_MINICPMV_POS_EMBD_K "resampler.pos_embed_k"
+#define TN_MINICPMV_QUERY "resampler.query"
+#define TN_MINICPMV_PROJ "resampler.proj.weight"
+#define TN_MINICPMV_KV_PROJ "resampler.kv.weight"
+#define TN_MINICPMV_ATTN "resampler.attn.%s.%s"
+#define TN_MINICPMV_LN "resampler.ln_%s.%s"
+
+
+enum projector_type {
+ PROJECTOR_TYPE_MLP,
+ PROJECTOR_TYPE_MLP_NORM,
+ PROJECTOR_TYPE_LDP,
+ PROJECTOR_TYPE_LDPV2,
+ PROJECTOR_TYPE_RESAMPLER,
+ PROJECTOR_TYPE_UNKNOWN,
+};
+
+static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
+ { PROJECTOR_TYPE_MLP, "mlp" },
+ { PROJECTOR_TYPE_LDP, "ldp" },
+ { PROJECTOR_TYPE_LDPV2, "ldpv2"},
+ { PROJECTOR_TYPE_RESAMPLER, "resampler"},
+};
+
+
+//
+// utilities to get data from a gguf file
+//
+
+static int get_key_idx(const gguf_context * ctx, const char * key) {
+ int i = gguf_find_key(ctx, key);
+ if (i == -1) {
+ LOG_TEE("key %s not found in file\n", key);
+ throw std::runtime_error(format("Missing required key: %s", key));
+ }
+
+ return i;
+}
+
+static uint32_t get_u32(const gguf_context * ctx, const std::string & key) {
+ const int i = get_key_idx(ctx, key.c_str());
+
+ return gguf_get_val_u32(ctx, i);
+}
+
+static float get_f32(const gguf_context * ctx, const std::string & key) {
+ const int i = get_key_idx(ctx, key.c_str());
+
+ return gguf_get_val_f32(ctx, i);
+}
+
+static struct ggml_tensor * get_tensor(struct ggml_context * ctx, const std::string & name) {
+ struct ggml_tensor * cur = ggml_get_tensor(ctx, name.c_str());
+ if (!cur) {
+ throw std::runtime_error(format("%s: unable to find tensor %s\n", __func__, name.c_str()));
+ }
+
+ return cur;
+}
+
+static std::string get_ftype(int ftype) {
+ return ggml_type_name(static_cast<ggml_type>(ftype));
+}
+
+static std::string gguf_data_to_str(enum gguf_type type, const void * data, int i) {
+ switch (type) {
+ case GGUF_TYPE_UINT8: return std::to_string(((const uint8_t *)data)[i]);
+ case GGUF_TYPE_INT8: return std::to_string(((const int8_t *)data)[i]);
+ case GGUF_TYPE_UINT16: return std::to_string(((const uint16_t *)data)[i]);
+ case GGUF_TYPE_INT16: return std::to_string(((const int16_t *)data)[i]);
+ case GGUF_TYPE_UINT32: return std::to_string(((const uint32_t *)data)[i]);
+ case GGUF_TYPE_INT32: return std::to_string(((const int32_t *)data)[i]);
+ case GGUF_TYPE_UINT64: return std::to_string(((const uint64_t *)data)[i]);
+ case GGUF_TYPE_INT64: return std::to_string(((const int64_t *)data)[i]);
+ case GGUF_TYPE_FLOAT32: return std::to_string(((const float *)data)[i]);
+ case GGUF_TYPE_FLOAT64: return std::to_string(((const double *)data)[i]);
+ case GGUF_TYPE_BOOL: return ((const bool *)data)[i] ? "true" : "false";
+ default: return format("unknown type %d", type);
+ }
+}
+
+static void replace_all(std::string & s, const std::string & search, const std::string & replace) {
+ if (search.empty()) {
+ return;
+ }
+ std::string builder;
+ builder.reserve(s.length());
+ size_t pos = 0;
+ size_t last_pos = 0;
+ while ((pos = s.find(search, last_pos)) != std::string::npos) {
+ builder.append(s, last_pos, pos - last_pos);
+ builder.append(replace);
+ last_pos = pos + search.length();
+ }
+ builder.append(s, last_pos, std::string::npos);
+ s = std::move(builder);
+}
+
+static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {
+ const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);
+
+ switch (type) {
+ case GGUF_TYPE_STRING:
+ return gguf_get_val_str(ctx_gguf, i);
+ case GGUF_TYPE_ARRAY:
+ {
+ const enum gguf_type arr_type = gguf_get_arr_type(ctx_gguf, i);
+ int arr_n = gguf_get_arr_n(ctx_gguf, i);
+ const void * data = gguf_get_arr_data(ctx_gguf, i);
+ std::stringstream ss;
+ ss << "[";
+ for (int j = 0; j < arr_n; j++) {
+ if (arr_type == GGUF_TYPE_STRING) {
+ std::string val = gguf_get_arr_str(ctx_gguf, i, j);
+ // escape quotes
+ replace_all(val, "\\", "\\\\");
+ replace_all(val, "\"", "\\\"");
+ ss << '"' << val << '"';
+ } else if (arr_type == GGUF_TYPE_ARRAY) {
+ ss << "???";
+ } else {
+ ss << gguf_data_to_str(arr_type, data, j);
+ }
+ if (j < arr_n - 1) {
+ ss << ", ";
+ }
+ }
+ ss << "]";
+ return ss.str();
+ }
+ default:
+ return gguf_data_to_str(type, gguf_get_val_data(ctx_gguf, i), 0);
+ }
+}
+
+static void print_tensor_info(const ggml_tensor * tensor, const char * prefix = "") {
+ size_t tensor_size = ggml_nbytes(tensor);
+ LOG_TEE("%s: n_dims = %d, name = %s, tensor_size=%zu, shape:[%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64 "], type = %s\n",
+ prefix, ggml_n_dims(tensor), tensor->name, tensor_size,
+ tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], ggml_type_name(tensor->type));
+}
+
+static projector_type clip_projector_type_from_string(const std::string & name) {
+ for (const auto & kv : PROJECTOR_TYPE_NAMES) { // NOLINT
+ if (kv.second == name) {
+ return kv.first;
+ }
+ }
+ return PROJECTOR_TYPE_UNKNOWN;
+}
+
+#ifdef CLIP_DEBUG_FUNCTIONS
+static void clip_image_write_image_to_ppm(const clip_image_u8& img, const std::string& filename) {
+ std::ofstream file(filename, std::ios::binary);
+ if (!file.is_open()) {
+ LOG_TEE("Failed to open file for writing: %s\n", filename.c_str());
+ return;
+ }
+
+ // PPM header: P6 format, width, height, and max color value
+ file << "P6\n" << img.nx << " " << img.ny << "\n255\n";
+
+ // Write pixel data
+ for (size_t i = 0; i < img.buf.size(); i += 3) {
+ // PPM expects binary data in RGB format, which matches our image buffer
+ file.write(reinterpret_cast<const char*>(&img.buf[i]), 3);
+ }
+
+ file.close();
+}
+
+static void clip_image_save_to_bmp(const clip_image_u8& img, const std::string& filename) {
+ std::ofstream file(filename, std::ios::binary);
+ if (!file.is_open()) {
+ LOG_TEE("Failed to open file for writing: %s\n", filename.c_str());
+ return;
+ }
+
+ int fileSize = 54 + 3 * img.nx * img.ny; // File header + info header + pixel data
+ int bytesPerPixel = 3;
+ int widthInBytes = img.nx * bytesPerPixel;
+ int paddingAmount = (4 - (widthInBytes % 4)) % 4;
+ int stride = widthInBytes + paddingAmount;
+
+ // Bitmap file header
+ unsigned char fileHeader[14] = {
+ 'B','M', // Signature
+ 0,0,0,0, // Image file size in bytes
+ 0,0,0,0, // Reserved
+ 54,0,0,0 // Start of pixel array
+ };
+
+ // Total file size
+ fileSize = 54 + (stride * img.ny);
+ fileHeader[2] = (unsigned char)(fileSize);
+ fileHeader[3] = (unsigned char)(fileSize >> 8);
+ fileHeader[4] = (unsigned char)(fileSize >> 16);
+ fileHeader[5] = (unsigned char)(fileSize >> 24);
+
+ // Bitmap information header (BITMAPINFOHEADER)
+ unsigned char infoHeader[40] = {
+ 40,0,0,0, // Size of this header (40 bytes)
+ 0,0,0,0, // Image width
+ 0,0,0,0, // Image height
+ 1,0, // Number of color planes
+ 24,0, // Bits per pixel
+ 0,0,0,0, // No compression
+ 0,0,0,0, // Image size (can be 0 for no compression)
+ 0,0,0,0, // X pixels per meter (not specified)
+ 0,0,0,0, // Y pixels per meter (not specified)
+ 0,0,0,0, // Total colors (color table not used)
+ 0,0,0,0 // Important colors (all are important)
+ };
+
+ // Width and height in the information header
+ infoHeader[4] = (unsigned char)(img.nx);
+ infoHeader[5] = (unsigned char)(img.nx >> 8);
+ infoHeader[6] = (unsigned char)(img.nx >> 16);
+ infoHeader[7] = (unsigned char)(img.nx >> 24);
+ infoHeader[8] = (unsigned char)(img.ny);
+ infoHeader[9] = (unsigned char)(img.ny >> 8);
+ infoHeader[10] = (unsigned char)(img.ny >> 16);
+ infoHeader[11] = (unsigned char)(img.ny >> 24);
+
+ // Write file headers
+ file.write(reinterpret_cast<char*>(fileHeader), sizeof(fileHeader));
+ file.write(reinterpret_cast<char*>(infoHeader), sizeof(infoHeader));
+
+ // Pixel data
+ std::vector<unsigned char> padding(3, 0); // Max padding size to be added to each row
+ for (int y = img.ny - 1; y >= 0; --y) { // BMP files are stored bottom-to-top
+ for (int x = 0; x < img.nx; ++x) {
+ // Each pixel
+ size_t pixelIndex = (y * img.nx + x) * 3;
+ unsigned char pixel[3] = {
+ img.buf[pixelIndex + 2], // BMP stores pixels in BGR format
+ img.buf[pixelIndex + 1],
+ img.buf[pixelIndex]
+ };
+ file.write(reinterpret_cast<char*>(pixel), 3);
+ }
+ // Write padding for the row
+ file.write(reinterpret_cast<char*>(padding.data()), paddingAmount);
+ }
+
+ file.close();
+}
+
+// debug function to convert f32 to u8
+static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u8& dst) {
+ dst.nx = src.nx;
+ dst.ny = src.ny;
+ dst.buf.resize(3 * src.nx * src.ny);
+ for (size_t i = 0; i < src.buf.size(); ++i) {
+ dst.buf[i] = static_cast<uint8_t>(std::min(std::max(int(src.buf[i] * 255.0f), 0), 255));
+ }
+}
+#endif
+
+
+//
+// clip layers
+//
+
+struct clip_hparams {
+ int32_t image_size;
+ int32_t patch_size;
+ int32_t hidden_size;
+ int32_t n_intermediate;
+ int32_t projection_dim;
+ int32_t n_head;
+ int32_t n_layer;
+
+ float eps;
+
+ char mm_patch_merge_type[32] = "flat"; // spatial_unpad or flat (default)
+
+ int32_t image_grid_pinpoints[32];
+ int32_t image_crop_resolution;
+};
+
+struct clip_layer {
+ // attention
+ struct ggml_tensor * k_w;
+ struct ggml_tensor * k_b;
+ struct ggml_tensor * q_w;
+ struct ggml_tensor * q_b;
+ struct ggml_tensor * v_w;
+ struct ggml_tensor * v_b;
+
+ struct ggml_tensor * o_w;
+ struct ggml_tensor * o_b;
+
+ // layernorm 1
+ struct ggml_tensor * ln_1_w;
+ struct ggml_tensor * ln_1_b;
+
+ // ff
+ struct ggml_tensor * ff_i_w;
+ struct ggml_tensor * ff_i_b;
+
+ struct ggml_tensor * ff_o_w;
+ struct ggml_tensor * ff_o_b;
+
+ // layernorm 2
+ struct ggml_tensor * ln_2_w;
+ struct ggml_tensor * ln_2_b;
+};
+
+struct clip_vision_model {
+ struct clip_hparams hparams;
+
+ // embeddings
+ struct ggml_tensor * class_embedding;
+ struct ggml_tensor * patch_embeddings;
+ struct ggml_tensor * patch_bias;
+ struct ggml_tensor * position_embeddings;
+
+ struct ggml_tensor * pre_ln_w;
+ struct ggml_tensor * pre_ln_b;
+
+ std::vector<clip_layer> layers;
+
+ struct ggml_tensor * post_ln_w;
+ struct ggml_tensor * post_ln_b;
+
+ struct ggml_tensor * projection;
+
+ // LLaVA projection
+ struct ggml_tensor * mm_0_w = NULL;
+ struct ggml_tensor * mm_0_b = NULL;
+ struct ggml_tensor * mm_2_w = NULL;
+ struct ggml_tensor * mm_2_b = NULL;
+
+ struct ggml_tensor * image_newline = NULL;
+
+ // Yi type models with mlp+normalization projection
+ struct ggml_tensor * mm_1_w = NULL; // Yi type models have 0, 1, 3, 4
+ struct ggml_tensor * mm_1_b = NULL;
+ struct ggml_tensor * mm_3_w = NULL;
+ struct ggml_tensor * mm_3_b = NULL;
+ struct ggml_tensor * mm_4_w = NULL;
+ struct ggml_tensor * mm_4_b = NULL;
+
+ // MobileVLM projection
+ struct ggml_tensor * mm_model_mlp_1_w;
+ struct ggml_tensor * mm_model_mlp_1_b;
+ struct ggml_tensor * mm_model_mlp_3_w;
+ struct ggml_tensor * mm_model_mlp_3_b;
+ struct ggml_tensor * mm_model_block_1_block_0_0_w;
+ struct ggml_tensor * mm_model_block_1_block_0_1_w;
+ struct ggml_tensor * mm_model_block_1_block_0_1_b;
+ struct ggml_tensor * mm_model_block_1_block_1_fc1_w;
+ struct ggml_tensor * mm_model_block_1_block_1_fc1_b;
+ struct ggml_tensor * mm_model_block_1_block_1_fc2_w;
+ struct ggml_tensor * mm_model_block_1_block_1_fc2_b;
+ struct ggml_tensor * mm_model_block_1_block_2_0_w;
+ struct ggml_tensor * mm_model_block_1_block_2_1_w;
+ struct ggml_tensor * mm_model_block_1_block_2_1_b;
+ struct ggml_tensor * mm_model_block_2_block_0_0_w;
+ struct ggml_tensor * mm_model_block_2_block_0_1_w;
+ struct ggml_tensor * mm_model_block_2_block_0_1_b;
+ struct ggml_tensor * mm_model_block_2_block_1_fc1_w;
+ struct ggml_tensor * mm_model_block_2_block_1_fc1_b;
+ struct ggml_tensor * mm_model_block_2_block_1_fc2_w;
+ struct ggml_tensor * mm_model_block_2_block_1_fc2_b;
+ struct ggml_tensor * mm_model_block_2_block_2_0_w;
+ struct ggml_tensor * mm_model_block_2_block_2_1_w;
+ struct ggml_tensor * mm_model_block_2_block_2_1_b;
+
+ // MobileVLM_V2 projection
+ struct ggml_tensor * mm_model_mlp_0_w;
+ struct ggml_tensor * mm_model_mlp_0_b;
+ struct ggml_tensor * mm_model_mlp_2_w;
+ struct ggml_tensor * mm_model_mlp_2_b;
+ struct ggml_tensor * mm_model_peg_0_w;
+ struct ggml_tensor * mm_model_peg_0_b;
+
+ // MINICPMV projection
+ struct ggml_tensor * mm_model_pos_embed_k;
+ struct ggml_tensor * mm_model_query;
+ struct ggml_tensor * mm_model_proj;
+ struct ggml_tensor * mm_model_kv_proj;
+ struct ggml_tensor * mm_model_attn_q_w;
+ struct ggml_tensor * mm_model_attn_q_b;
+ struct ggml_tensor * mm_model_attn_k_w;
+ struct ggml_tensor * mm_model_attn_k_b;
+ struct ggml_tensor * mm_model_attn_v_w;
+ struct ggml_tensor * mm_model_attn_v_b;
+ struct ggml_tensor * mm_model_attn_o_w;
+ struct ggml_tensor * mm_model_attn_o_b;
+ struct ggml_tensor * mm_model_ln_q_w;
+ struct ggml_tensor * mm_model_ln_q_b;
+ struct ggml_tensor * mm_model_ln_kv_w;
+ struct ggml_tensor * mm_model_ln_kv_b;
+ struct ggml_tensor * mm_model_ln_post_w;
+ struct ggml_tensor * mm_model_ln_post_b;
+};
+
+struct clip_ctx {
+ bool has_text_encoder = false;
+ bool has_vision_encoder = false;
+ bool has_llava_projector = false;
+ bool has_minicpmv_projector = false;
+ int minicpmv_version = 2;
+
+ struct clip_vision_model vision_model;
+ projector_type proj_type = PROJECTOR_TYPE_MLP;
+
+ float image_mean[3];
+ float image_std[3];
+ bool use_gelu = false;
+ int32_t ftype = 1;
+
+ bool has_class_embedding = true;
+ bool has_pre_norm = true;
+ bool has_post_norm = false;
+ bool has_patch_bias = false;
+
+ struct gguf_context * ctx_gguf;
+ struct ggml_context * ctx_data;
+
+ std::vector<uint8_t> buf_compute_meta;
+
+ // memory buffers to evaluate the model
+ ggml_backend_buffer_t params_buffer = NULL;
+
+ ggml_backend_t backend = NULL;
+ ggml_gallocr_t compute_alloc = NULL;
+
+ struct clip_image_size * load_image_size;
+};
+
+static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32_batch * imgs, struct clip_image_size * load_image_size, bool is_inf = false) {
+ if (!ctx->has_vision_encoder) {
+ LOG_TEE("This gguf file seems to have no vision encoder\n");
+ return nullptr;
+ }
+
+ const auto & model = ctx->vision_model;
+ const auto & hparams = model.hparams;
+
+ const int image_size = hparams.image_size;
+ int image_size_width = image_size;
+ int image_size_height = image_size;
+ if (ctx->has_minicpmv_projector) {
+ if (load_image_size == nullptr) {
+ load_image_size = clip_image_size_init();
+ }
+ LOG_TEE("%s: %d %d\n", __func__, load_image_size->width, load_image_size->height);
+ image_size_width = load_image_size->width;
+ image_size_height = load_image_size->height;
+ if (is_inf) {
+ image_size_width = imgs->data->nx;
+ image_size_height = imgs->data->ny;
+ }
+ }
+ const int patch_size = hparams.patch_size;
+ const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size));
+ const int num_positions = num_patches + (ctx->has_class_embedding ? 1 : 0);
+ const int hidden_size = hparams.hidden_size;
+ const int n_head = hparams.n_head;
+ const int d_head = hidden_size / n_head;
+ int n_layer = hparams.n_layer;
+ const float eps = hparams.eps;
+
+ const int batch_size = imgs->size;
+
+ if (ctx->has_llava_projector || ctx->has_minicpmv_projector) {
+ GGML_ASSERT(batch_size == 1);
+ }
+
+ struct ggml_init_params params = {
+ /*.mem_size =*/ ctx->buf_compute_meta.size(),
+ /*.mem_buffer =*/ ctx->buf_compute_meta.data(),
+ /*.no_alloc =*/ true,
+ };
+
+ struct ggml_context * ctx0 = ggml_init(params);
+ struct ggml_cgraph * gf = ggml_new_graph(ctx0);
+
+ struct ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, image_size_width, image_size_height, 3, batch_size);
+ ggml_set_name(inp_raw, "inp_raw");
+ ggml_set_input(inp_raw);
+
+ struct ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
+
+ inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size);
+ inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
+
+ if (ctx->has_patch_bias) {
+ // inp = ggml_add(ctx0, inp, ggml_repeat(ctx0, model.patch_bias, inp));
+ inp = ggml_add(ctx0, inp, model.patch_bias);
+ }
+ struct ggml_tensor * embeddings = inp;
+ struct ggml_tensor * pos_embed = nullptr;
+
+ if (ctx->has_llava_projector) {
+ // concat class_embeddings and patch_embeddings
+ if (ctx->has_class_embedding) {
+ embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size);
+ ggml_set_name(embeddings, "embeddings");
+ ggml_set_input(embeddings);
+ embeddings = ggml_acc(ctx0, embeddings, model.class_embedding,
+ embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], 0);
+ embeddings = ggml_acc(ctx0, embeddings, inp,
+ embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], model.class_embedding->nb[1]);
+ }
+ }
+
+ struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions);
+ ggml_set_name(positions, "positions");
+ ggml_set_input(positions);
+
+ embeddings =
+ ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, model.position_embeddings, positions));
+
+ if (ctx->has_minicpmv_projector) {
+ int pos_w = image_size_width/patch_size;
+ int pos_h = image_size_height/patch_size;
+ if (ctx->minicpmv_version == 2) {
+ pos_embed = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 4096, pos_w * pos_h, 1);
+ }
+ else if (ctx->minicpmv_version == 3) {
+ pos_embed = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 3584, pos_w * pos_h, 1);
+ }
+ ggml_set_name(pos_embed, "pos_embed");
+ ggml_set_input(pos_embed);
+ }
+
+ // pre-layernorm
+ if (ctx->has_pre_norm) {
+ embeddings = ggml_norm(ctx0, embeddings, eps);
+ ggml_set_name(embeddings, "pre_ln");
+
+ embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.pre_ln_w), model.pre_ln_b);
+ }
+
+ // loop over layers
+ if (ctx->has_minicpmv_projector) {
+ n_layer += 1;
+ }
+ for (int il = 0; il < n_layer - 1; il++) {
+ struct ggml_tensor * cur = embeddings; // embeddings = residual, cur = hidden_states
+
+ //const size_t nb_q_w = model.layers[il].q_w->nb[0];
+
+ // layernorm1
+ {
+ cur = ggml_norm(ctx0, cur, eps);
+
+ cur = ggml_add(ctx0, ggml_mul(ctx0, cur, model.layers[il].ln_1_w),
+ model.layers[il].ln_1_b);
+ }
+
+ // self-attention
+ {
+
+ struct ggml_tensor * Q =
+ ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].q_w, cur), model.layers[il].q_b);
+
+ Q = ggml_scale_inplace(ctx0, Q, 1.0f / sqrt((float)d_head));
+ Q = ggml_reshape_4d(ctx0, Q, d_head, n_head, num_positions, batch_size);
+ Q = ggml_cont(ctx0, ggml_permute(ctx0, Q, 0, 2, 1, 3));
+ Q = ggml_reshape_3d(ctx0, Q, d_head, num_positions, n_head * batch_size);
+
+ struct ggml_tensor * K =
+ ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].k_w, cur), model.layers[il].k_b);
+
+ K = ggml_reshape_4d(ctx0, K, d_head, n_head, num_positions, batch_size);
+ K = ggml_cont(ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3));
+ K = ggml_reshape_3d(ctx0, K, d_head, num_positions, n_head * batch_size);
+
+ struct ggml_tensor * V =
+ ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].v_w, cur), model.layers[il].v_b);
+
+ V = ggml_reshape_4d(ctx0, V, d_head, n_head, num_positions, batch_size);
+ V = ggml_cont(ctx0, ggml_permute(ctx0, V, 1, 2, 0, 3));
+ V = ggml_reshape_3d(ctx0, V, num_positions, d_head, n_head * batch_size);
+
+ struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
+ KQ = ggml_soft_max_inplace(ctx0, KQ);
+ struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ);
+ KQV = ggml_reshape_4d(ctx0, KQV, d_head, num_positions, n_head, batch_size);
+ KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
+
+ cur = ggml_cont_3d(ctx0, KQV, hidden_size, num_positions, batch_size);
+ }
+
+ // attention output
+ cur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].o_w, cur), model.layers[il].o_b);
+
+ // re-add the layer input, e.g., residual
+ cur = ggml_add(ctx0, cur, embeddings);
+
+ embeddings = cur; // embeddings = residual, cur = hidden_states
+
+ // layernorm2
+ {
+ cur = ggml_norm(ctx0, cur, eps);
+
+ cur = ggml_add(ctx0, ggml_mul(ctx0, cur, model.layers[il].ln_2_w), model.layers[il].ln_2_b);
+ }
+
+ cur = ggml_mul_mat(ctx0, model.layers[il].ff_i_w, cur);
+ cur = ggml_add(ctx0, cur, model.layers[il].ff_i_b);
+
+ if (ctx->use_gelu) {
+ cur = ggml_gelu_inplace(ctx0, cur);
+ } else {
+ cur = ggml_gelu_quick_inplace(ctx0, cur);
+ }
+
+ cur = ggml_mul_mat(ctx0, model.layers[il].ff_o_w, cur);
+ cur = ggml_add(ctx0, cur, model.layers[il].ff_o_b);
+
+ // residual 2
+ cur = ggml_add(ctx0, embeddings, cur);
+
+ embeddings = cur;
+ }
+
+ // post-layernorm
+ if (ctx->has_post_norm) {
+ embeddings = ggml_norm(ctx0, embeddings, eps);
+ ggml_set_name(embeddings, "post_ln");
+
+ embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.post_ln_w), model.post_ln_b);
+ }
+
+ // llava projector
+ if (ctx->has_llava_projector) {
+ embeddings = ggml_reshape_2d(ctx0, embeddings, embeddings->ne[0], embeddings->ne[1]);
+
+ struct ggml_tensor * patches = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_patches);
+ ggml_set_name(patches, "patches");
+ ggml_set_input(patches);
+
+ // shape [1, 576, 1024]
+ // ne is whcn, ne = [1024, 576, 1, 1]
+ embeddings = ggml_get_rows(ctx0, embeddings, patches);
+
+ // print_tensor_info(embeddings, "embeddings");
+
+ // llava projector
+ if (ctx->proj_type == PROJECTOR_TYPE_MLP) {
+ embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings);
+ embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);
+
+ embeddings = ggml_gelu(ctx0, embeddings);
+ embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);
+ embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);
+ }
+ else if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) {
+ embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings);
+ embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);
+ // ggml_tensor_printf(embeddings, "mm_0_w",0,true,false);
+ // First LayerNorm
+ embeddings = ggml_norm(ctx0, embeddings, eps);
+ embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.mm_1_w),
+ model.mm_1_b);
+
+ // GELU activation
+ embeddings = ggml_gelu(ctx0, embeddings);
+
+ // Second linear layer
+ embeddings = ggml_mul_mat(ctx0, model.mm_3_w, embeddings);
+ embeddings = ggml_add(ctx0, embeddings, model.mm_3_b);
+
+ // Second LayerNorm
+ embeddings = ggml_norm(ctx0, embeddings, eps);
+ embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.mm_4_w),
+ model.mm_4_b);
+ }
+ else if (ctx->proj_type == PROJECTOR_TYPE_LDP) {
+ // MobileVLM projector
+ int n_patch = 24;
+ struct ggml_tensor * mlp_1 = ggml_mul_mat(ctx0, model.mm_model_mlp_1_w, embeddings);
+ mlp_1 = ggml_add(ctx0, mlp_1, model.mm_model_mlp_1_b);
+ mlp_1 = ggml_gelu(ctx0, mlp_1);
+ struct ggml_tensor * mlp_3 = ggml_mul_mat(ctx0, model.mm_model_mlp_3_w, mlp_1);
+ mlp_3 = ggml_add(ctx0, mlp_3, model.mm_model_mlp_3_b);
+ // mlp_3 shape = [1, 576, 2048], ne = [2048, 576, 1, 1]
+
+ // block 1
+ struct ggml_tensor * block_1 = nullptr;
+ {
+ // transpose from [1, 576, 2048] --> [1, 2048, 576] --> [1, 2048, 24, 24]
+ mlp_3 = ggml_cont(ctx0, ggml_permute(ctx0, mlp_3, 1, 0, 2, 3));
+ mlp_3 = ggml_reshape_4d(ctx0, mlp_3, n_patch, n_patch, mlp_3->ne[1], mlp_3->ne[2]);
+ // stride = 1, padding = 1, bias is nullptr
+ block_1 = ggml_conv_depthwise_2d(ctx0, model.mm_model_block_1_block_0_0_w, mlp_3, 1, 1, 1, 1, 1, 1);
+
+ // layer norm
+ // // block_1 shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1]
+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 2, 0, 3));
+ // block_1 shape = [1, 24, 24, 2048], ne = [2048, 24, 24, 1]
+ block_1 = ggml_norm(ctx0, block_1, eps);
+ block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_1_block_0_1_w), model.mm_model_block_1_block_0_1_b);
+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 2, 0, 1, 3));
+
+ // block_1 shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1]
+ // hardswish
+ struct ggml_tensor * block_1_hw = ggml_hardswish(ctx0, block_1);
+
+ block_1 = ggml_pool_2d(ctx0, block_1_hw, GGML_OP_POOL_AVG, block_1_hw->ne[0], block_1_hw->ne[1], block_1_hw->ne[0], block_1_hw->ne[1], 0, 0);
+ // block_1 shape = [1, 2048, 1, 1], ne = [1, 1, 2048, 1]
+ // pointwise conv
+ block_1 = ggml_reshape_2d(ctx0, block_1, block_1->ne[0]*block_1->ne[1]*block_1->ne[2], block_1->ne[3]);
+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_1_block_1_fc1_w, block_1);
+ block_1 = ggml_add(ctx0, block_1, model.mm_model_block_1_block_1_fc1_b);
+ block_1 = ggml_relu(ctx0, block_1);
+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_1_block_1_fc2_w, block_1);
+ block_1 = ggml_add(ctx0, block_1, model.mm_model_block_1_block_1_fc2_b);
+ block_1 = ggml_hardsigmoid(ctx0, block_1);
+ // block_1_hw shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1], block_1 shape = [1, 2048], ne = [2048, 1, 1, 1]
+ block_1 = ggml_reshape_4d(ctx0, block_1, 1, 1, block_1->ne[0], block_1->ne[1]);
+ block_1 = ggml_mul(ctx0, block_1_hw, block_1);
+
+ int w = block_1->ne[0], h = block_1->ne[1];
+ block_1 = ggml_reshape_3d(ctx0, block_1, w*h, block_1->ne[2], block_1->ne[3]);
+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 0, 2, 3));
+
+ // block_1 shape = [1, 24*24, 2048], ne = [24*24, 2048, 1]
+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_1_block_2_0_w, block_1);
+ block_1 = ggml_reshape_4d(ctx0, block_1, block_1->ne[0], w, h, block_1->ne[3]);
+
+ // block_1 shape = [1, 24, 24, 2048], ne = [2048, 24, 24, 1]
+ block_1 = ggml_norm(ctx0, block_1, eps);
+ block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_1_block_2_1_w), model.mm_model_block_1_block_2_1_b);
+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 2, 0, 1, 3));
+ // block1 shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1]
+ // residual
+ block_1 = ggml_add(ctx0, mlp_3, block_1);
+ }
+
+ // block_2
+ {
+ // stride = 2
+ block_1 = ggml_conv_depthwise_2d(ctx0, model.mm_model_block_2_block_0_0_w, block_1, 2, 2, 1, 1, 1, 1);
+
+ // block_1 shape = [1, 2048, 12, 12], ne = [12, 12, 2048, 1]
+ // layer norm
+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 2, 0, 3));
+ // block_1 shape = [1, 12, 12, 2048], ne = [2048, 12, 12, 1]
+ block_1 = ggml_norm(ctx0, block_1, eps);
+ block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_2_block_0_1_w), model.mm_model_block_2_block_0_1_b);
+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 2, 0, 1, 3));
+ // block_1 shape = [1, 2048, 12, 12], ne = [12, 12, 2048, 1]
+ // hardswish
+ struct ggml_tensor * block_1_hw = ggml_hardswish(ctx0, block_1);
+
+ // not sure the parameters is right for globalAvgPooling
+ block_1 = ggml_pool_2d(ctx0, block_1_hw, GGML_OP_POOL_AVG, block_1_hw->ne[0], block_1_hw->ne[1], block_1_hw->ne[0], block_1_hw->ne[1], 0, 0);
+ // block_1 shape = [1, 2048, 1, 1], ne = [1, 1, 2048, 1]
+ // pointwise conv
+ block_1 = ggml_reshape_2d(ctx0, block_1, block_1->ne[0]*block_1->ne[1]*block_1->ne[2], block_1->ne[3]);
+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_2_block_1_fc1_w, block_1);
+ block_1 = ggml_add(ctx0, block_1, model.mm_model_block_2_block_1_fc1_b);
+ block_1 = ggml_relu(ctx0, block_1);
+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_2_block_1_fc2_w, block_1);
+ block_1 = ggml_add(ctx0, block_1, model.mm_model_block_2_block_1_fc2_b);
+ block_1 = ggml_hardsigmoid(ctx0, block_1);
+
+ // block_1_hw shape = [1, 2048, 12, 12], ne = [12, 12, 2048, 1], block_1 shape = [1, 2048, 1, 1], ne = [1, 1, 2048, 1]
+ block_1 = ggml_reshape_4d(ctx0, block_1, 1, 1, block_1->ne[0], block_1->ne[1]);
+ block_1 = ggml_mul(ctx0, block_1_hw, block_1);
+
+ int w = block_1->ne[0], h = block_1->ne[1];
+ block_1 = ggml_reshape_3d(ctx0, block_1, w*h, block_1->ne[2], block_1->ne[3]);
+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 0, 2, 3));
+ // block_1 shape = [1, 24*24, 2048], ne = [24*24, 2048, 1]
+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_2_block_2_0_w, block_1);
+ block_1 = ggml_reshape_4d(ctx0, block_1, block_1->ne[0], w, h, block_1->ne[3]);
+
+
+ // block_1 shape = [1, 12, 12, 2048], ne = [2048, 12, 12, 1]
+ block_1 = ggml_norm(ctx0, block_1, eps);
+ block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_2_block_2_1_w), model.mm_model_block_2_block_2_1_b);
+ block_1 = ggml_reshape_3d(ctx0, block_1, block_1->ne[0], block_1->ne[1] * block_1->ne[2], block_1->ne[3]);
+ // block_1 shape = [1, 144, 2048], ne = [2048, 144, 1]
+ }
+ embeddings = block_1;
+ }
+ else if (ctx->proj_type == PROJECTOR_TYPE_LDPV2)
+ {
+ int n_patch = 24;
+ struct ggml_tensor * mlp_0 = ggml_mul_mat(ctx0, model.mm_model_mlp_0_w, embeddings);
+ mlp_0 = ggml_add(ctx0, mlp_0, model.mm_model_mlp_0_b);
+ mlp_0 = ggml_gelu(ctx0, mlp_0);
+ struct ggml_tensor * mlp_2 = ggml_mul_mat(ctx0, model.mm_model_mlp_2_w, mlp_0);
+ mlp_2 = ggml_add(ctx0, mlp_2, model.mm_model_mlp_2_b);
+ // mlp_2 ne = [2048, 576, 1, 1]
+ // // AVG Pool Layer 2*2, strides = 2
+ mlp_2 = ggml_cont(ctx0, ggml_permute(ctx0, mlp_2, 1, 0, 2, 3));
+ // mlp_2 ne = [576, 2048, 1, 1]
+ mlp_2 = ggml_reshape_4d(ctx0, mlp_2, n_patch, n_patch, mlp_2->ne[1], mlp_2->ne[2]);
+ // mlp_2 ne [24, 24, 2048, 1]
+ mlp_2 = ggml_pool_2d(ctx0, mlp_2, GGML_OP_POOL_AVG, 2, 2, 2, 2, 0, 0);
+ // weight ne = [3, 3, 2048, 1]
+ struct ggml_tensor * peg_0 = ggml_conv_depthwise_2d(ctx0, model.mm_model_peg_0_w, mlp_2, 1, 1, 1, 1, 1, 1);
+ peg_0 = ggml_cont(ctx0, ggml_permute(ctx0, peg_0, 1, 2, 0, 3));
+ peg_0 = ggml_add(ctx0, peg_0, model.mm_model_peg_0_b);
+ mlp_2 = ggml_cont(ctx0, ggml_permute(ctx0, mlp_2, 1, 2, 0, 3));
+ peg_0 = ggml_add(ctx0, peg_0, mlp_2);
+ peg_0 = ggml_reshape_3d(ctx0, peg_0, peg_0->ne[0], peg_0->ne[1] * peg_0->ne[2], peg_0->ne[3]);
+ embeddings = peg_0;
+ }
+ else {
+ GGML_ABORT("fatal error");
+ }
+ }
+ // minicpmv projector
+ else if (ctx->has_minicpmv_projector)
+ {
+ if (ctx->proj_type == PROJECTOR_TYPE_RESAMPLER) {
+ struct ggml_tensor * q = model.mm_model_query;
+ { // layernorm
+ q = ggml_norm(ctx0, q, eps);
+ q = ggml_add(ctx0, ggml_mul(ctx0, q, model.mm_model_ln_q_w), model.mm_model_ln_q_b);
+ }
+ struct ggml_tensor * v = ggml_mul_mat(ctx0, model.mm_model_kv_proj, embeddings);
+ { // layernorm
+ v = ggml_norm(ctx0, v, eps);
+ v = ggml_add(ctx0, ggml_mul(ctx0, v, model.mm_model_ln_kv_w), model.mm_model_ln_kv_b);
+ }
+ struct ggml_tensor * k;
+ { // position
+ // q = ggml_add(ctx0, q, model.mm_model_pos_embed);
+ k = ggml_add(ctx0, v, pos_embed);
+ }
+
+ { // attention
+ int hidden_size = 4096;
+ const int d_head = 128;
+ int n_head = hidden_size/d_head;
+ int num_query = 96;
+ if (ctx->minicpmv_version == 2) {
+ hidden_size = 4096;
+ n_head = hidden_size/d_head;
+ num_query = 96;
+ }
+ else if (ctx->minicpmv_version == 3) {
+ hidden_size = 3584;
+ n_head = hidden_size/d_head;
+ num_query = 64;
+ }
+
+ struct ggml_tensor * Q = ggml_add(ctx0, ggml_mul_mat(ctx0, model.mm_model_attn_q_w, q), model.mm_model_attn_q_b);
+ Q = ggml_scale_inplace(ctx0, Q, 1.0f / sqrt((float)d_head));
+ struct ggml_tensor * K = ggml_add(ctx0, ggml_mul_mat(ctx0, model.mm_model_attn_k_w, k), model.mm_model_attn_k_b);
+ struct ggml_tensor * V = ggml_add(ctx0, ggml_mul_mat(ctx0, model.mm_model_attn_v_w, v), model.mm_model_attn_v_b);
+ // permute
+ Q = ggml_reshape_4d(ctx0, Q, d_head, n_head, num_query, batch_size);
+ Q = ggml_cont(ctx0, ggml_permute(ctx0, Q, 0, 2, 1, 3));
+ Q = ggml_reshape_3d(ctx0, Q, d_head, num_query, n_head * batch_size);
+ K = ggml_reshape_4d(ctx0, K, d_head, n_head, num_positions, batch_size);
+ K = ggml_cont(ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3));
+ K = ggml_reshape_3d(ctx0, K, d_head, num_positions, n_head * batch_size);
+ V = ggml_reshape_4d(ctx0, V, d_head, n_head, num_positions, batch_size);
+ V = ggml_cont(ctx0, ggml_permute(ctx0, V, 1, 2, 0, 3));
+ V = ggml_reshape_3d(ctx0, V, num_positions, d_head, n_head * batch_size);
+ struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
+ KQ = ggml_soft_max_inplace(ctx0, KQ);
+ struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ);
+ KQV = ggml_reshape_4d(ctx0, KQV, d_head, num_query, n_head, batch_size);
+ KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
+ KQV = ggml_cont_3d(ctx0, KQV, hidden_size, num_query, batch_size);
+
+ embeddings = ggml_add(ctx0, ggml_mul_mat(ctx0, model.mm_model_attn_o_w, KQV), model.mm_model_attn_o_b);
+ }
+ { // layernorm
+ embeddings = ggml_norm(ctx0, embeddings, eps);
+ embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.mm_model_ln_post_w), model.mm_model_ln_post_b);
+ }
+ embeddings = ggml_mul_mat(ctx0, model.mm_model_proj, embeddings);
+ }
+ else {
+ GGML_ASSERT(false);
+ }
+ }
+
+ // build the graph
+ ggml_build_forward_expand(gf, embeddings);
+
+ ggml_free(ctx0);
+
+ return gf;
+}
+
+// read and create ggml_context containing the tensors and their data
+struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
+ struct ggml_context * meta = NULL;
+
+ struct gguf_init_params params = {
+ /*.no_alloc = */ true,
+ /*.ctx = */ &meta,
+ };
+
+ struct gguf_context * ctx = gguf_init_from_file(fname, params);
+ if (!ctx) {
+ throw std::runtime_error(format("%s: failed to load CLIP model from %s. Does this file exist?\n", __func__, fname));
+ }
+
+ if (verbosity >= 1) {
+ const int n_tensors = gguf_get_n_tensors(ctx);
+ const int n_kv = gguf_get_n_kv(ctx);
+ const int ftype = get_u32(ctx, KEY_FTYPE);
+ const std::string ftype_str = get_ftype(ftype);
+ const int idx_desc = get_key_idx(ctx, KEY_DESCRIPTION);
+ const std::string description = gguf_get_val_str(ctx, idx_desc);
+ const int idx_name = gguf_find_key(ctx, KEY_NAME);
+ if (idx_name != -1) { // make name optional temporarily as some of the uploaded models missing it due to a bug
+ const std::string name = gguf_get_val_str(ctx, idx_name);
+ LOG_TEE("%s: model name: %s\n", __func__, name.c_str());
+ }
+ LOG_TEE("%s: description: %s\n", __func__, description.c_str());
+ LOG_TEE("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx));
+ LOG_TEE("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx));
+ LOG_TEE("%s: n_tensors: %d\n", __func__, n_tensors);
+ LOG_TEE("%s: n_kv: %d\n", __func__, n_kv);
+ LOG_TEE("%s: ftype: %s\n", __func__, ftype_str.c_str());
+ LOG_TEE("\n");
+ }
+ const int n_tensors = gguf_get_n_tensors(ctx);
+
+ // kv
+ const int n_kv = gguf_get_n_kv(ctx);
+ LOG_TEE("%s: loaded meta data with %d key-value pairs and %d tensors from %s\n",
+ __func__, n_kv, n_tensors, fname);
+ {
+ std::map<enum ggml_type, uint32_t> n_type;
+
+ for (int i = 0; i < n_tensors; i++) {
+ enum ggml_type type = gguf_get_tensor_type(ctx, i);
+
+ n_type[type]++;
+ }
+
+ LOG_TEE("%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", __func__);
+ for (int i = 0; i < n_kv; i++) {
+ const char * name = gguf_get_key(ctx, i);
+ const enum gguf_type type = gguf_get_kv_type(ctx, i);
+ const std::string type_name =
+ type == GGUF_TYPE_ARRAY
+ ? format("%s[%s,%d]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(ctx, i)), gguf_get_arr_n(ctx, i))
+ : gguf_type_name(type);
+
+ std::string value = gguf_kv_to_str(ctx, i);
+ const size_t MAX_VALUE_LEN = 40;
+ if (value.size() > MAX_VALUE_LEN) {
+ value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str());
+ }
+ replace_all(value, "\n", "\\n");
+
+ LOG_TEE("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), value.c_str());
+ }
+
+ // print type counts
+ for (auto & kv : n_type) {
+ if (kv.second == 0) {
+ continue;
+ }
+
+ LOG_TEE("%s: - type %4s: %4d tensors\n", __func__, ggml_type_name(kv.first), kv.second);
+ }
+ }
+
+ // data
+ size_t model_size = 0;
+ {
+ for (int i = 0; i < n_tensors; ++i) {
+ const char * name = gguf_get_tensor_name(ctx, i);
+ const size_t offset = gguf_get_tensor_offset(ctx, i);
+ enum ggml_type type = gguf_get_tensor_type(ctx, i);
+ struct ggml_tensor * cur = ggml_get_tensor(meta, name);
+ size_t tensor_size = ggml_nbytes(cur);
+ model_size += tensor_size;
+ if (verbosity >= 3) {
+ LOG_TEE("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n",
+ __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type));
+ }
+ }
+ }
+
+ clip_ctx * new_clip = new clip_ctx{};
+
+ // update projector type
+ {
+ int idx = gguf_find_key(ctx, KEY_PROJ_TYPE);
+ if (idx != -1) {
+ const std::string proj_type = gguf_get_val_str(ctx, idx);
+ new_clip->proj_type = clip_projector_type_from_string(proj_type);
+ } else {
+ new_clip->proj_type = PROJECTOR_TYPE_MLP;
+ }
+
+ if (new_clip->proj_type == PROJECTOR_TYPE_MLP) {
+ if (gguf_find_tensor(ctx, format(TN_LLAVA_PROJ, 3, "weight").c_str()) != -1) {
+ new_clip->proj_type = PROJECTOR_TYPE_MLP_NORM;
+ }
+ }
+ }
+
+#ifdef GGML_USE_CUDA
+ new_clip->backend = ggml_backend_cuda_init(0);
+ LOG_TEE("%s: CLIP using CUDA backend\n", __func__);
+#endif
+
+#ifdef GGML_USE_METAL
+ new_clip->backend = ggml_backend_metal_init();
+ LOG_TEE("%s: CLIP using Metal backend\n", __func__);
+#endif
+
+#ifdef GGML_USE_CANN
+ new_clip->backend = ggml_backend_cann_init(0);
+ LOG_TEE("%s: CLIP using CANN backend\n", __func__);
+#endif
+
+#ifdef GGML_USE_VULKAN
+ new_clip->backend = ggml_backend_vk_init(0);
+ LOG_TEE("%s: CLIP using Vulkan backend\n", __func__);
+#endif
+
+ if (!new_clip->backend) {
+ new_clip->backend = ggml_backend_cpu_init();
+ LOG_TEE("%s: CLIP using CPU backend\n", __func__);
+ }
+
+ // model size and capabilities
+ {
+ int idx = get_key_idx(ctx, KEY_HAS_TEXT_ENC);
+ new_clip->has_text_encoder = gguf_get_val_bool(ctx, idx);
+
+ idx = get_key_idx(ctx, KEY_HAS_VIS_ENC);
+ new_clip->has_vision_encoder = gguf_get_val_bool(ctx, idx);
+
+ idx = gguf_find_key(ctx, KEY_HAS_LLAVA_PROJ);
+ if (idx != -1) {
+ new_clip->has_llava_projector = gguf_get_val_bool(ctx, idx);
+ }
+
+ idx = gguf_find_key(ctx, KEY_HAS_MINICPMV_PROJ);
+ if (idx != -1) {
+ new_clip->has_minicpmv_projector = gguf_get_val_bool(ctx, idx);
+ }
+
+ idx = gguf_find_key(ctx, KEY_MINICPMV_VERSION);
+ if (idx != -1) {
+ new_clip->minicpmv_version = gguf_get_val_i32(ctx, idx);
+ }
+
+ // GGML_ASSERT(new_clip->has_llava_projector); // see monatis/clip.cpp for image and/or text encoding for semantic search
+
+ GGML_ASSERT(new_clip->has_vision_encoder);
+ GGML_ASSERT(!new_clip->has_text_encoder);
+
+ idx = get_key_idx(ctx, KEY_USE_GELU);
+ new_clip->use_gelu = gguf_get_val_bool(ctx, idx);
+
+ if (verbosity >= 1) {
+ LOG_TEE("%s: text_encoder: %d\n", __func__, new_clip->has_text_encoder);
+ LOG_TEE("%s: vision_encoder: %d\n", __func__, new_clip->has_vision_encoder);
+ LOG_TEE("%s: llava_projector: %d\n", __func__, new_clip->has_llava_projector);
+ LOG_TEE("%s: minicpmv_projector: %d\n", __func__, new_clip->has_minicpmv_projector);
+ LOG_TEE("%s: model size: %.2f MB\n", __func__, model_size / 1024.0 / 1024.0);
+ LOG_TEE("%s: metadata size: %.2f MB\n", __func__, ggml_get_mem_size(meta) / 1024.0 / 1024.0);
+ }
+ }
+
+ LOG_TEE("%s: params backend buffer size = % 6.2f MB (%i tensors)\n", __func__, model_size / (1024.0 * 1024.0), n_tensors);
+
+ // load tensors
+ {
+ std::vector<uint8_t> read_buf;
+ struct ggml_init_params params = {
+ /*.mem_size =*/ (n_tensors + 1) * ggml_tensor_overhead(),
+ /*.mem_buffer =*/ NULL,
+ /*.no_alloc =*/ true,
+ };
+
+ new_clip->ctx_data = ggml_init(params);
+ if (!new_clip->ctx_data) {
+ LOG_TEE("%s: ggml_init() failed\n", __func__);
+ clip_free(new_clip);
+ gguf_free(ctx);
+ return nullptr;
+ }
+
+#ifdef _WIN32
+ int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
+ if (!wlen) {
+ return NULL;
+ }
+ wchar_t * wbuf = (wchar_t *) malloc(wlen * sizeof(wchar_t));
+ wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, wbuf, wlen);
+ if (!wlen) {
+ free(wbuf);
+ return NULL;
+ }
+#if __GLIBCXX__
+ int fd = _wopen(wbuf, _O_RDONLY | _O_BINARY);
+ __gnu_cxx::stdio_filebuf<char> buffer(fd, std::ios_base::in);
+ std::istream fin(&buffer);
+#else // MSVC
+ // unused in our current build
+ auto fin = std::ifstream(wbuf, std::ios::binary);
+#endif
+ free(wbuf);
+#else
+ auto fin = std::ifstream(fname, std::ios::binary);
+#endif
+ if (!fin) {
+ LOG_TEE("cannot open model file for loading tensors\n");
+ clip_free(new_clip);
+ gguf_free(ctx);
+ return nullptr;
+ }
+
+ // add tensors to context
+ for (int i = 0; i < n_tensors; ++i) {
+ const char * name = gguf_get_tensor_name(ctx, i);
+ struct ggml_tensor * t = ggml_get_tensor(meta, name);
+ struct ggml_tensor * cur = ggml_dup_tensor(new_clip->ctx_data, t);
+ ggml_set_name(cur, name);
+ }
+
+ // alloc memory and offload data
+ new_clip->params_buffer = ggml_backend_alloc_ctx_tensors(new_clip->ctx_data, new_clip->backend);
+ for (int i = 0; i < n_tensors; ++i) {
+ const char * name = gguf_get_tensor_name(ctx, i);
+ struct ggml_tensor * cur = ggml_get_tensor(new_clip->ctx_data, name);
+ const size_t offset = gguf_get_data_offset(ctx) + gguf_get_tensor_offset(ctx, i);
+ fin.seekg(offset, std::ios::beg);
+ if (!fin) {
+ LOG_TEE("%s: failed to seek for tensor %s\n", __func__, name);
+ clip_free(new_clip);
+ gguf_free(ctx);
+ return nullptr;
+ }
+ int num_bytes = ggml_nbytes(cur);
+ if (ggml_backend_buffer_is_host(new_clip->params_buffer)) {
+ // for the CPU and Metal backend, we can read directly into the tensor
+ fin.read(reinterpret_cast<char *>(cur->data), num_bytes);
+ } else {
+ // read into a temporary buffer first, then copy to device memory
+ read_buf.resize(num_bytes);
+ fin.read(reinterpret_cast<char *>(read_buf.data()), num_bytes);
+ ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes);
+ }
+ }
+#if defined(_WIN32) && defined(__GLIBCXX__)
+ close(fd);
+#else
+ fin.close();
+#endif
+ }
+
+ // vision model
+ if (new_clip->has_vision_encoder) {
+ // load vision model
+ auto & vision_model = new_clip->vision_model;
+ auto & hparams = vision_model.hparams;
+ hparams.hidden_size = get_u32(ctx, format(KEY_N_EMBD, "vision"));
+ hparams.n_head = get_u32(ctx, format(KEY_N_HEAD, "vision"));
+ hparams.n_intermediate = get_u32(ctx, format(KEY_N_FF, "vision"));
+ hparams.n_layer = get_u32(ctx, format(KEY_N_BLOCK, "vision"));
+ hparams.image_size = get_u32(ctx, KEY_IMAGE_SIZE);
+ hparams.patch_size = get_u32(ctx, KEY_PATCH_SIZE);
+ hparams.projection_dim = get_u32(ctx, format(KEY_PROJ_DIM, "vision"));
+ hparams.eps = get_f32(ctx, format(KEY_LAYER_NORM_EPS, "vision"));
+
+ try {
+ int idx = get_key_idx(ctx, KEY_IMAGE_GRID_PINPOINTS);
+ int n = gguf_get_arr_n(ctx, idx);
+ const int32_t * pinpoints = (const int32_t *)gguf_get_arr_data(ctx, idx);
+ for (int i = 0; i < 32 && i < n && pinpoints[i] != 0; ++i) {
+ hparams.image_grid_pinpoints[i] = pinpoints[i];
+ }
+ if (n < 32)
+ hparams.image_grid_pinpoints[n] = 0;
+ } catch (std::runtime_error & /*e*/) {
+ hparams.image_grid_pinpoints[0]=0;
+ }
+
+ try {
+ int idx = get_key_idx(ctx, KEY_MM_PATCH_MERGE_TYPE);
+ strcpy(hparams.mm_patch_merge_type, gguf_get_val_str(ctx, idx));
+ } catch (std::runtime_error & /*e*/) {
+ strcpy(hparams.mm_patch_merge_type, "flat");
+ }
+
+ try {
+ hparams.image_crop_resolution = get_u32(ctx, KEY_IMAGE_CROP_RESOLUTION); // llava-1.6
+ } catch(const std::exception& /*e*/) {
+ hparams.image_crop_resolution = hparams.image_size;
+ }
+
+ int idx_mean = get_key_idx(ctx, KEY_IMAGE_MEAN);
+ int idx_std = get_key_idx(ctx, KEY_IMAGE_STD);
+
+ const float * mean_data = (const float *)gguf_get_arr_data(ctx, idx_mean);
+ const float * std_data = (const float *)gguf_get_arr_data(ctx, idx_std);
+
+ for (int i = 0; i < 3; ++i) {
+ new_clip->image_mean[i] = mean_data[i];
+ new_clip->image_std[i] = std_data[i];
+ }
+
+ if (verbosity >= 2) {
+ LOG_TEE("\n%s: vision model hparams\n", __func__);
+ LOG_TEE("image_size %d\n", hparams.image_size);
+ LOG_TEE("patch_size %d\n", hparams.patch_size);
+ LOG_TEE("v_hidden_size %d\n", hparams.hidden_size);
+ LOG_TEE("v_n_intermediate %d\n", hparams.n_intermediate);
+ LOG_TEE("v_projection_dim %d\n", hparams.projection_dim);
+ LOG_TEE("v_n_head %d\n", hparams.n_head);
+ LOG_TEE("v_n_layer %d\n", hparams.n_layer);
+ LOG_TEE("v_eps %f\n", hparams.eps);
+ LOG_TEE("v_image_mean %f %f %f\n", new_clip->image_mean[0], new_clip->image_mean[1], new_clip->image_mean[2]);
+ LOG_TEE("v_image_std %f %f %f\n", new_clip->image_std[0], new_clip->image_std[1], new_clip->image_std[2]);
+ LOG_TEE("v_image_grid_pinpoints: ");
+ for (int i = 0; i < 32 && (hparams.image_grid_pinpoints[i] != 0); ++i) {
+ LOG_TEE("%d ", hparams.image_grid_pinpoints[i]);
+ }
+ LOG_TEE("\n");
+ LOG_TEE("v_mm_patch_merge_type: %s\n", hparams.mm_patch_merge_type);
+
+ }
+
+ try {
+ vision_model.class_embedding = get_tensor(new_clip->ctx_data, TN_CLASS_EMBD);
+ new_clip->has_class_embedding = true;
+ } catch (const std::exception& /*e*/) {
+ new_clip->has_class_embedding = false;
+ }
+
+ try {
+ vision_model.pre_ln_w = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "weight"));
+ vision_model.pre_ln_b = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "bias"));
+ new_clip->has_pre_norm = true;
+ } catch (std::exception & /*e*/) {
+ new_clip->has_pre_norm = false;
+ }
+
+ try {
+ vision_model.post_ln_w = get_tensor(new_clip->ctx_data, format(TN_LN_POST, "v", "weight"));
+ vision_model.post_ln_b = get_tensor(new_clip->ctx_data, format(TN_LN_POST, "v", "bias"));
+ new_clip->has_post_norm = true;
+ } catch (std::exception & /*e*/) {
+ new_clip->has_post_norm = false;
+ }
+
+ try {
+ vision_model.patch_bias = get_tensor(new_clip->ctx_data, TN_PATCH_BIAS);
+ new_clip->has_patch_bias = true;
+ } catch (std::exception & /*e*/) {
+ new_clip->has_patch_bias = false;
+ }
+
+ try {
+ vision_model.patch_embeddings = get_tensor(new_clip->ctx_data, TN_PATCH_EMBD);
+ vision_model.position_embeddings = get_tensor(new_clip->ctx_data, format(TN_POS_EMBD, "v"));
+ } catch(const std::exception& /*e*/) {
+ LOG_TEE("%s: failed to load vision model tensors\n", __func__);
+ }
+
+ // LLaVA projection
+ if (new_clip->proj_type == PROJECTOR_TYPE_MLP || new_clip->proj_type == PROJECTOR_TYPE_MLP_NORM) {
+ vision_model.mm_0_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 0, "weight"));
+ vision_model.mm_0_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 0, "bias"));
+ try {
+ // Yi-type llava
+ vision_model.mm_1_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 1, "weight"));
+ vision_model.mm_1_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 1, "bias"));
+ } catch (std::runtime_error & /*e*/) { }
+ try {
+ // missing in Yi-type llava
+ vision_model.mm_2_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 2, "weight"));
+ vision_model.mm_2_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 2, "bias"));
+ } catch (std::runtime_error & /*e*/) { }
+ try {
+ // Yi-type llava
+ vision_model.mm_3_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 3, "weight"));
+ vision_model.mm_3_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 3, "bias"));
+ } catch (std::runtime_error & /*e*/) { }
+ try {
+ // Yi-type llava
+ vision_model.mm_4_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 4, "weight"));
+ vision_model.mm_4_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 4, "bias"));
+ } catch (std::runtime_error & /*e*/) { }
+ try {
+ vision_model.image_newline = get_tensor(new_clip->ctx_data, TN_IMAGE_NEWLINE);
+ // LOG_TEE("%s: image_newline tensor (llava-1.6) found\n", __func__);
+ } catch (std::runtime_error & /*e*/) { }
+ } else if (new_clip->proj_type == PROJECTOR_TYPE_LDP) {
+ // MobileVLM projection
+ vision_model.mm_model_mlp_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "weight"));
+ vision_model.mm_model_mlp_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "bias"));
+ vision_model.mm_model_mlp_3_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "weight"));
+ vision_model.mm_model_mlp_3_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "bias"));
+ vision_model.mm_model_block_1_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "0.weight"));
+ vision_model.mm_model_block_1_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.weight"));
+ vision_model.mm_model_block_1_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.bias"));
+ vision_model.mm_model_block_1_block_1_fc1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.weight"));
+ vision_model.mm_model_block_1_block_1_fc1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.bias"));
+ vision_model.mm_model_block_1_block_1_fc2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.weight"));
+ vision_model.mm_model_block_1_block_1_fc2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.bias"));
+ vision_model.mm_model_block_1_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "0.weight"));
+ vision_model.mm_model_block_1_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.weight"));
+ vision_model.mm_model_block_1_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.bias"));
+ vision_model.mm_model_block_2_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "0.weight"));
+ vision_model.mm_model_block_2_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.weight"));
+ vision_model.mm_model_block_2_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.bias"));
+ vision_model.mm_model_block_2_block_1_fc1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.weight"));
+ vision_model.mm_model_block_2_block_1_fc1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.bias"));
+ vision_model.mm_model_block_2_block_1_fc2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.weight"));
+ vision_model.mm_model_block_2_block_1_fc2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.bias"));
+ vision_model.mm_model_block_2_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight"));
+ vision_model.mm_model_block_2_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight"));
+ vision_model.mm_model_block_2_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias"));
+ }
+ else if (new_clip->proj_type == PROJECTOR_TYPE_LDPV2)
+ {
+ // MobilVLM_V2 projection
+ vision_model.mm_model_mlp_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 0, "weight"));
+ vision_model.mm_model_mlp_0_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 0, "bias"));
+ vision_model.mm_model_mlp_2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 2, "weight"));
+ vision_model.mm_model_mlp_2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 2, "bias"));
+ vision_model.mm_model_peg_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_PEG, 0, "weight"));
+ vision_model.mm_model_peg_0_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_PEG, 0, "bias"));
+ }
+ else if (new_clip->proj_type == PROJECTOR_TYPE_RESAMPLER) {
+ // vision_model.mm_model_pos_embed = get_tensor(new_clip->ctx_data, TN_MINICPMV_POS_EMBD);
+ vision_model.mm_model_pos_embed_k = get_tensor(new_clip->ctx_data, TN_MINICPMV_POS_EMBD_K);
+ vision_model.mm_model_query = get_tensor(new_clip->ctx_data, TN_MINICPMV_QUERY);
+ vision_model.mm_model_proj = get_tensor(new_clip->ctx_data, TN_MINICPMV_PROJ);
+ vision_model.mm_model_kv_proj = get_tensor(new_clip->ctx_data, TN_MINICPMV_KV_PROJ);
+ vision_model.mm_model_attn_q_w = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_ATTN, "q", "weight"));
+ vision_model.mm_model_attn_k_w = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_ATTN, "k", "weight"));
+ vision_model.mm_model_attn_v_w = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_ATTN, "v", "weight"));
+ vision_model.mm_model_attn_q_b = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_ATTN, "q", "bias"));
+ vision_model.mm_model_attn_k_b = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_ATTN, "k", "bias"));
+ vision_model.mm_model_attn_v_b = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_ATTN, "v", "bias"));
+ vision_model.mm_model_attn_o_w = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_ATTN, "out", "weight"));
+ vision_model.mm_model_attn_o_b = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_ATTN, "out", "bias"));
+ vision_model.mm_model_ln_q_w = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_LN, "q", "weight"));
+ vision_model.mm_model_ln_q_b = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_LN, "q", "bias"));
+ vision_model.mm_model_ln_kv_w = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_LN, "kv", "weight"));
+ vision_model.mm_model_ln_kv_b = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_LN, "kv", "bias"));
+ vision_model.mm_model_ln_post_w = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_LN, "post", "weight"));
+ vision_model.mm_model_ln_post_b = get_tensor(new_clip->ctx_data, format(TN_MINICPMV_LN, "post", "bias"));
+ }
+ else {
+ std::string proj_type = PROJECTOR_TYPE_NAMES[new_clip->proj_type];
+ throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str()));
+ }
+
+ vision_model.layers.resize(hparams.n_layer);
+
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & layer = vision_model.layers[il];
+ layer.k_w = get_tensor(new_clip->ctx_data, format(TN_ATTN_K, "v", il, "weight"));
+ layer.q_w = get_tensor(new_clip->ctx_data, format(TN_ATTN_Q, "v", il, "weight"));
+ layer.v_w = get_tensor(new_clip->ctx_data, format(TN_ATTN_V, "v", il, "weight"));
+ layer.o_w = get_tensor(new_clip->ctx_data, format(TN_ATTN_OUTPUT, "v", il, "weight"));
+ layer.ln_1_w = get_tensor(new_clip->ctx_data, format(TN_LN_1, "v", il, "weight"));
+ layer.ln_2_w = get_tensor(new_clip->ctx_data, format(TN_LN_2, "v", il, "weight"));
+ layer.ff_i_w = get_tensor(new_clip->ctx_data, format(TN_FFN_DOWN, "v", il, "weight"));
+ layer.ff_o_w = get_tensor(new_clip->ctx_data, format(TN_FFN_UP, "v", il, "weight"));
+ layer.k_b = get_tensor(new_clip->ctx_data, format(TN_ATTN_K, "v", il, "bias"));
+ layer.q_b = get_tensor(new_clip->ctx_data, format(TN_ATTN_Q, "v", il, "bias"));
+ layer.v_b = get_tensor(new_clip->ctx_data, format(TN_ATTN_V, "v", il, "bias"));
+ layer.o_b = get_tensor(new_clip->ctx_data, format(TN_ATTN_OUTPUT, "v", il, "bias"));
+ layer.ln_1_b = get_tensor(new_clip->ctx_data, format(TN_LN_1, "v", il, "bias"));
+ layer.ln_2_b = get_tensor(new_clip->ctx_data, format(TN_LN_2, "v", il, "bias"));
+ layer.ff_i_b = get_tensor(new_clip->ctx_data, format(TN_FFN_DOWN, "v", il, "bias"));
+ layer.ff_o_b = get_tensor(new_clip->ctx_data, format(TN_FFN_UP, "v", il, "bias"));
+ }
+ }
+
+ ggml_free(meta);
+
+ new_clip->ctx_gguf = ctx;
+
+ // measure mem requirement and allocate
+ {
+ new_clip->buf_compute_meta.resize(GGML_DEFAULT_GRAPH_SIZE * ggml_tensor_overhead() + ggml_graph_overhead());
+ new_clip->compute_alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(new_clip->backend));
+ clip_image_f32_batch batch;
+ batch.size = 1;
+ ggml_cgraph * gf = clip_image_build_graph(new_clip, &batch, nullptr, false);
+ ggml_gallocr_reserve(new_clip->compute_alloc, gf);
+ size_t compute_memory_buffer_size = ggml_gallocr_get_buffer_size(new_clip->compute_alloc, 0);
+ LOG_TEE("%s: compute allocated memory: %.2f MB\n", __func__, compute_memory_buffer_size /1024.0/1024.0);
+ }
+
+ return new_clip;
+}
+
+void clip_add_load_image_size(struct clip_ctx * ctx_clip, struct clip_image_size * load_image_size) {
+ ctx_clip->load_image_size = load_image_size;
+}
+
+struct clip_image_size * clip_image_size_init() {
+ struct clip_image_size * load_image_size = new struct clip_image_size();
+ load_image_size->width = 448;
+ load_image_size->height = 448;
+ return load_image_size;
+}
+
+struct clip_image_u8 * clip_image_u8_init() {
+ return new clip_image_u8();
+}
+
+struct clip_image_f32 * clip_image_f32_init() {
+ return new clip_image_f32();
+}
+
+void clip_image_u8_free(struct clip_image_u8 * img) { delete img; }
+void clip_image_f32_free(struct clip_image_f32 * img) { delete img; }
+void clip_image_u8_batch_free(struct clip_image_u8_batch * batch) {
+ if (batch->size > 0) {
+ delete[] batch->data;
+ batch->size = 0;
+ }
+}
+void clip_image_f32_batch_free(struct clip_image_f32_batch * batch) {
+ if (batch->size > 0) {
+ delete[] batch->data;
+ batch->size = 0;
+ }
+}
+
+static void build_clip_img_from_data(const stbi_uc * data, int nx, int ny, clip_image_u8 * img) {
+ img->nx = nx;
+ img->ny = ny;
+ img->buf.resize(3 * nx * ny);
+ memcpy(img->buf.data(), data, img->buf.size());
+}
+
+bool clip_image_load_from_file(const char * fname, clip_image_u8 * img) {
+ int nx, ny, nc;
+ auto * data = stbi_load(fname, &nx, &ny, &nc, 3);
+ if (!data) {
+ LOG_TEE("%s: failed to load image '%s'\n", __func__, fname);
+ return false;
+ }
+ build_clip_img_from_data(data, nx, ny, img);
+ stbi_image_free(data);
+ return true;
+}
+
+bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img) {
+ int nx, ny, nc;
+ auto * data = stbi_load_from_memory(bytes, bytes_length, &nx, &ny, &nc, 3);
+ if (!data) {
+ LOG_TEE("%s: failed to decode image bytes\n", __func__);
+ return false;
+ }
+ build_clip_img_from_data(data, nx, ny, img);
+ stbi_image_free(data);
+ return true;
+}
+
+// Linear interpolation between two points
+inline float clip_lerp(float s, float e, float t) {
+ return s + (e - s) * t;
+}
+// Bilinear resize function
+static void bilinear_resize(const clip_image_u8& src, clip_image_u8& dst, int target_width, int target_height) {
+ dst.nx = target_width;
+ dst.ny = target_height;
+ dst.buf.resize(3 * target_width * target_height);
+
+ float x_ratio = static_cast<float>(src.nx - 1) / target_width;
+ float y_ratio = static_cast<float>(src.ny - 1) / target_height;
+
+ for (int y = 0; y < target_height; y++) {
+ for (int x = 0; x < target_width; x++) {
+ float px = x_ratio * x;
+ float py = y_ratio * y;
+ int x_floor = static_cast<int>(px);
+ int y_floor = static_cast<int>(py);
+ float x_lerp = px - x_floor;
+ float y_lerp = py - y_floor;
+
+ for (int c = 0; c < 3; c++) {
+ float top = clip_lerp(
+ static_cast<float>(src.buf[3 * (y_floor * src.nx + x_floor) + c]),
+ static_cast<float>(src.buf[3 * (y_floor * src.nx + (x_floor + 1)) + c]),
+ x_lerp
+ );
+ float bottom = clip_lerp(
+ static_cast<float>(src.buf[3 * ((y_floor + 1) * src.nx + x_floor) + c]),
+ static_cast<float>(src.buf[3 * ((y_floor + 1) * src.nx + (x_floor + 1)) + c]),
+ x_lerp
+ );
+ dst.buf[3 * (y * target_width + x) + c] = static_cast<uint8_t>(clip_lerp(top, bottom, y_lerp));
+ }
+ }
+ }
+}
+
+// Normalize image to float32 - careful with pytorch .to(model.device, dtype=torch.float16) - this sometimes reduces precision (32>16>32), sometimes not
+static void normalize_image_u8_to_f32(const clip_image_u8* src, clip_image_f32* dst, const float mean[3], const float std[3]) {
+ dst->nx = src->nx;
+ dst->ny = src->ny;
+ dst->buf.resize(src->buf.size());
+
+ for (size_t i = 0; i < src->buf.size(); ++i) {
+ int c = i % 3; // rgb
+ dst->buf[i] = (static_cast<float>(src->buf[i]) / 255.0f - mean[c]) / std[c];
+ }
+}
+
+inline int clip(int x, int lower, int upper) {
+ return std::max(lower, std::min(x, upper));
+}
+
+static bool bicubic_resize(const clip_image_u8 &img, clip_image_u8 &dst, int target_width, int target_height) {
+ const int nx = img.nx;
+ const int ny = img.ny;
+
+ dst.nx = target_width;
+ dst.ny = target_height;
+ dst.buf.resize(3 * target_width * target_height);
+
+ float Cc;
+ float C[5];
+ float d0, d2, d3, a0, a1, a2, a3;
+ int i, j, k, jj;
+ int x, y;
+ float dx, dy;
+ float tx, ty;
+
+ tx = (float)nx / (float)target_width;
+ ty = (float)ny / (float)target_height;
+
+ // Bicubic interpolation; adapted from ViT.cpp, inspired from :
+ // -> https://github.com/yglukhov/bicubic-interpolation-image-processing/blob/master/libimage.c#L36
+ // -> https://en.wikipedia.org/wiki/Bicubic_interpolation
+
+ for (i = 0; i < target_height; i++) {
+ for (j = 0; j < target_width; j++) {
+ x = (int)(tx * j);
+ y = (int)(ty * i);
+
+ dx = tx * j - x;
+ dy = ty * i - y;
+
+ for (k = 0; k < 3; k++) {
+ for (jj = 0; jj <= 3; jj++) {
+ d0 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x - 1, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];
+ d2 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x + 1, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];
+ d3 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x + 2, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];
+ a0 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];
+
+ a1 = -1.0 / 3 * d0 + d2 - 1.0 / 6 * d3;
+ a2 = 1.0 / 2 * d0 + 1.0 / 2 * d2;
+ a3 = -1.0 / 6 * d0 - 1.0 / 2 * d2 + 1.0 / 6 * d3;
+
+ C[jj] = a0 + a1 * dx + a2 * dx * dx + a3 * dx * dx * dx;
+
+ d0 = C[0] - C[1];
+ d2 = C[2] - C[1];
+ d3 = C[3] - C[1];
+ a0 = C[1];
+ a1 = -1.0 / 3 * d0 + d2 - 1.0 / 6 * d3;
+ a2 = 1.0 / 2 * d0 + 1.0 / 2 * d2;
+ a3 = -1.0 / 6 * d0 - 1.0 / 2 * d2 + 1.0 / 6 * d3;
+ Cc = a0 + a1 * dy + a2 * dy * dy + a3 * dy * dy * dy;
+
+ const uint8_t Cc2 = std::min(std::max(std::round(Cc), 0.0f), 255.0f);
+ dst.buf[(i * target_width + j) * 3 + k] = float(Cc2);
+ }
+ }
+ }
+ }
+
+ return true;
+}
+
+// llava-1.6 type of resize_and_pad (black)
+static void resize_and_pad_image(const clip_image_u8& image, clip_image_u8 &image_output, const std::pair<int, int>& target_resolution) {
+ int target_width = target_resolution.first;
+ int target_height = target_resolution.second;
+
+ float scale_w = static_cast<float>(target_width) / image.nx;
+ float scale_h = static_cast<float>(target_height) / image.ny;
+
+ int new_width, new_height;
+
+ if (scale_w < scale_h) {
+ new_width = target_width;
+ new_height = std::min(static_cast<int>(std::ceil(image.ny * scale_w)), target_height);
+ } else {
+ new_height = target_height;
+ new_width = std::min(static_cast<int>(std::ceil(image.nx * scale_h)), target_width);
+ }
+
+ clip_image_u8 resized_image;
+ // bilinear_resize(image, resized_image, new_width, new_height);
+ bicubic_resize(image, resized_image, new_width, new_height);
+
+ clip_image_u8 padded_image;
+ padded_image.nx = target_width;
+ padded_image.ny = target_height;
+ padded_image.buf.resize(3 * target_width * target_height, 0); // Initialize with black
+
+ // Calculate padding offsets
+ int pad_x = (target_width - new_width) / 2;
+ int pad_y = (target_height - new_height) / 2;
+
+ // Copy the resized image into the center of the padded buffer
+ for (int y = 0; y < new_height; ++y) {
+ for (int x = 0; x < new_width; ++x) {
+ for (int c = 0; c < 3; ++c) {
+ padded_image.buf[3 * ((y + pad_y) * target_width + (x + pad_x)) + c] = resized_image.buf[3 * (y * new_width + x) + c];
+ }
+ }
+ }
+ image_output = std::move(padded_image);
+}
+
+/**
+ * Selects the best resolution from a list of possible resolutions based on the original size.
+ *
+ * @param original_size The original size of the image in the format (width, height).
+ * @param possible_resolutions A list of possible resolutions in the format [(width1, height1), (width2, height2), ...].
+ * @return The best fit resolution in the format (width, height).
+ */
+static std::pair<int, int> select_best_resolution(const std::pair<int, int> & original_size, const std::vector<std::pair<int, int>> & possible_resolutions) {
+ int original_width = original_size.first;
+ int original_height = original_size.second;
+ std::pair<int, int> best_fit;
+ int max_effective_resolution = 0;
+ int min_wasted_resolution = std::numeric_limits<int>::max();
+
+ for (const auto& resolution : possible_resolutions) {
+ int width = resolution.first;
+ int height = resolution.second;
+ float scale = std::min(static_cast<float>(width) / original_width, static_cast<float>(height) / original_height);
+ int downscaled_width = static_cast<int>(original_width * scale);
+ int downscaled_height = static_cast<int>(original_height * scale);
+ int effective_resolution = std::min(downscaled_width * downscaled_height, original_width * original_height);
+ int wasted_resolution = (width * height) - effective_resolution;
+ // LOG_TEE("resolution: %d %d, scale: %f, downscaled: %d %d, effective: %d, wasted: %d\n", width, height, scale, downscaled_width, downscaled_height, effective_resolution, wasted_resolution);
+ if (effective_resolution > max_effective_resolution || (effective_resolution == max_effective_resolution && wasted_resolution < min_wasted_resolution)) {
+ max_effective_resolution = effective_resolution;
+ min_wasted_resolution = wasted_resolution;
+ best_fit = resolution;
+ }
+ }
+
+ return best_fit;
+}
+
+static std::vector<clip_image_u8*> divide_to_patches_u8(const clip_image_u8 & image, int patch_size) {
+ std::vector<clip_image_u8*> patches;
+ int width = image.nx;
+ int height = image.ny;
+ for (int i = 0; i < height; i += patch_size) {
+ for (int j = 0; j < width; j += patch_size) {
+ clip_image_u8 *patch = clip_image_u8_init();
+ patch->nx = std::min(patch_size, width - j);
+ patch->ny = std::min(patch_size, height - i);
+ patch->buf.resize(3 * patch->nx * patch->ny);
+ for (int y = 0; y < patch->ny; ++y) {
+ for (int x = 0; x < patch->nx; ++x) {
+ for (int c = 0; c < 3; ++c) {
+ patch->buf[3 * (y * patch->nx + x) + c] = image.buf[3 * ((i + y) * width + (j + x)) + c];
+ }
+ }
+ }
+ patches.push_back(patch);
+ }
+ }
+ return patches;
+}
+
+static int ensure_divide(int length, int patch_size) {
+ return std::max(static_cast<int>(std::round(static_cast<float>(length) / patch_size) * patch_size), patch_size);
+}
+
+static std::pair<int, int> uhd_find_best_resize(std::pair<int, int> original_size, int scale_resolution, int patch_size, bool allow_upscale = false) {
+ int width = original_size.first;
+ int height = original_size.second;
+ if ((width * height > scale_resolution * scale_resolution) || allow_upscale) {
+ float r = static_cast<float>(width) / height;
+ height = static_cast<int>(scale_resolution / std::sqrt(r));
+ width = static_cast<int>(height * r);
+ }
+ int best_width = ensure_divide(width, patch_size);
+ int best_height = ensure_divide(height, patch_size);
+ return std::make_pair(best_width, best_height);
+}
+
+static std::pair<int, int> uhd_get_refine_size(std::pair<int, int> original_size, std::pair<int, int> grid, int scale_resolution, int patch_size, bool allow_upscale = false) {
+ int width, height;
+ std::tie(width, height) = original_size;
+ int grid_x, grid_y;
+ std::tie(grid_x, grid_y) = grid;
+
+ int refine_width = ensure_divide(width, grid_x);
+ int refine_height = ensure_divide(height, grid_y);
+
+ int grid_width = refine_width / grid_x;
+ int grid_height = refine_height / grid_y;
+
+ // auto best_grid_size = find_best_resize(std::make_tuple(grid_width, grid_height), scale_resolution, patch_size, allow_upscale); (old line)
+ auto best_grid_size = uhd_find_best_resize(std::make_pair(grid_width, grid_height), scale_resolution, patch_size, allow_upscale); // (new line) => fixes conversion for make_tuple to make_pair
+ int best_grid_width, best_grid_height;
+ std::tie(best_grid_width, best_grid_height) = best_grid_size;
+
+ // std::pair<int, int> refine_size = std::make_tuple(best_grid_width * grid_x, best_grid_height * grid_y); (old line)
+ std::pair<int, int> refine_size = std::make_pair(best_grid_width * grid_x, best_grid_height * grid_y); // (new line)
+ return refine_size;
+}
+
+static std::pair<int, int> uhd_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) {
+ std::vector<int> candidate_split_grids_nums;
+ for (int i : {multiple - 1, multiple, multiple + 1}) {
+ if (i == 1 || i > max_slice_nums) {
+ continue;
+ }
+ candidate_split_grids_nums.push_back(i);
+ }
+
+ std::vector<std::pair<int, int>> candidate_grids;
+ for (int split_grids_nums : candidate_split_grids_nums) {
+ int m = 1;
+ while (m <= split_grids_nums) {
+ if (split_grids_nums % m == 0) {
+ candidate_grids.emplace_back(m, split_grids_nums / m);
+ }
+ ++m;
+ }
+ }
+
+ std::pair<int, int> best_grid{1, 1};
+ float min_error = std::numeric_limits<float>::infinity();
+ for (const auto& grid : candidate_grids) {
+ float error = std::abs(log_ratio - std::log(1.0 * grid.first / grid.second));
+ if (error < min_error) {
+ best_grid = grid;
+ min_error = error;
+ }
+ }
+ return best_grid;
+}
+
+// inspired from LLaVA-UHD:
+// -> https://arxiv.org/pdf/2403.11703
+// -> https://github.com/thunlp/LLaVA-UHD
+// -> https://github.com/thunlp/LLaVA-UHD/blob/302301bc2175f7e717fb8548516188e89f649753/llava_uhd/train/llava-uhd/slice_logic.py#L118
+static std::vector<std::vector<clip_image_u8 *>> uhd_slice_image(const clip_image_u8 * img, const int max_slice_nums=9, const int scale_resolution=448, const int patch_size=14) {
+ const std::pair<int, int> original_size={img->nx,img->ny};
+ const int original_width = img->nx;
+ const int original_height = img->ny;
+ const float log_ratio = log(1.0*original_width/original_height);
+ const float ratio = 1.0 * original_width * original_height/ (scale_resolution * scale_resolution);
+ const int multiple = fmin(ceil(ratio), max_slice_nums);
+
+ std::vector<std::vector<clip_image_u8 *>> images;
+ LOG_TEE("%s: multiple %d\n", __func__, multiple);
+ images.push_back(std::vector<clip_image_u8 *>());
+
+ if (multiple <= 1) {
+ auto best_size = uhd_find_best_resize(original_size, scale_resolution, patch_size, true);
+ clip_image_u8 * source_image = clip_image_u8_init();
+ bicubic_resize(*img, *source_image, best_size.first, best_size.second);
+ // source_image = image.resize(best_size, Image.Resampling.BICUBIC)
+ images[images.size()-1].push_back(source_image);
+ }
+ else if (multiple > 1) {
+ auto best_size = uhd_find_best_resize(original_size, scale_resolution, patch_size);
+ clip_image_u8 * source_image = clip_image_u8_init();
+ bicubic_resize(*img, *source_image, best_size.first, best_size.second);
+ // source_image = image.copy().resize(best_resize, Image.Resampling.BICUBIC)
+ LOG_TEE("%s: image_size: %d %d; source_image size: %d %d\n", __func__, img->nx, img->ny, best_size.first, best_size.second);
+ images[images.size()-1].push_back(source_image);
+
+ std::pair<int, int> best_grid = uhd_best_grid(max_slice_nums, multiple, log_ratio);
+ LOG_TEE("%s: image_size: %d %d; best_grid: %d %d\n", __func__, img->nx, img->ny, best_grid.first, best_grid.second);
+
+ auto refine_size = uhd_get_refine_size(original_size, best_grid, scale_resolution, patch_size, true);
+ clip_image_u8 * refine_image = clip_image_u8_init();
+ bicubic_resize(*img, *refine_image, refine_size.first, refine_size.second);
+
+ LOG_TEE("%s: refine_image_size: %d %d; refine_size: %d %d\n", __func__, refine_image->nx, refine_image->ny, refine_size.first, refine_size.second);
+
+ // split_to_patches
+ int width = refine_image->nx;
+ int height = refine_image->ny;
+ int grid_x = int(width / best_grid.first);
+ int grid_y = int(height / best_grid.second);
+ for (int patches_i = 0, ic = 0; patches_i < height && ic < best_grid.second; patches_i += grid_y, ic += 1){
+ images.push_back(std::vector<clip_image_u8 *>());
+ for(int patches_j = 0, jc = 0; patches_j < width && jc < best_grid.first; patches_j += grid_x, jc += 1){
+ clip_image_u8 * patch = clip_image_u8_init();
+ patch->nx = grid_x;
+ patch->ny = grid_y;
+ patch->buf.resize(3 * patch->nx * patch->ny);
+ for (int y = patches_i; y < patches_i + grid_y; ++y) {
+ for (int x = patches_j; x < patches_j + grid_x; ++x) {
+ const int i = 3 * (y * refine_image->nx + x);
+ const int j = 3 * ((y-patches_i) * patch->nx + (x-patches_j));
+ patch->buf[j] = refine_image->buf[i];
+ patch->buf[j+1] = refine_image->buf[i+1];
+ patch->buf[j+2] = refine_image->buf[i+2];
+ }
+ }
+ images[images.size()-1].push_back(patch);
+ }
+ }
+ }
+ return images;
+}
+
+int clip_uhd_num_image_embeds_col(struct clip_ctx * ctx_clip) {
+ const int max_slice_nums=9;
+ const int scale_resolution=448;
+ const int original_width = ctx_clip->load_image_size->width;
+ const int original_height = ctx_clip->load_image_size->height;
+ const float log_ratio = log(1.0*original_width/original_height);
+ const float ratio = 1.0 * original_width * original_height/ (scale_resolution * scale_resolution);
+ const int multiple = fmin(ceil(ratio), max_slice_nums);
+ std::pair<int, int> best_grid = uhd_best_grid(max_slice_nums, multiple, log_ratio);
+ return best_grid.first;
+}
+
+// returns the normalized float tensor for llava-1.5, for spatial_unpad with anyres processing for llava-1.6 it returns the normalized image patch tensors as a vector
+// res_imgs memory is being allocated here, previous allocations will be freed if found
+bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch * res_imgs) {
+
+ if(clip_is_minicpmv(ctx)){
+ int max_slice_nums = 9;
+ std::vector<std::vector<clip_image_u8 *>> imgs = uhd_slice_image(img, max_slice_nums);
+ res_imgs->size = 0;
+ for (size_t i = 0; i < imgs.size(); ++i){
+ res_imgs->size += imgs[i].size();
+ }
+ res_imgs->data = new clip_image_f32[res_imgs->size];
+ int idx = 0;
+ for (size_t i = 0; i < imgs.size(); ++i) {
+ for (size_t j = 0; j < imgs[i].size(); ++j) {
+ LOG_TEE("%s: %d %d\n", __func__,imgs[i][j]->nx,imgs[i][j]->ny);
+ clip_image_f32 * res = clip_image_f32_init();
+ normalize_image_u8_to_f32(imgs[i][j], res, ctx->image_mean, ctx->image_std);
+ res_imgs->data[idx++] = *res;
+ clip_image_f32_free(res);
+ }
+ }
+ return true;
+ }
+
+ bool pad_to_square = true;
+ if (!ctx->has_vision_encoder) {
+ LOG_TEE("This gguf file seems to have no vision encoder\n");
+ return false;
+ }
+ auto & params = ctx->vision_model.hparams;
+ // The model config actually contains all we need to decide on how to preprocess, here we automatically switch to the new llava-1.6 preprocessing
+ if (strcmp(params.mm_patch_merge_type, "spatial_unpad") == 0) {
+ pad_to_square = false;
+ }
+ // free the previous res_imgs if any set
+ if (res_imgs->size > 0) {
+ clip_image_f32_batch_free(res_imgs);
+ }
+ res_imgs->data = nullptr;
+ res_imgs->size = 0;
+
+ // the logic below is to pad the shorter side to the longer side with a background color: rgb(122, 116, 104)
+ // see https://github.com/haotian-liu/LLaVA/blob/e854a2bf85118c504f6f16bf5c3c7c92f8fa8c6b/llava/conversation.py#L113-L156
+
+ clip_image_u8 * temp = clip_image_u8_init(); // we will keep the input image data here temporarily
+ if (pad_to_square && img->nx != img->ny) {
+ int longer_side = std::max(img->nx, img->ny);
+ temp->nx = longer_side;
+ temp->ny = longer_side;
+ temp->buf.resize(3 * longer_side * longer_side);
+ const uint8_t bc[3] = {122, 116, 104}; // background color in RGB from LLaVA (this is the mean rgb color * 255)
+
+ // fill with background color
+ for (size_t i = 0; i < temp->buf.size(); i++) {
+ temp->buf[i] = bc[i % 3];
+ }
+
+ // copy from the input image
+ for (int y = 0; y < img->ny; y++) {
+ for (int x = 0; x < img->nx; x++) {
+ const int i = 3 * (y * img->nx + x);
+ const int j = 3 * (y * temp->nx + x);
+ temp->buf[j] = img->buf[i];
+ temp->buf[j+1] = img->buf[i+1];
+ temp->buf[j+2] = img->buf[i+2];
+ }
+ }
+ } else {
+ if (params.image_grid_pinpoints[0] != 0) {
+ // "spatial_unpad" with "anyres" processing for llava-1.6
+ std::vector<std::pair<int, int>> possible_resolutions;
+ for (int i = 0; i < 32 && params.image_grid_pinpoints[i] != 0; i+=2) {
+ possible_resolutions.push_back({params.image_grid_pinpoints[i], params.image_grid_pinpoints[i+1]});
+ }
+ std::pair<int, int> best_resolution = select_best_resolution({img->nx, img->ny}, possible_resolutions);
+ // clip_image_save_to_bmp(*img, "input.bmp");
+ resize_and_pad_image(*img, *temp, best_resolution); // we do not pad with mean-bg color anymore in llava-1.6
+ // clip_image_save_to_bmp(*temp, "resized.bmp");
+ // visually verify normalized image:
+ // normalize_image_u8_to_f32(*temp, *res, ctx->image_mean, ctx->image_std);
+ // {
+ // clip_image_u8 * temp2 = clip_image_u8_init();
+ // clip_image_convert_f32_to_u8(*res, *temp2);
+ // clip_image_save_to_bmp(*temp2, "resized_normalized_f32.bmp");
+ // clip_image_u8_free(temp2);
+ // }
+
+ std::vector<clip_image_u8 *> patches = divide_to_patches_u8(*temp, params.image_size); // prepare spatial sorted main patches of image_size each (336 in llava-1.6)
+
+ clip_image_u8 *image_original_resize = clip_image_u8_init();
+ // bilinear_resize(*img, *image_original_resize, params.image_size, params.image_size); // in python this is "shortest_edge", but all CLIP are square
+ bicubic_resize(*img, *image_original_resize, params.image_size, params.image_size); // in python this is "shortest_edge", but all CLIP are square
+ patches.insert(patches.begin(), image_original_resize);
+ // clip_image_f32_batch_init(patches.size());
+ res_imgs->size = patches.size();
+ res_imgs->data = new clip_image_f32[res_imgs->size];
+ int num=0;
+ for (auto& patch : patches) {
+ normalize_image_u8_to_f32(patch, &res_imgs->data[num], ctx->image_mean, ctx->image_std);
+ num++;
+ }
+
+ for (size_t i = 0; i < patches.size(); i++) {
+ // LOG_TEE("patch %d: %d %d\n", i, patches[i]->nx, patches[i]->ny);
+ clip_image_u8_free(patches[i]);
+ }
+
+ clip_image_u8_free(temp);
+
+ return true;
+ } else {
+ temp->nx = img->nx;
+ temp->ny = img->ny;
+ temp->buf.resize(img->buf.size());
+ memcpy(temp->buf.data(), img->buf.data(), temp->buf.size());
+ }
+ }
+
+ const int nx = temp->nx;
+ const int ny = temp->ny;
+ // clip_image_save_to_bmp(*temp, "resized_vanilla.bmp");
+
+ const int nx2 = ctx->vision_model.hparams.image_size;
+ const int ny2 = ctx->vision_model.hparams.image_size;
+ clip_image_f32 * res = clip_image_f32_init();
+ res->nx = nx2;
+ res->ny = ny2;
+ res->buf.resize(3 * nx2 * ny2);
+
+ const float scale = std::max(nx, ny) / (float)ctx->vision_model.hparams.image_size;
+
+ const int nx3 = int(nx / scale + 0.5f);
+ const int ny3 = int(ny / scale + 0.5f);
+
+ const auto & m3 = ctx->image_mean; // {0.48145466f, 0.4578275f, 0.40821073f};
+ const auto & s3 = ctx->image_std; // {0.26862954f, 0.26130258f, 0.27577711f};
+
+ for (int y = 0; y < ny3; y++) {
+ for (int x = 0; x < nx3; x++) {
+ for (int c = 0; c < 3; c++) {
+ // linear interpolation
+ const float sx = (x + 0.5f) * scale - 0.5f;
+ const float sy = (y + 0.5f) * scale - 0.5f;
+
+ const int x0 = std::max(0, (int)std::floor(sx));
+ const int y0 = std::max(0, (int)std::floor(sy));
+
+ const int x1 = std::min(x0 + 1, nx - 1);
+ const int y1 = std::min(y0 + 1, ny - 1);
+
+ const float dx = sx - x0;
+ const float dy = sy - y0;
+
+ const int j00 = 3 * (y0 * nx + x0) + c;
+ const int j01 = 3 * (y0 * nx + x1) + c;
+ const int j10 = 3 * (y1 * nx + x0) + c;
+ const int j11 = 3 * (y1 * nx + x1) + c;
+
+ const float v00 = temp->buf[j00];
+ const float v01 = temp->buf[j01];
+ const float v10 = temp->buf[j10];
+ const float v11 = temp->buf[j11];
+
+ const float v0 = v00 * (1.0f - dx) + v01 * dx;
+ const float v1 = v10 * (1.0f - dx) + v11 * dx;
+
+ const float v = v0 * (1.0f - dy) + v1 * dy;
+
+ const uint8_t v2 = std::min(std::max(std::round(v), 0.0f), 255.0f);
+
+ const int i = 3 * (y * nx3 + x) + c;
+
+ res->buf[i] = ((float(v2) / 255.0f) - m3[c]) / s3[c];
+ }
+ }
+ }
+ clip_image_u8_free(temp);
+
+ // {
+ // clip_image_u8 * temp2 = clip_image_u8_init();
+ // clip_image_convert_f32_to_u8(*res, *temp2);
+ // clip_image_save_to_bmp(*temp2, "resized_normalized_f32_vanilla.bmp");
+ // clip_image_u8_free(temp2);
+ // }
+ // res_imgs.push_back(res);
+
+ res_imgs->size = 1;
+ res_imgs->data = new clip_image_f32[res_imgs->size];
+ res_imgs->data[0] = *res;
+ clip_image_f32_free(res);
+
+ return true;
+}
+
+ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx) {
+ return ctx->vision_model.image_newline;
+}
+
+void clip_free(clip_ctx * ctx) {
+ ggml_free(ctx->ctx_data);
+ gguf_free(ctx->ctx_gguf);
+
+ ggml_backend_buffer_free(ctx->params_buffer);
+ ggml_backend_free(ctx->backend);
+ ggml_gallocr_free(ctx->compute_alloc);
+ delete ctx;
+}
+
+size_t clip_embd_nbytes(const struct clip_ctx * ctx) {
+ return clip_n_patches(ctx) * clip_n_mmproj_embd(ctx) * sizeof(float);
+}
+
+int32_t clip_image_size(const struct clip_ctx * ctx) {
+ return ctx->vision_model.hparams.image_size;
+}
+
+int32_t clip_patch_size(const struct clip_ctx * ctx) {
+ return ctx->vision_model.hparams.patch_size;
+}
+
+int32_t clip_hidden_size(const struct clip_ctx * ctx) {
+ return ctx->vision_model.hparams.hidden_size;
+}
+
+const char * clip_patch_merge_type(const struct clip_ctx * ctx) {
+ return ctx->vision_model.hparams.mm_patch_merge_type;
+}
+
+const int32_t * clip_image_grid(const struct clip_ctx * ctx) {
+ return ctx->vision_model.hparams.image_grid_pinpoints;
+}
+
+int clip_n_patches(const struct clip_ctx * ctx) {
+ const auto & params = ctx->vision_model.hparams;
+
+ int n_patches = (params.image_size / params.patch_size) * (params.image_size / params.patch_size);
+
+ if (ctx->proj_type == PROJECTOR_TYPE_LDP || ctx->proj_type == PROJECTOR_TYPE_LDPV2) {
+ n_patches /= 4;
+ } else if (ctx->proj_type == PROJECTOR_TYPE_RESAMPLER) {
+ if (ctx->minicpmv_version == 2) {
+ n_patches = 96;
+ }
+ else if (ctx->minicpmv_version == 3) {
+ n_patches = 64;
+ }
+ }
+
+ return n_patches;
+}
+
+static std::vector<std::vector<std::vector<float>>> get_1d_sincos_pos_embed_from_grid_new(int embed_dim, const std::vector<std::vector<float>> & pos) {
+ assert(embed_dim % 2 == 0);
+ int H = pos.size();
+ int W = pos[0].size();
+
+ std::vector<float> omega(embed_dim / 2);
+ for (int i = 0; i < embed_dim / 2; ++i) {
+ omega[i] = 1.0 / pow(10000.0, static_cast<float>(i) / (embed_dim / 2));
+ }
+
+ std::vector<std::vector<std::vector<float>>> emb(H, std::vector<std::vector<float>>(W, std::vector<float>(embed_dim)));
+ for (int h = 0; h < H; ++h) {
+ for (int w = 0; w < W; ++w) {
+ for (int d = 0; d < embed_dim / 2; ++d) {
+ float out_value = pos[h][w] * omega[d];
+ emb[h][w][d] = sin(out_value);
+ emb[h][w][d + embed_dim / 2] = cos(out_value);
+ }
+ }
+ }
+
+ return emb;
+}
+
+static std::vector<std::vector<std::vector<float>>> get_2d_sincos_pos_embed_from_grid(int embed_dim, const std::vector<std::vector<std::vector<float>>> & grid) {
+ assert(embed_dim % 2 == 0);
+ std::vector<std::vector<std::vector<float>>> emb_h = get_1d_sincos_pos_embed_from_grid_new(embed_dim / 2, grid[0]); // (H, W, D/2)
+ std::vector<std::vector<std::vector<float>>> emb_w = get_1d_sincos_pos_embed_from_grid_new(embed_dim / 2, grid[1]); // (H, W, D/2)
+
+ int H = emb_h.size();
+ int W = emb_h[0].size();
+ std::vector<std::vector<std::vector<float>>> emb(H, std::vector<std::vector<float>>(W, std::vector<float>(embed_dim)));
+
+ for (int h = 0; h < H; ++h) {
+ for (int w = 0; w < W; ++w) {
+ for (int d = 0; d < embed_dim / 2; ++d) {
+ emb[h][w][d] = emb_h[h][w][d];
+ emb[h][w][d + embed_dim / 2] = emb_w[h][w][d];
+ }
+ }
+ }
+ return emb;
+}
+
+static std::vector<std::vector<float>> get_2d_sincos_pos_embed(int embed_dim, const std::pair<int, int> image_size) {
+ int grid_h_size = image_size.first;
+ int grid_w_size = image_size.second;
+
+ std::vector<float> grid_h(grid_h_size);
+ std::vector<float> grid_w(grid_w_size);
+
+ for (int i = 0; i < grid_h_size; ++i) {
+ grid_h[i] = static_cast<float>(i);
+ }
+ for (int i = 0; i < grid_w_size; ++i) {
+ grid_w[i] = static_cast<float>(i);
+ }
+
+ std::vector<std::vector<float>> grid(grid_h_size, std::vector<float>(grid_w_size));
+ for (int h = 0; h < grid_h_size; ++h) {
+ for (int w = 0; w < grid_w_size; ++w) {
+ grid[h][w] = grid_w[w];
+ }
+ }
+ std::vector<std::vector<std::vector<float>>> grid_2d = {grid, grid};
+ for (int h = 0; h < grid_h_size; ++h) {
+ for (int w = 0; w < grid_w_size; ++w) {
+ grid_2d[0][h][w] = grid_h[h];
+ grid_2d[1][h][w] = grid_w[w];
+ }
+ }
+
+ std::vector<std::vector<std::vector<float>>> pos_embed_3d = get_2d_sincos_pos_embed_from_grid(embed_dim, grid_2d);
+
+ int H = image_size.first;
+ int W = image_size.second;
+ std::vector<std::vector<float>> pos_embed_2d(H * W, std::vector<float>(embed_dim));
+ for (int h = 0; h < H; ++h) {
+ for (int w = 0; w < W; ++w) {
+ pos_embed_2d[w * H + h] = pos_embed_3d[h][w];
+ }
+ }
+
+ return pos_embed_2d;
+}
+
+bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f32 * img, float * vec) {
+ if (!ctx->has_vision_encoder) {
+ LOG_TEE("This gguf file seems to have no vision encoder\n");
+ return false;
+ }
+
+ clip_image_f32_batch imgs{};
+ imgs.size = 1;
+ imgs.data = img;
+ return clip_image_batch_encode(ctx, n_threads, &imgs, vec);
+}
+
+bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs, float * vec) {
+ if (!ctx->has_vision_encoder) {
+ LOG_TEE("This gguf file seems to have no vision encoder\n");
+ return false;
+ }
+
+ int batch_size = imgs->size;
+ if (ctx->has_llava_projector) {
+ GGML_ASSERT(batch_size == 1); // TODO: support multiple images
+ }
+ if (ctx->has_minicpmv_projector) {
+ GGML_ASSERT(batch_size == 1);
+ }
+
+ // build the inference graph
+ ggml_cgraph * gf = clip_image_build_graph(ctx, imgs, ctx->load_image_size, true);
+ ggml_gallocr_alloc_graph(ctx->compute_alloc, gf);
+
+ // set inputs
+ const auto & model = ctx->vision_model;
+ const auto & hparams = model.hparams;
+
+ const int image_size = hparams.image_size;
+ int image_size_width = image_size;
+ int image_size_height = image_size;
+ if (ctx->has_minicpmv_projector) {
+ image_size_width = imgs->data[0].nx;
+ image_size_height = imgs->data[0].ny;
+ }
+ const int patch_size = hparams.patch_size;
+ const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size));
+ const int num_positions = num_patches + (ctx->has_class_embedding ? 1 : 0);
+ if(ctx->load_image_size==nullptr){
+ ctx->load_image_size= clip_image_size_init();
+ }
+ const int pos_w = ctx->load_image_size->width/patch_size;
+ const int pos_h = ctx->load_image_size->height/patch_size;
+
+ {
+ struct ggml_tensor * inp_raw = ggml_graph_get_tensor(gf, "inp_raw");
+ float * data = (float *)malloc(ggml_nbytes(inp_raw));
+
+ for (size_t i = 0; i < imgs->size; i++) {
+ const int nx = imgs->data[i].nx;
+ const int ny = imgs->data[i].ny;
+ if (!ctx->has_minicpmv_projector) {
+ GGML_ASSERT(nx == image_size && ny == image_size);
+ }
+
+ const int n = nx * ny;
+
+ for (int b = 0; b < batch_size; b++) {
+ for (int k = 0; k < 3; k++) {
+ for (int y = 0; y < ny; y++) {
+ for (int x = 0; x < nx; x++) {
+ data[(b * 3 * n) + k * n + y * nx + x] = imgs->data[b].buf[3 * (y * nx + x) + k];
+ }
+ }
+ }
+ }
+ }
+ ggml_backend_tensor_set(inp_raw, data, 0, ggml_nbytes(inp_raw));
+ free(data);
+ }
+ if (ctx->has_minicpmv_projector) {
+ {
+ // inspired from siglip:
+ // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit
+ // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316
+ struct ggml_tensor * positions = ggml_graph_get_tensor(gf, "positions");
+ int* positions_data = (int*)malloc(ggml_nbytes(positions));
+ int bucket_coords_h[70];
+ int bucket_coords_w[70];
+ for (int i = 0; i < pos_h; i++){
+ bucket_coords_h[i] = std::floor(70.0*i/pos_h);
+ }
+ for (int i = 0; i < pos_w; i++){
+ bucket_coords_w[i] = std::floor(70.0*i/pos_w);
+ }
+ for (int i = 0, id = 0; i < pos_h; i++){
+ for (int j = 0; j < pos_w; j++){
+ positions_data[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j];
+ }
+ }
+ ggml_backend_tensor_set(positions, positions_data, 0, ggml_nbytes(positions));
+ free(positions_data);
+ }
+
+ {
+ // inspired from resampler of Qwen-VL:
+ // -> https://huggingface.co/Qwen/Qwen-VL/tree/main
+ // -> https://huggingface.co/Qwen/Qwen-VL/blob/0547ed36a86561e2e42fecec8fd0c4f6953e33c4/visual.py#L23
+ struct ggml_tensor * pos_embed = ggml_graph_get_tensor(gf, "pos_embed");
+ int embed_dim = 4096;
+ if (ctx->minicpmv_version == 2) {
+ embed_dim = 4096;
+ }
+ else if (ctx->minicpmv_version == 3) {
+ embed_dim = 3584;
+ }
+ auto pos_embed_t = get_2d_sincos_pos_embed(embed_dim, std::make_pair(pos_w, pos_h));
+
+ float * pos_embed_data = (float *)malloc(ggml_nbytes(pos_embed));
+ for(int i=0;i<pos_w * pos_h;++i){
+ for(int j=0;j<embed_dim;++j){
+ pos_embed_data[i*embed_dim+j]=pos_embed_t[i][j];
+ }
+ }
+
+ ggml_backend_tensor_set(pos_embed, pos_embed_data, 0, ggml_nbytes(pos_embed));
+ free(pos_embed_data);
+ }
+ }
+ else{
+ {
+ if (ctx->has_class_embedding) {
+ struct ggml_tensor * embeddings = ggml_graph_get_tensor(gf, "embeddings");
+
+ void* zero_mem = malloc(ggml_nbytes(embeddings));
+ memset(zero_mem, 0, ggml_nbytes(embeddings));
+ ggml_backend_tensor_set(embeddings, zero_mem, 0, ggml_nbytes(embeddings));
+ free(zero_mem);
+ }
+ }
+
+ {
+ struct ggml_tensor * positions = ggml_graph_get_tensor(gf, "positions");
+
+ int* positions_data = (int*)malloc(ggml_nbytes(positions));
+ for (int i = 0; i < num_positions; i++) {
+ positions_data[i] = i;
+ }
+ ggml_backend_tensor_set(positions, positions_data, 0, ggml_nbytes(positions));
+ free(positions_data);
+ }
+
+ {
+ struct ggml_tensor * patches = ggml_graph_get_tensor(gf, "patches");
+ int* patches_data = (int*)malloc(ggml_nbytes(patches));
+ for (int i = 0; i < num_patches; i++) {
+ patches_data[i] = i + 1;
+ }
+ ggml_backend_tensor_set(patches, patches_data, 0, ggml_nbytes(patches));
+ free(patches_data);
+ }
+ }
+
+ if (ggml_backend_is_cpu(ctx->backend)) {
+ ggml_backend_cpu_set_n_threads(ctx->backend, n_threads);
+ }
+
+#ifdef GGML_USE_METAL
+ if (ggml_backend_is_metal(ctx->backend)) {
+ ggml_backend_metal_set_n_cb(ctx->backend, n_threads);
+ }
+#endif
+
+ ggml_backend_graph_compute(ctx->backend, gf);
+
+ // the last node is the embedding tensor
+ struct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 1];
+
+ // copy the embeddings to the location passed by the user
+ ggml_backend_tensor_get(embeddings, vec, 0, ggml_nbytes(embeddings));
+
+ return true;
+}
+
+bool clip_model_quantize(const char * fname_inp, const char * fname_out, const int itype) {
+ ggml_type type = GGML_TYPE_Q4_1;
+
+ assert(itype < GGML_TYPE_COUNT);
+ type = static_cast<ggml_type>(itype);
+
+ auto * ctx_clip = clip_model_load(fname_inp, 2);
+
+ const auto & ctx_src = ctx_clip->ctx_gguf;
+ const auto & ctx_data = ctx_clip->ctx_data;
+
+ auto * ctx_out = gguf_init_empty();
+ gguf_set_kv(ctx_out, ctx_src);
+ gguf_set_val_u32(ctx_out, "general.quantization_version", GGML_QNT_VERSION);
+ gguf_set_val_u32(ctx_out, "general.file_type", itype);
+
+ auto fout = std::ofstream(fname_out, std::ios::binary);
+
+ const int n_tensors = gguf_get_n_tensors(ctx_src);
+
+ for (int i = 0; i < n_tensors; ++i) {
+ const char * name = gguf_get_tensor_name(ctx_src, i);
+ struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name);
+ gguf_add_tensor(ctx_out, cur);
+ }
+
+ const size_t meta_size = gguf_get_meta_size(ctx_out);
+ for (size_t i = 0; i < meta_size; ++i) {
+ fout.put(0);
+ }
+
+ // regexes of tensor names to be quantized
+ const std::vector<std::string> k_names = {
+ ".*weight",
+ };
+
+ std::vector<uint8_t> work(512);
+ std::vector<float> conv_buf(512);
+ size_t total_size_org = 0;
+ size_t total_size_new = 0;
+
+ for (int i = 0; i < n_tensors; ++i) {
+ const std::string name = gguf_get_tensor_name(ctx_src, i);
+ struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name.c_str());
+
+ enum ggml_type new_type;
+ void * new_data;
+ size_t new_size;
+
+ bool quantize = false;
+ for (const auto & s : k_names) {
+ if (std::regex_match(name, std::regex(s))) {
+ quantize = true;
+ break;
+ }
+ }
+
+ // quantize only 2D tensors
+ quantize &= (ggml_n_dims(cur) == 2);
+
+ if (quantize) {
+ new_type = type;
+ if (new_type >= GGML_TYPE_Q2_K && name.find("embd") != std::string::npos) {
+ new_type = GGML_TYPE_Q8_0; // ggml_get_rows needs non K type
+ // LOG_TEE("%s: quantizing %s to %s\n", __func__, name.c_str(), ggml_type_name(new_type));
+ }
+ const size_t n_elms = ggml_nelements(cur);
+ float * f32_data;
+
+ switch (cur->type) {
+ case GGML_TYPE_F32:
+ f32_data = (float *)cur->data;
+ break;
+ case GGML_TYPE_F16:
+ if (conv_buf.size() < n_elms) {
+ conv_buf.resize(n_elms);
+ }
+ for (size_t j = 0; j < n_elms; ++j) {
+ conv_buf[j] = ggml_fp16_to_fp32(((ggml_fp16_t *)cur->data)[j]);
+ }
+ f32_data = (float *)conv_buf.data();
+ break;
+ default:
+ LOG_TEE("Please use an input file in f32 or f16\n");
+ gguf_free(ctx_out);
+ return false;
+ }
+
+ if (work.size() < n_elms * 4) {
+ work.resize(n_elms * 4);
+ }
+ new_data = work.data();
+
+ new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, n_elms/cur->ne[0], cur->ne[0], nullptr);
+ } else {
+ new_type = cur->type;
+ new_data = cur->data;
+ new_size = ggml_nbytes(cur);
+ }
+ const size_t orig_size = ggml_nbytes(cur);
+ total_size_org += orig_size;
+ total_size_new += new_size;
+ gguf_set_tensor_type(ctx_out, name.c_str(), new_type);
+ gguf_set_tensor_data(ctx_out, name.c_str(), new_data, new_size);
+ fout.write((const char *)new_data, new_size);
+ size_t pad = GGML_PAD(new_size, gguf_get_alignment(ctx_out)) - new_size;
+ for (size_t j = 0; j < pad; ++j) {
+ fout.put(0);
+ }
+
+ LOG_TEE("%s: n_dims = %d | quantize=%d | size = %f MB -> %f MB\n", name.c_str(), ggml_n_dims(cur), quantize,
+ orig_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);
+ }
+
+ // go back to beginning of file and write the updated metadata
+ fout.seekp(0, std::ios::beg);
+ std::vector<uint8_t> meta(meta_size);
+ gguf_get_meta_data(ctx_out, meta.data());
+ fout.write((const char *)meta.data(), meta_size);
+
+ fout.close();
+
+ clip_free(ctx_clip);
+ gguf_free(ctx_out);
+
+ {
+ LOG_TEE("%s: original size = %8.2f MB\n", __func__, total_size_org / 1024.0 / 1024.0);
+ LOG_TEE("%s: quantized size = %8.2f MB\n", __func__, total_size_new / 1024.0 / 1024.0);
+ }
+
+ return true;
+}
+
+int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
+ if (ctx->proj_type == PROJECTOR_TYPE_LDP) {
+ return ctx->vision_model.mm_model_block_1_block_2_1_b->ne[0];
+ }
+ if (ctx->proj_type == PROJECTOR_TYPE_LDPV2) {
+ return ctx->vision_model.mm_model_peg_0_b->ne[0];
+ }
+ if (ctx->proj_type == PROJECTOR_TYPE_MLP) {
+ return ctx->vision_model.mm_2_b->ne[0];
+ }
+ if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) {
+ return ctx->vision_model.mm_3_b->ne[0];
+ }
+ if (ctx->proj_type == PROJECTOR_TYPE_RESAMPLER) {
+ if (ctx->minicpmv_version == 2) {
+ return 4096;
+ }
+ else if (ctx->minicpmv_version == 3) {
+ return 3584;
+ }
+ }
+
+ std::string proj_type = PROJECTOR_TYPE_NAMES[ctx->proj_type];
+ throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str()));
+}
+
+int clip_is_minicpmv(const struct clip_ctx * ctx) {
+ if (ctx->has_minicpmv_projector) {
+ return ctx->minicpmv_version;
+ }
+ return 0;
+}
diff --git a/llama/clip.h b/llama/clip.h
new file mode 100644
index 00000000..a064330c
--- /dev/null
+++ b/llama/clip.h
@@ -0,0 +1,120 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef CLIP_H
+#define CLIP_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef LLAMA_SHARED
+# if defined(_WIN32) && !defined(__MINGW32__)
+# ifdef LLAMA_BUILD
+# define CLIP_API __declspec(dllexport)
+# else
+# define CLIP_API __declspec(dllimport)
+# endif
+# else
+# define CLIP_API __attribute__ ((visibility ("default")))
+# endif
+#else
+# define CLIP_API
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct clip_ctx;
+
+struct clip_image_size {
+ int width;
+ int height;
+};
+
+struct clip_image_u8_batch {
+ struct clip_image_u8 * data;
+ size_t size;
+};
+
+struct clip_image_f32_batch {
+ struct clip_image_f32 * data;
+ size_t size;
+};
+
+CLIP_API struct clip_ctx * clip_model_load (const char * fname, int verbosity);
+CLIP_API struct clip_ctx * clip_model_load_cpu(const char * fname, int verbosity);
+
+CLIP_API void clip_free(struct clip_ctx * ctx);
+
+CLIP_API size_t clip_embd_nbytes(const struct clip_ctx * ctx);
+
+CLIP_API int32_t clip_image_size (const struct clip_ctx * ctx);
+CLIP_API int32_t clip_patch_size (const struct clip_ctx * ctx);
+CLIP_API int32_t clip_hidden_size(const struct clip_ctx * ctx);
+
+// TODO: should be enum, not string
+CLIP_API const char * clip_patch_merge_type(const struct clip_ctx * ctx);
+
+CLIP_API const int32_t * clip_image_grid(const struct clip_ctx * ctx);
+
+CLIP_API int clip_n_patches (const struct clip_ctx * ctx);
+CLIP_API int clip_n_mmproj_embd(const struct clip_ctx * ctx);
+
+CLIP_API int clip_uhd_num_image_embeds_col(struct clip_ctx * ctx_clip);
+CLIP_API void clip_add_load_image_size(struct clip_ctx * ctx_clip, struct clip_image_size * load_image_size);
+
+CLIP_API struct clip_image_size * clip_image_size_init();
+CLIP_API struct clip_image_u8 * clip_image_u8_init ();
+CLIP_API struct clip_image_f32 * clip_image_f32_init();
+
+CLIP_API void clip_image_u8_free (struct clip_image_u8 * img);
+CLIP_API void clip_image_f32_free(struct clip_image_f32 * img);
+CLIP_API void clip_image_u8_batch_free (struct clip_image_u8_batch * batch);
+CLIP_API void clip_image_f32_batch_free(struct clip_image_f32_batch * batch);
+
+CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img);
+
+/** interpret bytes as an image file with length bytes_length, and use the result to populate img */
+CLIP_API bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img);
+
+/** preprocess img and store the result in res_imgs, pad_to_square may be overridden to false depending on model configuration */
+CLIP_API bool clip_image_preprocess(struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32_batch * res_imgs );
+
+CLIP_API struct ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx);
+
+CLIP_API bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec);
+CLIP_API bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec);
+
+CLIP_API bool clip_model_quantize(const char * fname_inp, const char * fname_out, int itype);
+
+CLIP_API int clip_is_minicpmv(const struct clip_ctx * ctx);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // CLIP_H
diff --git a/llama/common.cpp b/llama/common.cpp
new file mode 100644
index 00000000..6cabaeab
--- /dev/null
+++ b/llama/common.cpp
@@ -0,0 +1,3688 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if defined(_MSC_VER)
+#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING
+#endif
+
+#include "common.h"
+// Change JSON_ASSERT from assert() to GGML_ASSERT:
+#define JSON_ASSERT GGML_ASSERT
+#include "json.hpp"
+#include "json-schema-to-grammar.h"
+#include "llama.h"
+
+#include <algorithm>
+#include <cinttypes>
+#include <cmath>
+#include <codecvt>
+#include <cstdarg>
+#include <cstring>
+#include <ctime>
+#include <fstream>
+#include <iostream>
+#include <iterator>
+#include <regex>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#if defined(__APPLE__) && defined(__MACH__)
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#endif
+
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+# define NOMINMAX
+#endif
+#include <locale>
+#include <windows.h>
+#include <fcntl.h>
+#include <io.h>
+#else
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#endif
+#if defined(LLAMA_USE_CURL)
+#include <curl/curl.h>
+#include <curl/easy.h>
+#include <thread>
+#include <future>
+#endif
+
+#if defined(_MSC_VER)
+#pragma warning(disable: 4244 4267) // possible loss of data
+#endif
+
+#if (defined(GGML_USE_CUDA) || defined(GGML_USE_SYCL))
+#define GGML_USE_CUDA_SYCL
+#endif
+
+#if (defined(GGML_USE_CUDA) || defined(GGML_USE_SYCL)) || defined(GGML_USE_VULKAN)
+#define GGML_USE_CUDA_SYCL_VULKAN
+#endif
+
+#if defined(LLAMA_USE_CURL)
+#ifdef __linux__
+#include <linux/limits.h>
+#elif defined(_WIN32)
+#define PATH_MAX MAX_PATH
+#else
+#include <sys/syslimits.h>
+#endif
+#define LLAMA_CURL_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083
+#endif // LLAMA_USE_CURL
+
+using json = nlohmann::ordered_json;
+
+//
+// Environment variable utils
+//
+
+template<typename T>
+static typename std::enable_if<std::is_same<T, std::string>::value, void>::type
+get_env(std::string name, T & target) {
+ char * value = std::getenv(name.c_str());
+ target = value ? std::string(value) : target;
+}
+
+template<typename T>
+static typename std::enable_if<!std::is_same<T, bool>::value && std::is_integral<T>::value, void>::type
+get_env(std::string name, T & target) {
+ char * value = std::getenv(name.c_str());
+ target = value ? std::stoi(value) : target;
+}
+
+template<typename T>
+static typename std::enable_if<std::is_floating_point<T>::value, void>::type
+get_env(std::string name, T & target) {
+ char * value = std::getenv(name.c_str());
+ target = value ? std::stof(value) : target;
+}
+
+template<typename T>
+static typename std::enable_if<std::is_same<T, bool>::value, void>::type
+get_env(std::string name, T & target) {
+ char * value = std::getenv(name.c_str());
+ if (value) {
+ std::string val(value);
+ target = val == "1" || val == "true";
+ }
+}
+
+//
+// CPU utils
+//
+
+int32_t cpu_get_num_physical_cores() {
+#ifdef __linux__
+ // enumerate the set of thread siblings, num entries is num cores
+ std::unordered_set<std::string> siblings;
+ for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {
+ std::ifstream thread_siblings("/sys/devices/system/cpu/cpu"
+ + std::to_string(cpu) + "/topology/thread_siblings");
+ if (!thread_siblings.is_open()) {
+ break; // no more cpus
+ }
+ std::string line;
+ if (std::getline(thread_siblings, line)) {
+ siblings.insert(line);
+ }
+ }
+ if (!siblings.empty()) {
+ return static_cast<int32_t>(siblings.size());
+ }
+#elif defined(__APPLE__) && defined(__MACH__)
+ int32_t num_physical_cores;
+ size_t len = sizeof(num_physical_cores);
+ int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);
+ if (result == 0) {
+ return num_physical_cores;
+ }
+ result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);
+ if (result == 0) {
+ return num_physical_cores;
+ }
+#elif defined(_WIN32) && (_WIN32_WINNT >= 0x0601) && !defined(__MINGW64__) // windows 7 and later
+ // TODO: windows + arm64 + mingw64
+ unsigned int n_threads_win = std::thread::hardware_concurrency();
+ unsigned int default_threads = n_threads_win > 0 ? (n_threads_win <= 4 ? n_threads_win : n_threads_win / 2) : 4;
+
+ DWORD buffer_size = 0;
+ if (!GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &buffer_size)) {
+ if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
+ return default_threads;
+ }
+ }
+
+ std::vector<char> buffer(buffer_size);
+ if (!GetLogicalProcessorInformationEx(RelationProcessorCore, reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(buffer.data()), &buffer_size)) {
+ return default_threads;
+ }
+
+ int32_t num_physical_cores = 0;
+ PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(buffer.data());
+ while (buffer_size > 0) {
+ if (info->Relationship == RelationProcessorCore) {
+ num_physical_cores += info->Processor.GroupCount;
+ }
+ buffer_size -= info->Size;
+ info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(reinterpret_cast<char*>(info) + info->Size);
+ }
+
+ return num_physical_cores > 0 ? num_physical_cores : default_threads;
+#endif
+ unsigned int n_threads = std::thread::hardware_concurrency();
+ return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;
+}
+
+#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)
+#include <pthread.h>
+
+static void cpuid(unsigned leaf, unsigned subleaf,
+ unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx) {
+ __asm__("movq\t%%rbx,%%rsi\n\t"
+ "cpuid\n\t"
+ "xchgq\t%%rbx,%%rsi"
+ : "=a"(*eax), "=S"(*ebx), "=c"(*ecx), "=d"(*edx)
+ : "0"(leaf), "2"(subleaf));
+}
+
+static int pin_cpu(int cpu) {
+ cpu_set_t mask;
+ CPU_ZERO(&mask);
+ CPU_SET(cpu, &mask);
+ return pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask);
+}
+
+static bool is_hybrid_cpu(void) {
+ unsigned eax, ebx, ecx, edx;
+ cpuid(7, 0, &eax, &ebx, &ecx, &edx);
+ return !!(edx & (1u << 15));
+}
+
+static bool is_running_on_efficiency_core(void) {
+ unsigned eax, ebx, ecx, edx;
+ cpuid(0x1a, 0, &eax, &ebx, &ecx, &edx);
+ int intel_atom = 0x20;
+ int core_type = (eax & 0xff000000u) >> 24;
+ return core_type == intel_atom;
+}
+
+static int cpu_count_math_cpus(int n_cpu) {
+ int result = 0;
+ for (int cpu = 0; cpu < n_cpu; ++cpu) {
+ if (pin_cpu(cpu)) {
+ return -1;
+ }
+ if (is_running_on_efficiency_core()) {
+ continue; // efficiency cores harm lockstep threading
+ }
+ ++cpu; // hyperthreading isn't useful for linear algebra
+ ++result;
+ }
+ return result;
+}
+
+#endif // __x86_64__ && __linux__
+
+/**
+ * Returns number of CPUs on system that are useful for math.
+ */
+int32_t cpu_get_num_math() {
+#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)
+ int n_cpu = sysconf(_SC_NPROCESSORS_ONLN);
+ if (n_cpu < 1) {
+ return cpu_get_num_physical_cores();
+ }
+ if (is_hybrid_cpu()) {
+ cpu_set_t affinity;
+ if (!pthread_getaffinity_np(pthread_self(), sizeof(affinity), &affinity)) {
+ int result = cpu_count_math_cpus(n_cpu);
+ pthread_setaffinity_np(pthread_self(), sizeof(affinity), &affinity);
+ if (result > 0) {
+ return result;
+ }
+ }
+ }
+#endif
+ return cpu_get_num_physical_cores();
+}
+
+// Helper for setting process priority
+
+#if defined(_WIN32)
+
+bool set_process_priority(enum ggml_sched_priority prio) {
+ if (prio == GGML_SCHED_PRIO_NORMAL) {
+ return true;
+ }
+
+ DWORD p = NORMAL_PRIORITY_CLASS;
+ switch (prio) {
+ case GGML_SCHED_PRIO_NORMAL: p = NORMAL_PRIORITY_CLASS; break;
+ case GGML_SCHED_PRIO_MEDIUM: p = ABOVE_NORMAL_PRIORITY_CLASS; break;
+ case GGML_SCHED_PRIO_HIGH: p = HIGH_PRIORITY_CLASS; break;
+ case GGML_SCHED_PRIO_REALTIME: p = REALTIME_PRIORITY_CLASS; break;
+ }
+
+ if (!SetPriorityClass(GetCurrentProcess(), p)) {
+ fprintf(stderr, "warn: failed to set process priority class %d : (%d)\n", prio, (int) GetLastError());
+ return false;
+ }
+
+ return true;
+}
+
+#else // MacOS and POSIX
+#include <sys/types.h>
+#include <sys/resource.h>
+
+bool set_process_priority(enum ggml_sched_priority prio) {
+ if (prio == GGML_SCHED_PRIO_NORMAL) {
+ return true;
+ }
+
+ int p = 0;
+ switch (prio) {
+ case GGML_SCHED_PRIO_NORMAL: p = 0; break;
+ case GGML_SCHED_PRIO_MEDIUM: p = -5; break;
+ case GGML_SCHED_PRIO_HIGH: p = -10; break;
+ case GGML_SCHED_PRIO_REALTIME: p = -20; break;
+ }
+
+ if (!setpriority(PRIO_PROCESS, 0, p)) {
+ fprintf(stderr, "warn: failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno);
+ return false;
+ }
+ return true;
+}
+
+#endif
+
+//
+// CLI argument parsing
+//
+
+void gpt_params_handle_model_default(gpt_params & params) {
+ if (!params.hf_repo.empty()) {
+ // short-hand to avoid specifying --hf-file -> default it to --model
+ if (params.hf_file.empty()) {
+ if (params.model.empty()) {
+ throw std::invalid_argument("error: --hf-repo requires either --hf-file or --model\n");
+ }
+ params.hf_file = params.model;
+ } else if (params.model.empty()) {
+ params.model = fs_get_cache_file(string_split(params.hf_file, '/').back());
+ }
+ } else if (!params.model_url.empty()) {
+ if (params.model.empty()) {
+ auto f = string_split(params.model_url, '#').front();
+ f = string_split(f, '?').front();
+ params.model = fs_get_cache_file(string_split(f, '/').back());
+ }
+ } else if (params.model.empty()) {
+ params.model = DEFAULT_MODEL_PATH;
+ }
+}
+
+void postprocess_cpu_params(cpu_params& cpuparams, const cpu_params* role_model) {
+ int32_t n_set = 0;
+
+ if (cpuparams.n_threads < 0) {
+ // Assuming everything about cpuparams is invalid
+ if (role_model != nullptr) {
+ cpuparams = *role_model;
+ } else {
+ cpuparams.n_threads = cpu_get_num_math();
+ }
+ }
+
+ for (int32_t i = 0; i < GGML_MAX_N_THREADS; i++) {
+ if (cpuparams.cpumask[i]) {
+ n_set++;
+ }
+ }
+
+ if (n_set && n_set < cpuparams.n_threads) {
+ // Not enough set bits, may experience performance issues.
+ fprintf(stderr, "warn: Not enough set bits in CPU mask (%d) to satisfy requested thread count: %d\n", n_set, cpuparams.n_threads);
+ }
+}
+
+bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {
+ bool invalid_param = false;
+ std::string arg;
+ const std::string arg_prefix = "--";
+ llama_sampling_params & sparams = params.sparams;
+
+ for (int i = 1; i < argc; i++) {
+ arg = argv[i];
+ if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {
+ std::replace(arg.begin(), arg.end(), '_', '-');
+ }
+ if (!gpt_params_find_arg(argc, argv, arg, params, i, invalid_param)) {
+ throw std::invalid_argument("error: unknown argument: " + arg);
+ }
+ if (invalid_param) {
+ throw std::invalid_argument("error: invalid parameter for argument: " + arg);
+ }
+ }
+
+ postprocess_cpu_params(params.cpuparams, nullptr);
+ postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams);
+ postprocess_cpu_params(params.draft_cpuparams, ¶ms.cpuparams);
+ postprocess_cpu_params(params.draft_cpuparams_batch, ¶ms.cpuparams_batch);
+
+ if (params.prompt_cache_all && (params.interactive || params.interactive_first)) {
+ throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");
+ }
+
+ gpt_params_handle_model_default(params);
+
+ if (params.hf_token.empty()) {
+ get_env("HF_TOKEN", params.hf_token);
+ }
+
+ if (params.escape) {
+ string_process_escapes(params.prompt);
+ string_process_escapes(params.input_prefix);
+ string_process_escapes(params.input_suffix);
+ string_process_escapes(sparams.cfg_negative_prompt);
+ for (auto & antiprompt : params.antiprompt) {
+ string_process_escapes(antiprompt);
+ }
+ }
+
+ if (!params.kv_overrides.empty()) {
+ params.kv_overrides.emplace_back();
+ params.kv_overrides.back().key[0] = 0;
+ }
+
+ return true;
+}
+
+void gpt_params_parse_from_env(gpt_params & params) {
+ // we only care about server-related params for now
+ get_env("LLAMA_ARG_MODEL", params.model);
+ get_env("LLAMA_ARG_MODEL_URL", params.model_url);
+ get_env("LLAMA_ARG_MODEL_ALIAS", params.model_alias);
+ get_env("LLAMA_ARG_HF_REPO", params.hf_repo);
+ get_env("LLAMA_ARG_HF_FILE", params.hf_file);
+ get_env("LLAMA_ARG_THREADS", params.cpuparams.n_threads);
+ get_env("LLAMA_ARG_CTX_SIZE", params.n_ctx);
+ get_env("LLAMA_ARG_N_PARALLEL", params.n_parallel);
+ get_env("LLAMA_ARG_BATCH", params.n_batch);
+ get_env("LLAMA_ARG_UBATCH", params.n_ubatch);
+ get_env("LLAMA_ARG_N_GPU_LAYERS", params.n_gpu_layers);
+ get_env("LLAMA_ARG_THREADS_HTTP", params.n_threads_http);
+ get_env("LLAMA_ARG_CHAT_TEMPLATE", params.chat_template);
+ get_env("LLAMA_ARG_N_PREDICT", params.n_predict);
+ get_env("LLAMA_ARG_ENDPOINT_METRICS", params.endpoint_metrics);
+ get_env("LLAMA_ARG_ENDPOINT_SLOTS", params.endpoint_slots);
+ get_env("LLAMA_ARG_EMBEDDINGS", params.embedding);
+ get_env("LLAMA_ARG_FLASH_ATTN", params.flash_attn);
+ get_env("LLAMA_ARG_DEFRAG_THOLD", params.defrag_thold);
+ get_env("LLAMA_ARG_CONT_BATCHING", params.cont_batching);
+ get_env("LLAMA_ARG_HOST", params.hostname);
+ get_env("LLAMA_ARG_PORT", params.port);
+}
+
+bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
+ const auto params_org = params; // the example can modify the default params
+
+ try {
+ if (!gpt_params_parse_ex(argc, argv, params) || params.usage) {
+ params = params_org;
+ params.usage = true;
+ return false;
+ }
+ } catch (const std::invalid_argument & ex) {
+ fprintf(stderr, "%s\n", ex.what());
+ params = params_org;
+ return false;
+ }
+
+ return true;
+}
+
+bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THREADS]) {
+ size_t dash_loc = range.find('-');
+ if (dash_loc == std::string::npos) {
+ fprintf(stderr, "Format of CPU range is invalid! Expected [<start>]-[<end>].\n");
+ return false;
+ }
+
+ size_t start_i;
+ size_t end_i;
+
+ if (dash_loc == 0) {
+ start_i = 0;
+ } else {
+ start_i = std::stoull(range.substr(0, dash_loc));
+ if (start_i >= GGML_MAX_N_THREADS) {
+ fprintf(stderr, "Start index out of bounds!\n");
+ return false;
+ }
+ }
+
+ if (dash_loc == range.length() - 1) {
+ end_i = GGML_MAX_N_THREADS - 1;
+ } else {
+ end_i = std::stoull(range.substr(dash_loc + 1));
+ if (end_i >= GGML_MAX_N_THREADS) {
+ fprintf(stderr, "End index out of bounds!\n");
+ return false;
+ }
+ }
+
+ for (size_t i = start_i; i <= end_i; i++) {
+ boolmask[i] = true;
+ }
+
+ return true;
+}
+
+bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREADS]) {
+ // Discard potential 0x prefix
+ size_t start_i = 0;
+ if (mask.length() >= 2 && mask.substr(0, 2) == "0x") {
+ start_i = 2;
+ }
+
+ size_t num_digits = mask.length() - start_i;
+ if (num_digits > 128) num_digits = 128;
+
+ size_t end_i = num_digits + start_i;
+
+ for (size_t i = start_i, n = (num_digits*4 - 1); i < end_i; i++, n-=4) {
+ char c = mask.at(i);
+ int8_t id = c;
+
+ if ((c >= '0' && c <= '9')) {
+ id -= '0';
+ } else if (c >= 'a' && c <= 'f') {
+ id -= 'a' - 10;
+ } else if (c >= 'A' && c <= 'F') {
+ id -= 'A' - 10;
+ } else {
+ fprintf(stderr, "Invalid hex character '%c' at position %d\n", c, int32_t(i));
+ return false;
+ }
+
+ boolmask[ n ] = boolmask[ n ] || ((id & 8) != 0);
+ boolmask[n - 1] = boolmask[n - 1] || ((id & 4) != 0);
+ boolmask[n - 2] = boolmask[n - 2] || ((id & 2) != 0);
+ boolmask[n - 3] = boolmask[n - 3] || ((id & 1) != 0);
+ }
+
+ return true;
+}
+
+#define CHECK_ARG if (++i >= argc) { invalid_param = true; return true; }
+
+bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_params & params, int & i, bool & invalid_param) {
+ const char split_delim = ',';
+
+ llama_sampling_params & sparams = params.sparams;
+
+ if (arg == "-s" || arg == "--seed") {
+ CHECK_ARG
+ // TODO: this is temporary, in the future the sampling state will be moved fully to llama_sampling_context.
+ params.seed = std::stoul(argv[i]);
+ sparams.seed = std::stoul(argv[i]);
+ return true;
+ }
+ if (arg == "-t" || arg == "--threads") {
+ CHECK_ARG
+ params.cpuparams.n_threads = std::stoi(argv[i]);
+ if (params.cpuparams.n_threads <= 0) {
+ params.cpuparams.n_threads = std::thread::hardware_concurrency();
+ }
+ return true;
+ }
+ if (arg == "-C" || arg == "--cpu-mask") {
+ CHECK_ARG
+ std::string mask = argv[i];
+ params.cpuparams.mask_valid = true;
+ invalid_param = !parse_cpu_mask(mask, params.cpuparams.cpumask);
+ return true;
+ }
+ if (arg == "-Cr" || arg == "--cpu-range") {
+ CHECK_ARG
+ std::string range = argv[i];
+ params.cpuparams.mask_valid = true;
+ invalid_param = !parse_cpu_range(range, params.cpuparams.cpumask);
+ return true;
+ }
+ if (arg == "--prio") {
+ CHECK_ARG
+ params.cpuparams.priority = (enum ggml_sched_priority) std::stoul(argv[i]);
+ return true;
+ }
+ if (arg == "--cpu-strict") {
+ CHECK_ARG
+ params.cpuparams.strict_cpu = std::stoul(argv[i]);
+ return true;
+ }
+ if (arg == "--poll") {
+ CHECK_ARG
+ params.cpuparams.poll = std::stoul(argv[i]);
+ return true;
+ }
+ if (arg == "-tb" || arg == "--threads-batch") {
+ CHECK_ARG
+ params.cpuparams_batch.n_threads = std::stoi(argv[i]);
+ if (params.cpuparams_batch.n_threads <= 0) {
+ params.cpuparams_batch.n_threads = std::thread::hardware_concurrency();
+ }
+ return true;
+ }
+ if (arg == "-Cb" || arg == "--cpu-mask-batch") {
+ CHECK_ARG
+ std::string mask = argv[i];
+ params.cpuparams_batch.mask_valid = true;
+ invalid_param = !parse_cpu_mask(mask, params.cpuparams_batch.cpumask);
+ return true;
+ }
+ if (arg == "-Crb" || arg == "--cpu-range_batch") {
+ CHECK_ARG
+ std::string range = argv[i];
+ params.cpuparams_batch.mask_valid = true;
+ invalid_param = !parse_cpu_range(range, params.cpuparams_batch.cpumask);
+ return true;
+ }
+ if (arg == "--prio-batch") {
+ CHECK_ARG
+ params.cpuparams_batch.priority = (enum ggml_sched_priority) std::stoul(argv[i]);
+ return true;
+ }
+ if (arg == "--cpu-strict-batch") {
+ params.cpuparams_batch.strict_cpu = true;
+ return true;
+ }
+ if (arg == "--poll-batch") {
+ CHECK_ARG
+ params.cpuparams_batch.poll = std::stoul(argv[i]);
+ return true;
+ }
+ if (arg == "-td" || arg == "--threads-draft") {
+ CHECK_ARG
+ params.draft_cpuparams.n_threads = std::stoi(argv[i]);
+ if (params.draft_cpuparams.n_threads <= 0) {
+ params.draft_cpuparams.n_threads = std::thread::hardware_concurrency();
+ }
+ return true;
+ }
+ if (arg == "-Cd" || arg == "--cpu-mask-draft") {
+ CHECK_ARG
+ std::string mask = argv[i];
+ params.draft_cpuparams.mask_valid = true;
+ invalid_param = !parse_cpu_mask(mask, params.draft_cpuparams.cpumask);
+ return true;
+ }
+ if (arg == "-Crd" || arg == "--cpu-range-draft") {
+ CHECK_ARG
+ std::string range = argv[i];
+ params.draft_cpuparams.mask_valid = true;
+ invalid_param = !parse_cpu_range(range, params.draft_cpuparams.cpumask);
+ return true;
+ }
+ if (arg == "--prio-draft") {
+ CHECK_ARG
+ params.draft_cpuparams.priority = (enum ggml_sched_priority) std::stoul(argv[i]);
+ return true;
+ }
+ if (arg == "--cpu-strict-draft") {
+ params.draft_cpuparams.strict_cpu = true;
+ return true;
+ }
+ if (arg == "--poll-draft") {
+ CHECK_ARG
+ params.draft_cpuparams.poll = std::stoul(argv[i]);
+ return true;
+ }
+ if (arg == "-tbd" || arg == "--threads-batch-draft") {
+ CHECK_ARG
+ params.draft_cpuparams_batch.n_threads = std::stoi(argv[i]);
+ if (params.draft_cpuparams_batch.n_threads <= 0) {
+ params.draft_cpuparams_batch.n_threads = std::thread::hardware_concurrency();
+ }
+ return true;
+ }
+ if (arg == "-Crbd" || arg == "--cpu-range-batch-draft") {
+ CHECK_ARG
+ std::string range = argv[i];
+ params.draft_cpuparams_batch.mask_valid = true;
+ invalid_param = !parse_cpu_range(range, params.draft_cpuparams_batch.cpumask);
+ return true;
+ }
+ if (arg == "--prio-batch-draft") {
+ CHECK_ARG
+ params.draft_cpuparams_batch.priority = (enum ggml_sched_priority) std::stoul(argv[i]);
+ return true;
+ }
+ if (arg == "--cpu-strict-batch-draft") {
+ params.draft_cpuparams_batch.strict_cpu = true;
+ return true;
+ }
+ if (arg == "--poll-batch-draft") {
+ CHECK_ARG
+ params.draft_cpuparams_batch.poll = std::stoul(argv[i]);
+ return true;
+ }
+ if (arg == "-p" || arg == "--prompt") {
+ CHECK_ARG
+ params.prompt = argv[i];
+ return true;
+ }
+ if (arg == "-e" || arg == "--escape") {
+ params.escape = true;
+ return true;
+ }
+ if (arg == "--no-escape") {
+ params.escape = false;
+ return true;
+ }
+ if (arg == "--prompt-cache") {
+ CHECK_ARG
+ params.path_prompt_cache = argv[i];
+ return true;
+ }
+ if (arg == "--prompt-cache-all") {
+ params.prompt_cache_all = true;
+ return true;
+ }
+ if (arg == "--prompt-cache-ro") {
+ params.prompt_cache_ro = true;
+ return true;
+ }
+ if (arg == "-bf" || arg == "--binary-file") {
+ CHECK_ARG
+ std::ifstream file(argv[i], std::ios::binary);
+ if (!file) {
+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+ invalid_param = true;
+ return true;
+ }
+ // store the external file name in params
+ params.prompt_file = argv[i];
+ std::ostringstream ss;
+ ss << file.rdbuf();
+ params.prompt = ss.str();
+ fprintf(stderr, "Read %zu bytes from binary file %s\n", params.prompt.size(), argv[i]);
+ return true;
+ }
+ if (arg == "-f" || arg == "--file") {
+ CHECK_ARG
+ std::ifstream file(argv[i]);
+ if (!file) {
+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+ invalid_param = true;
+ return true;
+ }
+ // store the external file name in params
+ params.prompt_file = argv[i];
+ std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));
+ if (!params.prompt.empty() && params.prompt.back() == '\n') {
+ params.prompt.pop_back();
+ }
+ return true;
+ }
+ if (arg == "--in-file") {
+ CHECK_ARG
+ std::ifstream file(argv[i]);
+ if (!file) {
+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+ invalid_param = true;
+ return true;
+ }
+ params.in_files.push_back(argv[i]);
+ return true;
+ }
+ if (arg == "-n" || arg == "--predict" || arg == "--n-predict") {
+ CHECK_ARG
+ params.n_predict = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--top-k") {
+ CHECK_ARG
+ sparams.top_k = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "-c" || arg == "--ctx-size") {
+ CHECK_ARG
+ params.n_ctx = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--grp-attn-n" || arg == "-gan") {
+ CHECK_ARG
+ params.grp_attn_n = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--grp-attn-w" || arg == "-gaw") {
+ CHECK_ARG
+ params.grp_attn_w = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--rope-freq-base") {
+ CHECK_ARG
+ params.rope_freq_base = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--rope-freq-scale") {
+ CHECK_ARG
+ params.rope_freq_scale = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--rope-scaling") {
+ CHECK_ARG
+ std::string value(argv[i]);
+ /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE; }
+ else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; }
+ else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; }
+ else { invalid_param = true; }
+ return true;
+ }
+ if (arg == "--rope-scale") {
+ CHECK_ARG
+ params.rope_freq_scale = 1.0f / std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--yarn-orig-ctx") {
+ CHECK_ARG
+ params.yarn_orig_ctx = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--yarn-ext-factor") {
+ CHECK_ARG
+ params.yarn_ext_factor = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--yarn-attn-factor") {
+ CHECK_ARG
+ params.yarn_attn_factor = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--yarn-beta-fast") {
+ CHECK_ARG
+ params.yarn_beta_fast = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--yarn-beta-slow") {
+ CHECK_ARG
+ params.yarn_beta_slow = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--pooling") {
+ CHECK_ARG
+ std::string value(argv[i]);
+ /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; }
+ else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; }
+ else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; }
+ else if (value == "last") { params.pooling_type = LLAMA_POOLING_TYPE_LAST; }
+ else { invalid_param = true; }
+ return true;
+ }
+ if (arg == "--attention") {
+ CHECK_ARG
+ std::string value(argv[i]);
+ /**/ if (value == "causal") { params.attention_type = LLAMA_ATTENTION_TYPE_CAUSAL; }
+ else if (value == "non-causal") { params.attention_type = LLAMA_ATTENTION_TYPE_NON_CAUSAL; }
+ else { invalid_param = true; }
+ return true;
+ }
+ if (arg == "--defrag-thold" || arg == "-dt") {
+ CHECK_ARG
+ params.defrag_thold = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--samplers") {
+ CHECK_ARG
+ const auto sampler_names = string_split(argv[i], ';');
+ sparams.samplers_sequence = llama_sampling_types_from_names(sampler_names, true);
+ return true;
+ }
+ if (arg == "--sampling-seq") {
+ CHECK_ARG
+ sparams.samplers_sequence = llama_sampling_types_from_chars(argv[i]);
+ return true;
+ }
+ if (arg == "--top-p") {
+ CHECK_ARG
+ sparams.top_p = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--min-p") {
+ CHECK_ARG
+ sparams.min_p = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--temp") {
+ CHECK_ARG
+ sparams.temp = std::stof(argv[i]);
+ sparams.temp = std::max(sparams.temp, 0.0f);
+ return true;
+ }
+ if (arg == "--tfs") {
+ CHECK_ARG
+ sparams.tfs_z = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--typical") {
+ CHECK_ARG
+ sparams.typical_p = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--repeat-last-n") {
+ CHECK_ARG
+ sparams.penalty_last_n = std::stoi(argv[i]);
+ sparams.n_prev = std::max(sparams.n_prev, sparams.penalty_last_n);
+ return true;
+ }
+ if (arg == "--repeat-penalty") {
+ CHECK_ARG
+ sparams.penalty_repeat = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--frequency-penalty") {
+ CHECK_ARG
+ sparams.penalty_freq = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--presence-penalty") {
+ CHECK_ARG
+ sparams.penalty_present = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--dynatemp-range") {
+ CHECK_ARG
+ sparams.dynatemp_range = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--dynatemp-exp") {
+ CHECK_ARG
+ sparams.dynatemp_exponent = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--mirostat") {
+ CHECK_ARG
+ sparams.mirostat = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--mirostat-lr") {
+ CHECK_ARG
+ sparams.mirostat_eta = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--mirostat-ent") {
+ CHECK_ARG
+ sparams.mirostat_tau = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "--cfg-negative-prompt") {
+ CHECK_ARG
+ sparams.cfg_negative_prompt = argv[i];
+ return true;
+ }
+ if (arg == "--cfg-negative-prompt-file") {
+ CHECK_ARG
+ std::ifstream file(argv[i]);
+ if (!file) {
+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+ invalid_param = true;
+ return true;
+ }
+ std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(sparams.cfg_negative_prompt));
+ if (!sparams.cfg_negative_prompt.empty() && sparams.cfg_negative_prompt.back() == '\n') {
+ sparams.cfg_negative_prompt.pop_back();
+ }
+ return true;
+ }
+ if (arg == "--cfg-scale") {
+ CHECK_ARG
+ sparams.cfg_scale = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "-b" || arg == "--batch-size") {
+ CHECK_ARG
+ params.n_batch = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "-ub" || arg == "--ubatch-size") {
+ CHECK_ARG
+ params.n_ubatch = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--keep") {
+ CHECK_ARG
+ params.n_keep = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--draft") {
+ CHECK_ARG
+ params.n_draft = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--chunks") {
+ CHECK_ARG
+ params.n_chunks = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "-np" || arg == "--parallel") {
+ CHECK_ARG
+ params.n_parallel = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "-ns" || arg == "--sequences") {
+ CHECK_ARG
+ params.n_sequences = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--p-split" || arg == "-ps") {
+ CHECK_ARG
+ params.p_split = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "-m" || arg == "--model") {
+ CHECK_ARG
+ params.model = argv[i];
+ return true;
+ }
+ if (arg == "-md" || arg == "--model-draft") {
+ CHECK_ARG
+ params.model_draft = argv[i];
+ return true;
+ }
+ if (arg == "-a" || arg == "--alias") {
+ CHECK_ARG
+ params.model_alias = argv[i];
+ return true;
+ }
+ if (arg == "-mu" || arg == "--model-url") {
+ CHECK_ARG
+ params.model_url = argv[i];
+ return true;
+ }
+ if (arg == "-hft" || arg == "--hf-token") {
+ if (++i >= argc) {
+ invalid_param = true;
+ return true;
+ }
+ params.hf_token = argv[i];
+ return true;
+ }
+ if (arg == "-hfr" || arg == "--hf-repo") {
+ CHECK_ARG
+ params.hf_repo = argv[i];
+ return true;
+ }
+ if (arg == "-hff" || arg == "--hf-file") {
+ CHECK_ARG
+ params.hf_file = argv[i];
+ return true;
+ }
+ if (arg == "--lora") {
+ CHECK_ARG
+ params.lora_adapters.push_back({
+ std::string(argv[i]),
+ 1.0,
+ });
+ return true;
+ }
+ if (arg == "--lora-scaled") {
+ CHECK_ARG
+ std::string lora_adapter = argv[i];
+ CHECK_ARG
+ params.lora_adapters.push_back({
+ lora_adapter,
+ std::stof(argv[i]),
+ });
+ return true;
+ }
+ if (arg == "--lora-init-without-apply") {
+ params.lora_init_without_apply = true;
+ return true;
+ }
+ if (arg == "--control-vector") {
+ CHECK_ARG
+ params.control_vectors.push_back({ 1.0f, argv[i], });
+ return true;
+ }
+ if (arg == "--control-vector-scaled") {
+ CHECK_ARG
+ const char* fname = argv[i];
+ CHECK_ARG
+ params.control_vectors.push_back({ std::stof(argv[i]), fname, });
+ return true;
+ }
+ if (arg == "--control-vector-layer-range") {
+ CHECK_ARG
+ params.control_vector_layer_start = std::stoi(argv[i]);
+ CHECK_ARG
+ params.control_vector_layer_end = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--mmproj") {
+ CHECK_ARG
+ params.mmproj = argv[i];
+ return true;
+ }
+ if (arg == "--image") {
+ CHECK_ARG
+ params.image.emplace_back(argv[i]);
+ return true;
+ }
+ if (arg == "-i" || arg == "--interactive") {
+ params.interactive = true;
+ return true;
+ }
+ if (arg == "-sp" || arg == "--special") {
+ params.special = true;
+ return true;
+ }
+ if (arg == "--embedding" || arg == "--embeddings") {
+ params.embedding = true;
+ return true;
+ }
+ if (arg == "--embd-normalize") {
+ CHECK_ARG
+ params.embd_normalize = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--embd-output-format") {
+ CHECK_ARG
+ params.embd_out = argv[i];
+ return true;
+ }
+ if (arg == "--embd-separator") {
+ CHECK_ARG
+ params.embd_sep = argv[i];
+ return true;
+ }
+ if (arg == "-if" || arg == "--interactive-first") {
+ params.interactive_first = true;
+ return true;
+ }
+ if (arg == "-cnv" || arg == "--conversation") {
+ params.conversation = true;
+ return true;
+ }
+ if (arg == "--infill") {
+ params.infill = true;
+ return true;
+ }
+ if (arg == "-dkvc" || arg == "--dump-kv-cache") {
+ params.dump_kv_cache = true;
+ return true;
+ }
+ if (arg == "-nkvo" || arg == "--no-kv-offload") {
+ params.no_kv_offload = true;
+ return true;
+ }
+ if (arg == "-ctk" || arg == "--cache-type-k") {
+ params.cache_type_k = argv[++i];
+ return true;
+ }
+ if (arg == "-ctv" || arg == "--cache-type-v") {
+ params.cache_type_v = argv[++i];
+ return true;
+ }
+ if (arg == "-mli" || arg == "--multiline-input") {
+ params.multiline_input = true;
+ return true;
+ }
+ if (arg == "--simple-io") {
+ params.simple_io = true;
+ return true;
+ }
+ if (arg == "-cb" || arg == "--cont-batching") {
+ params.cont_batching = true;
+ return true;
+ }
+ if (arg == "-nocb" || arg == "--no-cont-batching") {
+ params.cont_batching = false;
+ return true;
+ }
+ if (arg == "-fa" || arg == "--flash-attn") {
+ params.flash_attn = true;
+ return true;
+ }
+ if (arg == "-co" || arg == "--color") {
+ params.use_color = true;
+ return true;
+ }
+ if (arg == "--mlock") {
+ params.use_mlock = true;
+ return true;
+ }
+ if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") {
+ CHECK_ARG
+ params.n_gpu_layers = std::stoi(argv[i]);
+ if (!llama_supports_gpu_offload()) {
+ fprintf(stderr, "warning: not compiled with GPU offload support, --gpu-layers option will be ignored\n");
+ fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n");
+ }
+ return true;
+ }
+ if (arg == "-ngld" || arg == "--gpu-layers-draft" || arg == "--n-gpu-layers-draft") {
+ CHECK_ARG
+ params.n_gpu_layers_draft = std::stoi(argv[i]);
+ if (!llama_supports_gpu_offload()) {
+ fprintf(stderr, "warning: not compiled with GPU offload support, --gpu-layers-draft option will be ignored\n");
+ fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n");
+ }
+ return true;
+ }
+ if (arg == "--main-gpu" || arg == "-mg") {
+ CHECK_ARG
+ params.main_gpu = std::stoi(argv[i]);
+#ifndef GGML_USE_CUDA_SYCL_VULKAN
+ fprintf(stderr, "warning: llama.cpp was compiled without CUDA/SYCL/Vulkan. Setting the main GPU has no effect.\n");
+#endif // GGML_USE_CUDA_SYCL_VULKAN
+ return true;
+ }
+ if (arg == "--split-mode" || arg == "-sm") {
+ CHECK_ARG
+ std::string arg_next = argv[i];
+ if (arg_next == "none") {
+ params.split_mode = LLAMA_SPLIT_MODE_NONE;
+ }
+ else if (arg_next == "layer") {
+ params.split_mode = LLAMA_SPLIT_MODE_LAYER;
+ }
+ else if (arg_next == "row") {
+#ifdef GGML_USE_SYCL
+ fprintf(stderr, "warning: The split mode value:[row] is not supported by llama.cpp with SYCL. It's developing.\nExit!\n");
+ exit(1);
+#endif // GGML_USE_SYCL
+ params.split_mode = LLAMA_SPLIT_MODE_ROW;
+ }
+ else {
+ invalid_param = true;
+ return true;
+ }
+#ifndef GGML_USE_CUDA_SYCL_VULKAN
+ fprintf(stderr, "warning: llama.cpp was compiled without CUDA/SYCL/Vulkan. Setting the split mode has no effect.\n");
+#endif // GGML_USE_CUDA_SYCL_VULKAN
+ return true;
+ }
+ if (arg == "--tensor-split" || arg == "-ts") {
+ CHECK_ARG
+ std::string arg_next = argv[i];
+
+ // split string by , and /
+ const std::regex regex{ R"([,/]+)" };
+ std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
+ std::vector<std::string> split_arg{ it, {} };
+ if (split_arg.size() >= llama_max_devices()) {
+ invalid_param = true;
+ return true;
+ }
+ for (size_t i = 0; i < llama_max_devices(); ++i) {
+ if (i < split_arg.size()) {
+ params.tensor_split[i] = std::stof(split_arg[i]);
+ }
+ else {
+ params.tensor_split[i] = 0.0f;
+ }
+ }
+#ifndef GGML_USE_CUDA_SYCL_VULKAN
+ fprintf(stderr, "warning: llama.cpp was compiled without CUDA/SYCL/Vulkan. Setting a tensor split has no effect.\n");
+#endif // GGML_USE_CUDA_SYCL_VULKAN
+ return true;
+ }
+ if (arg == "--rpc") {
+ CHECK_ARG
+ params.rpc_servers = argv[i];
+ return true;
+ }
+ if (arg == "--no-mmap") {
+ params.use_mmap = false;
+ return true;
+ }
+ if (arg == "--numa") {
+ CHECK_ARG
+ std::string value(argv[i]);
+ /**/ if (value == "distribute" || value == "") { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; }
+ else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }
+ else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }
+ else { invalid_param = true; }
+ return true;
+ }
+ if (arg == "-v" || arg == "--verbose") {
+ params.verbosity = 1;
+ return true;
+ }
+ if (arg == "--verbosity") {
+ CHECK_ARG
+ params.verbosity = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--verbose-prompt") {
+ params.verbose_prompt = true;
+ return true;
+ }
+ if (arg == "--no-display-prompt") {
+ params.display_prompt = false;
+ return true;
+ }
+ if (arg == "-r" || arg == "--reverse-prompt") {
+ CHECK_ARG
+ params.antiprompt.emplace_back(argv[i]);
+ return true;
+ }
+ if (arg == "-ld" || arg == "--logdir") {
+ CHECK_ARG
+ params.logdir = argv[i];
+
+ if (params.logdir.back() != DIRECTORY_SEPARATOR) {
+ params.logdir += DIRECTORY_SEPARATOR;
+ }
+ return true;
+ }
+ if (arg == "-lcs" || arg == "--lookup-cache-static") {
+ CHECK_ARG
+ params.lookup_cache_static = argv[i];
+ return true;
+ }
+ if (arg == "-lcd" || arg == "--lookup-cache-dynamic") {
+ CHECK_ARG
+ params.lookup_cache_dynamic = argv[i];
+ return true;
+ }
+ if (arg == "--save-all-logits" || arg == "--kl-divergence-base") {
+ CHECK_ARG
+ params.logits_file = argv[i];
+ return true;
+ }
+ if (arg == "--perplexity" || arg == "--all-logits") {
+ params.logits_all = true;
+ return true;
+ }
+ if (arg == "--ppl-stride") {
+ CHECK_ARG
+ params.ppl_stride = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--ppl-output-type") {
+ CHECK_ARG
+ params.ppl_output_type = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "-ptc" || arg == "--print-token-count") {
+ CHECK_ARG
+ params.n_print = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--check-tensors") {
+ params.check_tensors = true;
+ return true;
+ }
+ if (arg == "--hellaswag") {
+ params.hellaswag = true;
+ return true;
+ }
+ if (arg == "--hellaswag-tasks") {
+ CHECK_ARG
+ params.hellaswag_tasks = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--winogrande") {
+ params.winogrande = true;
+ return true;
+ }
+ if (arg == "--winogrande-tasks") {
+ CHECK_ARG
+ params.winogrande_tasks = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--multiple-choice") {
+ params.multiple_choice = true;
+ return true;
+ }
+ if (arg == "--multiple-choice-tasks") {
+ CHECK_ARG
+ params.multiple_choice_tasks = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--kl-divergence") {
+ params.kl_divergence = true;
+ return true;
+ }
+ if (arg == "--ignore-eos") {
+ params.ignore_eos = true;
+ return true;
+ }
+ if (arg == "--penalize-nl") {
+ sparams.penalize_nl = true;
+ return true;
+ }
+ if (arg == "-l" || arg == "--logit-bias") {
+ CHECK_ARG
+ std::stringstream ss(argv[i]);
+ llama_token key;
+ char sign;
+ std::string value_str;
+ try {
+ if (ss >> key && ss >> sign && std::getline(ss, value_str) && (sign == '+' || sign == '-')) {
+ sparams.logit_bias[key] = std::stof(value_str) * ((sign == '-') ? -1.0f : 1.0f);
+ }
+ else {
+ throw std::exception();
+ }
+ }
+ catch (const std::exception&) {
+ invalid_param = true;
+ return true;
+ }
+ return true;
+ }
+ if (arg == "-h" || arg == "--help" || arg == "--usage" ) {
+ params.usage = true;
+ return true;
+ }
+ if (arg == "--version") {
+ fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT);
+ fprintf(stderr, "built with %s for %s\n", LLAMA_COMPILER, LLAMA_BUILD_TARGET);
+ exit(0);
+ }
+ if (arg == "--in-prefix-bos") {
+ params.input_prefix_bos = true;
+ params.enable_chat_template = false;
+ return true;
+ }
+ if (arg == "--in-prefix") {
+ CHECK_ARG
+ params.input_prefix = argv[i];
+ params.enable_chat_template = false;
+ return true;
+ }
+ if (arg == "--in-suffix") {
+ CHECK_ARG
+ params.input_suffix = argv[i];
+ params.enable_chat_template = false;
+ return true;
+ }
+ if (arg == "--spm-infill") {
+ params.spm_infill = true;
+ return true;
+ }
+ if (arg == "--grammar") {
+ CHECK_ARG
+ sparams.grammar = argv[i];
+ return true;
+ }
+ if (arg == "--grammar-file") {
+ CHECK_ARG
+ std::ifstream file(argv[i]);
+ if (!file) {
+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+ invalid_param = true;
+ return true;
+ }
+ std::copy(
+ std::istreambuf_iterator<char>(file),
+ std::istreambuf_iterator<char>(),
+ std::back_inserter(sparams.grammar)
+ );
+ return true;
+ }
+ if (arg == "-j" || arg == "--json-schema") {
+ CHECK_ARG
+ sparams.grammar = json_schema_to_grammar(json::parse(argv[i]));
+ return true;
+ }
+ if (arg == "--override-kv") {
+ CHECK_ARG
+ if (!string_parse_kv_override(argv[i], params.kv_overrides)) {
+ fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]);
+ invalid_param = true;
+ return true;
+ }
+ return true;
+ }
+ if (arg == "--host") {
+ CHECK_ARG
+ params.hostname = argv[i];
+ return true;
+ }
+ if (arg == "--port") {
+ CHECK_ARG
+ params.port = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--path") {
+ CHECK_ARG
+ params.public_path = argv[i];
+ return true;
+ }
+ if (arg == "--api-key") {
+ CHECK_ARG
+ params.api_keys.push_back(argv[i]);
+ return true;
+ }
+ if (arg == "--api-key-file") {
+ CHECK_ARG
+ std::ifstream key_file(argv[i]);
+ if (!key_file) {
+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+ invalid_param = true;
+ return true;
+ }
+ std::string key;
+ while (std::getline(key_file, key)) {
+ if (!key.empty()) {
+ params.api_keys.push_back(key);
+ }
+ }
+ key_file.close();
+ return true;
+ }
+ if (arg == "--ssl-key-file") {
+ CHECK_ARG
+ params.ssl_file_key = argv[i];
+ return true;
+ }
+ if (arg == "--ssl-cert-file") {
+ CHECK_ARG
+ params.ssl_file_cert = argv[i];
+ return true;
+ }
+ if (arg == "--timeout" || arg == "-to") {
+ CHECK_ARG
+ params.timeout_read = std::stoi(argv[i]);
+ params.timeout_write = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--threads-http") {
+ CHECK_ARG
+ params.n_threads_http = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "-spf" || arg == "--system-prompt-file") {
+ CHECK_ARG
+ std::ifstream file(argv[i]);
+ if (!file) {
+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+ invalid_param = true;
+ return true;
+ }
+ std::string system_prompt;
+ std::copy(
+ std::istreambuf_iterator<char>(file),
+ std::istreambuf_iterator<char>(),
+ std::back_inserter(system_prompt)
+ );
+ params.system_prompt = system_prompt;
+ return true;
+ }
+ if (arg == "--log-format") {
+ CHECK_ARG
+ if (std::strcmp(argv[i], "json") == 0) {
+ params.log_json = true;
+ } else if (std::strcmp(argv[i], "text") == 0) {
+ params.log_json = false;
+ } else {
+ invalid_param = true;
+ return true;
+ }
+ return true;
+ }
+ if (arg == "--no-slots") {
+ params.endpoint_slots = false;
+ return true;
+ }
+ if (arg == "--metrics") {
+ params.endpoint_metrics = true;
+ return true;
+ }
+ if (arg == "--slot-save-path") {
+ CHECK_ARG
+ params.slot_save_path = argv[i];
+ // if doesn't end with DIRECTORY_SEPARATOR, add it
+ if (!params.slot_save_path.empty() && params.slot_save_path[params.slot_save_path.size() - 1] != DIRECTORY_SEPARATOR) {
+ params.slot_save_path += DIRECTORY_SEPARATOR;
+ }
+ return true;
+ }
+ if (arg == "--chat-template") {
+ CHECK_ARG
+ if (!llama_chat_verify_template(argv[i])) {
+ fprintf(stderr, "error: the supplied chat template is not supported: %s\n", argv[i]);
+ fprintf(stderr, "note: llama.cpp does not use jinja parser, we only support commonly used templates\n");
+ invalid_param = true;
+ return true;
+ }
+ params.chat_template = argv[i];
+ return true;
+ }
+ if (arg == "--slot-prompt-similarity" || arg == "-sps") {
+ CHECK_ARG
+ params.slot_prompt_similarity = std::stof(argv[i]);
+ return true;
+ }
+ if (arg == "-pps") {
+ params.is_pp_shared = true;
+ return true;
+ }
+ if (arg == "-npp") {
+ CHECK_ARG
+ auto p = string_split<int>(argv[i], split_delim);
+ params.n_pp.insert(params.n_pp.end(), p.begin(), p.end());
+ return true;
+ }
+ if (arg == "-ntg") {
+ CHECK_ARG
+ auto p = string_split<int>(argv[i], split_delim);
+ params.n_tg.insert(params.n_tg.end(), p.begin(), p.end());
+ return true;
+ }
+ if (arg == "-npl") {
+ CHECK_ARG
+ auto p = string_split<int>(argv[i], split_delim);
+ params.n_pl.insert(params.n_pl.end(), p.begin(), p.end());
+ return true;
+ }
+ if (arg == "--context-file") {
+ CHECK_ARG
+ std::ifstream file(argv[i], std::ios::binary);
+ if (!file) {
+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+ invalid_param = true;
+ return true;
+ }
+ params.context_files.push_back(argv[i]);
+ return true;
+ }
+ if (arg == "--chunk-size") {
+ CHECK_ARG
+ params.chunk_size = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--chunk-separator") {
+ CHECK_ARG
+ params.chunk_separator = argv[i];
+ return true;
+ }
+ if (arg == "--junk") {
+ CHECK_ARG
+ params.n_junk = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--pos") {
+ CHECK_ARG
+ params.i_pos = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "-o" || arg == "--output" || arg == "--output-file") {
+ CHECK_ARG
+ params.out_file = argv[i];
+ params.cvector_outfile = argv[i];
+ params.lora_outfile = argv[i];
+ return true;
+ }
+ if (arg == "-ofreq" || arg == "--output-frequency") {
+ CHECK_ARG
+ params.n_out_freq = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--save-frequency") {
+ CHECK_ARG
+ params.n_save_freq = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--process-output") {
+ params.process_output = true;
+ return true;
+ }
+ if (arg == "--no-ppl") {
+ params.compute_ppl = false;
+ return true;
+ }
+ if (arg == "--chunk" || arg == "--from-chunk") {
+ CHECK_ARG
+ params.i_chunk = std::stoi(argv[i]);
+ return true;
+ }
+ // cvector params
+ if (arg == "--positive-file") {
+ CHECK_ARG
+ params.cvector_positive_file = argv[i];
+ return true;
+ }
+ if (arg == "--negative-file") {
+ CHECK_ARG
+ params.cvector_negative_file = argv[i];
+ return true;
+ }
+ if (arg == "--pca-batch") {
+ CHECK_ARG
+ params.n_pca_batch = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--pca-iter") {
+ CHECK_ARG
+ params.n_pca_iterations = std::stoi(argv[i]);
+ return true;
+ }
+ if (arg == "--method") {
+ CHECK_ARG
+ std::string value(argv[i]);
+ /**/ if (value == "pca") { params.cvector_dimre_method = DIMRE_METHOD_PCA; }
+ else if (value == "mean") { params.cvector_dimre_method = DIMRE_METHOD_MEAN; }
+ else { invalid_param = true; }
+ return true;
+ }
+ if (arg == "--no-warmup") {
+ params.warmup = false;
+ return true;
+ }
+#ifndef LOG_DISABLE_LOGS
+ // Parse args for logging parameters
+ if (log_param_single_parse(argv[i])) {
+ // Do nothing, log_param_single_parse automatically does it's thing
+ // and returns if a match was found and parsed.
+ return true;
+ }
+ if (log_param_pair_parse( /*check_but_dont_parse*/ true, argv[i])) {
+ // We have a matching known parameter requiring an argument,
+ // now we need to check if there is anything after this argv
+ // and flag invalid_param or parse it.
+ CHECK_ARG
+ if (!log_param_pair_parse( /*check_but_dont_parse*/ false, argv[i - 1], argv[i])) {
+ invalid_param = true;
+ return true;
+ }
+ return true;
+ }
+ // End of Parse args for logging parameters
+#endif // LOG_DISABLE_LOGS
+
+ return false;
+}
+
+#ifdef __GNUC__
+#ifdef __MINGW32__
+#define LLAMA_COMMON_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
+#else
+#define LLAMA_COMMON_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
+#endif
+#else
+#define LLAMA_COMMON_ATTRIBUTE_FORMAT(...)
+#endif
+
+void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
+ const llama_sampling_params & sparams = params.sparams;
+
+ std::string sampler_type_chars;
+ std::string sampler_type_names;
+ for (const auto sampler_type : sparams.samplers_sequence) {
+ sampler_type_chars += static_cast<char>(sampler_type);
+ sampler_type_names += llama_sampling_type_to_str(sampler_type) + ";";
+ }
+ sampler_type_names.pop_back();
+
+ struct option_info {
+ LLAMA_COMMON_ATTRIBUTE_FORMAT(4, 5)
+ option_info(const std::string & tags, const char * args, const char * desc, ...) : tags(tags), args(args), desc(desc) {
+ va_list args_list;
+ va_start(args_list, desc);
+ char buffer[1024];
+ vsnprintf(buffer, sizeof(buffer), desc, args_list);
+ va_end(args_list);
+ this->desc = buffer;
+ }
+
+ option_info(const std::string & grp) : grp(grp) {}
+
+ std::string tags;
+ std::string args;
+ std::string desc;
+ std::string grp;
+ };
+
+ std::vector<option_info> options;
+
+ // TODO: filter by tags
+
+ options.push_back({ "general" });
+ options.push_back({ "*", "-h, --help, --usage", "print usage and exit" });
+ options.push_back({ "*", " --version", "show version and build info" });
+ options.push_back({ "*", "-v, --verbose", "print verbose information" });
+ options.push_back({ "*", " --verbosity N", "set specific verbosity level (default: %d)", params.verbosity });
+ options.push_back({ "*", " --verbose-prompt", "print a verbose prompt before generation (default: %s)", params.verbose_prompt ? "true" : "false" });
+ options.push_back({ "*", " --no-display-prompt", "don't print prompt at generation (default: %s)", !params.display_prompt ? "true" : "false" });
+ options.push_back({ "*", "-co, --color", "colorise output to distinguish prompt and user input from generations (default: %s)", params.use_color ? "true" : "false" });
+ options.push_back({ "*", "-s, --seed SEED", "RNG seed (default: %d, use random seed for < 0)", params.seed });
+ options.push_back({ "*", "-t, --threads N", "number of threads to use during generation (default: %d)", params.cpuparams.n_threads });
+ options.push_back({ "*", "-tb, --threads-batch N", "number of threads to use during batch and prompt processing (default: same as --threads)" });
+ options.push_back({ "speculative", "-td, --threads-draft N", "number of threads to use during generation (default: same as --threads)" });
+ options.push_back({ "speculative", "-tbd, --threads-batch-draft N","number of threads to use during batch and prompt processing (default: same as --threads-draft)" });
+
+#ifndef GGML_USE_OPENMP
+ // these options are available only with the internal threadpool
+ options.push_back({ "*", "-C, --cpu-mask M", "CPU affinity mask: arbitrarily long hex. Complements cpu-range (default: \"\")"});
+ options.push_back({ "*", "-Cr, --cpu-range lo-hi", "range of CPUs for affinity. Complements --cpu-mask"});
+ options.push_back({ "*", " --cpu-strict <0|1>", "use strict CPU placement (default: %u)\n", (unsigned) params.cpuparams.strict_cpu});
+ options.push_back({ "*", " --priority N", "set process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.cpuparams.priority});
+ options.push_back({ "*", " --poll <0...100>", "use polling level to wait for work (0 - no polling, default: %u)\n", (unsigned) params.cpuparams.poll});
+
+ options.push_back({ "*", "-Cb, --cpu-mask-batch M", "CPU affinity mask: arbitrarily long hex. Complements cpu-range-batch (default: same as --cpu-mask)"});
+ options.push_back({ "*", "-Crb, --cpu-range-batch lo-hi", "ranges of CPUs for affinity. Complements --cpu-mask-batch"});
+ options.push_back({ "*", " --cpu-strict-batch <0|1>","use strict CPU placement (default: same as --cpu-strict)"});
+ options.push_back({ "*", " --priority-batch N", "set process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: --priority)"});
+ options.push_back({ "*", " --poll-batch <0|1>", "use polling to wait for work (default: same as --poll"});
+
+ options.push_back({ "speculative", "-Cd, --cpu-mask-draft M", "Draft model CPU affinity mask. Complements cpu-range-draft (default: same as --cpu-mask)"});
+ options.push_back({ "speculative", "-Crd, --cpu-range-draft lo-hi", "Ranges of CPUs for affinity. Complements --cpu-mask-draft"});
+ options.push_back({ "speculative", " --cpu-strict-draft <0|1>","Use strict CPU placement for draft model (default: same as --cpu-strict)"});
+ options.push_back({ "speculative", " --priority-draft N", "Set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: same as --priority)"});
+ options.push_back({ "speculative", " --poll-draft <0|1>", "Use polling to wait for draft model work (default: same as --poll])"});
+
+ options.push_back({ "speculative", "-Cbd, --cpu-mask-batch-draft M","Draft model CPU affinity mask. Complements cpu-range-draft-batch (default: same as --cpu-mask-draft)"});
+ options.push_back({ "speculative", "-Crbd, --cpu-range-batch-draft lo-hi",
+ "Ranges of CPUs for affinity. Complements --cpu-mask-draft-batch)"});
+ options.push_back({ "speculative", " --cpu-strict-batch-draft <0|1>",
+ "Use strict CPU placement for draft model (default: --cpu-strict-draft)"});
+ options.push_back({ "speculative", " --priority-batch-draft N","Set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: --priority-draft)"});
+ options.push_back({ "speculative", " --poll-batch-draft <0|1>","Use polling to wait for draft model work (default: --poll-draft)"});
+#endif // GGML_USE_OPENMP
+
+ options.push_back({ "speculative", " --draft N", "number of tokens to draft for speculative decoding (default: %d)", params.n_draft });
+ options.push_back({ "speculative", "-ps, --p-split N", "speculative decoding split probability (default: %.1f)", (double)params.p_split });
+ options.push_back({ "*", "-lcs, --lookup-cache-static FNAME",
+ "path to static lookup cache to use for lookup decoding (not updated by generation)" });
+ options.push_back({ "*", "-lcd, --lookup-cache-dynamic FNAME",
+ "path to dynamic lookup cache to use for lookup decoding (updated by generation)" });
+
+ options.push_back({ "*", "-c, --ctx-size N", "size of the prompt context (default: %d, 0 = loaded from model)", params.n_ctx });
+ options.push_back({ "*", "-n, --predict N", "number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)", params.n_predict });
+ options.push_back({ "*", "-b, --batch-size N", "logical maximum batch size (default: %d)", params.n_batch });
+ options.push_back({ "*", "-ub, --ubatch-size N", "physical maximum batch size (default: %d)", params.n_ubatch });
+ options.push_back({ "*", " --keep N", "number of tokens to keep from the initial prompt (default: %d, -1 = all)", params.n_keep });
+ options.push_back({ "*", " --chunks N", "max number of chunks to process (default: %d, -1 = all)", params.n_chunks });
+ options.push_back({ "*", "-fa, --flash-attn", "enable Flash Attention (default: %s)", params.flash_attn ? "enabled" : "disabled" });
+ options.push_back({ "*", "-p, --prompt PROMPT", "prompt to start generation with\n"
+ "in conversation mode, this will be used as system prompt\n"
+ "(default: '%s')", params.prompt.c_str() });
+ options.push_back({ "*", "-f, --file FNAME", "a file containing the prompt (default: none)" });
+ options.push_back({ "*", " --in-file FNAME", "an input file (repeat to specify multiple files)" });
+ options.push_back({ "*", "-bf, --binary-file FNAME", "binary file containing the prompt (default: none)" });
+ options.push_back({ "*", "-e, --escape", "process escapes sequences (\\n, \\r, \\t, \\', \\\", \\\\) (default: %s)", params.escape ? "true" : "false" });
+ options.push_back({ "*", " --no-escape", "do not process escape sequences" });
+ options.push_back({ "main", "-ptc, --print-token-count N", "print token count every N tokens (default: %d)", params.n_print });
+ options.push_back({ "main", " --prompt-cache FNAME", "file to cache prompt state for faster startup (default: none)" });
+ options.push_back({ "main", " --prompt-cache-all", "if specified, saves user input and generations to cache as well\n"
+ "not supported with --interactive or other interactive options" });
+ options.push_back({ "main", " --prompt-cache-ro", "if specified, uses the prompt cache but does not update it" });
+ options.push_back({ "main", "-r, --reverse-prompt PROMPT",
+ "halt generation at PROMPT, return control in interactive mode\n"
+ "can be specified more than once for multiple prompts" });
+ options.push_back({ "main", "-sp, --special", "special tokens output enabled (default: %s)", params.special ? "true" : "false" });
+ options.push_back({ "main", "-cnv, --conversation", "run in conversation mode, does not print special tokens and suffix/prefix\n"
+ "if suffix/prefix are not specified, default chat template will be used\n"
+ "(default: %s)", params.conversation ? "true" : "false" });
+ options.push_back({ "main infill", "-i, --interactive", "run in interactive mode (default: %s)", params.interactive ? "true" : "false" });
+ options.push_back({ "main infill", "-if, --interactive-first", "run in interactive mode and wait for input right away (default: %s)", params.interactive_first ? "true" : "false" });
+ options.push_back({ "main infill", "-mli, --multiline-input", "allows you to write or paste multiple lines without ending each in '\\'" });
+ options.push_back({ "main infill", " --in-prefix-bos", "prefix BOS to user inputs, preceding the `--in-prefix` string" });
+ options.push_back({ "main infill", " --in-prefix STRING", "string to prefix user inputs with (default: empty)" });
+ options.push_back({ "main infill", " --in-suffix STRING", "string to suffix after user inputs with (default: empty)" });
+ options.push_back({ "main", " --no-warmup", "skip warming up the model with an empty run" });
+ options.push_back({ "server infill",
+ " --spm-infill", "use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this. (default: %s)", params.spm_infill ? "enabled" : "disabled" });
+
+ options.push_back({ "sampling" });
+ options.push_back({ "*", " --samplers SAMPLERS", "samplers that will be used for generation in the order, separated by \';\'\n"
+ "(default: %s)", sampler_type_names.c_str() });
+ options.push_back({ "*", " --sampling-seq SEQUENCE",
+ "simplified sequence for samplers that will be used (default: %s)", sampler_type_chars.c_str() });
+ options.push_back({ "*", " --ignore-eos", "ignore end of stream token and continue generating (implies --logit-bias EOS-inf)" });
+ options.push_back({ "*", " --penalize-nl", "penalize newline tokens (default: %s)", sparams.penalize_nl ? "true" : "false" });
+ options.push_back({ "*", " --temp N", "temperature (default: %.1f)", (double)sparams.temp });
+ options.push_back({ "*", " --top-k N", "top-k sampling (default: %d, 0 = disabled)", sparams.top_k });
+ options.push_back({ "*", " --top-p N", "top-p sampling (default: %.1f, 1.0 = disabled)", (double)sparams.top_p });
+ options.push_back({ "*", " --min-p N", "min-p sampling (default: %.1f, 0.0 = disabled)", (double)sparams.min_p });
+ options.push_back({ "*", " --tfs N", "tail free sampling, parameter z (default: %.1f, 1.0 = disabled)", (double)sparams.tfs_z });
+ options.push_back({ "*", " --typical N", "locally typical sampling, parameter p (default: %.1f, 1.0 = disabled)", (double)sparams.typical_p });
+ options.push_back({ "*", " --repeat-last-n N", "last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size)", sparams.penalty_last_n });
+ options.push_back({ "*", " --repeat-penalty N", "penalize repeat sequence of tokens (default: %.1f, 1.0 = disabled)", (double)sparams.penalty_repeat });
+ options.push_back({ "*", " --presence-penalty N", "repeat alpha presence penalty (default: %.1f, 0.0 = disabled)", (double)sparams.penalty_present });
+ options.push_back({ "*", " --frequency-penalty N", "repeat alpha frequency penalty (default: %.1f, 0.0 = disabled)", (double)sparams.penalty_freq });
+ options.push_back({ "*", " --dynatemp-range N", "dynamic temperature range (default: %.1f, 0.0 = disabled)", (double)sparams.dynatemp_range });
+ options.push_back({ "*", " --dynatemp-exp N", "dynamic temperature exponent (default: %.1f)", (double)sparams.dynatemp_exponent });
+ options.push_back({ "*", " --mirostat N", "use Mirostat sampling.\n"
+ "Top K, Nucleus, Tail Free and Locally Typical samplers are ignored if used.\n"
+ "(default: %d, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)", sparams.mirostat });
+ options.push_back({ "*", " --mirostat-lr N", "Mirostat learning rate, parameter eta (default: %.1f)", (double)sparams.mirostat_eta });
+ options.push_back({ "*", " --mirostat-ent N", "Mirostat target entropy, parameter tau (default: %.1f)", (double)sparams.mirostat_tau });
+ options.push_back({ "*", " -l TOKEN_ID(+/-)BIAS", "modifies the likelihood of token appearing in the completion,\n"
+ "i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',\n"
+ "or `--logit-bias 15043-1` to decrease likelihood of token ' Hello'" });
+ options.push_back({ "main", " --cfg-negative-prompt PROMPT",
+ "negative prompt to use for guidance (default: '%s')", sparams.cfg_negative_prompt.c_str() });
+ options.push_back({ "main", " --cfg-negative-prompt-file FNAME",
+ "negative prompt file to use for guidance" });
+ options.push_back({ "main", " --cfg-scale N", "strength of guidance (default: %.1f, 1.0 = disable)", (double)sparams.cfg_scale });
+ options.push_back({ "main", " --chat-template JINJA_TEMPLATE",
+ "set custom jinja chat template (default: template taken from model's metadata)\n"
+ "if suffix/prefix are specified, template will be disabled\n"
+ "only commonly used templates are accepted:\n"
+ "https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template" });
+ options.push_back({ "grammar" });
+ options.push_back({ "*", " --grammar GRAMMAR", "BNF-like grammar to constrain generations (see samples in grammars/ dir) (default: '%s')", sparams.grammar.c_str() });
+ options.push_back({ "*", " --grammar-file FNAME", "file to read grammar from" });
+ options.push_back({ "*", "-j, --json-schema SCHEMA",
+ "JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\n"
+ "For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead" });
+
+ options.push_back({ "embedding" });
+ options.push_back({ "embedding", " --pooling {none,mean,cls,last}",
+ "pooling type for embeddings, use model default if unspecified" });
+ options.push_back({ "embedding", " --attention {causal,non-causal}",
+ "attention type for embeddings, use model default if unspecified" });
+
+ options.push_back({ "context hacking" });
+ options.push_back({ "*", " --rope-scaling {none,linear,yarn}",
+ "RoPE frequency scaling method, defaults to linear unless specified by the model" });
+ options.push_back({ "*", " --rope-scale N", "RoPE context scaling factor, expands context by a factor of N" });
+ options.push_back({ "*", " --rope-freq-base N", "RoPE base frequency, used by NTK-aware scaling (default: loaded from model)" });
+ options.push_back({ "*", " --rope-freq-scale N", "RoPE frequency scaling factor, expands context by a factor of 1/N" });
+ options.push_back({ "*", " --yarn-orig-ctx N", "YaRN: original context size of model (default: %d = model training context size)", params.yarn_orig_ctx });
+ options.push_back({ "*", " --yarn-ext-factor N", "YaRN: extrapolation mix factor (default: %.1f, 0.0 = full interpolation)", (double)params.yarn_ext_factor });
+ options.push_back({ "*", " --yarn-attn-factor N", "YaRN: scale sqrt(t) or attention magnitude (default: %.1f)", (double)params.yarn_attn_factor });
+ options.push_back({ "*", " --yarn-beta-slow N", "YaRN: high correction dim or alpha (default: %.1f)", (double)params.yarn_beta_slow });
+ options.push_back({ "*", " --yarn-beta-fast N", "YaRN: low correction dim or beta (default: %.1f)", (double)params.yarn_beta_fast });
+ options.push_back({ "*", "-gan, --grp-attn-n N", "group-attention factor (default: %d)", params.grp_attn_n });
+ options.push_back({ "*", "-gaw, --grp-attn-w N", "group-attention width (default: %.1f)", (double)params.grp_attn_w });
+ options.push_back({ "*", "-dkvc, --dump-kv-cache", "verbose print of the KV cache" });
+ options.push_back({ "*", "-nkvo, --no-kv-offload", "disable KV offload" });
+ options.push_back({ "*", "-ctk, --cache-type-k TYPE", "KV cache data type for K (default: %s)", params.cache_type_k.c_str() });
+ options.push_back({ "*", "-ctv, --cache-type-v TYPE", "KV cache data type for V (default: %s)", params.cache_type_v.c_str() });
+
+ options.push_back({ "perplexity" });
+ options.push_back({ "perplexity", " --all-logits", "return logits for all tokens in the batch (default: %s)", params.logits_all ? "true" : "false" });
+ options.push_back({ "perplexity", " --hellaswag", "compute HellaSwag score over random tasks from datafile supplied with -f" });
+ options.push_back({ "perplexity", " --hellaswag-tasks N", "number of tasks to use when computing the HellaSwag score (default: %zu)", params.hellaswag_tasks });
+ options.push_back({ "perplexity", " --winogrande", "compute Winogrande score over random tasks from datafile supplied with -f" });
+ options.push_back({ "perplexity", " --winogrande-tasks N", "number of tasks to use when computing the Winogrande score (default: %zu)", params.winogrande_tasks });
+ options.push_back({ "perplexity", " --multiple-choice", "compute multiple choice score over random tasks from datafile supplied with -f" });
+ options.push_back({ "perplexity", " --multiple-choice-tasks N",
+ "number of tasks to use when computing the multiple choice score (default: %zu)", params.multiple_choice_tasks });
+ options.push_back({ "perplexity", " --kl-divergence", "computes KL-divergence to logits provided via --kl-divergence-base" });
+ options.push_back({ "perplexity", " --ppl-stride N", "stride for perplexity calculation (default: %d)", params.ppl_stride });
+ options.push_back({ "perplexity", " --ppl-output-type {0,1}",
+ "output type for perplexity calculation (default: %d)", params.ppl_output_type });
+
+ options.push_back({ "parallel" });
+ options.push_back({ "*", "-dt, --defrag-thold N", "KV cache defragmentation threshold (default: %.1f, < 0 - disabled)", (double)params.defrag_thold });
+ options.push_back({ "*", "-np, --parallel N", "number of parallel sequences to decode (default: %d)", params.n_parallel });
+ options.push_back({ "*", "-ns, --sequences N", "number of sequences to decode (default: %d)", params.n_sequences });
+ options.push_back({ "*", "-cb, --cont-batching", "enable continuous batching (a.k.a dynamic batching) (default: %s)", params.cont_batching ? "enabled" : "disabled" });
+ options.push_back({ "*", "-nocb, --no-cont-batching", "disable continuous batching" });
+
+ options.push_back({ "multi-modality" });
+ options.push_back({ "*", " --mmproj FILE", "path to a multimodal projector file for LLaVA. see examples/llava/README.md" });
+ options.push_back({ "*", " --image FILE", "path to an image file. use with multimodal models. Specify multiple times for batching" });
+
+ options.push_back({ "backend" });
+ options.push_back({ "*", " --rpc SERVERS", "comma separated list of RPC servers" });
+
+ if (llama_supports_mlock()) {
+ options.push_back({ "*", " --mlock", "force system to keep model in RAM rather than swapping or compressing" });
+ }
+ if (llama_supports_mmap()) {
+ options.push_back({ "*", " --no-mmap", "do not memory-map model (slower load but may reduce pageouts if not using mlock)" });
+ }
+ options.push_back({ "*", " --numa TYPE", "attempt optimizations that help on some NUMA systems\n"
+ " - distribute: spread execution evenly over all nodes\n"
+ " - isolate: only spawn threads on CPUs on the node that execution started on\n"
+ " - numactl: use the CPU map provided by numactl\n"
+ "if run without this previously, it is recommended to drop the system page cache before using this\n"
+ "see https://github.com/ggerganov/llama.cpp/issues/1437" });
+
+ if (llama_supports_gpu_offload()) {
+ options.push_back({ "*", "-ngl, --gpu-layers N",
+ "number of layers to store in VRAM" });
+ options.push_back({ "*", "-ngld, --gpu-layers-draft N",
+ "number of layers to store in VRAM for the draft model" });
+ options.push_back({ "*", "-sm, --split-mode SPLIT_MODE",
+ "how to split the model across multiple GPUs, one of:\n"
+ " - none: use one GPU only\n"
+ " - layer (default): split layers and KV across GPUs\n"
+ " - row: split rows across GPUs" });
+ options.push_back({ "*", "-ts, --tensor-split SPLIT",
+ "fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1" });
+ options.push_back({ "*", "-mg, --main-gpu i", "the GPU to use for the model (with split-mode = none),\n"
+ "or for intermediate results and KV (with split-mode = row) (default: %d)", params.main_gpu });
+ }
+
+ options.push_back({ "model" });
+ options.push_back({ "*", " --check-tensors", "check model tensor data for invalid values (default: %s)", params.check_tensors ? "true" : "false" });
+ options.push_back({ "*", " --override-kv KEY=TYPE:VALUE",
+ "advanced option to override model metadata by key. may be specified multiple times.\n"
+ "types: int, float, bool, str. example: --override-kv tokenizer.ggml.add_bos_token=bool:false" });
+ options.push_back({ "*", " --lora FNAME", "apply LoRA adapter (can be repeated to use multiple adapters)" });
+ options.push_back({ "*", " --lora-scaled FNAME S", "apply LoRA adapter with user defined scaling S (can be repeated to use multiple adapters)" });
+ options.push_back({ "*", " --control-vector FNAME", "add a control vector\n"
+ "note: this argument can be repeated to add multiple control vectors" });
+ options.push_back({ "*", " --control-vector-scaled FNAME SCALE",
+ "add a control vector with user defined scaling SCALE\n"
+ "note: this argument can be repeated to add multiple scaled control vectors" });
+ options.push_back({ "*", " --control-vector-layer-range START END",
+ "layer range to apply the control vector(s) to, start and end inclusive" });
+ options.push_back({ "*", "-m, --model FNAME", "model path (default: models/$filename with filename from --hf-file\n"
+ "or --model-url if set, otherwise %s)", DEFAULT_MODEL_PATH });
+ options.push_back({ "*", "-md, --model-draft FNAME", "draft model for speculative decoding (default: unused)" });
+ options.push_back({ "*", "-mu, --model-url MODEL_URL", "model download url (default: unused)" });
+ options.push_back({ "*", "-hfr, --hf-repo REPO", "Hugging Face model repository (default: unused)" });
+ options.push_back({ "*", "-hff, --hf-file FILE", "Hugging Face model file (default: unused)" });
+ options.push_back({ "*", "-hft, --hf-token TOKEN", "Hugging Face access token (default: value from HF_TOKEN environment variable)" });
+
+ options.push_back({ "retrieval" });
+ options.push_back({ "retrieval", " --context-file FNAME", "file to load context from (repeat to specify multiple files)" });
+ options.push_back({ "retrieval", " --chunk-size N", "minimum length of embedded text chunks (default: %d)", params.chunk_size });
+ options.push_back({ "retrieval", " --chunk-separator STRING",
+ "separator between chunks (default: '%s')", params.chunk_separator.c_str() });
+
+ options.push_back({ "passkey" });
+ options.push_back({ "passkey", " --junk N", "number of times to repeat the junk text (default: %d)", params.n_junk });
+ options.push_back({ "passkey", " --pos N", "position of the passkey in the junk text (default: %d)", params.i_pos });
+
+ options.push_back({ "imatrix" });
+ options.push_back({ "imatrix", "-o, --output FNAME", "output file (default: '%s')", params.out_file.c_str() });
+ options.push_back({ "imatrix", " --output-frequency N", "output the imatrix every N iterations (default: %d)", params.n_out_freq });
+ options.push_back({ "imatrix", " --save-frequency N", "save an imatrix copy every N iterations (default: %d)", params.n_save_freq });
+ options.push_back({ "imatrix", " --process-output", "collect data for the output tensor (default: %s)", params.process_output ? "true" : "false" });
+ options.push_back({ "imatrix", " --no-ppl", "do not compute perplexity (default: %s)", params.compute_ppl ? "true" : "false" });
+ options.push_back({ "imatrix", " --chunk N", "start processing the input from chunk N (default: %d)", params.i_chunk });
+
+ options.push_back({ "bench" });
+ options.push_back({ "bench", "-pps", "is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false" });
+ options.push_back({ "bench", "-npp n0,n1,...", "number of prompt tokens" });
+ options.push_back({ "bench", "-ntg n0,n1,...", "number of text generation tokens" });
+ options.push_back({ "bench", "-npl n0,n1,...", "number of parallel prompts" });
+
+ options.push_back({ "embedding" });
+ options.push_back({ "embedding", " --embd-normalize", "normalisation for embendings (default: %d) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)", params.embd_normalize });
+ options.push_back({ "embedding", " --embd-output-format", "empty = default, \"array\" = [[],[]...], \"json\" = openai style, \"json+\" = same \"json\" + cosine similarity matrix" });
+ options.push_back({ "embedding", " --embd-separator", "separator of embendings (default \\n) for example \"<#sep#>\"" });
+
+ options.push_back({ "server" });
+ options.push_back({ "server", " --host HOST", "ip address to listen (default: %s)", params.hostname.c_str() });
+ options.push_back({ "server", " --port PORT", "port to listen (default: %d)", params.port });
+ options.push_back({ "server", " --path PATH", "path to serve static files from (default: %s)", params.public_path.c_str() });
+ options.push_back({ "server", " --embedding(s)", "restrict to only support embedding use case; use only with dedicated embedding models (default: %s)", params.embedding ? "enabled" : "disabled" });
+ options.push_back({ "server", " --api-key KEY", "API key to use for authentication (default: none)" });
+ options.push_back({ "server", " --api-key-file FNAME", "path to file containing API keys (default: none)" });
+ options.push_back({ "server", " --ssl-key-file FNAME", "path to file a PEM-encoded SSL private key" });
+ options.push_back({ "server", " --ssl-cert-file FNAME", "path to file a PEM-encoded SSL certificate" });
+ options.push_back({ "server", " --timeout N", "server read/write timeout in seconds (default: %d)", params.timeout_read });
+ options.push_back({ "server", " --threads-http N", "number of threads used to process HTTP requests (default: %d)", params.n_threads_http });
+ options.push_back({ "server", " --system-prompt-file FNAME",
+ "set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications" });
+ options.push_back({ "server", " --log-format {text,json}",
+ "log output format: json or text (default: json)" });
+ options.push_back({ "server", " --metrics", "enable prometheus compatible metrics endpoint (default: %s)", params.endpoint_metrics ? "enabled" : "disabled" });
+ options.push_back({ "server", " --no-slots", "disables slots monitoring endpoint (default: %s)", params.endpoint_slots ? "enabled" : "disabled" });
+ options.push_back({ "server", " --slot-save-path PATH", "path to save slot kv cache (default: disabled)" });
+ options.push_back({ "server", " --chat-template JINJA_TEMPLATE",
+ "set custom jinja chat template (default: template taken from model's metadata)\n"
+ "only commonly used templates are accepted:\n"
+ "https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template" });
+ options.push_back({ "server", "-sps, --slot-prompt-similarity SIMILARITY",
+ "how much the prompt of a request must match the prompt of a slot in order to use that slot (default: %.2f, 0.0 = disabled)\n", params.slot_prompt_similarity });
+ options.push_back({ "server", " --lora-init-without-apply", "load LoRA adapters without applying them (apply later via POST /lora-adapters) (default: %s)", params.lora_init_without_apply ? "enabled" : "disabled"});
+
+#ifndef LOG_DISABLE_LOGS
+ options.push_back({ "logging" });
+ options.push_back({ "*", " --simple-io", "use basic IO for better compatibility in subprocesses and limited consoles" });
+ options.push_back({ "*", "-ld, --logdir LOGDIR", "path under which to save YAML logs (no logging if unset)" });
+ options.push_back({ "logging", " --log-test", "Run simple logging test" });
+ options.push_back({ "logging", " --log-disable", "Disable trace logs" });
+ options.push_back({ "logging", " --log-enable", "Enable trace logs" });
+ options.push_back({ "logging", " --log-file FNAME", "Specify a log filename (without extension)" });
+ options.push_back({ "logging", " --log-new", "Create a separate new log file on start. "
+ "Each log file will have unique name: \"<name>.<ID>.log\"" });
+ options.push_back({ "logging", " --log-append", "Don't truncate the old log file." });
+#endif // LOG_DISABLE_LOGS
+
+ options.push_back({ "cvector" });
+ options.push_back({ "cvector", "-o, --output FNAME", "output file (default: '%s')", params.cvector_outfile.c_str() });
+ options.push_back({ "cvector", " --positive-file FNAME", "positive prompts file, one prompt per line (default: '%s')", params.cvector_positive_file.c_str() });
+ options.push_back({ "cvector", " --negative-file FNAME", "negative prompts file, one prompt per line (default: '%s')", params.cvector_negative_file.c_str() });
+ options.push_back({ "cvector", " --pca-batch N", "batch size used for PCA. Larger batch runs faster, but uses more memory (default: %d)", params.n_pca_batch });
+ options.push_back({ "cvector", " --pca-iter N", "number of iterations used for PCA (default: %d)", params.n_pca_iterations });
+ options.push_back({ "cvector", " --method {pca,mean}", "dimensionality reduction method to be used (default: pca)" });
+
+ options.push_back({ "export-lora" });
+ options.push_back({ "export-lora", "-m, --model", "model path from which to load base model (default '%s')", params.model.c_str() });
+ options.push_back({ "export-lora", " --lora FNAME", "path to LoRA adapter (can be repeated to use multiple adapters)" });
+ options.push_back({ "export-lora", " --lora-scaled FNAME S", "path to LoRA adapter with user defined scaling S (can be repeated to use multiple adapters)" });
+ options.push_back({ "export-lora", "-o, --output FNAME", "output file (default: '%s')", params.lora_outfile.c_str() });
+
+ printf("usage: %s [options]\n", argv[0]);
+
+ for (const auto & o : options) {
+ if (!o.grp.empty()) {
+ printf("\n%s:\n\n", o.grp.c_str());
+ continue;
+ }
+ printf(" %-32s", o.args.c_str());
+ if (o.args.length() > 30) {
+ printf("\n%34s", "");
+ }
+
+ const auto desc = o.desc;
+ size_t start = 0;
+ size_t end = desc.find('\n');
+ while (end != std::string::npos) {
+ printf("%s\n%34s", desc.substr(start, end - start).c_str(), "");
+ start = end + 1;
+ end = desc.find('\n', start);
+ }
+
+ printf("%s\n", desc.substr(start).c_str());
+ }
+ printf("\n");
+}
+
+std::string gpt_params_get_system_info(const gpt_params & params) {
+ std::ostringstream os;
+
+ os << "system_info: n_threads = " << params.cpuparams.n_threads;
+ if (params.cpuparams_batch.n_threads != -1) {
+ os << " (n_threads_batch = " << params.cpuparams_batch.n_threads << ")";
+ }
+#if defined(_WIN32) && (_WIN32_WINNT >= 0x0601) && !defined(__MINGW64__) // windows 7 and later
+ // TODO: windows + arm64 + mingw64
+ DWORD logicalProcessorCount = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
+ os << " / " << logicalProcessorCount << " | " << llama_print_system_info();
+#else
+ os << " / " << std::thread::hardware_concurrency() << " | " << llama_print_system_info();
+#endif
+
+ return os.str();
+}
+
+//
+// String utils
+//
+
+std::vector<std::string> string_split(std::string input, char separator) {
+ std::vector<std::string> parts;
+ size_t separator_pos = input.find(separator);
+ while (separator_pos != std::string::npos) {
+ std::string part = input.substr(0, separator_pos);
+ parts.emplace_back(part);
+ input = input.substr(separator_pos + 1);
+ separator_pos = input.find(separator);
+ }
+ parts.emplace_back(input);
+ return parts;
+}
+
+std::string string_strip(const std::string & str) {
+ size_t start = 0;
+ size_t end = str.size();
+ while (start < end && std::isspace(str[start])) {
+ start++;
+ }
+ while (end > start && std::isspace(str[end - 1])) {
+ end--;
+ }
+ return str.substr(start, end - start);
+}
+
+std::string string_get_sortable_timestamp() {
+ using clock = std::chrono::system_clock;
+
+ const clock::time_point current_time = clock::now();
+ const time_t as_time_t = clock::to_time_t(current_time);
+ char timestamp_no_ns[100];
+ std::strftime(timestamp_no_ns, 100, "%Y_%m_%d-%H_%M_%S", std::localtime(&as_time_t));
+
+ const int64_t ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
+ current_time.time_since_epoch() % 1000000000).count();
+ char timestamp_ns[11];
+ snprintf(timestamp_ns, 11, "%09" PRId64, ns);
+
+ return std::string(timestamp_no_ns) + "." + std::string(timestamp_ns);
+}
+
+void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {
+ if (search.empty()) {
+ return;
+ }
+ std::string builder;
+ builder.reserve(s.length());
+ size_t pos = 0;
+ size_t last_pos = 0;
+ while ((pos = s.find(search, last_pos)) != std::string::npos) {
+ builder.append(s, last_pos, pos - last_pos);
+ builder.append(replace);
+ last_pos = pos + search.length();
+ }
+ builder.append(s, last_pos, std::string::npos);
+ s = std::move(builder);
+}
+
+void string_process_escapes(std::string & input) {
+ std::size_t input_len = input.length();
+ std::size_t output_idx = 0;
+
+ for (std::size_t input_idx = 0; input_idx < input_len; ++input_idx) {
+ if (input[input_idx] == '\\' && input_idx + 1 < input_len) {
+ switch (input[++input_idx]) {
+ case 'n': input[output_idx++] = '\n'; break;
+ case 'r': input[output_idx++] = '\r'; break;
+ case 't': input[output_idx++] = '\t'; break;
+ case '\'': input[output_idx++] = '\''; break;
+ case '\"': input[output_idx++] = '\"'; break;
+ case '\\': input[output_idx++] = '\\'; break;
+ case 'x':
+ // Handle \x12, etc
+ if (input_idx + 2 < input_len) {
+ const char x[3] = { input[input_idx + 1], input[input_idx + 2], 0 };
+ char *err_p = nullptr;
+ const long val = std::strtol(x, &err_p, 16);
+ if (err_p == x + 2) {
+ input_idx += 2;
+ input[output_idx++] = char(val);
+ break;
+ }
+ }
+ // fall through
+ default: input[output_idx++] = '\\';
+ input[output_idx++] = input[input_idx]; break;
+ }
+ } else {
+ input[output_idx++] = input[input_idx];
+ }
+ }
+
+ input.resize(output_idx);
+}
+
+bool string_parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides) {
+ const char * sep = strchr(data, '=');
+ if (sep == nullptr || sep - data >= 128) {
+ fprintf(stderr, "%s: malformed KV override '%s'\n", __func__, data);
+ return false;
+ }
+ llama_model_kv_override kvo;
+ std::strncpy(kvo.key, data, sep - data);
+ kvo.key[sep - data] = 0;
+ sep++;
+ if (strncmp(sep, "int:", 4) == 0) {
+ sep += 4;
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;
+ kvo.val_i64 = std::atol(sep);
+ } else if (strncmp(sep, "float:", 6) == 0) {
+ sep += 6;
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;
+ kvo.val_f64 = std::atof(sep);
+ } else if (strncmp(sep, "bool:", 5) == 0) {
+ sep += 5;
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;
+ if (std::strcmp(sep, "true") == 0) {
+ kvo.val_bool = true;
+ } else if (std::strcmp(sep, "false") == 0) {
+ kvo.val_bool = false;
+ } else {
+ fprintf(stderr, "%s: invalid boolean value for KV override '%s'\n", __func__, data);
+ return false;
+ }
+ } else if (strncmp(sep, "str:", 4) == 0) {
+ sep += 4;
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;
+ if (strlen(sep) > 127) {
+ fprintf(stderr, "%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data);
+ return false;
+ }
+ strncpy(kvo.val_str, sep, 127);
+ kvo.val_str[127] = '\0';
+ } else {
+ fprintf(stderr, "%s: invalid type for KV override '%s'\n", __func__, data);
+ return false;
+ }
+ overrides.emplace_back(std::move(kvo));
+ return true;
+}
+
+//
+// Filesystem utils
+//
+
+// Validate if a filename is safe to use
+// To validate a full path, split the path by the OS-specific path separator, and validate each part with this function
+bool fs_validate_filename(const std::string & filename) {
+ if (!filename.length()) {
+ // Empty filename invalid
+ return false;
+ }
+ if (filename.length() > 255) {
+ // Limit at common largest possible filename on Linux filesystems
+ // to avoid unnecessary further validation
+ // (On systems with smaller limits it will be caught by the OS)
+ return false;
+ }
+
+ std::u32string filename_utf32;
+ try {
+ std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> converter;
+ filename_utf32 = converter.from_bytes(filename);
+
+ // If the reverse conversion mismatches, it means overlong UTF-8 sequences were used,
+ // or invalid encodings were encountered. Reject such attempts
+ std::string filename_reencoded = converter.to_bytes(filename_utf32);
+ if (filename_reencoded != filename) {
+ return false;
+ }
+ } catch (const std::exception &) {
+ return false;
+ }
+
+ // Check for forbidden codepoints:
+ // - Control characters
+ // - Unicode equivalents of illegal characters
+ // - UTF-16 surrogate pairs
+ // - UTF-8 replacement character
+ // - Byte order mark (BOM)
+ // - Illegal characters: / \ : * ? " < > |
+ for (char32_t c : filename_utf32) {
+ if (c <= 0x1F // Control characters (C0)
+ || c == 0x7F // Control characters (DEL)
+ || (c >= 0x80 && c <= 0x9F) // Control characters (C1)
+ || c == 0xFF0E // Fullwidth Full Stop (period equivalent)
+ || c == 0x2215 // Division Slash (forward slash equivalent)
+ || c == 0x2216 // Set Minus (backslash equivalent)
+ || (c >= 0xD800 && c <= 0xDFFF) // UTF-16 surrogate pairs
+ || c == 0xFFFD // Replacement Character (UTF-8)
+ || c == 0xFEFF // Byte Order Mark (BOM)
+ || c == '/' || c == '\\' || c == ':' || c == '*' // Illegal characters
+ || c == '?' || c == '"' || c == '<' || c == '>' || c == '|') {
+ return false;
+ }
+ }
+
+ // Reject any leading or trailing ' ', or any trailing '.', these are stripped on Windows and will cause a different filename
+ // Unicode and other whitespace is not affected, only 0x20 space
+ if (filename.front() == ' ' || filename.back() == ' ' || filename.back() == '.') {
+ return false;
+ }
+
+ // Reject any ".." (currently stricter than necessary, it should be fine to just check for == ".." instead)
+ if (filename.find("..") != std::string::npos) {
+ return false;
+ }
+
+ // Reject "."
+ if (filename == ".") {
+ return false;
+ }
+
+ return true;
+}
+
+// returns true if successful, false otherwise
+bool fs_create_directory_with_parents(const std::string & path) {
+#ifdef _WIN32
+ std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
+ std::wstring wpath = converter.from_bytes(path);
+
+ // if the path already exists, check whether it's a directory
+ const DWORD attributes = GetFileAttributesW(wpath.c_str());
+ if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {
+ return true;
+ }
+
+ size_t pos_slash = 0;
+
+ // process path from front to back, procedurally creating directories
+ while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) {
+ const std::wstring subpath = wpath.substr(0, pos_slash);
+ const wchar_t * test = subpath.c_str();
+
+ const bool success = CreateDirectoryW(test, NULL);
+ if (!success) {
+ const DWORD error = GetLastError();
+
+ // if the path already exists, ensure that it's a directory
+ if (error == ERROR_ALREADY_EXISTS) {
+ const DWORD attributes = GetFileAttributesW(subpath.c_str());
+ if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+
+ pos_slash += 1;
+ }
+
+ return true;
+#else
+ // if the path already exists, check whether it's a directory
+ struct stat info;
+ if (stat(path.c_str(), &info) == 0) {
+ return S_ISDIR(info.st_mode);
+ }
+
+ size_t pos_slash = 1; // skip leading slashes for directory creation
+
+ // process path from front to back, procedurally creating directories
+ while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) {
+ const std::string subpath = path.substr(0, pos_slash);
+ struct stat info;
+
+ // if the path already exists, ensure that it's a directory
+ if (stat(subpath.c_str(), &info) == 0) {
+ if (!S_ISDIR(info.st_mode)) {
+ return false;
+ }
+ } else {
+ // create parent directories
+ const int ret = mkdir(subpath.c_str(), 0755);
+ if (ret != 0) {
+ return false;
+ }
+ }
+
+ pos_slash += 1;
+ }
+
+ return true;
+#endif // _WIN32
+}
+
+std::string fs_get_cache_directory() {
+ std::string cache_directory = "";
+ auto ensure_trailing_slash = [](std::string p) {
+ // Make sure to add trailing slash
+ if (p.back() != DIRECTORY_SEPARATOR) {
+ p += DIRECTORY_SEPARATOR;
+ }
+ return p;
+ };
+ if (getenv("LLAMA_CACHE")) {
+ cache_directory = std::getenv("LLAMA_CACHE");
+ } else {
+#ifdef __linux__
+ if (std::getenv("XDG_CACHE_HOME")) {
+ cache_directory = std::getenv("XDG_CACHE_HOME");
+ } else {
+ cache_directory = std::getenv("HOME") + std::string("/.cache/");
+ }
+#elif defined(__APPLE__)
+ cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");
+#elif defined(_WIN32)
+ cache_directory = std::getenv("LOCALAPPDATA");
+#endif // __linux__
+ cache_directory = ensure_trailing_slash(cache_directory);
+ cache_directory += "llama.cpp";
+ }
+ return ensure_trailing_slash(cache_directory);
+}
+
+std::string fs_get_cache_file(const std::string & filename) {
+ GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);
+ std::string cache_directory = fs_get_cache_directory();
+ const bool success = fs_create_directory_with_parents(cache_directory);
+ if (!success) {
+ throw std::runtime_error("failed to create cache directory: " + cache_directory);
+ }
+ return cache_directory + filename;
+}
+
+
+//
+// Model utils
+//
+struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
+ llama_init_result iparams;
+ auto mparams = llama_model_params_from_gpt_params(params);
+
+ llama_model * model = nullptr;
+
+ if (!params.hf_repo.empty() && !params.hf_file.empty()) {
+ model = llama_load_model_from_hf(params.hf_repo.c_str(), params.hf_file.c_str(), params.model.c_str(), params.hf_token.c_str(), mparams);
+ } else if (!params.model_url.empty()) {
+ model = llama_load_model_from_url(params.model_url.c_str(), params.model.c_str(), params.hf_token.c_str(), mparams);
+ } else {
+ model = llama_load_model_from_file(params.model.c_str(), mparams);
+ }
+
+ if (model == NULL) {
+ fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str());
+ return iparams;
+ }
+
+ auto cparams = llama_context_params_from_gpt_params(params);
+
+ llama_context * lctx = llama_new_context_with_model(model, cparams);
+ if (lctx == NULL) {
+ fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, params.model.c_str());
+ llama_free_model(model);
+ return iparams;
+ }
+
+ if (!params.control_vectors.empty()) {
+ if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;
+ if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_n_layer(model);
+
+ const auto cvec = llama_control_vector_load(params.control_vectors);
+ if (cvec.n_embd == -1) {
+ llama_free(lctx);
+ llama_free_model(model);
+ return iparams;
+ }
+
+ int err = llama_control_vector_apply(lctx,
+ cvec.data.data(),
+ cvec.data.size(),
+ cvec.n_embd,
+ params.control_vector_layer_start,
+ params.control_vector_layer_end);
+ if (err) {
+ llama_free(lctx);
+ llama_free_model(model);
+ return iparams;
+ }
+ }
+
+ // load and optionally apply lora adapters
+ for (auto & la : params.lora_adapters) {
+ llama_lora_adapter_container loaded_la;
+ loaded_la.path = la.path;
+ loaded_la.scale = la.scale;
+ loaded_la.adapter = llama_lora_adapter_init(model, la.path.c_str());
+ if (loaded_la.adapter == nullptr) {
+ fprintf(stderr, "%s: error: failed to apply lora adapter '%s'\n", __func__, la.path.c_str());
+ llama_free(lctx);
+ llama_free_model(model);
+ return iparams;
+ }
+ iparams.lora_adapters.push_back(loaded_la); // copy to list of loaded adapters
+ }
+ if (!params.lora_init_without_apply) {
+ llama_lora_adapters_apply(lctx, iparams.lora_adapters);
+ }
+
+ if (params.ignore_eos) {
+ params.sparams.logit_bias[llama_token_eos(model)] = -INFINITY;
+ }
+
+ if (params.warmup) {
+ LOG("warming up the model with an empty run\n");
+
+ std::vector<llama_token> tmp;
+ llama_token bos = llama_token_bos(model);
+ llama_token eos = llama_token_eos(model);
+ // some models (e.g. T5) don't have a BOS token
+ if (bos != -1) {
+ tmp.push_back(bos);
+ }
+ tmp.push_back(eos);
+
+ if (llama_model_has_encoder(model)) {
+ llama_encode(lctx, llama_batch_get_one(tmp.data(), tmp.size(), 0, 0));
+ llama_token decoder_start_token_id = llama_model_decoder_start_token(model);
+ if (decoder_start_token_id == -1) {
+ decoder_start_token_id = bos;
+ }
+ tmp.clear();
+ tmp.push_back(decoder_start_token_id);
+ }
+ if (llama_model_has_decoder(model)) {
+ llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch), 0, 0));
+ }
+ llama_kv_cache_clear(lctx);
+ llama_synchronize(lctx);
+ llama_reset_timings(lctx);
+ }
+
+ iparams.model = model;
+ iparams.context = lctx;
+ return iparams;
+}
+
+void llama_lora_adapters_apply(struct llama_context * ctx, std::vector<llama_lora_adapter_container> & lora_adapters) {
+ llama_lora_adapter_clear(ctx);
+ for (auto & la : lora_adapters) {
+ if (la.scale != 0.0f) {
+ llama_lora_adapter_set(ctx, la.adapter, la.scale);
+ }
+ }
+}
+
+struct llama_model_params llama_model_params_from_gpt_params(const gpt_params & params) {
+ auto mparams = llama_model_default_params();
+
+ if (params.n_gpu_layers != -1) {
+ mparams.n_gpu_layers = params.n_gpu_layers;
+ }
+ mparams.rpc_servers = params.rpc_servers.c_str();
+ mparams.main_gpu = params.main_gpu;
+ mparams.split_mode = params.split_mode;
+ mparams.tensor_split = params.tensor_split;
+ mparams.use_mmap = params.use_mmap;
+ mparams.use_mlock = params.use_mlock;
+ mparams.check_tensors = params.check_tensors;
+ if (params.kv_overrides.empty()) {
+ mparams.kv_overrides = NULL;
+ } else {
+ GGML_ASSERT(params.kv_overrides.back().key[0] == 0 && "KV overrides not terminated with empty key");
+ mparams.kv_overrides = params.kv_overrides.data();
+ }
+
+ return mparams;
+}
+
+static ggml_type kv_cache_type_from_str(const std::string & s) {
+ if (s == "f32") {
+ return GGML_TYPE_F32;
+ }
+ if (s == "f16") {
+ return GGML_TYPE_F16;
+ }
+ if (s == "q8_0") {
+ return GGML_TYPE_Q8_0;
+ }
+ if (s == "q4_0") {
+ return GGML_TYPE_Q4_0;
+ }
+ if (s == "q4_1") {
+ return GGML_TYPE_Q4_1;
+ }
+ if (s == "iq4_nl") {
+ return GGML_TYPE_IQ4_NL;
+ }
+ if (s == "q5_0") {
+ return GGML_TYPE_Q5_0;
+ }
+ if (s == "q5_1") {
+ return GGML_TYPE_Q5_1;
+ }
+
+ throw std::runtime_error("Invalid cache type: " + s);
+}
+
+struct llama_context_params llama_context_params_from_gpt_params(const gpt_params & params) {
+ auto cparams = llama_context_default_params();
+
+ cparams.n_ctx = params.n_ctx;
+ cparams.n_seq_max = params.n_parallel;
+ cparams.n_batch = params.n_batch;
+ cparams.n_ubatch = params.n_ubatch;
+ cparams.n_threads = params.cpuparams.n_threads;
+ cparams.n_threads_batch = params.cpuparams_batch.n_threads == -1 ?
+ params.cpuparams.n_threads : params.cpuparams_batch.n_threads;
+ cparams.seed = params.seed;
+ cparams.logits_all = params.logits_all;
+ cparams.embeddings = params.embedding;
+ cparams.rope_scaling_type = params.rope_scaling_type;
+ cparams.rope_freq_base = params.rope_freq_base;
+ cparams.rope_freq_scale = params.rope_freq_scale;
+ cparams.yarn_ext_factor = params.yarn_ext_factor;
+ cparams.yarn_attn_factor = params.yarn_attn_factor;
+ cparams.yarn_beta_fast = params.yarn_beta_fast;
+ cparams.yarn_beta_slow = params.yarn_beta_slow;
+ cparams.yarn_orig_ctx = params.yarn_orig_ctx;
+ cparams.pooling_type = params.pooling_type;
+ cparams.attention_type = params.attention_type;
+ cparams.defrag_thold = params.defrag_thold;
+ cparams.cb_eval = params.cb_eval;
+ cparams.cb_eval_user_data = params.cb_eval_user_data;
+ cparams.offload_kqv = !params.no_kv_offload;
+ cparams.flash_attn = params.flash_attn;
+
+ cparams.type_k = kv_cache_type_from_str(params.cache_type_k);
+ cparams.type_v = kv_cache_type_from_str(params.cache_type_v);
+
+ return cparams;
+}
+
+struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const cpu_params & params) {
+ struct ggml_threadpool_params tpp;
+
+ ggml_threadpool_params_init(&tpp, params.n_threads); // setup the defaults
+
+ if (params.mask_valid) {
+ std::memcpy(&tpp.cpumask, ¶ms.cpumask, GGML_MAX_N_THREADS);
+ }
+
+ tpp.prio = params.priority;
+ tpp.poll = params.poll;
+ tpp.strict_cpu = params.strict_cpu;
+
+ return tpp;
+}
+
+#ifdef LLAMA_USE_CURL
+
+static bool starts_with(const std::string & str, const std::string & prefix) {
+ // While we wait for C++20's std::string::starts_with...
+ return str.rfind(prefix, 0) == 0;
+}
+
+static bool llama_download_file(const std::string & url, const std::string & path, const std::string & hf_token) {
+
+ // Initialize libcurl
+ std::unique_ptr<CURL, decltype(&curl_easy_cleanup)> curl(curl_easy_init(), &curl_easy_cleanup);
+ if (!curl) {
+ fprintf(stderr, "%s: error initializing libcurl\n", __func__);
+ return false;
+ }
+
+ bool force_download = false;
+
+ // Set the URL, allow to follow http redirection
+ curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
+ curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L);
+
+ // Check if hf-token or bearer-token was specified
+ if (!hf_token.empty()) {
+ std::string auth_header = "Authorization: Bearer ";
+ auth_header += hf_token.c_str();
+ struct curl_slist *http_headers = NULL;
+ http_headers = curl_slist_append(http_headers, auth_header.c_str());
+ curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, http_headers);
+ }
+
+#if defined(_WIN32)
+ // CURLSSLOPT_NATIVE_CA tells libcurl to use standard certificate store of
+ // operating system. Currently implemented under MS-Windows.
+ curl_easy_setopt(curl.get(), CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA);
+#endif
+
+ // Check if the file already exists locally
+ struct stat model_file_info;
+ auto file_exists = (stat(path.c_str(), &model_file_info) == 0);
+
+ // If the file exists, check its JSON metadata companion file.
+ std::string metadata_path = path + ".json";
+ nlohmann::json metadata;
+ std::string etag;
+ std::string last_modified;
+
+ if (file_exists) {
+ // Try and read the JSON metadata file (note: stream autoclosed upon exiting this block).
+ std::ifstream metadata_in(metadata_path);
+ if (metadata_in.good()) {
+ try {
+ metadata_in >> metadata;
+ fprintf(stderr, "%s: previous metadata file found %s: %s\n", __func__, metadata_path.c_str(), metadata.dump().c_str());
+ if (metadata.contains("url") && metadata.at("url").is_string()) {
+ auto previous_url = metadata.at("url").get<std::string>();
+ if (previous_url != url) {
+ fprintf(stderr, "%s: Model URL mismatch: %s != %s\n", __func__, url.c_str(), previous_url.c_str());
+ return false;
+ }
+ }
+ if (metadata.contains("etag") && metadata.at("etag").is_string()) {
+ etag = metadata.at("etag");
+ }
+ if (metadata.contains("lastModified") && metadata.at("lastModified").is_string()) {
+ last_modified = metadata.at("lastModified");
+ }
+ } catch (const nlohmann::json::exception & e) {
+ fprintf(stderr, "%s: error reading metadata file %s: %s\n", __func__, metadata_path.c_str(), e.what());
+ return false;
+ }
+ }
+ } else {
+ fprintf(stderr, "%s: no previous model file found %s\n", __func__, path.c_str());
+ }
+
+ // Send a HEAD request to retrieve the etag and last-modified headers
+ struct llama_load_model_from_url_headers {
+ std::string etag;
+ std::string last_modified;
+ };
+ llama_load_model_from_url_headers headers;
+ {
+ typedef size_t(*CURLOPT_HEADERFUNCTION_PTR)(char *, size_t, size_t, void *);
+ auto header_callback = [](char * buffer, size_t /*size*/, size_t n_items, void * userdata) -> size_t {
+ llama_load_model_from_url_headers *headers = (llama_load_model_from_url_headers *) userdata;
+
+ static std::regex header_regex("([^:]+): (.*)\r\n");
+ static std::regex etag_regex("ETag", std::regex_constants::icase);
+ static std::regex last_modified_regex("Last-Modified", std::regex_constants::icase);
+
+ std::string header(buffer, n_items);
+ std::smatch match;
+ if (std::regex_match(header, match, header_regex)) {
+ const std::string & key = match[1];
+ const std::string & value = match[2];
+ if (std::regex_match(key, match, etag_regex)) {
+ headers->etag = value;
+ } else if (std::regex_match(key, match, last_modified_regex)) {
+ headers->last_modified = value;
+ }
+ }
+ return n_items;
+ };
+
+ curl_easy_setopt(curl.get(), CURLOPT_NOBODY, 1L); // will trigger the HEAD verb
+ curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 1L); // hide head request progress
+ curl_easy_setopt(curl.get(), CURLOPT_HEADERFUNCTION, static_cast<CURLOPT_HEADERFUNCTION_PTR>(header_callback));
+ curl_easy_setopt(curl.get(), CURLOPT_HEADERDATA, &headers);
+
+ CURLcode res = curl_easy_perform(curl.get());
+ if (res != CURLE_OK) {
+ fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res));
+ return false;
+ }
+
+ long http_code = 0;
+ curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &http_code);
+ if (http_code != 200) {
+ // HEAD not supported, we don't know if the file has changed
+ // force trigger downloading
+ force_download = true;
+ fprintf(stderr, "%s: HEAD invalid http status code received: %ld\n", __func__, http_code);
+ }
+ }
+
+ bool should_download = !file_exists || force_download;
+ if (!should_download) {
+ if (!etag.empty() && etag != headers.etag) {
+ fprintf(stderr, "%s: ETag header is different (%s != %s): triggering a new download\n", __func__, etag.c_str(), headers.etag.c_str());
+ should_download = true;
+ } else if (!last_modified.empty() && last_modified != headers.last_modified) {
+ fprintf(stderr, "%s: Last-Modified header is different (%s != %s): triggering a new download\n", __func__, last_modified.c_str(), headers.last_modified.c_str());
+ should_download = true;
+ }
+ }
+ if (should_download) {
+ std::string path_temporary = path + ".downloadInProgress";
+ if (file_exists) {
+ fprintf(stderr, "%s: deleting previous downloaded file: %s\n", __func__, path.c_str());
+ if (remove(path.c_str()) != 0) {
+ fprintf(stderr, "%s: unable to delete file: %s\n", __func__, path.c_str());
+ return false;
+ }
+ }
+
+ // Set the output file
+
+ struct FILE_deleter {
+ void operator()(FILE * f) const {
+ fclose(f);
+ }
+ };
+
+ std::unique_ptr<FILE, FILE_deleter> outfile(fopen(path_temporary.c_str(), "wb"));
+ if (!outfile) {
+ fprintf(stderr, "%s: error opening local file for writing: %s\n", __func__, path.c_str());
+ return false;
+ }
+
+ typedef size_t(*CURLOPT_WRITEFUNCTION_PTR)(void * data, size_t size, size_t nmemb, void * fd);
+ auto write_callback = [](void * data, size_t size, size_t nmemb, void * fd) -> size_t {
+ return fwrite(data, size, nmemb, (FILE *)fd);
+ };
+ curl_easy_setopt(curl.get(), CURLOPT_NOBODY, 0L);
+ curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, static_cast<CURLOPT_WRITEFUNCTION_PTR>(write_callback));
+ curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, outfile.get());
+
+ // display download progress
+ curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 0L);
+
+ // helper function to hide password in URL
+ auto llama_download_hide_password_in_url = [](const std::string & url) -> std::string {
+ std::size_t protocol_pos = url.find("://");
+ if (protocol_pos == std::string::npos) {
+ return url; // Malformed URL
+ }
+
+ std::size_t at_pos = url.find('@', protocol_pos + 3);
+ if (at_pos == std::string::npos) {
+ return url; // No password in URL
+ }
+
+ return url.substr(0, protocol_pos + 3) + "********" + url.substr(at_pos);
+ };
+
+ // start the download
+ fprintf(stderr, "%s: downloading from %s to %s (server_etag:%s, server_last_modified:%s)...\n", __func__,
+ llama_download_hide_password_in_url(url).c_str(), path.c_str(), headers.etag.c_str(), headers.last_modified.c_str());
+ auto res = curl_easy_perform(curl.get());
+ if (res != CURLE_OK) {
+ fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res));
+ return false;
+ }
+
+ long http_code = 0;
+ curl_easy_getinfo (curl.get(), CURLINFO_RESPONSE_CODE, &http_code);
+ if (http_code < 200 || http_code >= 400) {
+ fprintf(stderr, "%s: invalid http status code received: %ld\n", __func__, http_code);
+ return false;
+ }
+
+ // Causes file to be closed explicitly here before we rename it.
+ outfile.reset();
+
+ // Write the updated JSON metadata file.
+ metadata.update({
+ {"url", url},
+ {"etag", headers.etag},
+ {"lastModified", headers.last_modified}
+ });
+ std::ofstream(metadata_path) << metadata.dump(4);
+ fprintf(stderr, "%s: file metadata saved: %s\n", __func__, metadata_path.c_str());
+
+ if (rename(path_temporary.c_str(), path.c_str()) != 0) {
+ fprintf(stderr, "%s: unable to rename file: %s to %s\n", __func__, path_temporary.c_str(), path.c_str());
+ return false;
+ }
+ }
+
+ return true;
+}
+
+struct llama_model * llama_load_model_from_url(
+ const char * model_url,
+ const char * path_model,
+ const char * hf_token,
+ const struct llama_model_params & params) {
+ // Basic validation of the model_url
+ if (!model_url || strlen(model_url) == 0) {
+ fprintf(stderr, "%s: invalid model_url\n", __func__);
+ return NULL;
+ }
+
+ if (!llama_download_file(model_url, path_model, hf_token)) {
+ return NULL;
+ }
+
+ // check for additional GGUFs split to download
+ int n_split = 0;
+ {
+ struct gguf_init_params gguf_params = {
+ /*.no_alloc = */ true,
+ /*.ctx = */ NULL,
+ };
+ auto * ctx_gguf = gguf_init_from_file(path_model, gguf_params);
+ if (!ctx_gguf) {
+ fprintf(stderr, "\n%s: failed to load input GGUF from %s\n", __func__, path_model);
+ return NULL;
+ }
+
+ auto key_n_split = gguf_find_key(ctx_gguf, LLM_KV_SPLIT_COUNT);
+ if (key_n_split >= 0) {
+ n_split = gguf_get_val_u16(ctx_gguf, key_n_split);
+ }
+
+ gguf_free(ctx_gguf);
+ }
+
+ if (n_split > 1) {
+ char split_prefix[PATH_MAX] = {0};
+ char split_url_prefix[LLAMA_CURL_MAX_URL_LENGTH] = {0};
+
+ // Verify the first split file format
+ // and extract split URL and PATH prefixes
+ {
+ if (!llama_split_prefix(split_prefix, sizeof(split_prefix), path_model, 0, n_split)) {
+ fprintf(stderr, "\n%s: unexpected model file name: %s"
+ " n_split=%d\n", __func__, path_model, n_split);
+ return NULL;
+ }
+
+ if (!llama_split_prefix(split_url_prefix, sizeof(split_url_prefix), model_url, 0, n_split)) {
+ fprintf(stderr, "\n%s: unexpected model url: %s"
+ " n_split=%d\n", __func__, model_url, n_split);
+ return NULL;
+ }
+ }
+
+ // Prepare download in parallel
+ std::vector<std::future<bool>> futures_download;
+ for (int idx = 1; idx < n_split; idx++) {
+ futures_download.push_back(std::async(std::launch::async, [&split_prefix, &split_url_prefix, &n_split, hf_token](int download_idx) -> bool {
+ char split_path[PATH_MAX] = {0};
+ llama_split_path(split_path, sizeof(split_path), split_prefix, download_idx, n_split);
+
+ char split_url[LLAMA_CURL_MAX_URL_LENGTH] = {0};
+ llama_split_path(split_url, sizeof(split_url), split_url_prefix, download_idx, n_split);
+
+ return llama_download_file(split_url, split_path, hf_token);
+ }, idx));
+ }
+
+ // Wait for all downloads to complete
+ for (auto & f : futures_download) {
+ if (!f.get()) {
+ return NULL;
+ }
+ }
+ }
+
+ return llama_load_model_from_file(path_model, params);
+}
+
+struct llama_model * llama_load_model_from_hf(
+ const char * repo,
+ const char * model,
+ const char * path_model,
+ const char * hf_token,
+ const struct llama_model_params & params) {
+ // construct hugging face model url:
+ //
+ // --repo ggml-org/models --file tinyllama-1.1b/ggml-model-f16.gguf
+ // https://huggingface.co/ggml-org/models/resolve/main/tinyllama-1.1b/ggml-model-f16.gguf
+ //
+ // --repo TheBloke/Mixtral-8x7B-v0.1-GGUF --file mixtral-8x7b-v0.1.Q4_K_M.gguf
+ // https://huggingface.co/TheBloke/Mixtral-8x7B-v0.1-GGUF/resolve/main/mixtral-8x7b-v0.1.Q4_K_M.gguf
+ //
+
+ std::string model_url = "https://huggingface.co/";
+ model_url += repo;
+ model_url += "/resolve/main/";
+ model_url += model;
+
+ return llama_load_model_from_url(model_url.c_str(), path_model, hf_token, params);
+}
+
+#else
+
+struct llama_model * llama_load_model_from_url(
+ const char * /*model_url*/,
+ const char * /*path_model*/,
+ const char * /*hf_token*/,
+ const struct llama_model_params & /*params*/) {
+ fprintf(stderr, "%s: llama.cpp built without libcurl, downloading from an url not supported.\n", __func__);
+ return nullptr;
+}
+
+struct llama_model * llama_load_model_from_hf(
+ const char * /*repo*/,
+ const char * /*model*/,
+ const char * /*path_model*/,
+ const char * /*hf_token*/,
+ const struct llama_model_params & /*params*/) {
+ fprintf(stderr, "%s: llama.cpp built without libcurl, downloading from Hugging Face not supported.\n", __func__);
+ return nullptr;
+}
+
+#endif // LLAMA_USE_CURL
+
+//
+// Batch utils
+//
+
+void llama_batch_clear(struct llama_batch & batch) {
+ batch.n_tokens = 0;
+}
+
+void llama_batch_add(
+ struct llama_batch & batch,
+ llama_token id,
+ llama_pos pos,
+ const std::vector<llama_seq_id> & seq_ids,
+ bool logits) {
+ batch.token [batch.n_tokens] = id;
+ batch.pos [batch.n_tokens] = pos;
+ batch.n_seq_id[batch.n_tokens] = seq_ids.size();
+ for (size_t i = 0; i < seq_ids.size(); ++i) {
+ batch.seq_id[batch.n_tokens][i] = seq_ids[i];
+ }
+ batch.logits [batch.n_tokens] = logits;
+
+ batch.n_tokens++;
+}
+
+//
+// Vocab utils
+//
+
+std::vector<llama_token> llama_tokenize(
+ const struct llama_context * ctx,
+ const std::string & text,
+ bool add_special,
+ bool parse_special) {
+ return llama_tokenize(llama_get_model(ctx), text, add_special, parse_special);
+}
+
+std::vector<llama_token> llama_tokenize(
+ const struct llama_model * model,
+ const std::string & text,
+ bool add_special,
+ bool parse_special) {
+ // upper limit for the number of tokens
+ int n_tokens = text.length() + 2 * add_special;
+ std::vector<llama_token> result(n_tokens);
+ n_tokens = llama_tokenize(model, text.data(), text.length(), result.data(), result.size(), add_special, parse_special);
+ if (n_tokens < 0) {
+ result.resize(-n_tokens);
+ int check = llama_tokenize(model, text.data(), text.length(), result.data(), result.size(), add_special, parse_special);
+ GGML_ASSERT(check == -n_tokens);
+ } else {
+ result.resize(n_tokens);
+ }
+ return result;
+}
+
+std::string llama_token_to_piece(const struct llama_context * ctx, llama_token token, bool special) {
+ std::string piece;
+ piece.resize(piece.capacity()); // using string internal cache, 15 bytes + '\n'
+ const int n_chars = llama_token_to_piece(llama_get_model(ctx), token, &piece[0], piece.size(), 0, special);
+ if (n_chars < 0) {
+ piece.resize(-n_chars);
+ int check = llama_token_to_piece(llama_get_model(ctx), token, &piece[0], piece.size(), 0, special);
+ GGML_ASSERT(check == -n_chars);
+ }
+ else {
+ piece.resize(n_chars);
+ }
+
+ return piece;
+}
+
+std::string llama_detokenize(llama_context * ctx, const std::vector<llama_token> & tokens, bool special) {
+ std::string text;
+ text.resize(std::max(text.capacity(), tokens.size()));
+ int32_t n_chars = llama_detokenize(llama_get_model(ctx), tokens.data(), (int32_t)tokens.size(), &text[0], (int32_t)text.size(), false, special);
+ if (n_chars < 0) {
+ text.resize(-n_chars);
+ n_chars = llama_detokenize(llama_get_model(ctx), tokens.data(), (int32_t)tokens.size(), &text[0], (int32_t)text.size(), false, special);
+ GGML_ASSERT(n_chars <= (int32_t)text.size()); // whitespace trimming is performed after per-token detokenization
+ }
+
+ text.resize(n_chars);
+
+ // NOTE: the original tokenizer decodes bytes after collecting the pieces.
+ return text;
+}
+
+//
+// Chat template utils
+//
+
+bool llama_chat_verify_template(const std::string & tmpl) {
+ llama_chat_message chat[] = {{"user", "test"}};
+ int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, nullptr, 0);
+ return res >= 0;
+}
+
+std::string llama_chat_apply_template(const struct llama_model * model,
+ const std::string & tmpl,
+ const std::vector<llama_chat_msg> & msgs,
+ bool add_ass) {
+ int alloc_size = 0;
+ bool fallback = false; // indicate if we must fallback to default chatml
+ std::vector<llama_chat_message> chat;
+ for (auto & msg : msgs) {
+ chat.push_back({msg.role.c_str(), msg.content.c_str()});
+ alloc_size += (msg.role.size() + msg.content.size()) * 1.25;
+ }
+
+ const char * ptr_tmpl = tmpl.empty() ? nullptr : tmpl.c_str();
+ std::vector<char> buf(alloc_size);
+
+ // run the first time to get the total output length
+ int32_t res = llama_chat_apply_template(model, ptr_tmpl, chat.data(), chat.size(), add_ass, buf.data(), buf.size());
+
+ // error: chat template is not supported
+ if (res < 0) {
+ if (ptr_tmpl != nullptr) {
+ // if the custom "tmpl" is not supported, we throw an error
+ // this is a bit redundant (for good), since we're not sure if user validated the custom template with llama_chat_verify_template()
+ throw std::runtime_error("this custom template is not supported");
+ } else {
+ // If the built-in template is not supported, we default to chatml
+ res = llama_chat_apply_template(nullptr, "chatml", chat.data(), chat.size(), add_ass, buf.data(), buf.size());
+ fallback = true;
+ }
+ }
+
+ // if it turns out that our buffer is too small, we resize it
+ if ((size_t) res > buf.size()) {
+ buf.resize(res);
+ res = llama_chat_apply_template(
+ fallback ? nullptr : model,
+ fallback ? "chatml" : ptr_tmpl,
+ chat.data(), chat.size(), add_ass, buf.data(), buf.size());
+ }
+
+ std::string formatted_chat(buf.data(), res);
+ return formatted_chat;
+}
+
+std::string llama_chat_format_single(const struct llama_model * model,
+ const std::string & tmpl,
+ const std::vector<llama_chat_msg> & past_msg,
+ const llama_chat_msg & new_msg,
+ bool add_ass) {
+ std::ostringstream ss;
+ auto fmt_past_msg = past_msg.empty() ? "" : llama_chat_apply_template(model, tmpl, past_msg, false);
+ std::vector<llama_chat_msg> chat_new(past_msg);
+ // if the past_msg ends with a newline, we must preserve it in the formatted version
+ if (add_ass && !fmt_past_msg.empty() && fmt_past_msg.back() == '\n') {
+ ss << "\n";
+ };
+ // format chat with new_msg
+ chat_new.push_back(new_msg);
+ auto fmt_new_msg = llama_chat_apply_template(model, tmpl, chat_new, add_ass);
+ // get the diff part
+ ss << fmt_new_msg.substr(fmt_past_msg.size(), fmt_new_msg.size() - fmt_past_msg.size());
+ return ss.str();
+}
+
+std::string llama_chat_format_example(const struct llama_model * model,
+ const std::string & tmpl) {
+ std::vector<llama_chat_msg> msgs = {
+ {"system", "You are a helpful assistant"},
+ {"user", "Hello"},
+ {"assistant", "Hi there"},
+ {"user", "How are you?"},
+ };
+ return llama_chat_apply_template(model, tmpl, msgs, true);
+}
+
+//
+// KV cache utils
+//
+
+void llama_kv_cache_dump_view(const llama_kv_cache_view & view, int row_size) {
+ static const char slot_chars[] = ".123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+";
+
+ printf("=== Dumping KV cache. total cells %d, max sequences per cell %d, populated cells %d, total tokens in cache %d, largest empty slot=%d @ %d",
+ view.n_cells, view.n_seq_max, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx);
+
+ llama_kv_cache_view_cell * c_curr = view.cells;
+ llama_seq_id * cs_curr = view.cells_sequences;
+
+ for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_seq_max) {
+ if (i % row_size == 0) {
+ printf("\n%5d: ", i);
+ }
+ int seq_count = 0;
+ for (int j = 0; j < view.n_seq_max; j++) {
+ if (cs_curr[j] >= 0) { seq_count++; }
+ }
+ putchar(slot_chars[std::min(sizeof(slot_chars) - 2, size_t(seq_count))]);
+ }
+
+ printf("\n=== Done dumping\n");
+}
+
+void llama_kv_cache_dump_view_seqs(const llama_kv_cache_view & view, int row_size) {
+ static const char slot_chars[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+ printf("=== Dumping KV cache. total cells %d, max sequences per cell %d, populated cells %d, total tokens in cache %d, largest empty slot=%d @ %d\n",
+ view.n_cells, view.n_seq_max, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx);
+
+ std::unordered_map<llama_seq_id, size_t> seqs;
+ llama_kv_cache_view_cell * c_curr = view.cells;
+ llama_seq_id * cs_curr = view.cells_sequences;
+
+ for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_seq_max) {
+ for (int j = 0; j < view.n_seq_max; j++) {
+ if (cs_curr[j] < 0) { continue; }
+ if (seqs.find(cs_curr[j]) == seqs.end()) {
+ if (seqs.size() + 1 >= sizeof(slot_chars)) { break; }
+ const size_t sz = seqs.size();
+ seqs[cs_curr[j]] = sz;
+ }
+ }
+ if (seqs.size() + 1 >= sizeof(slot_chars)) { break; }
+ }
+
+ printf("=== Sequence legend: ");
+ for (const auto & it : seqs) {
+ printf("%zu=%d, ", it.second, it.first);
+ }
+ printf("'+'=other sequence ids");
+
+ c_curr = view.cells;
+ cs_curr = view.cells_sequences;
+ for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_seq_max) {
+ if (i % row_size == 0) {
+ printf("\n%5d: ", i);
+ }
+ for (int j = 0; j < view.n_seq_max; j++) {
+ if (cs_curr[j] >= 0) {
+ const auto & it = seqs.find(cs_curr[j]);
+ putchar(it != seqs.end() ? int(slot_chars[it->second]) : '+');
+ } else {
+ putchar('.');
+ }
+ }
+ putchar(' ');
+ }
+
+ printf("\n=== Done dumping\n");
+}
+
+//
+// Embedding utils
+//
+
+void llama_embd_normalize(const float * inp, float * out, int n, int embd_norm) {
+ double sum = 0.0;
+
+ switch (embd_norm) {
+ case -1: // no normalisation
+ sum = 1.0;
+ break;
+ case 0: // max absolute
+ for (int i = 0; i < n; i++) {
+ if (sum < std::abs(inp[i])) sum = std::abs(inp[i]);
+ }
+ sum /= 32760.0; // make an int16 range
+ break;
+ case 2: // euclidean
+ for (int i = 0; i < n; i++) {
+ sum += inp[i] * inp[i];
+ }
+ sum = std::sqrt(sum);
+ break;
+ default: // p-norm (euclidean is p-norm p=2)
+ for (int i = 0; i < n; i++) {
+ sum += std::pow(std::abs(inp[i]), embd_norm);
+ }
+ sum = std::pow(sum, 1.0 / embd_norm);
+ break;
+ }
+
+ const float norm = sum > 0.0 ? 1.0 / sum : 0.0f;
+
+ for (int i = 0; i < n; i++) {
+ out[i] = inp[i] * norm;
+ }
+}
+
+float llama_embd_similarity_cos(const float * embd1, const float * embd2, int n){
+ double sum = 0.0;
+ double sum1 = 0.0;
+ double sum2 = 0.0;
+
+ for (int i = 0; i < n; i++) {
+ sum += embd1[i] * embd2[i];
+ sum1 += embd1[i] * embd1[i];
+ sum2 += embd2[i] * embd2[i];
+ }
+
+ // Handle the case where one or both vectors are zero vectors
+ if (sum1 == 0.0 || sum2 == 0.0) {
+ if (sum1 == 0.0 && sum2 == 0.0) {
+ return 1.0f; // two zero vectors are similar
+ }
+ return 0.0f;
+ }
+
+ return sum / (sqrt(sum1) * sqrt(sum2));
+}
+
+//
+// Control vector utils
+//
+
+static llama_control_vector_data llama_control_vector_load_one(const llama_control_vector_load_info & load_info) {
+ llama_control_vector_data result = { -1, {} };
+
+ ggml_context * ctx = nullptr;
+ struct gguf_init_params meta_gguf_params = {
+ /* .no_alloc = */ false,
+ /* .ctx = */ &ctx,
+ };
+ struct gguf_context * ctx_gguf = gguf_init_from_file(load_info.fname.c_str(), meta_gguf_params);
+ if (!ctx_gguf) {
+ fprintf(stderr, "%s: failed to load control vector file from %s\n", __func__, load_info.fname.c_str());
+ return result;
+ }
+
+ int32_t n_tensors = gguf_get_n_tensors(ctx_gguf);
+ if (n_tensors == 0) {
+ fprintf(stderr, "%s: no direction tensors found in %s\n", __func__, load_info.fname.c_str());
+ }
+
+ for (int i = 0; i < n_tensors; i++) {
+ std::string name = gguf_get_tensor_name(ctx_gguf, i);
+
+ int layer_idx = -1;
+
+ // split on '.'
+ size_t dotpos = name.find('.');
+ if (dotpos != std::string::npos && name.substr(0, dotpos) == "direction") {
+ try {
+ layer_idx = std::stoi(name.substr(dotpos + 1));
+ } catch (...) {
+ layer_idx = -1;
+ }
+ }
+ if (layer_idx < 0) {
+ fprintf(stderr, "%s: invalid/unparsable direction tensor layer index in %s\n", __func__, load_info.fname.c_str());
+ result.n_embd = -1;
+ break;
+ } else if (layer_idx == 0) {
+ fprintf(stderr, "%s: invalid (zero) direction tensor layer index in %s\n", __func__, load_info.fname.c_str());
+ result.n_embd = -1;
+ break;
+ }
+
+ struct ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str());
+ if (tensor->type != GGML_TYPE_F32) {
+ fprintf(stderr, "%s: invalid (non-F32) direction tensor type in %s\n", __func__, load_info.fname.c_str());
+ result.n_embd = -1;
+ break;
+ }
+ if (ggml_n_dims(tensor) != 1) {
+ fprintf(stderr, "%s: invalid (non-1D) direction tensor shape in %s\n", __func__, load_info.fname.c_str());
+ result.n_embd = -1;
+ break;
+ }
+
+ if (result.n_embd == -1) {
+ result.n_embd = ggml_nelements(tensor);
+ } else if (ggml_nelements(tensor) != result.n_embd) {
+ fprintf(stderr, "%s: direction tensor in %s does not match previous dimensions\n", __func__, load_info.fname.c_str());
+ result.n_embd = -1;
+ break;
+ }
+
+ // extend if necessary - do not store data for layer 0 (it's not used)
+ result.data.resize(std::max(result.data.size(), static_cast<size_t>(result.n_embd * layer_idx)), 0.0f);
+
+ const float * src = (const float *) tensor->data;
+ float * dst = result.data.data() + result.n_embd * (layer_idx - 1); // layer 1 at [0]
+ for (int j = 0; j < result.n_embd; j++) {
+ dst[j] += src[j] * load_info.strength; // allows multiple directions for same layer in same file
+ }
+
+ }
+
+ if (result.n_embd == -1) {
+ fprintf(stderr, "%s: skipping %s due to invalid direction tensors\n", __func__, load_info.fname.c_str());
+ result.data.clear();
+ }
+
+ gguf_free(ctx_gguf);
+ ggml_free(ctx);
+
+ return result;
+}
+
+llama_control_vector_data llama_control_vector_load(const std::vector<llama_control_vector_load_info> & load_infos) {
+ llama_control_vector_data result = { -1, {} };
+
+ for (const auto & info : load_infos) {
+ auto cur = llama_control_vector_load_one(info);
+
+ if (cur.n_embd == -1) {
+ result.n_embd = -1;
+ break;
+ }
+ if (result.n_embd != -1 && result.n_embd != cur.n_embd) {
+ fprintf(stderr, "%s: control vectors in %s does not match previous dimensions\n", __func__, info.fname.c_str());
+ result.n_embd = -1;
+ break;
+ }
+
+ if (result.n_embd == -1) {
+ result = std::move(cur);
+ } else {
+ result.data.resize(std::max(result.data.size(), cur.data.size()), 0.0f); // extend if necessary
+ for (size_t i = 0; i < cur.data.size(); i++) {
+ result.data[i] += cur.data[i];
+ }
+ }
+ }
+
+ if (result.n_embd == -1) {
+ fprintf(stderr, "%s: no valid control vector files passed\n", __func__);
+ result.data.clear();
+ }
+
+ return result;
+}
+
+//
+// YAML utils
+//
+
+void yaml_dump_vector_float(FILE * stream, const char * prop_name, const std::vector<float> & data) {
+ if (data.empty()) {
+ fprintf(stream, "%s:\n", prop_name);
+ return;
+ }
+
+ fprintf(stream, "%s: [", prop_name);
+ for (size_t i = 0; i < data.size() - 1; ++i) {
+ fprintf(stream, "%e, ", data[i]);
+ }
+ fprintf(stream, "%e]\n", data.back());
+}
+
+void yaml_dump_vector_int(FILE * stream, const char * prop_name, const std::vector<int> & data) {
+ if (data.empty()) {
+ fprintf(stream, "%s:\n", prop_name);
+ return;
+ }
+
+ fprintf(stream, "%s: [", prop_name);
+ for (size_t i = 0; i < data.size() - 1; ++i) {
+ fprintf(stream, "%d, ", data[i]);
+ }
+ fprintf(stream, "%d]\n", data.back());
+}
+
+void yaml_dump_string_multiline(FILE * stream, const char * prop_name, const char * data) {
+ std::string data_str(data == NULL ? "" : data);
+
+ if (data_str.empty()) {
+ fprintf(stream, "%s:\n", prop_name);
+ return;
+ }
+
+ size_t pos_start = 0;
+ size_t pos_found = 0;
+
+ if (std::isspace(data_str[0]) || std::isspace(data_str.back())) {
+ data_str = std::regex_replace(data_str, std::regex("\n"), "\\n");
+ data_str = std::regex_replace(data_str, std::regex("\""), "\\\"");
+ data_str = std::regex_replace(data_str, std::regex(R"(\\[^n"])"), R"(\$&)");
+ data_str = "\"" + data_str + "\"";
+ fprintf(stream, "%s: %s\n", prop_name, data_str.c_str());
+ return;
+ }
+
+ if (data_str.find('\n') == std::string::npos) {
+ fprintf(stream, "%s: %s\n", prop_name, data_str.c_str());
+ return;
+ }
+
+ fprintf(stream, "%s: |\n", prop_name);
+ while ((pos_found = data_str.find('\n', pos_start)) != std::string::npos) {
+ fprintf(stream, " %s\n", data_str.substr(pos_start, pos_found-pos_start).c_str());
+ pos_start = pos_found + 1;
+ }
+}
+
+void yaml_dump_non_result_info(FILE * stream, const gpt_params & params, const llama_context * lctx,
+ const std::string & timestamp, const std::vector<int> & prompt_tokens, const char * model_desc) {
+ const llama_sampling_params & sparams = params.sparams;
+
+ fprintf(stream, "build_commit: %s\n", LLAMA_COMMIT);
+ fprintf(stream, "build_number: %d\n", LLAMA_BUILD_NUMBER);
+ fprintf(stream, "cpu_has_arm_fma: %s\n", ggml_cpu_has_arm_fma() ? "true" : "false");
+ fprintf(stream, "cpu_has_avx: %s\n", ggml_cpu_has_avx() ? "true" : "false");
+ fprintf(stream, "cpu_has_avx_vnni: %s\n", ggml_cpu_has_avx_vnni() ? "true" : "false");
+ fprintf(stream, "cpu_has_avx2: %s\n", ggml_cpu_has_avx2() ? "true" : "false");
+ fprintf(stream, "cpu_has_avx512: %s\n", ggml_cpu_has_avx512() ? "true" : "false");
+ fprintf(stream, "cpu_has_avx512_vbmi: %s\n", ggml_cpu_has_avx512_vbmi() ? "true" : "false");
+ fprintf(stream, "cpu_has_avx512_vnni: %s\n", ggml_cpu_has_avx512_vnni() ? "true" : "false");
+ fprintf(stream, "cpu_has_cuda: %s\n", ggml_cpu_has_cuda() ? "true" : "false");
+ fprintf(stream, "cpu_has_vulkan: %s\n", ggml_cpu_has_vulkan() ? "true" : "false");
+ fprintf(stream, "cpu_has_kompute: %s\n", ggml_cpu_has_kompute() ? "true" : "false");
+ fprintf(stream, "cpu_has_fma: %s\n", ggml_cpu_has_fma() ? "true" : "false");
+ fprintf(stream, "cpu_has_gpublas: %s\n", ggml_cpu_has_gpublas() ? "true" : "false");
+ fprintf(stream, "cpu_has_neon: %s\n", ggml_cpu_has_neon() ? "true" : "false");
+ fprintf(stream, "cpu_has_sve: %s\n", ggml_cpu_has_sve() ? "true" : "false");
+ fprintf(stream, "cpu_has_f16c: %s\n", ggml_cpu_has_f16c() ? "true" : "false");
+ fprintf(stream, "cpu_has_fp16_va: %s\n", ggml_cpu_has_fp16_va() ? "true" : "false");
+ fprintf(stream, "cpu_has_wasm_simd: %s\n", ggml_cpu_has_wasm_simd() ? "true" : "false");
+ fprintf(stream, "cpu_has_blas: %s\n", ggml_cpu_has_blas() ? "true" : "false");
+ fprintf(stream, "cpu_has_sse3: %s\n", ggml_cpu_has_sse3() ? "true" : "false");
+ fprintf(stream, "cpu_has_vsx: %s\n", ggml_cpu_has_vsx() ? "true" : "false");
+ fprintf(stream, "cpu_has_matmul_int8: %s\n", ggml_cpu_has_matmul_int8() ? "true" : "false");
+
+#ifdef NDEBUG
+ fprintf(stream, "debug: false\n");
+#else
+ fprintf(stream, "debug: true\n");
+#endif // NDEBUG
+
+ fprintf(stream, "model_desc: %s\n", model_desc);
+ fprintf(stream, "n_vocab: %d # output size of the final layer, 32001 for some models\n", llama_n_vocab(llama_get_model(lctx)));
+
+#ifdef __OPTIMIZE__
+ fprintf(stream, "optimize: true\n");
+#else
+ fprintf(stream, "optimize: false\n");
+#endif // __OPTIMIZE__
+
+ fprintf(stream, "time: %s\n", timestamp.c_str());
+
+ fprintf(stream, "\n");
+ fprintf(stream, "###############\n");
+ fprintf(stream, "# User Inputs #\n");
+ fprintf(stream, "###############\n");
+ fprintf(stream, "\n");
+
+ fprintf(stream, "alias: %s # default: unknown\n", params.model_alias.c_str());
+ fprintf(stream, "batch_size: %d # default: 512\n", params.n_batch);
+ yaml_dump_string_multiline(stream, "cfg_negative_prompt", sparams.cfg_negative_prompt.c_str());
+ fprintf(stream, "cfg_scale: %f # default: 1.0\n", sparams.cfg_scale);
+ fprintf(stream, "chunks: %d # default: -1 (unlimited)\n", params.n_chunks);
+ fprintf(stream, "color: %s # default: false\n", params.use_color ? "true" : "false");
+ fprintf(stream, "ctx_size: %d # default: 512\n", params.n_ctx);
+ fprintf(stream, "escape: %s # default: false\n", params.escape ? "true" : "false");
+ fprintf(stream, "file: # never logged, see prompt instead. Can still be specified for input.\n");
+ fprintf(stream, "frequency_penalty: %f # default: 0.0 \n", sparams.penalty_freq);
+ yaml_dump_string_multiline(stream, "grammar", sparams.grammar.c_str());
+ fprintf(stream, "grammar-file: # never logged, see grammar instead. Can still be specified for input.\n");
+ fprintf(stream, "hellaswag: %s # default: false\n", params.hellaswag ? "true" : "false");
+ fprintf(stream, "hellaswag_tasks: %zu # default: 400\n", params.hellaswag_tasks);
+
+ const auto logit_bias_eos = sparams.logit_bias.find(llama_token_eos(llama_get_model(lctx)));
+ const bool ignore_eos = logit_bias_eos != sparams.logit_bias.end() && logit_bias_eos->second == -INFINITY;
+ fprintf(stream, "ignore_eos: %s # default: false\n", ignore_eos ? "true" : "false");
+
+ yaml_dump_string_multiline(stream, "in_prefix", params.input_prefix.c_str());
+ fprintf(stream, "in_prefix_bos: %s # default: false\n", params.input_prefix_bos ? "true" : "false");
+ yaml_dump_string_multiline(stream, "in_suffix", params.input_prefix.c_str());
+ fprintf(stream, "interactive: %s # default: false\n", params.interactive ? "true" : "false");
+ fprintf(stream, "interactive_first: %s # default: false\n", params.interactive_first ? "true" : "false");
+ fprintf(stream, "keep: %d # default: 0\n", params.n_keep);
+ fprintf(stream, "logdir: %s # default: unset (no logging)\n", params.logdir.c_str());
+
+ fprintf(stream, "logit_bias:\n");
+ for (std::pair<llama_token, float> lb : sparams.logit_bias) {
+ if (ignore_eos && lb.first == logit_bias_eos->first) {
+ continue;
+ }
+ fprintf(stream, " %d: %f", lb.first, lb.second);
+ }
+
+ fprintf(stream, "lora:\n");
+ for (auto & la : params.lora_adapters) {
+ if (la.scale == 1.0f) {
+ fprintf(stream, " - %s\n", la.path.c_str());
+ }
+ }
+ fprintf(stream, "lora_scaled:\n");
+ for (auto & la : params.lora_adapters) {
+ if (la.scale != 1.0f) {
+ fprintf(stream, " - %s: %f\n", la.path.c_str(), la.scale);
+ }
+ }
+ fprintf(stream, "lora_init_without_apply: %s # default: false\n", params.lora_init_without_apply ? "true" : "false");
+ fprintf(stream, "main_gpu: %d # default: 0\n", params.main_gpu);
+ fprintf(stream, "min_keep: %d # default: 0 (disabled)\n", sparams.min_keep);
+ fprintf(stream, "mirostat: %d # default: 0 (disabled)\n", sparams.mirostat);
+ fprintf(stream, "mirostat_ent: %f # default: 5.0\n", sparams.mirostat_tau);
+ fprintf(stream, "mirostat_lr: %f # default: 0.1\n", sparams.mirostat_eta);
+ fprintf(stream, "mlock: %s # default: false\n", params.use_mlock ? "true" : "false");
+ fprintf(stream, "model: %s # default: %s\n", params.model.c_str(), DEFAULT_MODEL_PATH);
+ fprintf(stream, "model_draft: %s # default:\n", params.model_draft.c_str());
+ fprintf(stream, "multiline_input: %s # default: false\n", params.multiline_input ? "true" : "false");
+ fprintf(stream, "n_gpu_layers: %d # default: -1\n", params.n_gpu_layers);
+ fprintf(stream, "n_predict: %d # default: -1 (unlimited)\n", params.n_predict);
+ fprintf(stream, "n_probs: %d # only used by server binary, default: 0\n", sparams.n_probs);
+ fprintf(stream, "no_mmap: %s # default: false\n", !params.use_mmap ? "true" : "false");
+ fprintf(stream, "penalize_nl: %s # default: false\n", sparams.penalize_nl ? "true" : "false");
+ fprintf(stream, "ppl_output_type: %d # default: 0\n", params.ppl_output_type);
+ fprintf(stream, "ppl_stride: %d # default: 0\n", params.ppl_stride);
+ fprintf(stream, "presence_penalty: %f # default: 0.0\n", sparams.penalty_present);
+ yaml_dump_string_multiline(stream, "prompt", params.prompt.c_str());
+ fprintf(stream, "prompt_cache: %s\n", params.path_prompt_cache.c_str());
+ fprintf(stream, "prompt_cache_all: %s # default: false\n", params.prompt_cache_all ? "true" : "false");
+ fprintf(stream, "prompt_cache_ro: %s # default: false\n", params.prompt_cache_ro ? "true" : "false");
+ yaml_dump_vector_int(stream, "prompt_tokens", prompt_tokens);
+ fprintf(stream, "repeat_penalty: %f # default: 1.1\n", sparams.penalty_repeat);
+
+ fprintf(stream, "reverse_prompt:\n");
+ for (std::string ap : params.antiprompt) {
+ size_t pos = 0;
+ while ((pos = ap.find('\n', pos)) != std::string::npos) {
+ ap.replace(pos, 1, "\\n");
+ pos += 1;
+ }
+
+ fprintf(stream, " - %s\n", ap.c_str());
+ }
+
+ fprintf(stream, "rope_freq_base: %f # default: 10000.0\n", params.rope_freq_base);
+ fprintf(stream, "rope_freq_scale: %f # default: 1.0\n", params.rope_freq_scale);
+ fprintf(stream, "seed: %u # default: -1 (random seed)\n", params.seed);
+ fprintf(stream, "simple_io: %s # default: false\n", params.simple_io ? "true" : "false");
+ fprintf(stream, "cont_batching: %s # default: false\n", params.cont_batching ? "true" : "false");
+ fprintf(stream, "flash_attn: %s # default: false\n", params.flash_attn ? "true" : "false");
+ fprintf(stream, "temp: %f # default: 0.8\n", sparams.temp);
+
+ const std::vector<float> tensor_split_vector(params.tensor_split, params.tensor_split + llama_max_devices());
+ yaml_dump_vector_float(stream, "tensor_split", tensor_split_vector);
+
+ fprintf(stream, "tfs: %f # default: 1.0\n", sparams.tfs_z);
+ fprintf(stream, "threads: %d # default: %u\n", params.cpuparams.n_threads, std::thread::hardware_concurrency());
+ fprintf(stream, "top_k: %d # default: 40\n", sparams.top_k);
+ fprintf(stream, "top_p: %f # default: 0.95\n", sparams.top_p);
+ fprintf(stream, "min_p: %f # default: 0.0\n", sparams.min_p);
+ fprintf(stream, "typical_p: %f # default: 1.0\n", sparams.typical_p);
+ fprintf(stream, "verbose_prompt: %s # default: false\n", params.verbose_prompt ? "true" : "false");
+ fprintf(stream, "display_prompt: %s # default: true\n", params.display_prompt ? "true" : "false");
+}
diff --git a/llama/common.h b/llama/common.h
new file mode 100644
index 00000000..6ac94243
--- /dev/null
+++ b/llama/common.h
@@ -0,0 +1,514 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// Various helper functions and utilities
+
+#pragma once
+
+#include "llama.h"
+
+#include "sampling.h"
+
+#define LOG_NO_FILE_LINE_FUNCTION
+#include "log.h"
+
+#include <cmath>
+#include <string>
+#include <vector>
+#include <random>
+#include <thread>
+#include <unordered_map>
+#include <tuple>
+
+#ifdef _WIN32
+#define DIRECTORY_SEPARATOR '\\'
+#else
+#define DIRECTORY_SEPARATOR '/'
+#endif // _WIN32
+
+#define die(msg) do { fputs("error: " msg "\n", stderr); exit(1); } while (0)
+#define die_fmt(fmt, ...) do { fprintf(stderr, "error: " fmt "\n", __VA_ARGS__); exit(1); } while (0)
+
+#define print_build_info() do { \
+ fprintf(stderr, "%s: build = %d (%s)\n", __func__, LLAMA_BUILD_NUMBER, LLAMA_COMMIT); \
+ fprintf(stderr, "%s: built with %s for %s\n", __func__, LLAMA_COMPILER, LLAMA_BUILD_TARGET); \
+} while(0)
+
+#define DEFAULT_MODEL_PATH "models/7B/ggml-model-f16.gguf"
+
+struct llama_lora_adapter_info {
+ std::string path;
+ float scale;
+};
+
+struct llama_lora_adapter_container : llama_lora_adapter_info {
+ struct llama_lora_adapter * adapter;
+};
+
+// build info
+extern int LLAMA_BUILD_NUMBER;
+extern char const * LLAMA_COMMIT;
+extern char const * LLAMA_COMPILER;
+extern char const * LLAMA_BUILD_TARGET;
+
+struct llama_control_vector_load_info;
+
+//
+// CPU utils
+//
+
+int32_t cpu_get_num_physical_cores();
+int32_t cpu_get_num_math();
+
+//
+// CLI argument parsing
+//
+
+// dimensionality reduction methods, used by cvector-generator
+enum dimre_method {
+ DIMRE_METHOD_PCA,
+ DIMRE_METHOD_MEAN,
+};
+
+struct cpu_params {
+ int n_threads = -1;
+ bool cpumask[GGML_MAX_N_THREADS] = {false}; // CPU affinity mask.
+ bool mask_valid = false; // Default: any CPU
+ enum ggml_sched_priority priority = GGML_SCHED_PRIO_NORMAL; // Scheduling prio : (0 - normal, 1 - medium, 2 - high, 3 - realtime)
+ bool strict_cpu = false; // Use strict CPU placement
+ uint32_t poll = 50; // Polling (busywait) level (0 - no polling, 100 - mostly polling)
+};
+
+struct gpt_params {
+ uint32_t seed = LLAMA_DEFAULT_SEED; // RNG seed
+
+ int32_t n_predict = -1; // new tokens to predict
+ int32_t n_ctx = 0; // context size
+ int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
+ int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS)
+ int32_t n_keep = 0; // number of tokens to keep from initial prompt
+ int32_t n_draft = 5; // number of tokens to draft during speculative decoding
+ int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited)
+ int32_t n_parallel = 1; // number of parallel sequences to decode
+ int32_t n_sequences = 1; // number of sequences to decode
+ float p_split = 0.1f; // speculative decoding split probability
+ int32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default)
+ int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default)
+ int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors
+ float tensor_split[128] = {0}; // how split tensors should be distributed across GPUs
+ int32_t grp_attn_n = 1; // group-attention factor
+ int32_t grp_attn_w = 512; // group-attention width
+ int32_t n_print = -1; // print token count every n tokens (-1 = disabled)
+ float rope_freq_base = 0.0f; // RoPE base frequency
+ float rope_freq_scale = 0.0f; // RoPE frequency scaling factor
+ float yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor
+ float yarn_attn_factor = 1.0f; // YaRN magnitude scaling factor
+ float yarn_beta_fast = 32.0f; // YaRN low correction dim
+ float yarn_beta_slow = 1.0f; // YaRN high correction dim
+ int32_t yarn_orig_ctx = 0; // YaRN original context length
+ float defrag_thold = -1.0f; // KV cache defragmentation threshold
+
+ struct cpu_params cpuparams;
+ struct cpu_params cpuparams_batch;
+ struct cpu_params draft_cpuparams;
+ struct cpu_params draft_cpuparams_batch;
+
+ ggml_backend_sched_eval_callback cb_eval = nullptr;
+ void * cb_eval_user_data = nullptr;
+
+ ggml_numa_strategy numa = GGML_NUMA_STRATEGY_DISABLED;
+
+ enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
+ enum llama_rope_scaling_type rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;
+ enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED; // pooling type for embeddings
+ enum llama_attention_type attention_type = LLAMA_ATTENTION_TYPE_UNSPECIFIED; // attention type for embeddings
+
+ // // sampling parameters
+ struct llama_sampling_params sparams;
+
+ std::string model = ""; // model path
+ std::string model_draft = ""; // draft model for speculative decoding
+ std::string model_alias = "unknown"; // model alias
+ std::string model_url = ""; // model url to download
+ std::string hf_token = ""; // HF token
+ std::string hf_repo = ""; // HF repo
+ std::string hf_file = ""; // HF file
+ std::string prompt = "";
+ std::string prompt_file = ""; // store the external prompt file name
+ std::string path_prompt_cache = ""; // path to file for saving/loading prompt eval state
+ std::string input_prefix = ""; // string to prefix user inputs with
+ std::string input_suffix = ""; // string to suffix user inputs with
+ std::string logdir = ""; // directory in which to save YAML log files
+ std::string lookup_cache_static = ""; // path of static ngram cache file for lookup decoding
+ std::string lookup_cache_dynamic = ""; // path of dynamic ngram cache file for lookup decoding
+ std::string logits_file = ""; // file for saving *all* logits
+ std::string rpc_servers = ""; // comma separated list of RPC servers
+
+ std::vector<std::string> in_files; // all input files
+ std::vector<std::string> antiprompt; // strings upon which more user input is prompted (a.k.a. reverse prompts)
+ std::vector<llama_model_kv_override> kv_overrides;
+
+ bool lora_init_without_apply = false; // only load lora to memory, but do not apply it to ctx (user can manually apply lora later using llama_lora_adapter_apply)
+ std::vector<llama_lora_adapter_info> lora_adapters; // lora adapter path with user defined scale
+
+ std::vector<llama_control_vector_load_info> control_vectors; // control vector with user defined scale
+
+ int32_t verbosity = 0;
+ int32_t control_vector_layer_start = -1; // layer range for control vector
+ int32_t control_vector_layer_end = -1; // layer range for control vector
+
+ int32_t ppl_stride = 0; // stride for perplexity calculations. If left at 0, the pre-existing approach will be used.
+ int32_t ppl_output_type = 0; // = 0 -> ppl output is as usual, = 1 -> ppl output is num_tokens, ppl, one per line
+ // (which is more convenient to use for plotting)
+ //
+ bool hellaswag = false; // compute HellaSwag score over random tasks from datafile supplied in prompt
+ size_t hellaswag_tasks = 400; // number of tasks to use when computing the HellaSwag score
+
+ bool winogrande = false; // compute Winogrande score over random tasks from datafile supplied in prompt
+ size_t winogrande_tasks = 0; // number of tasks to use when computing the Winogrande score. If 0, all tasks will be computed
+
+ bool multiple_choice = false; // compute TruthfulQA score over random tasks from datafile supplied in prompt
+ size_t multiple_choice_tasks = 0; // number of tasks to use when computing the TruthfulQA score. If 0, all tasks will be computed
+
+ bool kl_divergence = false; // compute KL divergence
+
+ bool usage = false; // print usage
+ bool use_color = false; // use color to distinguish generations and inputs
+ bool special = false; // enable special token output
+ bool interactive = false; // interactive mode
+ bool interactive_first = false; // wait for user input immediately
+ bool conversation = false; // conversation mode (does not print special tokens and suffix/prefix)
+ bool prompt_cache_all = false; // save user input and generations to prompt cache
+ bool prompt_cache_ro = false; // open the prompt cache read-only and do not update it
+
+ bool escape = true; // escape "\n", "\r", "\t", "\'", "\"", and "\\"
+ bool multiline_input = false; // reverse the usage of `\`
+ bool simple_io = false; // improves compatibility with subprocesses and limited consoles
+ bool cont_batching = true; // insert new sequences for decoding on-the-fly
+ bool flash_attn = false; // flash attention
+
+ bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix
+ bool ignore_eos = false; // ignore generated EOS tokens
+ bool logits_all = false; // return logits for all tokens in the batch
+ bool use_mmap = true; // use mmap for faster loads
+ bool use_mlock = false; // use mlock to keep model in memory
+ bool verbose_prompt = false; // print prompt tokens before generation
+ bool display_prompt = true; // print prompt before generation
+ bool infill = false; // use infill mode
+ bool dump_kv_cache = false; // dump the KV cache contents for debugging purposes
+ bool no_kv_offload = false; // disable KV offloading
+ bool warmup = true; // warmup run
+ bool check_tensors = false; // validate tensor data
+
+ std::string cache_type_k = "f16"; // KV cache data type for the K
+ std::string cache_type_v = "f16"; // KV cache data type for the V
+
+ // multimodal models (see examples/llava)
+ std::string mmproj = ""; // path to multimodal projector
+ std::vector<std::string> image; // path to image file(s)
+
+ // embedding
+ bool embedding = false; // get only sentence embedding
+ int32_t embd_normalize = 2; // normalisation for embendings (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)
+ std::string embd_out = ""; // empty = default, "array" = [[],[]...], "json" = openai style, "json+" = same "json" + cosine similarity matrix
+ std::string embd_sep = "\n"; // separator of embendings
+
+ // server params
+ int32_t port = 8080; // server listens on this network port
+ int32_t timeout_read = 600; // http read timeout in seconds
+ int32_t timeout_write = timeout_read; // http write timeout in seconds
+ int n_threads_http = -1; // number of threads to process HTTP requests (TODO: support threadpool)
+
+ std::string hostname = "127.0.0.1";
+ std::string public_path = "";
+ std::string chat_template = "";
+ std::string system_prompt = "";
+ bool enable_chat_template = true;
+
+ std::vector<std::string> api_keys;
+
+ std::string ssl_file_key = "";
+ std::string ssl_file_cert = "";
+
+ bool endpoint_slots = true;
+ bool endpoint_metrics = false;
+
+ bool log_json = false;
+
+ std::string slot_save_path;
+
+ float slot_prompt_similarity = 0.5f;
+
+ // batched-bench params
+ bool is_pp_shared = false;
+
+ std::vector<int32_t> n_pp;
+ std::vector<int32_t> n_tg;
+ std::vector<int32_t> n_pl;
+
+ // retrieval params
+ std::vector<std::string> context_files; // context files to embed
+
+ int32_t chunk_size = 64; // chunk size for context embedding
+
+ std::string chunk_separator = "\n"; // chunk separator for context embedding
+
+ // passkey params
+ int32_t n_junk = 250; // number of times to repeat the junk text
+ int32_t i_pos = -1; // position of the passkey in the junk text
+
+ // imatrix params
+ std::string out_file = "imatrix.dat"; // save the resulting imatrix to this file
+
+ int32_t n_out_freq = 10; // output the imatrix every n_out_freq iterations
+ int32_t n_save_freq = 0; // save the imatrix every n_save_freq iterations
+ int32_t i_chunk = 0; // start processing from this chunk
+
+ bool process_output = false; // collect data for the output tensor
+ bool compute_ppl = true; // whether to compute perplexity
+
+ // cvector-generator params
+ int n_pca_batch = 100;
+ int n_pca_iterations = 1000;
+ dimre_method cvector_dimre_method = DIMRE_METHOD_PCA;
+ std::string cvector_outfile = "control_vector.gguf";
+ std::string cvector_positive_file = "examples/cvector-generator/positive.txt";
+ std::string cvector_negative_file = "examples/cvector-generator/negative.txt";
+
+ bool spm_infill = false; // suffix/prefix/middle pattern for infill
+
+ std::string lora_outfile = "ggml-lora-merged-f16.gguf";
+};
+
+void gpt_params_parse_from_env(gpt_params & params);
+void gpt_params_handle_model_default(gpt_params & params);
+
+bool gpt_params_parse_ex (int argc, char ** argv, gpt_params & params);
+bool gpt_params_parse (int argc, char ** argv, gpt_params & params);
+bool gpt_params_find_arg (int argc, char ** argv, const std::string & arg, gpt_params & params, int & i, bool & invalid_param);
+void gpt_params_print_usage(int argc, char ** argv, const gpt_params & params);
+
+std::string gpt_params_get_system_info(const gpt_params & params);
+
+bool parse_cpu_range(const std::string& range, bool(&boolmask)[GGML_MAX_N_THREADS]);
+bool parse_cpu_mask(const std::string& mask, bool(&boolmask)[GGML_MAX_N_THREADS]);
+void postprocess_cpu_params(cpu_params& cpuparams, const cpu_params* role_model = nullptr);
+bool set_process_priority(enum ggml_sched_priority prio);
+
+//
+// String utils
+//
+
+std::vector<std::string> string_split(std::string input, char separator);
+
+std::string string_strip(const std::string & str);
+std::string string_get_sortable_timestamp();
+
+void string_replace_all(std::string & s, const std::string & search, const std::string & replace);
+
+template<class T>
+static std::vector<T> string_split(const std::string & str, char delim) {
+ std::vector<T> values;
+ std::istringstream str_stream(str);
+ std::string token;
+ while (std::getline(str_stream, token, delim)) {
+ T value;
+ std::istringstream token_stream(token);
+ token_stream >> value;
+ values.push_back(value);
+ }
+ return values;
+}
+
+bool string_parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides);
+void string_process_escapes(std::string & input);
+
+//
+// Filesystem utils
+//
+
+bool fs_validate_filename(const std::string & filename);
+bool fs_create_directory_with_parents(const std::string & path);
+
+std::string fs_get_cache_directory();
+std::string fs_get_cache_file(const std::string & filename);
+
+//
+// Model utils
+//
+
+struct llama_init_result {
+ struct llama_model * model = nullptr;
+ struct llama_context * context = nullptr;
+ std::vector<llama_lora_adapter_container> lora_adapters;
+};
+
+struct llama_init_result llama_init_from_gpt_params(gpt_params & params);
+
+struct llama_model_params llama_model_params_from_gpt_params (const gpt_params & params);
+struct llama_context_params llama_context_params_from_gpt_params (const gpt_params & params);
+struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const cpu_params & params);
+
+struct llama_model * llama_load_model_from_url(const char * model_url, const char * path_model, const char * hf_token, const struct llama_model_params & params);
+struct llama_model * llama_load_model_from_hf(const char * repo, const char * file, const char * path_model, const char * hf_token, const struct llama_model_params & params);
+
+// clear LoRA adapters from context, then apply new list of adapters
+void llama_lora_adapters_apply(struct llama_context * ctx, std::vector<llama_lora_adapter_container> & lora_adapters);
+
+// Batch utils
+
+void llama_batch_clear(struct llama_batch & batch);
+
+void llama_batch_add(
+ struct llama_batch & batch,
+ llama_token id,
+ llama_pos pos,
+ const std::vector<llama_seq_id> & seq_ids,
+ bool logits);
+
+//
+// Vocab utils
+//
+
+// tokenizes a string into a vector of tokens
+// should work similar to Python's `tokenizer.encode`
+std::vector<llama_token> llama_tokenize(
+ const struct llama_context * ctx,
+ const std::string & text,
+ bool add_special,
+ bool parse_special = false);
+
+std::vector<llama_token> llama_tokenize(
+ const struct llama_model * model,
+ const std::string & text,
+ bool add_special,
+ bool parse_special = false);
+
+// tokenizes a token into a piece, optionally renders special/control tokens
+// should work similar to Python's `tokenizer.id_to_piece`
+std::string llama_token_to_piece(
+ const struct llama_context * ctx,
+ llama_token token,
+ bool special = true);
+
+// detokenizes a vector of tokens into a string
+// should work similar to Python's `tokenizer.decode`
+// optionally renders special/control tokens
+std::string llama_detokenize(
+ llama_context * ctx,
+ const std::vector<llama_token> & tokens,
+ bool special = true);
+
+//
+// Chat template utils
+//
+
+// same with llama_chat_message, but uses std::string
+struct llama_chat_msg {
+ std::string role;
+ std::string content;
+};
+
+// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid
+bool llama_chat_verify_template(const std::string & tmpl);
+
+// CPP wrapper for llama_chat_apply_template
+// If the built-in template is not supported, we default to chatml
+// If the custom "tmpl" is not supported, we throw an error
+std::string llama_chat_apply_template(const struct llama_model * model,
+ const std::string & tmpl,
+ const std::vector<llama_chat_msg> & chat,
+ bool add_ass);
+
+// Format single message, while taking into account the position of that message in chat history
+std::string llama_chat_format_single(const struct llama_model * model,
+ const std::string & tmpl,
+ const std::vector<llama_chat_msg> & past_msg,
+ const llama_chat_msg & new_msg,
+ bool add_ass);
+
+// Returns an example of formatted chat
+std::string llama_chat_format_example(const struct llama_model * model,
+ const std::string & tmpl);
+
+//
+// KV cache utils
+//
+
+// Dump the KV cache view with the number of sequences per cell.
+void llama_kv_cache_dump_view(const llama_kv_cache_view & view, int row_size = 80);
+
+// Dump the KV cache view showing individual sequences in each cell (long output).
+void llama_kv_cache_dump_view_seqs(const llama_kv_cache_view & view, int row_size = 40);
+
+//
+// Embedding utils
+//
+
+void llama_embd_normalize(const float * inp, float * out, int n, int embd_norm = 2);
+
+float llama_embd_similarity_cos(const float * embd1, const float * embd2, int n);
+
+//
+// Control vector utils
+//
+
+struct llama_control_vector_data {
+ int n_embd;
+
+ // stores data for layers [1, n_layer] where n_layer = data.size() / n_embd
+ std::vector<float> data;
+};
+
+struct llama_control_vector_load_info {
+ float strength;
+
+ std::string fname;
+};
+
+// Load control vectors, scale each by strength, and add them together.
+// On error, returns {-1, empty}
+llama_control_vector_data llama_control_vector_load(const std::vector<llama_control_vector_load_info> & load_infos);
+
+//
+// Split utils
+//
+
+static const char * const LLM_KV_SPLIT_NO = "split.no";
+static const char * const LLM_KV_SPLIT_COUNT = "split.count";
+static const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count";
+
+//
+// YAML utils
+//
+
+void yaml_dump_vector_float (FILE * stream, const char * prop_name, const std::vector<float> & data);
+void yaml_dump_vector_int (FILE * stream, const char * prop_name, const std::vector<int> & data);
+void yaml_dump_string_multiline(FILE * stream, const char * prop_name, const char * data);
+
+void yaml_dump_non_result_info(
+ FILE * stream, const gpt_params & params, const llama_context * lctx,
+ const std::string & timestamp, const std::vector<int> & prompt_tokens, const char * model_desc);
diff --git a/llama/ggml-aarch64.c b/llama/ggml-aarch64.c
new file mode 100644
index 00000000..5ffad03c
--- /dev/null
+++ b/llama/ggml-aarch64.c
@@ -0,0 +1,2206 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// SPDX-FileCopyrightText: Copyright 2024 Arm Ltd.
+#define GGML_COMMON_IMPL_C
+#include "ggml-common.h"
+
+#include "ggml-quants.h"
+#include "ggml-impl.h"
+
+#include <math.h>
+#include <string.h>
+#include <assert.h>
+#include <float.h>
+#include <stdlib.h> // for qsort
+#include <stdio.h> // for GGML_ASSERT
+
+#include "ggml-aarch64.h"
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Woverlength-strings"
+#elif defined(_MSC_VER)
+#pragma warning(disable: 4244 4267) // possible loss of data
+#endif
+
+#define UNUSED GGML_UNUSED
+
+// Functions to create the interleaved data layout formats
+
+// interleave 4 block_q4_0s in blocks of blck_size_interleave
+// returns an interleaved block_q4_0x4
+// in the interleaved block_q4_0x4, place deltas for 4 block_q4_0 blocks
+// first, then interleave quants from 4 block_q4_0s in blocks of blck_size_interleave
+//
+// - in : an array of block_q4_0 pointers
+// - blck_size_interleave : the block_q4_0 quants bytes are interleaved in blocks of
+// blck_size_interleave bytes
+// - xor_mask : the mask to convert the nibbles in block_q4_0 quants bytes
+// from bias offset form to pure sign form (this saves subtract
+// operations durin unpacking)
+//
+static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, unsigned int blck_size_interleave, unsigned int xor_mask) {
+ block_q4_0x4 out;
+
+ for (int i = 0; i < 4; i++) {
+ out.d[i] = in[i].d;
+ }
+
+ for (int i = 0; i < QK4_0 * 2; i++) {
+ int src_offset = (i / (4 * blck_size_interleave)) * blck_size_interleave;
+ int src_id = (i % (4 * blck_size_interleave)) / blck_size_interleave;
+ src_offset += (i % blck_size_interleave);
+
+ out.qs[i] = in[src_id].qs[src_offset] ^ xor_mask;
+ }
+
+ return out;
+}
+
+// interleave 8 block_q4_0s in blocks of blck_size_interleave
+// returns an interleaved block_q4_0x8
+// in the interleaved block_q4_0x8, place deltas for 8 block_q4_0 blocks
+// first, then interleave quants from 8 block_q4_0s in blocks of blck_size_interleave
+static block_q4_0x8 make_block_q4_0x8(block_q4_0 * in, unsigned int blck_size_interleave, unsigned int xor_mask) {
+ block_q4_0x8 out;
+
+ for (int i = 0; i < 8; i++) {
+ out.d[i] = in[i].d;
+ }
+
+ for (int i = 0; i < QK4_0 * 4; i++) {
+ int src_offset = (i / (8 * blck_size_interleave)) * blck_size_interleave;
+ int src_id = (i % (8 * blck_size_interleave)) / blck_size_interleave;
+ src_offset += (i % blck_size_interleave);
+
+ out.qs[i] = in[src_id].qs[src_offset] ^ xor_mask;
+ }
+
+ return out;
+}
+
+void quantize_q8_0_4x4(const float * restrict x, void * restrict vy, int64_t k) {
+ assert(QK8_0 == 32);
+ assert(k % QK8_0 == 0);
+ const int nb = k / QK8_0;
+
+ block_q8_0x4 * restrict y = (block_q8_0x4 *) vy;
+
+#if defined(__ARM_NEON)
+ float32x4_t srcv[4][8];
+ float id[4];
+
+ for (int i = 0; i < nb; i++) {
+ float32x4_t asrcv[8];
+ float32x4_t amaxv[8];
+
+ for (int row_iter = 0; row_iter < 4; row_iter++) {
+ for (int j = 0; j < 8; j++) srcv[row_iter][j] = vld1q_f32(x + row_iter * k + i * 32 + 4 * j);
+ for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[row_iter][j]);
+
+ for (int j = 0; j < 4; j++) amaxv[2 * j] = vmaxq_f32(asrcv[2 * j], asrcv[2 * j + 1]);
+ for (int j = 0; j < 2; j++) amaxv[4 * j] = vmaxq_f32(amaxv[4 * j], amaxv[4 * j + 2]);
+ for (int j = 0; j < 1; j++) amaxv[8 * j] = vmaxq_f32(amaxv[8 * j], amaxv[8 * j + 4]);
+
+ const float amax = vmaxvq_f32(amaxv[0]);
+
+ const float d = amax / ((1 << 7) - 1);
+ id[row_iter] = d ? 1.0f / d : 0.0f;
+
+ y[i].d[row_iter] = GGML_FP32_TO_FP16(d);
+ }
+
+ for (int j = 0; j < 8; j++) {
+ float32x4_t v = vmulq_n_f32(srcv[0][j], id[0]);
+ int32x4_t vi = vcvtnq_s32_f32(v);
+ y[i].qs[16 * j + 0] = vgetq_lane_s32(vi, 0);
+ y[i].qs[16 * j + 1] = vgetq_lane_s32(vi, 1);
+ y[i].qs[16 * j + 2] = vgetq_lane_s32(vi, 2);
+ y[i].qs[16 * j + 3] = vgetq_lane_s32(vi, 3);
+
+ v = vmulq_n_f32(srcv[1][j], id[1]);
+ vi = vcvtnq_s32_f32(v);
+ y[i].qs[16 * j + 4] = vgetq_lane_s32(vi, 0);
+ y[i].qs[16 * j + 5] = vgetq_lane_s32(vi, 1);
+ y[i].qs[16 * j + 6] = vgetq_lane_s32(vi, 2);
+ y[i].qs[16 * j + 7] = vgetq_lane_s32(vi, 3);
+
+ v = vmulq_n_f32(srcv[2][j], id[2]);
+ vi = vcvtnq_s32_f32(v);
+ y[i].qs[16 * j + 8] = vgetq_lane_s32(vi, 0);
+ y[i].qs[16 * j + 9] = vgetq_lane_s32(vi, 1);
+ y[i].qs[16 * j + 10] = vgetq_lane_s32(vi, 2);
+ y[i].qs[16 * j + 11] = vgetq_lane_s32(vi, 3);
+
+ v = vmulq_n_f32(srcv[3][j], id[3]);
+ vi = vcvtnq_s32_f32(v);
+ y[i].qs[16 * j + 12] = vgetq_lane_s32(vi, 0);
+ y[i].qs[16 * j + 13] = vgetq_lane_s32(vi, 1);
+ y[i].qs[16 * j + 14] = vgetq_lane_s32(vi, 2);
+ y[i].qs[16 * j + 15] = vgetq_lane_s32(vi, 3);
+ }
+ }
+#else
+ // scalar
+ const int blck_size_interleave = 4;
+ float srcv[4][QK8_0];
+ float id[4];
+
+ for (int i = 0; i < nb; i++) {
+ for (int row_iter = 0; row_iter < 4; row_iter++) {
+ float amax = 0.0f; // absolute max
+
+ for (int j = 0; j < QK8_0; j++) {
+ srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j];
+ amax = MAX(amax, fabsf(srcv[row_iter][j]));
+ }
+
+ const float d = amax / ((1 << 7) - 1);
+ id[row_iter] = d ? 1.0f / d : 0.0f;
+
+ y[i].d[row_iter] = GGML_FP32_TO_FP16(d);
+ }
+
+ for (int j = 0; j < QK8_0 * 4; j++) {
+ int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;
+ int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;
+ src_offset += (j % blck_size_interleave);
+
+ float x0 = srcv[src_id][src_offset] * id[src_id];
+ y[i].qs[j] = roundf(x0);
+ }
+ }
+#endif
+}
+
+void quantize_q8_0_4x8(const float * restrict x, void * restrict vy, int64_t k) {
+ assert(QK8_0 == 32);
+ assert(k % QK8_0 == 0);
+ const int nb = k / QK8_0;
+
+ block_q8_0x4 * restrict y = (block_q8_0x4 *) vy;
+
+#if defined(__ARM_NEON)
+ float32x4_t srcv[4][8];
+ float id[4];
+
+ for (int i = 0; i < nb; i++) {
+ float32x4_t asrcv[8];
+ float32x4_t amaxv[8];
+
+ for (int row_iter = 0; row_iter < 4; row_iter++) {
+ for (int j = 0; j < 8; j++) srcv[row_iter][j] = vld1q_f32(x + row_iter * k + i * 32 + 4 * j);
+ for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[row_iter][j]);
+
+ for (int j = 0; j < 4; j++) amaxv[2 * j] = vmaxq_f32(asrcv[2 * j], asrcv[2 * j + 1]);
+ for (int j = 0; j < 2; j++) amaxv[4 * j] = vmaxq_f32(amaxv[4 * j], amaxv[4 * j + 2]);
+ for (int j = 0; j < 1; j++) amaxv[8 * j] = vmaxq_f32(amaxv[8 * j], amaxv[8 * j + 4]);
+
+ const float amax = vmaxvq_f32(amaxv[0]);
+
+ const float d = amax / ((1 << 7) - 1);
+ id[row_iter] = d ? 1.0f / d : 0.0f;
+
+ y[i].d[row_iter] = GGML_FP32_TO_FP16(d);
+ }
+
+ for (int j = 0; j < 4; j++) {
+ float32x4_t v = vmulq_n_f32(srcv[0][2 * j], id[0]);
+ int32x4_t vi = vcvtnq_s32_f32(v);
+ y[i].qs[32 * j + 0] = vgetq_lane_s32(vi, 0);
+ y[i].qs[32 * j + 1] = vgetq_lane_s32(vi, 1);
+ y[i].qs[32 * j + 2] = vgetq_lane_s32(vi, 2);
+ y[i].qs[32 * j + 3] = vgetq_lane_s32(vi, 3);
+ v = vmulq_n_f32(srcv[0][2 * j + 1], id[0]);
+ vi = vcvtnq_s32_f32(v);
+ y[i].qs[32 * j + 4] = vgetq_lane_s32(vi, 0);
+ y[i].qs[32 * j + 5] = vgetq_lane_s32(vi, 1);
+ y[i].qs[32 * j + 6] = vgetq_lane_s32(vi, 2);
+ y[i].qs[32 * j + 7] = vgetq_lane_s32(vi, 3);
+
+ v = vmulq_n_f32(srcv[1][2 * j], id[1]);
+ vi = vcvtnq_s32_f32(v);
+ y[i].qs[32 * j + 8] = vgetq_lane_s32(vi, 0);
+ y[i].qs[32 * j + 9] = vgetq_lane_s32(vi, 1);
+ y[i].qs[32 * j + 10] = vgetq_lane_s32(vi, 2);
+ y[i].qs[32 * j + 11] = vgetq_lane_s32(vi, 3);
+ v = vmulq_n_f32(srcv[1][2 * j + 1], id[1]);
+ vi = vcvtnq_s32_f32(v);
+ y[i].qs[32 * j + 12] = vgetq_lane_s32(vi, 0);
+ y[i].qs[32 * j + 13] = vgetq_lane_s32(vi, 1);
+ y[i].qs[32 * j + 14] = vgetq_lane_s32(vi, 2);
+ y[i].qs[32 * j + 15] = vgetq_lane_s32(vi, 3);
+
+ v = vmulq_n_f32(srcv[2][2 * j], id[2]);
+ vi = vcvtnq_s32_f32(v);
+ y[i].qs[32 * j + 16] = vgetq_lane_s32(vi, 0);
+ y[i].qs[32 * j + 17] = vgetq_lane_s32(vi, 1);
+ y[i].qs[32 * j + 18] = vgetq_lane_s32(vi, 2);
+ y[i].qs[32 * j + 19] = vgetq_lane_s32(vi, 3);
+ v = vmulq_n_f32(srcv[2][2 * j + 1], id[2]);
+ vi = vcvtnq_s32_f32(v);
+ y[i].qs[32 * j + 20] = vgetq_lane_s32(vi, 0);
+ y[i].qs[32 * j + 21] = vgetq_lane_s32(vi, 1);
+ y[i].qs[32 * j + 22] = vgetq_lane_s32(vi, 2);
+ y[i].qs[32 * j + 23] = vgetq_lane_s32(vi, 3);
+
+ v = vmulq_n_f32(srcv[3][2 * j], id[3]);
+ vi = vcvtnq_s32_f32(v);
+ y[i].qs[32 * j + 24] = vgetq_lane_s32(vi, 0);
+ y[i].qs[32 * j + 25] = vgetq_lane_s32(vi, 1);
+ y[i].qs[32 * j + 26] = vgetq_lane_s32(vi, 2);
+ y[i].qs[32 * j + 27] = vgetq_lane_s32(vi, 3);
+ v = vmulq_n_f32(srcv[3][2 * j + 1], id[3]);
+ vi = vcvtnq_s32_f32(v);
+ y[i].qs[32 * j + 28] = vgetq_lane_s32(vi, 0);
+ y[i].qs[32 * j + 29] = vgetq_lane_s32(vi, 1);
+ y[i].qs[32 * j + 30] = vgetq_lane_s32(vi, 2);
+ y[i].qs[32 * j + 31] = vgetq_lane_s32(vi, 3);
+ }
+ }
+#else
+ // scalar
+ const int blck_size_interleave = 8;
+ float srcv[4][QK8_0];
+ float id[4];
+
+ for (int i = 0; i < nb; i++) {
+ for (int row_iter = 0; row_iter < 4; row_iter++) {
+ float amax = 0.0f; // absolute max
+
+ for (int j = 0; j < QK8_0; j++) {
+ srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j];
+ amax = MAX(amax, fabsf(srcv[row_iter][j]));
+ }
+
+ const float d = amax / ((1 << 7) - 1);
+ id[row_iter] = d ? 1.0f / d : 0.0f;
+
+ y[i].d[row_iter] = GGML_FP32_TO_FP16(d);
+ }
+
+ for (int j = 0; j < QK8_0 * 4; j++) {
+ int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;
+ int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;
+ src_offset += (j % blck_size_interleave);
+
+ float x0 = srcv[src_id][src_offset] * id[src_id];
+ y[i].qs[j] = roundf(x0);
+ }
+ }
+#endif
+}
+
+void quantize_mat_q8_0(const float * restrict x, void * restrict vy, int64_t nrow, int64_t n_per_row, int64_t blck_size_interleave) {
+ assert(nrow == 4);
+ UNUSED(nrow);
+ if (blck_size_interleave == 4) {
+ quantize_q8_0_4x4(x, vy, n_per_row);
+ } else if (blck_size_interleave == 8) {
+ quantize_q8_0_4x8(x, vy, n_per_row);
+ } else {
+ assert(false);
+ }
+}
+
+static size_t quantize_q4_0_nr_bl(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, int nrows_interleaved, int blck_size_interleave) {
+ assert(n_per_row % QK4_0 == 0);
+ const int nb = n_per_row / QK4_0;
+
+ void * out_ptr = NULL;
+ if (nrows_interleaved == 8) {
+ out_ptr = (block_q4_0x8 *) dst;
+ }
+ else if (nrows_interleaved == 4) {
+ out_ptr = (block_q4_0x4 *) dst;
+ }
+ assert(nrows_interleaved <= 8);
+ block_q4_0 dst_tmp[8];
+
+ for (int b = 0; b < (nrow * n_per_row); b += nrows_interleaved * n_per_row) {
+
+ for (int64_t x = 0; x < nb; x++) {
+
+ for (int i = 0; i < nrows_interleaved; i++ ) {
+ quantize_row_q4_0_ref(src + b + i * n_per_row + x * QK4_0, (block_q4_0 *) dst_tmp + i, QK4_0);
+ }
+
+ if (nrows_interleaved == 8) {
+ *(block_q4_0x8 *) out_ptr = make_block_q4_0x8(dst_tmp, blck_size_interleave, 0x88);
+ out_ptr = (block_q4_0x8 *) out_ptr + 1;
+ }
+ else if (nrows_interleaved == 4) {
+ *(block_q4_0x4 *) out_ptr = make_block_q4_0x4(dst_tmp, blck_size_interleave, 0x88);
+ out_ptr = (block_q4_0x4 *) out_ptr + 1;
+ }
+ }
+ }
+
+ return ((nrow * n_per_row) / QK4_0 * sizeof(block_q4_0));
+}
+
+size_t quantize_q4_0_4x4(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ UNUSED(quant_weights);
+ return quantize_q4_0_nr_bl(src, dst, nrow, n_per_row, 4, 4);
+}
+
+size_t quantize_q4_0_4x8(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ UNUSED(quant_weights);
+ return quantize_q4_0_nr_bl(src, dst, nrow, n_per_row, 4, 8);
+}
+
+size_t quantize_q4_0_8x8(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ UNUSED(quant_weights);
+ return quantize_q4_0_nr_bl(src, dst, nrow, n_per_row, 8, 8);
+}
+
+void ggml_gemv_q4_0_4x4_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
+ const int qk = QK8_0;
+ const int nb = n / qk;
+ const int ncols_interleaved = 4;
+ const int blocklen = 4;
+
+ assert (n % qk == 0);
+ assert (nc % ncols_interleaved == 0);
+
+ UNUSED(s);
+ UNUSED(bs);
+ UNUSED(vx);
+ UNUSED(vy);
+ UNUSED(nr);
+ UNUSED(nc);
+ UNUSED(nb);
+ UNUSED(ncols_interleaved);
+ UNUSED(blocklen);
+
+#if defined(__ARM_FEATURE_SVE)
+ if (ggml_sve_cnt_b == QK8_0) {
+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
+ "__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
+ }
+#endif
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
+ GGML_ASSERT(!(ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) &&
+ "__ARM_NEON and __ARM_FEATURE_MATMUL_INT8 defined, use the Q4_0_4_8 quantization format for optimal performance");
+#elif defined(__ARM_NEON) && defined(__aarch64__) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
+ const void * b_ptr = vx;
+ const void * a_ptr = vy;
+ float * res_ptr = s;
+
+ __asm__ __volatile__(
+ "movi v31.16b, #0x4\n"
+ "movi v30.16b, #0xf0\n"
+ "add %x[b_ptr], %x[b_ptr], #0x8\n"
+ "1:" // Column loop
+ "add x22, %x[a_ptr], #0x2\n"
+ "movi v29.16b, #0x0\n"
+ "mov x21, %x[nb]\n"
+ "2:" // Block loop
+ "ldr q28, [%x[b_ptr], #0x0]\n"
+ "ldr q27, [x22, #0x0]\n"
+ "movi v26.4s, #0x0\n"
+ "sub x20, x22, #0x2\n"
+ "ldr q25, [x22, #0x10]\n"
+ "ldr q24, [%x[b_ptr], #0x10]\n"
+ "sub x21, x21, #0x1\n"
+ "add x22, x22, #0x22\n"
+ "ldr q23, [%x[b_ptr], #0x20]\n"
+ "ldr q22, [%x[b_ptr], #0x30]\n"
+ "ld1r { v21.8h }, [x20]\n"
+ "ldr q20, [%x[b_ptr], #-0x8]\n"
+ "sshl v16.16b, v28.16b, v31.16b\n"
+ "and v28.16b, v28.16b, v30.16b\n"
+ "sshl v19.16b, v24.16b, v31.16b\n"
+ "and v24.16b, v24.16b, v30.16b\n"
+ "add %x[b_ptr], %x[b_ptr], #0x48\n"
+ "sshl v18.16b, v23.16b, v31.16b\n"
+ "and v23.16b, v23.16b, v30.16b\n"
+ ".inst 0x4f9be21a // sdot v26.4s, v16.16b, v27.4b[0]\n"
+ "sshl v17.16b, v22.16b, v31.16b\n"
+ "and v22.16b, v22.16b, v30.16b\n"
+ "fcvtl v21.4s, v21.4h\n"
+ "fcvtl v16.4s, v20.4h\n"
+ ".inst 0x4f99e39a // sdot v26.4s, v28.16b, v25.4b[0]\n"
+ "fmul v16.4s, v16.4s, v21.4s\n"
+ ".inst 0x4fbbe27a // sdot v26.4s, v19.16b, v27.4b[1]\n"
+ ".inst 0x4fb9e31a // sdot v26.4s, v24.16b, v25.4b[1]\n"
+ ".inst 0x4f9bea5a // sdot v26.4s, v18.16b, v27.4b[2]\n"
+ ".inst 0x4f99eafa // sdot v26.4s, v23.16b, v25.4b[2]\n"
+ ".inst 0x4fbbea3a // sdot v26.4s, v17.16b, v27.4b[3]\n"
+ ".inst 0x4fb9eada // sdot v26.4s, v22.16b, v25.4b[3]\n"
+ "scvtf v26.4s, v26.4s, #0x4\n"
+ "fmla v29.4s, v26.4s, v16.4s\n"
+ "cbnz x21, 2b\n"
+ "sub %x[nc], %x[nc], #0x4\n"
+ "str q29, [%x[res_ptr], #0x0]\n"
+ "add %x[res_ptr], %x[res_ptr], #0x10\n"
+ "cbnz %x[nc], 1b\n"
+ : [b_ptr] "+&r" (b_ptr), [res_ptr] "+&r" (res_ptr), [nc] "+&r" (nc)
+ : [a_ptr] "r" (a_ptr), [nb] "r" (nb)
+ : "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22"
+ );
+#else
+ float sumf[4];
+ int sumi;
+
+ const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
+ for (int x = 0; x < nc / ncols_interleaved; x++) {
+ const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);
+
+ for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;
+ for (int l = 0; l < nb; l++) {
+ for (int k = 0; k < (qk / (2 * blocklen)); k++) {
+ for (int j = 0; j < ncols_interleaved; j++) {
+ sumi = 0;
+ for (int i = 0; i < blocklen; ++i) {
+ const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
+ const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
+ sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;
+ }
+ sumf[j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d);
+ }
+ }
+ }
+ for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];
+ }
+#endif
+}
+
+void ggml_gemv_q4_0_4x8_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
+ const int qk = QK8_0;
+ const int nb = n / qk;
+ const int ncols_interleaved = 4;
+ const int blocklen = 8;
+
+ assert (n % qk == 0);
+ assert (nc % ncols_interleaved == 0);
+
+ UNUSED(s);
+ UNUSED(bs);
+ UNUSED(vx);
+ UNUSED(vy);
+ UNUSED(nr);
+ UNUSED(nc);
+ UNUSED(nb);
+ UNUSED(ncols_interleaved);
+ UNUSED(blocklen);
+
+#if defined(__ARM_FEATURE_SVE)
+ if (ggml_sve_cnt_b == QK8_0) {
+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
+ "__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
+ }
+#endif
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
+ const void * b_ptr = vx;
+ const void * a_ptr = vy;
+ float * res_ptr = s;
+
+ __asm__ __volatile__(
+ "movi v2.16b, #0x4\n"
+ "movi v1.16b, #0xf0\n"
+ "add %x[b_ptr], %x[b_ptr], #0x8\n"
+ "1:" // Column loop
+ "add x23, %x[a_ptr], #0x2\n"
+ "movi v0.16b, #0x0\n"
+ "mov x22, %x[nb]\n"
+ "2:" // Block loop
+ "ldr q31, [%x[b_ptr], #0x0]\n"
+ "ldr q30, [%x[b_ptr], #0x10]\n"
+ "mov x21, x23\n"
+ "movi v29.4s, #0x0\n"
+ "ldr q28, [%x[b_ptr], #0x20]\n"
+ "ldr q27, [%x[b_ptr], #0x30]\n"
+ "movi v26.4s, #0x0\n"
+ "sub x20, x23, #0x2\n"
+ "ld1r { v25.8h }, [x20]\n"
+ "ldr q24, [%x[b_ptr], #-0x8]\n"
+ "sub x22, x22, #0x1\n"
+ "add x23, x23, #0x22\n"
+ "ld1r { v23.2d }, [x21], #0x8\n"
+ "sshl v22.16b, v31.16b, v2.16b\n"
+ "sshl v16.16b, v30.16b, v2.16b\n"
+ "add %x[b_ptr], %x[b_ptr], #0x48\n"
+ "ld1r { v21.2d }, [x21], #0x8\n"
+ "sshl v20.16b, v28.16b, v2.16b\n"
+ "sshl v19.16b, v27.16b, v2.16b\n"
+ "ld1r { v18.2d }, [x21], #0x8\n"
+ "ld1r { v17.2d }, [x21], #0x8\n"
+ "and v31.16b, v31.16b, v1.16b\n"
+ "and v30.16b, v30.16b, v1.16b\n"
+ ".inst 0x4e9796dd // sdot v29.4s, v22.16b, v23.16b\n"
+ ".inst 0x4e97961a // sdot v26.4s, v16.16b, v23.16b\n"
+ "and v28.16b, v28.16b, v1.16b\n"
+ "and v27.16b, v27.16b, v1.16b\n"
+ "fcvtl v25.4s, v25.4h\n"
+ "fcvtl v16.4s, v24.4h\n"
+ ".inst 0x4e95969d // sdot v29.4s, v20.16b, v21.16b\n"
+ ".inst 0x4e95967a // sdot v26.4s, v19.16b, v21.16b\n"
+ "fmul v16.4s, v16.4s, v25.4s\n"
+ ".inst 0x4e9297fd // sdot v29.4s, v31.16b, v18.16b\n"
+ ".inst 0x4e9297da // sdot v26.4s, v30.16b, v18.16b\n"
+ ".inst 0x4e91979d // sdot v29.4s, v28.16b, v17.16b\n"
+ ".inst 0x4e91977a // sdot v26.4s, v27.16b, v17.16b\n"
+ "addp v29.4s, v29.4s, v26.4s\n"
+ "scvtf v29.4s, v29.4s, #0x4\n"
+ "fmla v0.4s, v29.4s, v16.4s\n"
+ "cbnz x22, 2b\n"
+ "sub %x[nc], %x[nc], #0x4\n"
+ "str q0, [%x[res_ptr], #0x0]\n"
+ "add %x[res_ptr], %x[res_ptr], #0x10\n"
+ "cbnz %x[nc], 1b\n"
+ : [b_ptr] "+&r" (b_ptr), [res_ptr] "+&r" (res_ptr), [nc] "+&r" (nc)
+ : [a_ptr] "r" (a_ptr), [nb] "r" (nb)
+ : "memory", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23"
+ );
+#elif defined(__ARM_NEON) && defined(__aarch64__)
+ GGML_ASSERT((ggml_cpu_has_sve() || ggml_cpu_has_matmul_int8()) &&
+ "__ARM_FEATURE_SVE and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 quantization format for optimal "
+ "performance");
+#else
+ float sumf[4];
+ int sumi;
+
+ const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
+ for (int x = 0; x < nc / ncols_interleaved; x++) {
+ const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);
+
+ for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;
+ for (int l = 0; l < nb; l++) {
+ for (int k = 0; k < (qk / (2 * blocklen)); k++) {
+ for (int j = 0; j < ncols_interleaved; j++) {
+ sumi = 0;
+ for (int i = 0; i < blocklen; ++i) {
+ const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
+ const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
+ sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;
+ }
+ sumf[j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d);
+ }
+ }
+ }
+ for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];
+ }
+#endif
+}
+
+void ggml_gemv_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
+ const int qk = QK8_0;
+ const int nb = n / qk;
+ const int ncols_interleaved = 8;
+ const int blocklen = 8;
+
+ assert (n % qk == 0);
+ assert (nc % ncols_interleaved == 0);
+
+ UNUSED(s);
+ UNUSED(bs);
+ UNUSED(vx);
+ UNUSED(vy);
+ UNUSED(nr);
+ UNUSED(nc);
+ UNUSED(nb);
+ UNUSED(ncols_interleaved);
+ UNUSED(blocklen);
+
+#if defined(__ARM_FEATURE_SVE) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
+ if (ggml_sve_cnt_b == QK8_0) {
+ const void * b_ptr = vx;
+ const void * a_ptr = vy;
+ float * res_ptr = s;
+
+ __asm__ __volatile__(
+ "ptrue p0.b\n"
+ "add %x[b_ptr], %x[b_ptr], #0x10\n"
+ "1:" // Column loop
+ "add x22, %x[a_ptr], #0x2\n"
+ "mov z31.b, #0x0\n"
+ "mov x21, %x[nb]\n"
+ "2:" // Block loop
+ "ld1b { z30.b }, p0/Z, [%x[b_ptr]]\n"
+ "ld1b { z29.b }, p0/Z, [%x[b_ptr], #1, MUL VL]\n"
+ "mov z28.s, #0x0\n"
+ "mov z27.s, #0x0\n"
+ "ld1rd { z26.d }, p0/Z, [x22]\n"
+ "ld1b { z25.b }, p0/Z, [%x[b_ptr], #2, MUL VL]\n"
+ "sub x20, x22, #0x2\n"
+ "sub x21, x21, #0x1\n"
+ "ld1b { z24.b }, p0/Z, [%x[b_ptr], #3, MUL VL]\n"
+ "ld1rd { z23.d }, p0/Z, [x22, #8]\n"
+ "lsl z22.b, z30.b, #0x4\n"
+ "lsl z16.b, z29.b, #0x4\n"
+ "and z30.b, z30.b, #0xf0\n"
+ "and z29.b, z29.b, #0xf0\n"
+ "ld1rd { z21.d }, p0/Z, [x22, #16]\n"
+ "ld1rd { z20.d }, p0/Z, [x22, #24]\n"
+ "lsl z19.b, z25.b, #0x4\n"
+ "and z25.b, z25.b, #0xf0\n"
+ "ld1rh { z17.h }, p0/Z, [x20]\n"
+ "ld1h { z18.s }, p0/Z, [%x[b_ptr], #-1, MUL VL]\n"
+ "sdot z28.s, z22.b, z26.b\n"
+ "sdot z27.s, z16.b, z26.b\n"
+ "lsl z16.b, z24.b, #0x4\n"
+ "add x22, x22, #0x22\n"
+ "and z24.b, z24.b, #0xf0\n"
+ "add %x[b_ptr], %x[b_ptr], #0x90\n"
+ "fcvt z17.s, p0/m, z17.h\n"
+ "fcvt z18.s, p0/m, z18.h\n"
+ "sdot z28.s, z19.b, z23.b\n"
+ "sdot z27.s, z16.b, z23.b\n"
+ "fmul z18.s, z18.s, z17.s\n"
+ "sdot z28.s, z30.b, z21.b\n"
+ "sdot z27.s, z29.b, z21.b\n"
+ "sdot z28.s, z25.b, z20.b\n"
+ "sdot z27.s, z24.b, z20.b\n"
+ "uzp1 z17.s, z28.s, z27.s\n"
+ "uzp2 z16.s, z28.s, z27.s\n"
+ "add z17.s, z17.s, z16.s\n"
+ "asr z17.s, z17.s, #0x4\n"
+ "scvtf z17.s, p0/m, z17.s\n"
+ "fmla z31.s, p0/M, z17.s, z18.s\n"
+ "cbnz x21, 2b\n"
+ "sub %x[nc], %x[nc], #0x8\n"
+ "st1w { z31.s }, p0, [%x[res_ptr]]\n"
+ "add %x[res_ptr], %x[res_ptr], #0x20\n"
+ "cbnz %x[nc], 1b\n"
+ : [b_ptr] "+&r" (b_ptr), [res_ptr] "+&r" (res_ptr), [nc] "+&r" (nc)
+ : [a_ptr] "r" (a_ptr), [nb] "r" (nb)
+ : "memory", "p0", "x20", "x21", "x22", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+ return;
+ }
+ else if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {
+ GGML_ASSERT((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
+ "__ARM_FEATURE_SVE for vector size of 256-bits not defined, use the Q4_0_4_8 quantization format for optimal "
+ "performance");
+ }
+ else if (ggml_cpu_has_neon()) {
+ GGML_ASSERT(((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) || ggml_cpu_has_matmul_int8()) &&
+ "__ARM_FEATURE_SVE for vector size of 256-bits and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 "
+ "quantization format for optimal performance");
+ }
+#endif
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
+ GGML_ASSERT(ggml_cpu_has_sve() &&
+ "__ARM_FEATURE_SVE not defined, use the Q4_0_4_8 quantization format for optimal performance");
+#elif defined(__ARM_NEON) && defined(__aarch64__)
+ GGML_ASSERT((ggml_cpu_has_sve() || ggml_cpu_has_matmul_int8()) &&
+ "__ARM_FEATURE_SVE and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 quantization format for optimal "
+ "performance");
+#else
+ float sumf[8];
+ int sumi;
+
+ const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
+ for (int x = 0; x < nc / ncols_interleaved; x++) {
+ const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);
+
+ for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;
+ for (int l = 0; l < nb; l++) {
+ for (int k = 0; k < (qk / (2 * blocklen)); k++) {
+ for (int j = 0; j < ncols_interleaved; j++) {
+ sumi = 0;
+ for (int i = 0; i < blocklen; ++i) {
+ const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
+ const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
+ sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;
+ }
+ sumf[j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d);
+ }
+ }
+ }
+ for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];
+ }
+#endif
+}
+
+void ggml_gemm_q4_0_4x4_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
+ const int qk = QK8_0;
+ const int nb = n / qk;
+ const int ncols_interleaved = 4;
+ const int blocklen = 4;
+
+ assert (n % qk == 0);
+ assert (nr % 4 == 0);
+ assert (nc % ncols_interleaved == 0);
+
+ UNUSED(s);
+ UNUSED(bs);
+ UNUSED(vx);
+ UNUSED(vy);
+ UNUSED(nr);
+ UNUSED(nc);
+ UNUSED(nb);
+ UNUSED(ncols_interleaved);
+ UNUSED(blocklen);
+
+#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
+ if (ggml_sve_cnt_b == QK8_0) {
+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
+ "__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
+ }
+#endif
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
+ GGML_ASSERT(!(ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) &&
+ "__ARM_NEON and __ARM_FEATURE_MATMUL_INT8 defined, use the Q4_0_4_8 quantization format for optimal performance");
+#elif defined(__ARM_NEON) && defined(__aarch64__) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
+ const void * b_ptr = vx;
+ const void * a_ptr = vy;
+ float * res_ptr = s;
+ size_t res_stride = bs * sizeof(float);
+
+ __asm__ __volatile__(
+ "mov x10, %x[nr]\n"
+ "mov x9, #0x88\n"
+ "cmp x10, #0x10\n"
+ "mul x9, %x[nb], x9\n"
+ "blt 4f\n"
+ "1:" // Row loop
+ "add x28, %x[b_ptr], #0x8\n"
+ "mov x27, %x[nc]\n"
+ "add x26, %x[res_ptr], %x[res_stride], LSL #4\n"
+ "2:" // Column loop
+ "add x25, %x[a_ptr], #0x8\n"
+ "movi v15.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "mov x24, %x[nb]\n"
+ "add x23, x25, x9\n"
+ "movi v18.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "add x22, x23, x9\n"
+ "movi v11.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "add x21, x22, x9\n"
+ "movi v23.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v7.16b, #0x0\n"
+ "movi v0.16b, #0x0\n"
+ "movi v4.16b, #0x0\n"
+ "movi v5.16b, #0x0\n"
+ "movi v21.16b, #0x0\n"
+ "movi v8.16b, #0x0\n"
+ "movi v1.16b, #0x0\n"
+ "3:" // Block loop
+ "ldr q3, [x28, #0x0]\n"
+ "ldr q31, [x25, #0x0]\n"
+ "movi v28.16b, #0x4\n"
+ "movi v10.4s, #0x0\n"
+ "ldr q22, [x28, #0x10]\n"
+ "ldr q6, [x25, #0x10]\n"
+ "movi v29.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "ldr q27, [x28, #0x20]\n"
+ "ldr q30, [x28, #0x30]\n"
+ "movi v20.4s, #0x0\n"
+ "movi v24.16b, #0xf0\n"
+ "ldr d2, [x25, #-0x8]\n"
+ "ldr d26, [x23, #-0x8]\n"
+ "sshl v12.16b, v3.16b, v28.16b\n"
+ "sub x20, x28, #0x8\n"
+ "ldr d17, [x20, #0x0]\n"
+ "and v3.16b, v3.16b, v24.16b\n"
+ "subs x24, x24, #0x1\n"
+ "add x28, x28, #0x48\n"
+ ".inst 0x4f9fe18a // sdot v10.4s, v12.16b, v31.4b[0]\n"
+ ".inst 0x4fbfe19d // sdot v29.4s, v12.16b, v31.4b[1]\n"
+ ".inst 0x4f9fe989 // sdot v9.4s, v12.16b, v31.4b[2]\n"
+ ".inst 0x4fbfe994 // sdot v20.4s, v12.16b, v31.4b[3]\n"
+ "sshl v31.16b, v22.16b, v28.16b\n"
+ "and v22.16b, v22.16b, v24.16b\n"
+ "fcvtl v17.4s, v17.4h\n"
+ "fcvtl v2.4s, v2.4h\n"
+ "fcvtl v26.4s, v26.4h\n"
+ ".inst 0x4f86e3ea // sdot v10.4s, v31.16b, v6.4b[0]\n"
+ ".inst 0x4fa6e3fd // sdot v29.4s, v31.16b, v6.4b[1]\n"
+ ".inst 0x4f86ebe9 // sdot v9.4s, v31.16b, v6.4b[2]\n"
+ ".inst 0x4fa6ebf4 // sdot v20.4s, v31.16b, v6.4b[3]\n"
+ "sshl v6.16b, v27.16b, v28.16b\n"
+ "sshl v28.16b, v30.16b, v28.16b\n"
+ "and v27.16b, v27.16b, v24.16b\n"
+ "and v30.16b, v30.16b, v24.16b\n"
+ "ldr q24, [x25, #0x20]\n"
+ ".inst 0x4f98e0ca // sdot v10.4s, v6.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e0dd // sdot v29.4s, v6.16b, v24.4b[1]\n"
+ ".inst 0x4f98e8c9 // sdot v9.4s, v6.16b, v24.4b[2]\n"
+ ".inst 0x4fb8e8d4 // sdot v20.4s, v6.16b, v24.4b[3]\n"
+ "ldr q24, [x25, #0x30]\n"
+ ".inst 0x4f98e38a // sdot v10.4s, v28.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e39d // sdot v29.4s, v28.16b, v24.4b[1]\n"
+ ".inst 0x4f98eb89 // sdot v9.4s, v28.16b, v24.4b[2]\n"
+ ".inst 0x4fb8eb94 // sdot v20.4s, v28.16b, v24.4b[3]\n"
+ "ldr q24, [x25, #0x40]\n"
+ ".inst 0x4f98e06a // sdot v10.4s, v3.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e07d // sdot v29.4s, v3.16b, v24.4b[1]\n"
+ ".inst 0x4f98e869 // sdot v9.4s, v3.16b, v24.4b[2]\n"
+ ".inst 0x4fb8e874 // sdot v20.4s, v3.16b, v24.4b[3]\n"
+ "ldr q24, [x25, #0x50]\n"
+ ".inst 0x4f98e2ca // sdot v10.4s, v22.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e2dd // sdot v29.4s, v22.16b, v24.4b[1]\n"
+ ".inst 0x4f98eac9 // sdot v9.4s, v22.16b, v24.4b[2]\n"
+ ".inst 0x4fb8ead4 // sdot v20.4s, v22.16b, v24.4b[3]\n"
+ "ldr q24, [x25, #0x60]\n"
+ ".inst 0x4f98e36a // sdot v10.4s, v27.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e37d // sdot v29.4s, v27.16b, v24.4b[1]\n"
+ ".inst 0x4f98eb69 // sdot v9.4s, v27.16b, v24.4b[2]\n"
+ ".inst 0x4fb8eb74 // sdot v20.4s, v27.16b, v24.4b[3]\n"
+ "ldr q24, [x25, #0x70]\n"
+ "add x25, x25, #0x88\n"
+ ".inst 0x4f98e3ca // sdot v10.4s, v30.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e3dd // sdot v29.4s, v30.16b, v24.4b[1]\n"
+ ".inst 0x4f98ebc9 // sdot v9.4s, v30.16b, v24.4b[2]\n"
+ ".inst 0x4fb8ebd4 // sdot v20.4s, v30.16b, v24.4b[3]\n"
+ "fmul v24.4s, v17.4s, v2.s[0]\n"
+ "scvtf v10.4s, v10.4s, #0x4\n"
+ "scvtf v29.4s, v29.4s, #0x4\n"
+ "scvtf v9.4s, v9.4s, #0x4\n"
+ "scvtf v20.4s, v20.4s, #0x4\n"
+ "fmla v15.4s, v10.4s, v24.4s\n"
+ "ldr q24, [x23, #0x0]\n"
+ "fmul v10.4s, v17.4s, v2.s[1]\n"
+ "fmla v19.4s, v29.4s, v10.4s\n"
+ "ldr q10, [x23, #0x10]\n"
+ "fmul v29.4s, v17.4s, v2.s[2]\n"
+ "fmul v2.4s, v17.4s, v2.s[3]\n"
+ "fmla v18.4s, v9.4s, v29.4s\n"
+ "movi v9.4s, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ ".inst 0x4f98e189 // sdot v9.4s, v12.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e19d // sdot v29.4s, v12.16b, v24.4b[1]\n"
+ "fmla v14.4s, v20.4s, v2.4s\n"
+ "movi v20.4s, #0x0\n"
+ "movi v2.4s, #0x0\n"
+ ".inst 0x4f98e994 // sdot v20.4s, v12.16b, v24.4b[2]\n"
+ ".inst 0x4fb8e982 // sdot v2.4s, v12.16b, v24.4b[3]\n"
+ "ldr q24, [x23, #0x20]\n"
+ ".inst 0x4f8ae3e9 // sdot v9.4s, v31.16b, v10.4b[0]\n"
+ ".inst 0x4faae3fd // sdot v29.4s, v31.16b, v10.4b[1]\n"
+ ".inst 0x4f8aebf4 // sdot v20.4s, v31.16b, v10.4b[2]\n"
+ ".inst 0x4faaebe2 // sdot v2.4s, v31.16b, v10.4b[3]\n"
+ "ldr q10, [x23, #0x30]\n"
+ ".inst 0x4f98e0c9 // sdot v9.4s, v6.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e0dd // sdot v29.4s, v6.16b, v24.4b[1]\n"
+ ".inst 0x4f98e8d4 // sdot v20.4s, v6.16b, v24.4b[2]\n"
+ ".inst 0x4fb8e8c2 // sdot v2.4s, v6.16b, v24.4b[3]\n"
+ "ldr q24, [x23, #0x40]\n"
+ ".inst 0x4f8ae389 // sdot v9.4s, v28.16b, v10.4b[0]\n"
+ ".inst 0x4faae39d // sdot v29.4s, v28.16b, v10.4b[1]\n"
+ ".inst 0x4f8aeb94 // sdot v20.4s, v28.16b, v10.4b[2]\n"
+ ".inst 0x4faaeb82 // sdot v2.4s, v28.16b, v10.4b[3]\n"
+ "ldr q10, [x23, #0x50]\n"
+ ".inst 0x4f98e069 // sdot v9.4s, v3.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e07d // sdot v29.4s, v3.16b, v24.4b[1]\n"
+ ".inst 0x4f98e874 // sdot v20.4s, v3.16b, v24.4b[2]\n"
+ ".inst 0x4fb8e862 // sdot v2.4s, v3.16b, v24.4b[3]\n"
+ "ldr q24, [x23, #0x60]\n"
+ ".inst 0x4f8ae2c9 // sdot v9.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4faae2dd // sdot v29.4s, v22.16b, v10.4b[1]\n"
+ ".inst 0x4f8aead4 // sdot v20.4s, v22.16b, v10.4b[2]\n"
+ ".inst 0x4faaeac2 // sdot v2.4s, v22.16b, v10.4b[3]\n"
+ "ldr q10, [x23, #0x70]\n"
+ "add x23, x23, #0x88\n"
+ ".inst 0x4f98e369 // sdot v9.4s, v27.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e37d // sdot v29.4s, v27.16b, v24.4b[1]\n"
+ ".inst 0x4f98eb74 // sdot v20.4s, v27.16b, v24.4b[2]\n"
+ ".inst 0x4fb8eb62 // sdot v2.4s, v27.16b, v24.4b[3]\n"
+ "ldr q24, [x22, #0x0]\n"
+ ".inst 0x4f8ae3c9 // sdot v9.4s, v30.16b, v10.4b[0]\n"
+ ".inst 0x4faae3dd // sdot v29.4s, v30.16b, v10.4b[1]\n"
+ ".inst 0x4f8aebd4 // sdot v20.4s, v30.16b, v10.4b[2]\n"
+ ".inst 0x4faaebc2 // sdot v2.4s, v30.16b, v10.4b[3]\n"
+ "fmul v10.4s, v17.4s, v26.s[0]\n"
+ "scvtf v9.4s, v9.4s, #0x4\n"
+ "scvtf v29.4s, v29.4s, #0x4\n"
+ "scvtf v20.4s, v20.4s, #0x4\n"
+ "scvtf v2.4s, v2.4s, #0x4\n"
+ "fmla v11.4s, v9.4s, v10.4s\n"
+ "ldr q9, [x22, #0x10]\n"
+ "fmul v10.4s, v17.4s, v26.s[1]\n"
+ "fmla v13.4s, v29.4s, v10.4s\n"
+ "ldr d29, [x22, #-0x8]\n"
+ "fmul v10.4s, v17.4s, v26.s[2]\n"
+ "fmul v26.4s, v17.4s, v26.s[3]\n"
+ "fcvtl v29.4s, v29.4h\n"
+ "fmla v23.4s, v20.4s, v10.4s\n"
+ "movi v20.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "fmla v16.4s, v2.4s, v26.4s\n"
+ "movi v26.4s, #0x0\n"
+ "movi v2.4s, #0x0\n"
+ ".inst 0x4f98e194 // sdot v20.4s, v12.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e18a // sdot v10.4s, v12.16b, v24.4b[1]\n"
+ ".inst 0x4f98e99a // sdot v26.4s, v12.16b, v24.4b[2]\n"
+ ".inst 0x4fb8e982 // sdot v2.4s, v12.16b, v24.4b[3]\n"
+ "ldr q24, [x22, #0x20]\n"
+ ".inst 0x4f89e3f4 // sdot v20.4s, v31.16b, v9.4b[0]\n"
+ ".inst 0x4fa9e3ea // sdot v10.4s, v31.16b, v9.4b[1]\n"
+ ".inst 0x4f89ebfa // sdot v26.4s, v31.16b, v9.4b[2]\n"
+ ".inst 0x4fa9ebe2 // sdot v2.4s, v31.16b, v9.4b[3]\n"
+ "ldr q9, [x22, #0x30]\n"
+ ".inst 0x4f98e0d4 // sdot v20.4s, v6.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e0ca // sdot v10.4s, v6.16b, v24.4b[1]\n"
+ ".inst 0x4f98e8da // sdot v26.4s, v6.16b, v24.4b[2]\n"
+ ".inst 0x4fb8e8c2 // sdot v2.4s, v6.16b, v24.4b[3]\n"
+ "ldr q24, [x22, #0x40]\n"
+ ".inst 0x4f89e394 // sdot v20.4s, v28.16b, v9.4b[0]\n"
+ ".inst 0x4fa9e38a // sdot v10.4s, v28.16b, v9.4b[1]\n"
+ ".inst 0x4f89eb9a // sdot v26.4s, v28.16b, v9.4b[2]\n"
+ ".inst 0x4fa9eb82 // sdot v2.4s, v28.16b, v9.4b[3]\n"
+ "ldr q9, [x22, #0x50]\n"
+ ".inst 0x4f98e074 // sdot v20.4s, v3.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e06a // sdot v10.4s, v3.16b, v24.4b[1]\n"
+ ".inst 0x4f98e87a // sdot v26.4s, v3.16b, v24.4b[2]\n"
+ ".inst 0x4fb8e862 // sdot v2.4s, v3.16b, v24.4b[3]\n"
+ "ldr q24, [x22, #0x60]\n"
+ ".inst 0x4f89e2d4 // sdot v20.4s, v22.16b, v9.4b[0]\n"
+ ".inst 0x4fa9e2ca // sdot v10.4s, v22.16b, v9.4b[1]\n"
+ ".inst 0x4f89eada // sdot v26.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4fa9eac2 // sdot v2.4s, v22.16b, v9.4b[3]\n"
+ "ldr q9, [x22, #0x70]\n"
+ "add x22, x22, #0x88\n"
+ ".inst 0x4f98e374 // sdot v20.4s, v27.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e36a // sdot v10.4s, v27.16b, v24.4b[1]\n"
+ ".inst 0x4f98eb7a // sdot v26.4s, v27.16b, v24.4b[2]\n"
+ ".inst 0x4fb8eb62 // sdot v2.4s, v27.16b, v24.4b[3]\n"
+ "ldr q24, [x21, #0x0]\n"
+ ".inst 0x4f89e3d4 // sdot v20.4s, v30.16b, v9.4b[0]\n"
+ ".inst 0x4fa9e3ca // sdot v10.4s, v30.16b, v9.4b[1]\n"
+ ".inst 0x4f89ebda // sdot v26.4s, v30.16b, v9.4b[2]\n"
+ ".inst 0x4fa9ebc2 // sdot v2.4s, v30.16b, v9.4b[3]\n"
+ "fmul v9.4s, v17.4s, v29.s[0]\n"
+ "scvtf v20.4s, v20.4s, #0x4\n"
+ "scvtf v10.4s, v10.4s, #0x4\n"
+ "scvtf v26.4s, v26.4s, #0x4\n"
+ "scvtf v2.4s, v2.4s, #0x4\n"
+ "fmla v25.4s, v20.4s, v9.4s\n"
+ "ldr q9, [x21, #0x10]\n"
+ "fmul v20.4s, v17.4s, v29.s[1]\n"
+ "fmla v7.4s, v10.4s, v20.4s\n"
+ "ldr d20, [x21, #-0x8]\n"
+ "fmul v10.4s, v17.4s, v29.s[2]\n"
+ "fmul v29.4s, v17.4s, v29.s[3]\n"
+ "fcvtl v20.4s, v20.4h\n"
+ "fmla v0.4s, v26.4s, v10.4s\n"
+ "movi v26.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "fmla v4.4s, v2.4s, v29.4s\n"
+ "movi v2.4s, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ ".inst 0x4f98e19a // sdot v26.4s, v12.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e18a // sdot v10.4s, v12.16b, v24.4b[1]\n"
+ ".inst 0x4f98e982 // sdot v2.4s, v12.16b, v24.4b[2]\n"
+ ".inst 0x4fb8e99d // sdot v29.4s, v12.16b, v24.4b[3]\n"
+ "ldr q12, [x21, #0x20]\n"
+ "fmul v24.4s, v17.4s, v20.s[0]\n"
+ ".inst 0x4f89e3fa // sdot v26.4s, v31.16b, v9.4b[0]\n"
+ ".inst 0x4fa9e3ea // sdot v10.4s, v31.16b, v9.4b[1]\n"
+ ".inst 0x4f89ebe2 // sdot v2.4s, v31.16b, v9.4b[2]\n"
+ ".inst 0x4fa9ebfd // sdot v29.4s, v31.16b, v9.4b[3]\n"
+ "ldr q9, [x21, #0x30]\n"
+ "fmul v31.4s, v17.4s, v20.s[1]\n"
+ ".inst 0x4f8ce0da // sdot v26.4s, v6.16b, v12.4b[0]\n"
+ ".inst 0x4face0ca // sdot v10.4s, v6.16b, v12.4b[1]\n"
+ ".inst 0x4f8ce8c2 // sdot v2.4s, v6.16b, v12.4b[2]\n"
+ ".inst 0x4face8dd // sdot v29.4s, v6.16b, v12.4b[3]\n"
+ "ldr q12, [x21, #0x40]\n"
+ "fmul v6.4s, v17.4s, v20.s[2]\n"
+ "fmul v20.4s, v17.4s, v20.s[3]\n"
+ ".inst 0x4f89e39a // sdot v26.4s, v28.16b, v9.4b[0]\n"
+ ".inst 0x4fa9e38a // sdot v10.4s, v28.16b, v9.4b[1]\n"
+ ".inst 0x4f89eb82 // sdot v2.4s, v28.16b, v9.4b[2]\n"
+ ".inst 0x4fa9eb9d // sdot v29.4s, v28.16b, v9.4b[3]\n"
+ "ldr q9, [x21, #0x50]\n"
+ ".inst 0x4f8ce07a // sdot v26.4s, v3.16b, v12.4b[0]\n"
+ ".inst 0x4face06a // sdot v10.4s, v3.16b, v12.4b[1]\n"
+ ".inst 0x4f8ce862 // sdot v2.4s, v3.16b, v12.4b[2]\n"
+ ".inst 0x4face87d // sdot v29.4s, v3.16b, v12.4b[3]\n"
+ "ldr q12, [x21, #0x60]\n"
+ ".inst 0x4f89e2da // sdot v26.4s, v22.16b, v9.4b[0]\n"
+ ".inst 0x4fa9e2ca // sdot v10.4s, v22.16b, v9.4b[1]\n"
+ ".inst 0x4f89eac2 // sdot v2.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4fa9eadd // sdot v29.4s, v22.16b, v9.4b[3]\n"
+ "ldr q17, [x21, #0x70]\n"
+ "add x21, x21, #0x88\n"
+ ".inst 0x4f8ce37a // sdot v26.4s, v27.16b, v12.4b[0]\n"
+ ".inst 0x4face36a // sdot v10.4s, v27.16b, v12.4b[1]\n"
+ ".inst 0x4f8ceb62 // sdot v2.4s, v27.16b, v12.4b[2]\n"
+ ".inst 0x4faceb7d // sdot v29.4s, v27.16b, v12.4b[3]\n"
+ ".inst 0x4f91e3da // sdot v26.4s, v30.16b, v17.4b[0]\n"
+ ".inst 0x4fb1e3ca // sdot v10.4s, v30.16b, v17.4b[1]\n"
+ ".inst 0x4f91ebc2 // sdot v2.4s, v30.16b, v17.4b[2]\n"
+ ".inst 0x4fb1ebdd // sdot v29.4s, v30.16b, v17.4b[3]\n"
+ "scvtf v26.4s, v26.4s, #0x4\n"
+ "scvtf v10.4s, v10.4s, #0x4\n"
+ "fmla v5.4s, v26.4s, v24.4s\n"
+ "scvtf v2.4s, v2.4s, #0x4\n"
+ "scvtf v29.4s, v29.4s, #0x4\n"
+ "fmla v21.4s, v10.4s, v31.4s\n"
+ "fmla v8.4s, v2.4s, v6.4s\n"
+ "fmla v1.4s, v29.4s, v20.4s\n"
+ "bgt 3b\n"
+ "mov x20, %x[res_ptr]\n"
+ "subs x27, x27, #0x4\n"
+ "add %x[res_ptr], %x[res_ptr], #0x10\n"
+ "str q15, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q19, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q18, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q14, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q11, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q13, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q23, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q16, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q25, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q7, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q0, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q4, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q5, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q21, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q8, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q1, [x20, #0x0]\n"
+ "bne 2b\n"
+ "mov x20, #0x4\n"
+ "sub x10, x10, #0x10\n"
+ "cmp x10, #0x10\n"
+ "mov %x[res_ptr], x26\n"
+ "madd %x[a_ptr], x20, x9, %x[a_ptr]\n"
+ "bge 1b\n"
+ "4:" // Row loop skip
+ "cbz x10, 9f\n"
+ "5:" // Row tail: Row loop
+ "add x24, %x[b_ptr], #0x8\n"
+ "mov x23, %x[nc]\n"
+ "add x22, %x[res_ptr], %x[res_stride], LSL #2\n"
+ "6:" // Row tail: Column loop
+ "movi v15.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "add x25, %x[a_ptr], #0x8\n"
+ "mov x21, %x[nb]\n"
+ "movi v18.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "7:" // Row tail: Block loop
+ "ldr q7, [x24, #0x0]\n"
+ "ldr q5, [x25, #0x0]\n"
+ "movi v9.16b, #0x4\n"
+ "movi v4.4s, #0x0\n"
+ "ldr q3, [x24, #0x10]\n"
+ "ldr q2, [x25, #0x10]\n"
+ "movi v1.4s, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "ldr q13, [x24, #0x20]\n"
+ "ldr q31, [x25, #0x20]\n"
+ "movi v30.4s, #0x0\n"
+ "movi v29.16b, #0xf0\n"
+ "ldr q28, [x24, #0x30]\n"
+ "ldr q27, [x25, #0x30]\n"
+ "sshl v20.16b, v7.16b, v9.16b\n"
+ "sub x20, x24, #0x8\n"
+ "ldr q26, [x25, #0x40]\n"
+ "ldr q25, [x25, #0x50]\n"
+ "sshl v17.16b, v3.16b, v9.16b\n"
+ "and v7.16b, v7.16b, v29.16b\n"
+ "ldr q24, [x25, #0x60]\n"
+ "ldr q16, [x25, #0x70]\n"
+ "sshl v22.16b, v13.16b, v9.16b\n"
+ "and v3.16b, v3.16b, v29.16b\n"
+ "ldr d21, [x20, #0x0]\n"
+ "ldr d12, [x25, #-0x8]\n"
+ ".inst 0x4f85e284 // sdot v4.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4fa5e281 // sdot v1.4s, v20.16b, v5.4b[1]\n"
+ ".inst 0x4f85ea80 // sdot v0.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4fa5ea9e // sdot v30.4s, v20.16b, v5.4b[3]\n"
+ "sshl v9.16b, v28.16b, v9.16b\n"
+ "subs x21, x21, #0x1\n"
+ "and v13.16b, v13.16b, v29.16b\n"
+ "and v28.16b, v28.16b, v29.16b\n"
+ "add x25, x25, #0x88\n"
+ "add x24, x24, #0x48\n"
+ "fcvtl v21.4s, v21.4h\n"
+ "fcvtl v12.4s, v12.4h\n"
+ ".inst 0x4f82e224 // sdot v4.4s, v17.16b, v2.4b[0]\n"
+ ".inst 0x4fa2e221 // sdot v1.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4f82ea20 // sdot v0.4s, v17.16b, v2.4b[2]\n"
+ ".inst 0x4fa2ea3e // sdot v30.4s, v17.16b, v2.4b[3]\n"
+ "fmul v11.4s, v21.4s, v12.s[0]\n"
+ "fmul v23.4s, v21.4s, v12.s[1]\n"
+ "fmul v17.4s, v21.4s, v12.s[2]\n"
+ ".inst 0x4f9fe2c4 // sdot v4.4s, v22.16b, v31.4b[0]\n"
+ "fmul v6.4s, v21.4s, v12.s[3]\n"
+ ".inst 0x4fbfe2c1 // sdot v1.4s, v22.16b, v31.4b[1]\n"
+ ".inst 0x4f9feac0 // sdot v0.4s, v22.16b, v31.4b[2]\n"
+ ".inst 0x4fbfeade // sdot v30.4s, v22.16b, v31.4b[3]\n"
+ ".inst 0x4f9be124 // sdot v4.4s, v9.16b, v27.4b[0]\n"
+ ".inst 0x4fbbe121 // sdot v1.4s, v9.16b, v27.4b[1]\n"
+ ".inst 0x4f9be920 // sdot v0.4s, v9.16b, v27.4b[2]\n"
+ ".inst 0x4fbbe93e // sdot v30.4s, v9.16b, v27.4b[3]\n"
+ ".inst 0x4f9ae0e4 // sdot v4.4s, v7.16b, v26.4b[0]\n"
+ ".inst 0x4fbae0e1 // sdot v1.4s, v7.16b, v26.4b[1]\n"
+ ".inst 0x4f9ae8e0 // sdot v0.4s, v7.16b, v26.4b[2]\n"
+ ".inst 0x4fbae8fe // sdot v30.4s, v7.16b, v26.4b[3]\n"
+ ".inst 0x4f99e064 // sdot v4.4s, v3.16b, v25.4b[0]\n"
+ ".inst 0x4fb9e061 // sdot v1.4s, v3.16b, v25.4b[1]\n"
+ ".inst 0x4f99e860 // sdot v0.4s, v3.16b, v25.4b[2]\n"
+ ".inst 0x4fb9e87e // sdot v30.4s, v3.16b, v25.4b[3]\n"
+ ".inst 0x4f98e1a4 // sdot v4.4s, v13.16b, v24.4b[0]\n"
+ ".inst 0x4fb8e1a1 // sdot v1.4s, v13.16b, v24.4b[1]\n"
+ ".inst 0x4f98e9a0 // sdot v0.4s, v13.16b, v24.4b[2]\n"
+ ".inst 0x4fb8e9be // sdot v30.4s, v13.16b, v24.4b[3]\n"
+ ".inst 0x4f90e384 // sdot v4.4s, v28.16b, v16.4b[0]\n"
+ ".inst 0x4fb0e381 // sdot v1.4s, v28.16b, v16.4b[1]\n"
+ ".inst 0x4f90eb80 // sdot v0.4s, v28.16b, v16.4b[2]\n"
+ ".inst 0x4fb0eb9e // sdot v30.4s, v28.16b, v16.4b[3]\n"
+ "scvtf v4.4s, v4.4s, #0x4\n"
+ "scvtf v1.4s, v1.4s, #0x4\n"
+ "scvtf v0.4s, v0.4s, #0x4\n"
+ "fmla v15.4s, v4.4s, v11.4s\n"
+ "scvtf v30.4s, v30.4s, #0x4\n"
+ "fmla v19.4s, v1.4s, v23.4s\n"
+ "fmla v18.4s, v0.4s, v17.4s\n"
+ "fmla v14.4s, v30.4s, v6.4s\n"
+ "bgt 7b\n"
+ "mov x20, %x[res_ptr]\n"
+ "cmp x10, #0x1\n"
+ "str q15, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "ble 8f\n"
+ "cmp x10, #0x2\n"
+ "str q19, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "ble 8f\n"
+ "cmp x10, #0x3\n"
+ "str q18, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "ble 8f\n"
+ "str q14, [x20, #0x0]\n"
+ "8:" // Row tail: Accumulator store skip
+ "subs x23, x23, #0x4\n"
+ "add %x[res_ptr], %x[res_ptr], #0x10\n"
+ "bne 6b\n"
+ "subs x10, x10, #0x4\n"
+ "add %x[a_ptr], %x[a_ptr], x9\n"
+ "mov %x[res_ptr], x22\n"
+ "bgt 5b\n"
+ "9:" // Row tail: Row loop skip
+ : [a_ptr] "+&r" (a_ptr), [res_ptr] "+&r" (res_ptr)
+ : [b_ptr] "r" (b_ptr), [nr] "r" (nr), [nb] "r" (nb), [res_stride] "r" (res_stride), [nc] "r" (nc)
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+#else
+ float sumf[4][4];
+ int sumi;
+
+ for (int y = 0; y < nr / 4; y++) {
+ const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);
+ for (int x = 0; x < nc / ncols_interleaved; x++) {
+ const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;
+ }
+ for (int l = 0; l < nb; l++) {
+ for (int k = 0; k < (qk / (2 * blocklen)); k++) {
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++) {
+ sumi = 0;
+ for (int i = 0; i < blocklen; ++i) {
+ const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
+ const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
+ sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +
+ (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;
+ }
+ sumf[m][j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d[m]);
+ }
+ }
+ }
+ }
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++)
+ s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];
+ }
+ }
+ }
+#endif
+}
+
+void ggml_gemm_q4_0_4x8_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
+ const int qk = QK8_0;
+ const int nb = n / qk;
+ const int ncols_interleaved = 4;
+ const int blocklen = 8;
+
+ assert (n % qk == 0);
+ assert (nr % 4 == 0);
+ assert (nc % ncols_interleaved == 0);
+
+ UNUSED(s);
+ UNUSED(bs);
+ UNUSED(vx);
+ UNUSED(vy);
+ UNUSED(nr);
+ UNUSED(nc);
+ UNUSED(nb);
+ UNUSED(ncols_interleaved);
+ UNUSED(blocklen);
+
+#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
+ if (ggml_sve_cnt_b == QK8_0) {
+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
+ "__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
+ }
+#endif
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
+ const void * b_ptr = vx;
+ const void * a_ptr = vy;
+ float * res_ptr = s;
+ size_t res_stride = bs * sizeof(float);
+
+ __asm__ __volatile__(
+ "mov x10, %x[nr]\n"
+ "mov x9, #0x88\n"
+ "cmp x10, #0x10\n"
+ "mul x9, %x[nb], x9\n"
+ "blt 4f\n"
+ "1:" // Row loop
+ "add x28, %x[b_ptr], #0x8\n"
+ "mov x27, %x[nc]\n"
+ "add x26, %x[res_ptr], %x[res_stride], LSL #4\n"
+ "2:" // Column loop
+ "add x25, %x[a_ptr], #0x8\n"
+ "movi v2.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "mov x24, %x[nb]\n"
+ "add x23, x25, x9\n"
+ "movi v12.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "add x22, x23, x9\n"
+ "movi v11.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "add x21, x22, x9\n"
+ "movi v22.16b, #0x0\n"
+ "movi v23.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v5.16b, #0x0\n"
+ "movi v7.16b, #0x0\n"
+ "movi v4.16b, #0x0\n"
+ "movi v6.16b, #0x0\n"
+ "movi v30.16b, #0x0\n"
+ "movi v24.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "3:" // Block loop
+ "ldr q21, [x28, #0x0]\n"
+ "ldr q16, [x28, #0x10]\n"
+ "movi v1.16b, #0x4\n"
+ "movi v19.4s, #0x0\n"
+ "ldr q27, [x25, #0x0]\n"
+ "ldr q15, [x25, #0x10]\n"
+ "movi v26.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "ldr q29, [x28, #0x20]\n"
+ "ldr q3, [x28, #0x30]\n"
+ "movi v17.4s, #0x0\n"
+ "movi v0.16b, #0xf0\n"
+ "ldr d20, [x25, #-0x8]\n"
+ "ldr d9, [x23, #-0x8]\n"
+ "sshl v8.16b, v21.16b, v1.16b\n"
+ "sshl v31.16b, v16.16b, v1.16b\n"
+ "and v21.16b, v21.16b, v0.16b\n"
+ "and v16.16b, v16.16b, v0.16b\n"
+ "sub x20, x28, #0x8\n"
+ "subs x24, x24, #0x1\n"
+ "add x28, x28, #0x48\n"
+ ".inst 0x4e88a773 // smmla v19.4s, v27.16b, v8.16b\n"
+ ".inst 0x4e9fa77a // smmla v26.4s, v27.16b, v31.16b\n"
+ "ldr q27, [x25, #0x20]\n"
+ ".inst 0x4e88a5f2 // smmla v18.4s, v15.16b, v8.16b\n"
+ ".inst 0x4e9fa5f1 // smmla v17.4s, v15.16b, v31.16b\n"
+ "sshl v15.16b, v29.16b, v1.16b\n"
+ "sshl v1.16b, v3.16b, v1.16b\n"
+ "and v29.16b, v29.16b, v0.16b\n"
+ "and v3.16b, v3.16b, v0.16b\n"
+ "ldr q0, [x25, #0x30]\n"
+ "fcvtl v20.4s, v20.4h\n"
+ ".inst 0x4e8fa773 // smmla v19.4s, v27.16b, v15.16b\n"
+ "fcvtl v9.4s, v9.4h\n"
+ ".inst 0x4e81a77a // smmla v26.4s, v27.16b, v1.16b\n"
+ "ldr q27, [x25, #0x40]\n"
+ ".inst 0x4e8fa412 // smmla v18.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e81a411 // smmla v17.4s, v0.16b, v1.16b\n"
+ "ldr q0, [x25, #0x50]\n"
+ ".inst 0x4e95a773 // smmla v19.4s, v27.16b, v21.16b\n"
+ ".inst 0x4e90a77a // smmla v26.4s, v27.16b, v16.16b\n"
+ "ldr q27, [x25, #0x60]\n"
+ ".inst 0x4e95a412 // smmla v18.4s, v0.16b, v21.16b\n"
+ ".inst 0x4e90a411 // smmla v17.4s, v0.16b, v16.16b\n"
+ "ldr q0, [x25, #0x70]\n"
+ "add x25, x25, #0x88\n"
+ ".inst 0x4e9da773 // smmla v19.4s, v27.16b, v29.16b\n"
+ ".inst 0x4e83a77a // smmla v26.4s, v27.16b, v3.16b\n"
+ "ldr d27, [x20, #0x0]\n"
+ ".inst 0x4e9da412 // smmla v18.4s, v0.16b, v29.16b\n"
+ ".inst 0x4e83a411 // smmla v17.4s, v0.16b, v3.16b\n"
+ "fcvtl v27.4s, v27.4h\n"
+ "uzp1 v0.2d, v19.2d, v26.2d\n"
+ "uzp2 v26.2d, v19.2d, v26.2d\n"
+ "fmul v19.4s, v27.4s, v20.s[0]\n"
+ "scvtf v0.4s, v0.4s, #0x4\n"
+ "scvtf v26.4s, v26.4s, #0x4\n"
+ "fmla v2.4s, v0.4s, v19.4s\n"
+ "ldr q19, [x23, #0x0]\n"
+ "uzp1 v0.2d, v18.2d, v17.2d\n"
+ "uzp2 v18.2d, v18.2d, v17.2d\n"
+ "fmul v17.4s, v27.4s, v20.s[1]\n"
+ "scvtf v0.4s, v0.4s, #0x4\n"
+ "scvtf v18.4s, v18.4s, #0x4\n"
+ "fmla v10.4s, v26.4s, v17.4s\n"
+ "ldr q17, [x23, #0x10]\n"
+ "fmul v26.4s, v27.4s, v20.s[2]\n"
+ "fmul v20.4s, v27.4s, v20.s[3]\n"
+ "fmla v12.4s, v0.4s, v26.4s\n"
+ "ldr d0, [x22, #-0x8]\n"
+ "ldr d26, [x21, #-0x8]\n"
+ "fcvtl v0.4s, v0.4h\n"
+ "fmla v28.4s, v18.4s, v20.4s\n"
+ "movi v20.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ ".inst 0x4e88a674 // smmla v20.4s, v19.16b, v8.16b\n"
+ ".inst 0x4e9fa672 // smmla v18.4s, v19.16b, v31.16b\n"
+ "ldr q19, [x23, #0x20]\n"
+ "fcvtl v26.4s, v26.4h\n"
+ ".inst 0x4e8fa674 // smmla v20.4s, v19.16b, v15.16b\n"
+ ".inst 0x4e81a672 // smmla v18.4s, v19.16b, v1.16b\n"
+ "ldr q19, [x23, #0x40]\n"
+ ".inst 0x4e95a674 // smmla v20.4s, v19.16b, v21.16b\n"
+ ".inst 0x4e90a672 // smmla v18.4s, v19.16b, v16.16b\n"
+ "ldr q19, [x23, #0x60]\n"
+ ".inst 0x4e9da674 // smmla v20.4s, v19.16b, v29.16b\n"
+ ".inst 0x4e83a672 // smmla v18.4s, v19.16b, v3.16b\n"
+ "uzp1 v19.2d, v20.2d, v18.2d\n"
+ "scvtf v19.4s, v19.4s, #0x4\n"
+ "uzp2 v20.2d, v20.2d, v18.2d\n"
+ "fmul v18.4s, v27.4s, v9.s[0]\n"
+ "scvtf v20.4s, v20.4s, #0x4\n"
+ "fmla v11.4s, v19.4s, v18.4s\n"
+ "ldr q18, [x22, #0x0]\n"
+ "fmul v19.4s, v27.4s, v9.s[1]\n"
+ "fmla v13.4s, v20.4s, v19.4s\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ ".inst 0x4e88a633 // smmla v19.4s, v17.16b, v8.16b\n"
+ ".inst 0x4e9fa634 // smmla v20.4s, v17.16b, v31.16b\n"
+ "ldr q17, [x23, #0x30]\n"
+ ".inst 0x4e8fa633 // smmla v19.4s, v17.16b, v15.16b\n"
+ ".inst 0x4e81a634 // smmla v20.4s, v17.16b, v1.16b\n"
+ "ldr q17, [x23, #0x50]\n"
+ ".inst 0x4e95a633 // smmla v19.4s, v17.16b, v21.16b\n"
+ ".inst 0x4e90a634 // smmla v20.4s, v17.16b, v16.16b\n"
+ "ldr q17, [x23, #0x70]\n"
+ "add x23, x23, #0x88\n"
+ ".inst 0x4e9da633 // smmla v19.4s, v17.16b, v29.16b\n"
+ ".inst 0x4e83a634 // smmla v20.4s, v17.16b, v3.16b\n"
+ "uzp1 v17.2d, v19.2d, v20.2d\n"
+ "scvtf v17.4s, v17.4s, #0x4\n"
+ "uzp2 v20.2d, v19.2d, v20.2d\n"
+ "fmul v19.4s, v27.4s, v9.s[2]\n"
+ "fmul v9.4s, v27.4s, v9.s[3]\n"
+ "scvtf v20.4s, v20.4s, #0x4\n"
+ "fmla v22.4s, v17.4s, v19.4s\n"
+ "ldr q17, [x22, #0x10]\n"
+ "movi v19.4s, #0x0\n"
+ ".inst 0x4e88a653 // smmla v19.4s, v18.16b, v8.16b\n"
+ "fmla v23.4s, v20.4s, v9.4s\n"
+ "movi v20.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ ".inst 0x4e9fa654 // smmla v20.4s, v18.16b, v31.16b\n"
+ "ldr q18, [x22, #0x20]\n"
+ ".inst 0x4e88a629 // smmla v9.4s, v17.16b, v8.16b\n"
+ ".inst 0x4e8fa653 // smmla v19.4s, v18.16b, v15.16b\n"
+ ".inst 0x4e81a654 // smmla v20.4s, v18.16b, v1.16b\n"
+ "ldr q18, [x22, #0x40]\n"
+ ".inst 0x4e95a653 // smmla v19.4s, v18.16b, v21.16b\n"
+ ".inst 0x4e90a654 // smmla v20.4s, v18.16b, v16.16b\n"
+ "ldr q18, [x22, #0x60]\n"
+ ".inst 0x4e9da653 // smmla v19.4s, v18.16b, v29.16b\n"
+ ".inst 0x4e83a654 // smmla v20.4s, v18.16b, v3.16b\n"
+ "movi v18.4s, #0x0\n"
+ ".inst 0x4e9fa632 // smmla v18.4s, v17.16b, v31.16b\n"
+ "ldr q17, [x22, #0x30]\n"
+ ".inst 0x4e8fa629 // smmla v9.4s, v17.16b, v15.16b\n"
+ ".inst 0x4e81a632 // smmla v18.4s, v17.16b, v1.16b\n"
+ "ldr q17, [x22, #0x50]\n"
+ ".inst 0x4e95a629 // smmla v9.4s, v17.16b, v21.16b\n"
+ ".inst 0x4e90a632 // smmla v18.4s, v17.16b, v16.16b\n"
+ "ldr q17, [x22, #0x70]\n"
+ "add x22, x22, #0x88\n"
+ ".inst 0x4e9da629 // smmla v9.4s, v17.16b, v29.16b\n"
+ ".inst 0x4e83a632 // smmla v18.4s, v17.16b, v3.16b\n"
+ "uzp1 v17.2d, v19.2d, v20.2d\n"
+ "uzp2 v20.2d, v19.2d, v20.2d\n"
+ "fmul v19.4s, v27.4s, v0.s[0]\n"
+ "scvtf v17.4s, v17.4s, #0x4\n"
+ "scvtf v20.4s, v20.4s, #0x4\n"
+ "fmla v25.4s, v17.4s, v19.4s\n"
+ "ldr q19, [x21, #0x0]\n"
+ "fmul v17.4s, v27.4s, v0.s[1]\n"
+ "fmla v5.4s, v20.4s, v17.4s\n"
+ "ldr q17, [x21, #0x10]\n"
+ "uzp1 v20.2d, v9.2d, v18.2d\n"
+ "uzp2 v9.2d, v9.2d, v18.2d\n"
+ "fmul v18.4s, v27.4s, v0.s[2]\n"
+ "fmul v0.4s, v27.4s, v0.s[3]\n"
+ "scvtf v20.4s, v20.4s, #0x4\n"
+ "scvtf v9.4s, v9.4s, #0x4\n"
+ "fmla v7.4s, v20.4s, v18.4s\n"
+ "movi v20.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ ".inst 0x4e88a674 // smmla v20.4s, v19.16b, v8.16b\n"
+ ".inst 0x4e9fa672 // smmla v18.4s, v19.16b, v31.16b\n"
+ "ldr q19, [x21, #0x20]\n"
+ "fmla v4.4s, v9.4s, v0.4s\n"
+ "movi v9.4s, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ ".inst 0x4e88a629 // smmla v9.4s, v17.16b, v8.16b\n"
+ "fmul v8.4s, v27.4s, v26.s[0]\n"
+ ".inst 0x4e9fa620 // smmla v0.4s, v17.16b, v31.16b\n"
+ "ldr q17, [x21, #0x30]\n"
+ ".inst 0x4e8fa674 // smmla v20.4s, v19.16b, v15.16b\n"
+ "fmul v31.4s, v27.4s, v26.s[1]\n"
+ ".inst 0x4e81a672 // smmla v18.4s, v19.16b, v1.16b\n"
+ "ldr q19, [x21, #0x40]\n"
+ ".inst 0x4e8fa629 // smmla v9.4s, v17.16b, v15.16b\n"
+ "fmul v15.4s, v27.4s, v26.s[2]\n"
+ "fmul v27.4s, v27.4s, v26.s[3]\n"
+ ".inst 0x4e81a620 // smmla v0.4s, v17.16b, v1.16b\n"
+ "ldr q1, [x21, #0x50]\n"
+ ".inst 0x4e95a674 // smmla v20.4s, v19.16b, v21.16b\n"
+ ".inst 0x4e90a672 // smmla v18.4s, v19.16b, v16.16b\n"
+ "ldr q26, [x21, #0x60]\n"
+ ".inst 0x4e95a429 // smmla v9.4s, v1.16b, v21.16b\n"
+ ".inst 0x4e90a420 // smmla v0.4s, v1.16b, v16.16b\n"
+ "ldr q21, [x21, #0x70]\n"
+ "add x21, x21, #0x88\n"
+ ".inst 0x4e9da754 // smmla v20.4s, v26.16b, v29.16b\n"
+ ".inst 0x4e83a752 // smmla v18.4s, v26.16b, v3.16b\n"
+ ".inst 0x4e9da6a9 // smmla v9.4s, v21.16b, v29.16b\n"
+ ".inst 0x4e83a6a0 // smmla v0.4s, v21.16b, v3.16b\n"
+ "uzp1 v29.2d, v20.2d, v18.2d\n"
+ "uzp2 v21.2d, v20.2d, v18.2d\n"
+ "scvtf v29.4s, v29.4s, #0x4\n"
+ "uzp1 v18.2d, v9.2d, v0.2d\n"
+ "uzp2 v16.2d, v9.2d, v0.2d\n"
+ "scvtf v21.4s, v21.4s, #0x4\n"
+ "fmla v6.4s, v29.4s, v8.4s\n"
+ "scvtf v18.4s, v18.4s, #0x4\n"
+ "scvtf v16.4s, v16.4s, #0x4\n"
+ "fmla v30.4s, v21.4s, v31.4s\n"
+ "fmla v24.4s, v18.4s, v15.4s\n"
+ "fmla v14.4s, v16.4s, v27.4s\n"
+ "bgt 3b\n"
+ "mov x20, %x[res_ptr]\n"
+ "subs x27, x27, #0x4\n"
+ "add %x[res_ptr], %x[res_ptr], #0x10\n"
+ "str q2, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q10, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q12, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q28, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q11, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q13, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q22, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q23, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q25, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q5, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q7, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q4, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q6, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q30, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q24, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "str q14, [x20, #0x0]\n"
+ "bne 2b\n"
+ "mov x20, #0x4\n"
+ "sub x10, x10, #0x10\n"
+ "cmp x10, #0x10\n"
+ "mov %x[res_ptr], x26\n"
+ "madd %x[a_ptr], x20, x9, %x[a_ptr]\n"
+ "bge 1b\n"
+ "4:" // Row loop skip
+ "cbz x10, 9f\n"
+ "5:" // Row tail: Row loop
+ "add x24, %x[b_ptr], #0x8\n"
+ "mov x23, %x[nc]\n"
+ "add x22, %x[res_ptr], %x[res_stride], LSL #2\n"
+ "6:" // Row tail: Column loop
+ "movi v2.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "add x25, %x[a_ptr], #0x8\n"
+ "mov x21, %x[nb]\n"
+ "movi v12.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "7:" // Row tail: Block loop
+ "ldr q6, [x24, #0x0]\n"
+ "ldr q5, [x24, #0x10]\n"
+ "movi v17.16b, #0x4\n"
+ "movi v8.4s, #0x0\n"
+ "ldr q4, [x25, #0x0]\n"
+ "ldr q13, [x25, #0x10]\n"
+ "movi v27.4s, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "ldr q31, [x24, #0x20]\n"
+ "ldr q14, [x24, #0x30]\n"
+ "movi v29.4s, #0x0\n"
+ "movi v22.16b, #0xf0\n"
+ "ldr q11, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "sshl v21.16b, v6.16b, v17.16b\n"
+ "sshl v16.16b, v5.16b, v17.16b\n"
+ "ldr q20, [x25, #0x40]\n"
+ "ldr q26, [x25, #0x50]\n"
+ "and v6.16b, v6.16b, v22.16b\n"
+ "and v5.16b, v5.16b, v22.16b\n"
+ "ldr q25, [x25, #0x60]\n"
+ "ldr q3, [x25, #0x70]\n"
+ "sshl v19.16b, v31.16b, v17.16b\n"
+ "sshl v18.16b, v14.16b, v17.16b\n"
+ "ldr d17, [x25, #-0x8]\n"
+ ".inst 0x4e95a488 // smmla v8.4s, v4.16b, v21.16b\n"
+ ".inst 0x4e90a49b // smmla v27.4s, v4.16b, v16.16b\n"
+ "and v31.16b, v31.16b, v22.16b\n"
+ ".inst 0x4e95a5a0 // smmla v0.4s, v13.16b, v21.16b\n"
+ ".inst 0x4e90a5bd // smmla v29.4s, v13.16b, v16.16b\n"
+ "and v14.16b, v14.16b, v22.16b\n"
+ "sub x20, x24, #0x8\n"
+ "ldr d16, [x20, #0x0]\n"
+ "subs x21, x21, #0x1\n"
+ "add x25, x25, #0x88\n"
+ "fcvtl v17.4s, v17.4h\n"
+ "add x24, x24, #0x48\n"
+ ".inst 0x4e93a568 // smmla v8.4s, v11.16b, v19.16b\n"
+ ".inst 0x4e92a57b // smmla v27.4s, v11.16b, v18.16b\n"
+ ".inst 0x4e93a6e0 // smmla v0.4s, v23.16b, v19.16b\n"
+ ".inst 0x4e92a6fd // smmla v29.4s, v23.16b, v18.16b\n"
+ "fcvtl v16.4s, v16.4h\n"
+ ".inst 0x4e86a688 // smmla v8.4s, v20.16b, v6.16b\n"
+ ".inst 0x4e85a69b // smmla v27.4s, v20.16b, v5.16b\n"
+ "fmul v23.4s, v16.4s, v17.s[0]\n"
+ "fmul v21.4s, v16.4s, v17.s[1]\n"
+ "fmul v1.4s, v16.4s, v17.s[2]\n"
+ "fmul v20.4s, v16.4s, v17.s[3]\n"
+ ".inst 0x4e86a740 // smmla v0.4s, v26.16b, v6.16b\n"
+ ".inst 0x4e85a75d // smmla v29.4s, v26.16b, v5.16b\n"
+ ".inst 0x4e9fa728 // smmla v8.4s, v25.16b, v31.16b\n"
+ ".inst 0x4e8ea73b // smmla v27.4s, v25.16b, v14.16b\n"
+ ".inst 0x4e9fa460 // smmla v0.4s, v3.16b, v31.16b\n"
+ ".inst 0x4e8ea47d // smmla v29.4s, v3.16b, v14.16b\n"
+ "uzp1 v19.2d, v8.2d, v27.2d\n"
+ "uzp2 v18.2d, v8.2d, v27.2d\n"
+ "scvtf v19.4s, v19.4s, #0x4\n"
+ "uzp1 v17.2d, v0.2d, v29.2d\n"
+ "uzp2 v16.2d, v0.2d, v29.2d\n"
+ "scvtf v18.4s, v18.4s, #0x4\n"
+ "fmla v2.4s, v19.4s, v23.4s\n"
+ "scvtf v17.4s, v17.4s, #0x4\n"
+ "scvtf v16.4s, v16.4s, #0x4\n"
+ "fmla v10.4s, v18.4s, v21.4s\n"
+ "fmla v12.4s, v17.4s, v1.4s\n"
+ "fmla v28.4s, v16.4s, v20.4s\n"
+ "bgt 7b\n"
+ "mov x20, %x[res_ptr]\n"
+ "cmp x10, #0x1\n"
+ "str q2, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "ble 8f\n"
+ "cmp x10, #0x2\n"
+ "str q10, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "ble 8f\n"
+ "cmp x10, #0x3\n"
+ "str q12, [x20, #0x0]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "ble 8f\n"
+ "str q28, [x20, #0x0]\n"
+ "8:" // Row tail: Accumulator store skip
+ "subs x23, x23, #0x4\n"
+ "add %x[res_ptr], %x[res_ptr], #0x10\n"
+ "bne 6b\n"
+ "subs x10, x10, #0x4\n"
+ "add %x[a_ptr], %x[a_ptr], x9\n"
+ "mov %x[res_ptr], x22\n"
+ "bgt 5b\n"
+ "9:" // Row tail: Row loop skip
+ : [a_ptr] "+&r" (a_ptr), [res_ptr] "+&r" (res_ptr)
+ : [b_ptr] "r" (b_ptr), [nr] "r" (nr), [nb] "r" (nb), [res_stride] "r" (res_stride), [nc] "r" (nc)
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+#elif defined(__ARM_NEON) && defined(__aarch64__)
+ GGML_ASSERT((ggml_cpu_has_sve() || ggml_cpu_has_matmul_int8()) &&
+ "__ARM_FEATURE_SVE and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 quantization format for optimal "
+ "performance");
+#else
+ float sumf[4][4];
+ int sumi;
+
+ for (int y = 0; y < nr / 4; y++) {
+ const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);
+ for (int x = 0; x < nc / ncols_interleaved; x++) {
+ const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;
+ }
+ for (int l = 0; l < nb; l++) {
+ for (int k = 0; k < (qk / (2 * blocklen)); k++) {
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++) {
+ sumi = 0;
+ for (int i = 0; i < blocklen; ++i) {
+ const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
+ const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
+ sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +
+ (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;
+ }
+ sumf[m][j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d[m]);
+ }
+ }
+ }
+ }
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++)
+ s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];
+ }
+ }
+ }
+#endif
+}
+
+void ggml_gemm_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
+ const int qk = QK8_0;
+ const int nb = n / qk;
+ const int ncols_interleaved = 8;
+ const int blocklen = 8;
+
+ assert (n % qk == 0);
+ assert (nr % 4 == 0);
+ assert (nc % ncols_interleaved == 0);
+
+ UNUSED(s);
+ UNUSED(bs);
+ UNUSED(vx);
+ UNUSED(vy);
+ UNUSED(nr);
+ UNUSED(nc);
+ UNUSED(nb);
+ UNUSED(ncols_interleaved);
+ UNUSED(blocklen);
+
+#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
+ if (ggml_sve_cnt_b == QK8_0) {
+ const void * b_ptr = vx;
+ const void * a_ptr = vy;
+ float * res_ptr = s;
+ size_t res_stride = bs * sizeof(float);
+
+ __asm__ __volatile__(
+ "mov x20, #0x4\n"
+ "mov x13, %x[nr]\n"
+ "mov z28.s, #-0x4\n"
+ "mov x12, #0x88\n"
+ "ptrue p1.b\n"
+ "whilelt p0.s, XZR, x20\n"
+ "cmp x13, #0x10\n"
+ "mul x12, %x[nb], x12\n"
+ "blt 4f\n"
+ "1:" // Row loop
+ "add x11, %x[b_ptr], #0x10\n"
+ "mov x10, %x[nc]\n"
+ "add x9, %x[res_ptr], %x[res_stride], LSL #4\n"
+ "2:" // Column loop
+ "add x28, %x[a_ptr], #0x8\n"
+ "mov z24.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov x27, %x[nb]\n"
+ "add x26, x28, x12\n"
+ "mov z12.b, #0x0\n"
+ "mov z0.b, #0x0\n"
+ "add x25, x26, x12\n"
+ "mov z13.b, #0x0\n"
+ "mov z1.b, #0x0\n"
+ "add x24, x25, x12\n"
+ "mov z20.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z8.b, #0x0\n"
+ "mov z29.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "3:" // Block loop
+ "ld1b { z30.b }, p1/Z, [x11]\n"
+ "ld1b { z21.b }, p1/Z, [x11, #1, MUL VL]\n"
+ "mov z18.s, #0x0\n"
+ "mov z7.s, #0x0\n"
+ "ld1rqb { z3.b }, p1/Z, [x28]\n"
+ "ld1rqb { z5.b }, p1/Z, [x28, #16]\n"
+ "mov z9.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "ld1b { z4.b }, p1/Z, [x11, #2, MUL VL]\n"
+ "ld1b { z17.b }, p1/Z, [x11, #3, MUL VL]\n"
+ "sub x20, x11, #0x10\n"
+ "sub x23, x28, #0x8\n"
+ "lsl z31.b, z30.b, #0x4\n"
+ "lsl z6.b, z21.b, #0x4\n"
+ "ld1h { z23.s }, p1/Z, [x20]\n"
+ "sub x22, x26, #0x8\n"
+ "and z30.b, z30.b, #0xf0\n"
+ "and z21.b, z21.b, #0xf0\n"
+ "sub x21, x25, #0x8\n"
+ "sub x20, x24, #0x8\n"
+ "lsl z14.b, z4.b, #0x4\n"
+ "lsl z2.b, z17.b, #0x4\n"
+ "subs x27, x27, #0x1\n"
+ "add x11, x11, #0x90\n"
+ ".inst 0x451f9872 // smmla z18.s, z3.b, z31.b\n"
+ ".inst 0x45069867 // smmla z7.s, z3.b, z6.b\n"
+ "ld1rqb { z3.b }, p1/Z, [x28, #32]\n"
+ "and z4.b, z4.b, #0xf0\n"
+ ".inst 0x451f98a9 // smmla z9.s, z5.b, z31.b\n"
+ ".inst 0x450698b6 // smmla z22.s, z5.b, z6.b\n"
+ "ld1rqb { z5.b }, p1/Z, [x28, #48]\n"
+ "and z17.b, z17.b, #0xf0\n"
+ "fcvt z23.s, p1/m, z23.h\n"
+ ".inst 0x450e9872 // smmla z18.s, z3.b, z14.b\n"
+ ".inst 0x45029867 // smmla z7.s, z3.b, z2.b\n"
+ "ld1rqb { z3.b }, p1/Z, [x28, #64]\n"
+ ".inst 0x450e98a9 // smmla z9.s, z5.b, z14.b\n"
+ ".inst 0x450298b6 // smmla z22.s, z5.b, z2.b\n"
+ "ld1rqb { z5.b }, p1/Z, [x28, #80]\n"
+ "fscale z23.s, p1/m, z23.s, z28.s\n"
+ ".inst 0x451e9872 // smmla z18.s, z3.b, z30.b\n"
+ ".inst 0x45159867 // smmla z7.s, z3.b, z21.b\n"
+ "ld1rqb { z3.b }, p1/Z, [x28, #96]\n"
+ ".inst 0x451e98a9 // smmla z9.s, z5.b, z30.b\n"
+ ".inst 0x451598b6 // smmla z22.s, z5.b, z21.b\n"
+ "ld1rqb { z5.b }, p1/Z, [x28, #112]\n"
+ "add x28, x28, #0x88\n"
+ ".inst 0x45049872 // smmla z18.s, z3.b, z4.b\n"
+ ".inst 0x45119867 // smmla z7.s, z3.b, z17.b\n"
+ "ld1h { z3.s }, p0/Z, [x23]\n"
+ ".inst 0x450498a9 // smmla z9.s, z5.b, z4.b\n"
+ ".inst 0x451198b6 // smmla z22.s, z5.b, z17.b\n"
+ "fcvt z3.s, p1/m, z3.h\n"
+ "uzp1 z5.d, z18.d, z7.d\n"
+ "uzp2 z18.d, z18.d, z7.d\n"
+ "mov z3.q, z3.q[0]\n"
+ "uzp1 z7.d, z9.d, z22.d\n"
+ "uzp2 z22.d, z9.d, z22.d\n"
+ "fmul z9.s, z23.s, z3.s[0]\n"
+ "scvtf z5.s, p1/m, z5.s\n"
+ "scvtf z18.s, p1/m, z18.s\n"
+ "scvtf z7.s, p1/m, z7.s\n"
+ "scvtf z22.s, p1/m, z22.s\n"
+ "fmla z24.s, p1/M, z5.s, z9.s\n"
+ "ld1rqb { z5.b }, p1/Z, [x26]\n"
+ "fmul z9.s, z23.s, z3.s[1]\n"
+ "fmla z15.s, p1/M, z18.s, z9.s\n"
+ "ld1rqb { z18.b }, p1/Z, [x26, #16]\n"
+ "fmul z9.s, z23.s, z3.s[2]\n"
+ "fmul z3.s, z23.s, z3.s[3]\n"
+ "fmla z12.s, p1/M, z7.s, z9.s\n"
+ "mov z9.s, #0x0\n"
+ "ld1h { z7.s }, p0/Z, [x22]\n"
+ ".inst 0x451f98a9 // smmla z9.s, z5.b, z31.b\n"
+ "fmla z0.s, p1/M, z22.s, z3.s\n"
+ "mov z22.s, #0x0\n"
+ "ld1h { z3.s }, p0/Z, [x21]\n"
+ ".inst 0x450698b6 // smmla z22.s, z5.b, z6.b\n"
+ "ld1rqb { z5.b }, p1/Z, [x26, #32]\n"
+ "fcvt z7.s, p1/m, z7.h\n"
+ "fcvt z3.s, p1/m, z3.h\n"
+ ".inst 0x450e98a9 // smmla z9.s, z5.b, z14.b\n"
+ ".inst 0x450298b6 // smmla z22.s, z5.b, z2.b\n"
+ "ld1rqb { z5.b }, p1/Z, [x26, #64]\n"
+ "mov z7.q, z7.q[0]\n"
+ "mov z3.q, z3.q[0]\n"
+ ".inst 0x451e98a9 // smmla z9.s, z5.b, z30.b\n"
+ ".inst 0x451598b6 // smmla z22.s, z5.b, z21.b\n"
+ "ld1rqb { z5.b }, p1/Z, [x26, #96]\n"
+ ".inst 0x450498a9 // smmla z9.s, z5.b, z4.b\n"
+ ".inst 0x451198b6 // smmla z22.s, z5.b, z17.b\n"
+ "uzp1 z5.d, z9.d, z22.d\n"
+ "scvtf z5.s, p1/m, z5.s\n"
+ "uzp2 z22.d, z9.d, z22.d\n"
+ "fmul z9.s, z23.s, z7.s[0]\n"
+ "scvtf z22.s, p1/m, z22.s\n"
+ "fmla z13.s, p1/M, z5.s, z9.s\n"
+ "ld1rqb { z9.b }, p1/Z, [x25]\n"
+ "fmul z5.s, z23.s, z7.s[1]\n"
+ "fmla z1.s, p1/M, z22.s, z5.s\n"
+ "mov z5.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ ".inst 0x451f9a45 // smmla z5.s, z18.b, z31.b\n"
+ ".inst 0x45069a56 // smmla z22.s, z18.b, z6.b\n"
+ "ld1rqb { z18.b }, p1/Z, [x26, #48]\n"
+ ".inst 0x450e9a45 // smmla z5.s, z18.b, z14.b\n"
+ ".inst 0x45029a56 // smmla z22.s, z18.b, z2.b\n"
+ "ld1rqb { z18.b }, p1/Z, [x26, #80]\n"
+ ".inst 0x451e9a45 // smmla z5.s, z18.b, z30.b\n"
+ ".inst 0x45159a56 // smmla z22.s, z18.b, z21.b\n"
+ "ld1rqb { z18.b }, p1/Z, [x26, #112]\n"
+ "add x26, x26, #0x88\n"
+ ".inst 0x45049a45 // smmla z5.s, z18.b, z4.b\n"
+ ".inst 0x45119a56 // smmla z22.s, z18.b, z17.b\n"
+ "uzp1 z18.d, z5.d, z22.d\n"
+ "scvtf z18.s, p1/m, z18.s\n"
+ "uzp2 z22.d, z5.d, z22.d\n"
+ "fmul z5.s, z23.s, z7.s[2]\n"
+ "fmul z7.s, z23.s, z7.s[3]\n"
+ "scvtf z22.s, p1/m, z22.s\n"
+ "fmla z20.s, p1/M, z18.s, z5.s\n"
+ "ld1rqb { z18.b }, p1/Z, [x25, #16]\n"
+ "ld1h { z5.s }, p0/Z, [x20]\n"
+ "fcvt z5.s, p1/m, z5.h\n"
+ "fmla z25.s, p1/M, z22.s, z7.s\n"
+ "mov z22.s, #0x0\n"
+ "mov z7.s, #0x0\n"
+ ".inst 0x451f9936 // smmla z22.s, z9.b, z31.b\n"
+ ".inst 0x45069927 // smmla z7.s, z9.b, z6.b\n"
+ "ld1rqb { z9.b }, p1/Z, [x25, #32]\n"
+ "mov z5.q, z5.q[0]\n"
+ ".inst 0x450e9936 // smmla z22.s, z9.b, z14.b\n"
+ ".inst 0x45029927 // smmla z7.s, z9.b, z2.b\n"
+ "ld1rqb { z9.b }, p1/Z, [x25, #64]\n"
+ ".inst 0x451e9936 // smmla z22.s, z9.b, z30.b\n"
+ ".inst 0x45159927 // smmla z7.s, z9.b, z21.b\n"
+ "ld1rqb { z9.b }, p1/Z, [x25, #96]\n"
+ ".inst 0x45049936 // smmla z22.s, z9.b, z4.b\n"
+ ".inst 0x45119927 // smmla z7.s, z9.b, z17.b\n"
+ "uzp1 z9.d, z22.d, z7.d\n"
+ "scvtf z9.s, p1/m, z9.s\n"
+ "uzp2 z22.d, z22.d, z7.d\n"
+ "fmul z7.s, z23.s, z3.s[0]\n"
+ "scvtf z22.s, p1/m, z22.s\n"
+ "fmla z11.s, p1/M, z9.s, z7.s\n"
+ "ld1rqb { z9.b }, p1/Z, [x24]\n"
+ "fmul z7.s, z23.s, z3.s[1]\n"
+ "fmla z16.s, p1/M, z22.s, z7.s\n"
+ "mov z22.s, #0x0\n"
+ "mov z7.s, #0x0\n"
+ ".inst 0x451f9a56 // smmla z22.s, z18.b, z31.b\n"
+ ".inst 0x45069a47 // smmla z7.s, z18.b, z6.b\n"
+ "ld1rqb { z18.b }, p1/Z, [x25, #48]\n"
+ ".inst 0x450e9a56 // smmla z22.s, z18.b, z14.b\n"
+ ".inst 0x45029a47 // smmla z7.s, z18.b, z2.b\n"
+ "ld1rqb { z18.b }, p1/Z, [x25, #80]\n"
+ ".inst 0x451e9a56 // smmla z22.s, z18.b, z30.b\n"
+ ".inst 0x45159a47 // smmla z7.s, z18.b, z21.b\n"
+ "ld1rqb { z18.b }, p1/Z, [x25, #112]\n"
+ "add x25, x25, #0x88\n"
+ ".inst 0x45049a56 // smmla z22.s, z18.b, z4.b\n"
+ ".inst 0x45119a47 // smmla z7.s, z18.b, z17.b\n"
+ "uzp1 z18.d, z22.d, z7.d\n"
+ "scvtf z18.s, p1/m, z18.s\n"
+ "uzp2 z7.d, z22.d, z7.d\n"
+ "fmul z22.s, z23.s, z3.s[2]\n"
+ "fmul z3.s, z23.s, z3.s[3]\n"
+ "scvtf z7.s, p1/m, z7.s\n"
+ "fmla z19.s, p1/M, z18.s, z22.s\n"
+ "ld1rqb { z18.b }, p1/Z, [x24, #16]\n"
+ "fmul z22.s, z23.s, z5.s[0]\n"
+ "fmla z26.s, p1/M, z7.s, z3.s\n"
+ "mov z3.s, #0x0\n"
+ "mov z7.s, #0x0\n"
+ ".inst 0x451f9923 // smmla z3.s, z9.b, z31.b\n"
+ ".inst 0x45069927 // smmla z7.s, z9.b, z6.b\n"
+ "ld1rqb { z9.b }, p1/Z, [x24, #32]\n"
+ ".inst 0x450e9923 // smmla z3.s, z9.b, z14.b\n"
+ ".inst 0x45029927 // smmla z7.s, z9.b, z2.b\n"
+ "mov z9.s, #0x0\n"
+ ".inst 0x451f9a49 // smmla z9.s, z18.b, z31.b\n"
+ "mov z31.s, #0x0\n"
+ ".inst 0x45069a5f // smmla z31.s, z18.b, z6.b\n"
+ "ld1rqb { z6.b }, p1/Z, [x24, #48]\n"
+ "ld1rqb { z18.b }, p1/Z, [x24, #64]\n"
+ ".inst 0x450e98c9 // smmla z9.s, z6.b, z14.b\n"
+ "fmul z14.s, z23.s, z5.s[1]\n"
+ ".inst 0x450298df // smmla z31.s, z6.b, z2.b\n"
+ "ld1rqb { z6.b }, p1/Z, [x24, #80]\n"
+ "fmul z2.s, z23.s, z5.s[2]\n"
+ "fmul z23.s, z23.s, z5.s[3]\n"
+ ".inst 0x451e9a43 // smmla z3.s, z18.b, z30.b\n"
+ ".inst 0x45159a47 // smmla z7.s, z18.b, z21.b\n"
+ "ld1rqb { z5.b }, p1/Z, [x24, #96]\n"
+ ".inst 0x451e98c9 // smmla z9.s, z6.b, z30.b\n"
+ ".inst 0x451598df // smmla z31.s, z6.b, z21.b\n"
+ "ld1rqb { z18.b }, p1/Z, [x24, #112]\n"
+ "add x24, x24, #0x88\n"
+ ".inst 0x450498a3 // smmla z3.s, z5.b, z4.b\n"
+ ".inst 0x451198a7 // smmla z7.s, z5.b, z17.b\n"
+ ".inst 0x45049a49 // smmla z9.s, z18.b, z4.b\n"
+ ".inst 0x45119a5f // smmla z31.s, z18.b, z17.b\n"
+ "uzp1 z18.d, z3.d, z7.d\n"
+ "uzp2 z5.d, z3.d, z7.d\n"
+ "scvtf z18.s, p1/m, z18.s\n"
+ "uzp1 z6.d, z9.d, z31.d\n"
+ "uzp2 z9.d, z9.d, z31.d\n"
+ "scvtf z5.s, p1/m, z5.s\n"
+ "fmla z8.s, p1/M, z18.s, z22.s\n"
+ "scvtf z6.s, p1/m, z6.s\n"
+ "scvtf z9.s, p1/m, z9.s\n"
+ "fmla z29.s, p1/M, z5.s, z14.s\n"
+ "fmla z27.s, p1/M, z6.s, z2.s\n"
+ "fmla z10.s, p1/M, z9.s, z23.s\n"
+ "bgt 3b\n"
+ "mov x20, %x[res_ptr]\n"
+ "subs x10, x10, #0x8\n"
+ "add %x[res_ptr], %x[res_ptr], #0x20\n"
+ "st1w { z24.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z15.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z12.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z0.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z13.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z1.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z20.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z25.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z11.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z16.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z19.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z26.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z8.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z29.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z27.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "st1w { z10.s }, p1, [x20]\n"
+ "bne 2b\n"
+ "mov x20, #0x4\n"
+ "sub x13, x13, #0x10\n"
+ "cmp x13, #0x10\n"
+ "mov %x[res_ptr], x9\n"
+ "madd %x[a_ptr], x20, x12, %x[a_ptr]\n"
+ "bge 1b\n"
+ "4:" // Row loop skip
+ "cbz x13, 9f\n"
+ "5:" // Row tail: Row loop
+ "add x25, %x[b_ptr], #0x10\n"
+ "mov x24, %x[nc]\n"
+ "add x23, %x[res_ptr], %x[res_stride], LSL #2\n"
+ "6:" // Row tail: Column loop
+ "mov z24.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "add x28, %x[a_ptr], #0x8\n"
+ "mov x22, %x[nb]\n"
+ "mov z12.b, #0x0\n"
+ "mov z0.b, #0x0\n"
+ "7:" // Row tail: Block loop
+ "ld1b { z3.b }, p1/Z, [x25]\n"
+ "ld1b { z6.b }, p1/Z, [x25, #1, MUL VL]\n"
+ "mov z2.s, #0x0\n"
+ "mov z25.s, #0x0\n"
+ "ld1rqb { z26.b }, p1/Z, [x28]\n"
+ "ld1rqb { z21.b }, p1/Z, [x28, #16]\n"
+ "mov z27.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "ld1b { z29.b }, p1/Z, [x25, #2, MUL VL]\n"
+ "ld1b { z16.b }, p1/Z, [x25, #3, MUL VL]\n"
+ "sub x21, x25, #0x10\n"
+ "sub x20, x28, #0x8\n"
+ "lsl z20.b, z3.b, #0x4\n"
+ "lsl z4.b, z6.b, #0x4\n"
+ "ld1rqb { z10.b }, p1/Z, [x28, #32]\n"
+ "ld1rqb { z23.b }, p1/Z, [x28, #48]\n"
+ "and z3.b, z3.b, #0xf0\n"
+ "and z6.b, z6.b, #0xf0\n"
+ "ld1rqb { z11.b }, p1/Z, [x28, #64]\n"
+ "ld1rqb { z7.b }, p1/Z, [x28, #80]\n"
+ "lsl z8.b, z29.b, #0x4\n"
+ "lsl z14.b, z16.b, #0x4\n"
+ "ld1rqb { z18.b }, p1/Z, [x28, #96]\n"
+ "ld1rqb { z30.b }, p1/Z, [x28, #112]\n"
+ ".inst 0x45149b42 // smmla z2.s, z26.b, z20.b\n"
+ ".inst 0x45049b59 // smmla z25.s, z26.b, z4.b\n"
+ "and z29.b, z29.b, #0xf0\n"
+ "ld1h { z17.s }, p1/Z, [x21]\n"
+ ".inst 0x45149abb // smmla z27.s, z21.b, z20.b\n"
+ ".inst 0x45049ab3 // smmla z19.s, z21.b, z4.b\n"
+ "and z16.b, z16.b, #0xf0\n"
+ "ld1h { z4.s }, p0/Z, [x20]\n"
+ "subs x22, x22, #0x1\n"
+ "add x28, x28, #0x88\n"
+ "fcvt z17.s, p1/m, z17.h\n"
+ "add x25, x25, #0x90\n"
+ ".inst 0x45089942 // smmla z2.s, z10.b, z8.b\n"
+ ".inst 0x450e9959 // smmla z25.s, z10.b, z14.b\n"
+ "fcvt z4.s, p1/m, z4.h\n"
+ ".inst 0x45089afb // smmla z27.s, z23.b, z8.b\n"
+ ".inst 0x450e9af3 // smmla z19.s, z23.b, z14.b\n"
+ "fscale z17.s, p1/m, z17.s, z28.s\n"
+ "mov z4.q, z4.q[0]\n"
+ ".inst 0x45039962 // smmla z2.s, z11.b, z3.b\n"
+ ".inst 0x45069979 // smmla z25.s, z11.b, z6.b\n"
+ "fmul z23.s, z17.s, z4.s[0]\n"
+ "fmul z9.s, z17.s, z4.s[1]\n"
+ "fmul z21.s, z17.s, z4.s[2]\n"
+ "fmul z4.s, z17.s, z4.s[3]\n"
+ ".inst 0x450398fb // smmla z27.s, z7.b, z3.b\n"
+ ".inst 0x450698f3 // smmla z19.s, z7.b, z6.b\n"
+ ".inst 0x451d9a42 // smmla z2.s, z18.b, z29.b\n"
+ ".inst 0x45109a59 // smmla z25.s, z18.b, z16.b\n"
+ ".inst 0x451d9bdb // smmla z27.s, z30.b, z29.b\n"
+ ".inst 0x45109bd3 // smmla z19.s, z30.b, z16.b\n"
+ "uzp1 z31.d, z2.d, z25.d\n"
+ "uzp2 z13.d, z2.d, z25.d\n"
+ "scvtf z31.s, p1/m, z31.s\n"
+ "uzp1 z17.d, z27.d, z19.d\n"
+ "uzp2 z18.d, z27.d, z19.d\n"
+ "scvtf z13.s, p1/m, z13.s\n"
+ "fmla z24.s, p1/M, z31.s, z23.s\n"
+ "scvtf z17.s, p1/m, z17.s\n"
+ "scvtf z18.s, p1/m, z18.s\n"
+ "fmla z15.s, p1/M, z13.s, z9.s\n"
+ "fmla z12.s, p1/M, z17.s, z21.s\n"
+ "fmla z0.s, p1/M, z18.s, z4.s\n"
+ "bgt 7b\n"
+ "mov x20, %x[res_ptr]\n"
+ "cmp x13, #0x1\n"
+ "st1w { z24.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "ble 8f\n"
+ "cmp x13, #0x2\n"
+ "st1w { z15.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "ble 8f\n"
+ "cmp x13, #0x3\n"
+ "st1w { z12.s }, p1, [x20]\n"
+ "add x20, x20, %x[res_stride]\n"
+ "ble 8f\n"
+ "st1w { z0.s }, p1, [x20]\n"
+ "8:" // Row tail: Accumulator store skip
+ "subs x24, x24, #0x8\n"
+ "add %x[res_ptr], %x[res_ptr], #0x20\n"
+ "bne 6b\n"
+ "subs x13, x13, #0x4\n"
+ "add %x[a_ptr], %x[a_ptr], x12\n"
+ "mov %x[res_ptr], x23\n"
+ "bgt 5b\n"
+ "9:" // Row tail: Row loop skip
+ : [a_ptr] "+&r" (a_ptr), [res_ptr] "+&r" (res_ptr)
+ : [b_ptr] "r" (b_ptr), [nr] "r" (nr), [nb] "r" (nb), [res_stride] "r" (res_stride), [nc] "r" (nc)
+ : "cc", "memory", "p0", "p1", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+ return;
+ }
+ else if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {
+ GGML_ASSERT((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
+ "__ARM_FEATURE_SVE for vector size of 256-bits not defined, use the Q4_0_4_8 quantization format for optimal "
+ "performance");
+ }
+ else if (ggml_cpu_has_neon()) {
+ GGML_ASSERT(((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) || ggml_cpu_has_matmul_int8()) &&
+ "__ARM_FEATURE_SVE for vector size of 256-bits and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 "
+ "quantization format for optimal performance");
+ }
+#endif
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
+ GGML_ASSERT(ggml_cpu_has_sve() &&
+ "__ARM_FEATURE_SVE not defined, use the Q4_0_4_8 quantization format for optimal performance");
+#elif defined(__ARM_NEON) && defined(__aarch64__)
+ GGML_ASSERT((ggml_cpu_has_sve() || ggml_cpu_has_matmul_int8()) &&
+ "__ARM_FEATURE_SVE and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 quantization format for optimal "
+ "performance");
+#else
+ float sumf[4][8];
+ int sumi;
+
+ for (int y = 0; y < nr / 4; y++) {
+ const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);
+ for (int x = 0; x < nc / ncols_interleaved; x++) {
+ const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;
+ }
+ for (int l = 0; l < nb; l++) {
+ for (int k = 0; k < (qk / (2 * blocklen)); k++) {
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++) {
+ sumi = 0;
+ for (int i = 0; i < blocklen; ++i) {
+ const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
+ const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
+ sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +
+ (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;
+ }
+ sumf[m][j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d[m]);
+ }
+ }
+ }
+ }
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++)
+ s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];
+ }
+ }
+ }
+#endif
+}
diff --git a/llama/ggml-aarch64.h b/llama/ggml-aarch64.h
new file mode 100644
index 00000000..0fc24e80
--- /dev/null
+++ b/llama/ggml-aarch64.h
@@ -0,0 +1,65 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// SPDX-FileCopyrightText: Copyright 2024 Arm Ltd.
+#pragma once
+
+#define GGML_COMMON_DECL_C
+#include "ggml-common.h"
+
+#include "ggml.h"
+
+// GGML internal header
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Quantization
+void quantize_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+void quantize_mat_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t nrows, int64_t n_per_row, int64_t blck_size_interleave);
+
+// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")
+size_t quantize_q4_0_4x4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q4_0_4x8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q4_0_8x8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+// GEMV
+void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+
+// GEMM
+void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/llama/ggml-alloc.c b/llama/ggml-alloc.c
new file mode 100644
index 00000000..edd33746
--- /dev/null
+++ b/llama/ggml-alloc.c
@@ -0,0 +1,1062 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "ggml-alloc.h"
+#include "ggml-backend-impl.h"
+#include "ggml.h"
+#include "ggml-impl.h"
+#include <assert.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MAX_FREE_BLOCKS 256
+
+//#define GGML_ALLOCATOR_DEBUG
+
+//#define AT_PRINTF(...) fprintf(stderr, __VA_ARGS__)
+#define AT_PRINTF(...)
+
+
+static bool ggml_is_view(const struct ggml_tensor * t) {
+ return t->view_src != NULL;
+}
+
+static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {
+ if (a->type != b->type) {
+ return false;
+ }
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
+ if (a->ne[i] != b->ne[i]) {
+ return false;
+ }
+ if (a->nb[i] != b->nb[i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool ggml_op_can_inplace(enum ggml_op op) {
+ switch (op) {
+ case GGML_OP_SCALE:
+ case GGML_OP_DIAG_MASK_ZERO:
+ case GGML_OP_DIAG_MASK_INF:
+ case GGML_OP_ADD:
+ case GGML_OP_ADD1:
+ case GGML_OP_SUB:
+ case GGML_OP_MUL:
+ case GGML_OP_DIV:
+ case GGML_OP_SQR:
+ case GGML_OP_SQRT:
+ case GGML_OP_LOG:
+ case GGML_OP_UNARY:
+ case GGML_OP_ROPE:
+ case GGML_OP_RMS_NORM:
+ case GGML_OP_SOFT_MAX:
+ return true;
+
+ default:
+ return false;
+ }
+}
+
+static size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) {
+ assert(alignment && !(alignment & (alignment - 1))); // power of 2
+ size_t align = (alignment - (((uintptr_t)buffer + offset) % alignment)) % alignment;
+ return offset + align;
+}
+
+// tallocr
+
+struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer) {
+ void * base = ggml_backend_buffer_get_base(buffer);
+ size_t align = ggml_backend_buffer_get_alignment(buffer);
+
+ assert(align && !(align & (align - 1))); // power of 2
+
+ struct ggml_tallocr talloc = (struct ggml_tallocr) {
+ /*.buffer = */ buffer,
+ /*.base = */ base,
+ /*.alignment = */ align,
+ /*.offset = */ aligned_offset(base, 0, align),
+ };
+ return talloc;
+}
+
+void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor) {
+ size_t size = ggml_backend_buffer_get_alloc_size(talloc->buffer, tensor);
+ size = GGML_PAD(size, talloc->alignment);
+
+ if (talloc->offset + size > ggml_backend_buffer_get_size(talloc->buffer)) {
+ fprintf(stderr, "%s: not enough space in the buffer to allocate %s (needed %zu, available %zu)\n",
+ __func__, tensor->name, size, ggml_backend_buffer_get_size(talloc->buffer) - talloc->offset);
+ GGML_ABORT("not enough space in the buffer");
+ }
+
+ void * addr = (char *)ggml_backend_buffer_get_base(talloc->buffer) + talloc->offset;
+ talloc->offset += size;
+
+ assert(((uintptr_t)addr % talloc->alignment) == 0);
+
+ ggml_backend_tensor_alloc(talloc->buffer, tensor, addr);
+}
+
+// dynamic tensor allocator
+
+struct free_block {
+ size_t offset;
+ size_t size;
+};
+
+struct ggml_dyn_tallocr {
+ size_t alignment;
+ int n_free_blocks;
+ struct free_block free_blocks[MAX_FREE_BLOCKS];
+ size_t max_size;
+
+#ifdef GGML_ALLOCATOR_DEBUG
+ struct {
+ const struct ggml_tensor * tensor;
+ size_t offset;
+ } allocated_tensors[1024];
+#endif
+};
+
+#ifdef GGML_ALLOCATOR_DEBUG
+static void add_allocated_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, const struct ggml_tensor * tensor) {
+ for (int i = 0; i < 1024; i++) {
+ if (alloc->allocated_tensors[i].tensor == NULL) {
+ alloc->allocated_tensors[i].tensor = tensor;
+ alloc->allocated_tensors[i].offset = offset;
+ return;
+ }
+ }
+ GGML_ABORT("out of allocated_tensors");
+}
+static void remove_allocated_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, const struct ggml_tensor * tensor) {
+ for (int i = 0; i < 1024; i++) {
+ if (alloc->allocated_tensors[i].offset == offset) {
+ alloc->allocated_tensors[i].tensor = NULL;
+ return;
+ }
+ }
+ GGML_ABORT("tried to free tensor %s not found\n", tensor->name);
+}
+#endif
+
+static size_t ggml_dyn_tallocr_alloc(struct ggml_dyn_tallocr * alloc, size_t size, const struct ggml_tensor * tensor) {
+ size = aligned_offset(NULL, size, alloc->alignment);
+
+ AT_PRINTF("%s: allocating %s (%zu bytes) - ", __func__, tensor->name, size);
+
+ size_t max_avail = 0;
+
+ // find the best fitting free block besides the last block
+ int best_fit_block = -1;
+ size_t best_fit_size = SIZE_MAX;
+ for (int i = 0; i < alloc->n_free_blocks - 1; i++) {
+ struct free_block * block = &alloc->free_blocks[i];
+ max_avail = MAX(max_avail, block->size);
+ if (block->size >= size && block->size <= best_fit_size) {
+ best_fit_block = i;
+ best_fit_size = block->size;
+ }
+ }
+
+ if (best_fit_block == -1) {
+ // the last block is our last resort
+ struct free_block * block = &alloc->free_blocks[alloc->n_free_blocks - 1];
+ max_avail = MAX(max_avail, block->size);
+ if (block->size >= size) {
+ best_fit_block = alloc->n_free_blocks - 1;
+ } else {
+ // this should never happen
+ fprintf(stderr, "%s: not enough space in the buffer to allocate %zu bytes, largest block available %zu bytes\n",
+ __func__, size, max_avail);
+ GGML_ABORT("not enough space in the buffer");
+ }
+ }
+
+ struct free_block * block = &alloc->free_blocks[best_fit_block];
+ size_t offset = block->offset;
+ block->offset = offset + size;
+ block->size -= size;
+ if (block->size == 0) {
+ // remove block if empty
+ alloc->n_free_blocks--;
+ for (int j = best_fit_block; j < alloc->n_free_blocks; j++) {
+ alloc->free_blocks[j] = alloc->free_blocks[j+1];
+ }
+ }
+
+ AT_PRINTF("block %d, offset %zu\n", best_fit_block, offset);
+
+#ifdef GGML_ALLOCATOR_DEBUG
+ add_allocated_tensor(alloc, offset, tensor);
+ size_t cur_max = offset + size;
+ if (cur_max > alloc->max_size) {
+ // sort allocated_tensors by offset
+ for (int i = 0; i < 1024; i++) {
+ for (int j = i + 1; j < 1024; j++) {
+ if (alloc->allocated_tensors[i].offset > alloc->allocated_tensors[j].offset) {
+ const struct ggml_tensor * tmp_tensor = alloc->allocated_tensors[i].tensor;
+ size_t tmp_offset = alloc->allocated_tensors[i].offset;
+ alloc->allocated_tensors[i].tensor = alloc->allocated_tensors[j].tensor;
+ alloc->allocated_tensors[i].offset = alloc->allocated_tensors[j].offset;
+ alloc->allocated_tensors[j].tensor = tmp_tensor;
+ alloc->allocated_tensors[j].offset = tmp_offset;
+ }
+ }
+ }
+ fprintf(stderr, "max_size = %.2f MB: tensors: ", cur_max / 1024.0 / 1024.0);
+ for (int i = 0; i < 1024; i++) {
+ if (alloc->allocated_tensors[i].tensor) {
+ fprintf(stderr, "%s [%zx-%zx] (%.2f MB) ", alloc->allocated_tensors[i].tensor->name,
+ alloc->allocated_tensors[i].offset,
+ alloc->allocated_tensors[i].offset + ggml_nbytes(alloc->allocated_tensors[i].tensor),
+ ggml_nbytes(alloc->allocated_tensors[i].tensor) / 1024.0 / 1024.0);
+ }
+ }
+ fprintf(stderr, "\n");
+ }
+#endif
+
+ alloc->max_size = MAX(alloc->max_size, offset + size);
+
+ return offset;
+
+ GGML_UNUSED(tensor);
+}
+
+// this is a very naive implementation, but for our case the number of free blocks should be very small
+static void ggml_dyn_tallocr_free_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, size_t size, const struct ggml_tensor * tensor) {
+ size = aligned_offset(NULL, size, alloc->alignment);
+
+ AT_PRINTF("%s: freeing %s at %zu (%zu bytes) - n_free_blocks = %d\n", __func__, tensor->name, offset, size, alloc->n_free_blocks);
+
+#ifdef GGML_ALLOCATOR_DEBUG
+ remove_allocated_tensor(alloc, offset, tensor);
+#endif
+
+ // see if we can merge with an existing block
+ for (int i = 0; i < alloc->n_free_blocks; i++) {
+ struct free_block * block = &alloc->free_blocks[i];
+ // check if ptr is at the end of the block
+ if (block->offset + block->size == offset) {
+ block->size += size;
+ // check if we can merge with the next block
+ if (i < alloc->n_free_blocks - 1 && block->offset + block->size == alloc->free_blocks[i+1].offset) {
+ block->size += alloc->free_blocks[i+1].size;
+ alloc->n_free_blocks--;
+ for (int j = i+1; j < alloc->n_free_blocks; j++) {
+ alloc->free_blocks[j] = alloc->free_blocks[j+1];
+ }
+ }
+ return;
+ }
+ // check if ptr is at the beginning of the block
+ if (offset + size == block->offset) {
+ block->offset = offset;
+ block->size += size;
+ // check if we can merge with the previous block
+ if (i > 0 && alloc->free_blocks[i-1].offset + alloc->free_blocks[i-1].size == block->offset) {
+ alloc->free_blocks[i-1].size += block->size;
+ alloc->n_free_blocks--;
+ for (int j = i; j < alloc->n_free_blocks; j++) {
+ alloc->free_blocks[j] = alloc->free_blocks[j+1];
+ }
+ }
+ return;
+ }
+ }
+ // otherwise, add a new block
+ GGML_ASSERT(alloc->n_free_blocks < MAX_FREE_BLOCKS && "out of free blocks");
+ // insert the new block in the correct position to keep the array sorted by address (to make merging blocks faster)
+ int insert_pos = 0;
+ while (insert_pos < alloc->n_free_blocks && alloc->free_blocks[insert_pos].offset < offset) {
+ insert_pos++;
+ }
+ // shift all blocks from insert_pos onward to make room for the new block
+ for (int i = alloc->n_free_blocks; i > insert_pos; i--) {
+ alloc->free_blocks[i] = alloc->free_blocks[i-1];
+ }
+ // insert the new block
+ alloc->free_blocks[insert_pos].offset = offset;
+ alloc->free_blocks[insert_pos].size = size;
+ alloc->n_free_blocks++;
+
+ GGML_UNUSED(tensor);
+}
+
+static void ggml_dyn_tallocr_reset(struct ggml_dyn_tallocr * alloc) {
+ alloc->n_free_blocks = 1;
+ alloc->free_blocks[0].offset = 0;
+ alloc->free_blocks[0].size = SIZE_MAX/2; // restrict maximum size of a measure allocator to half size_t max to avoid overflows
+ alloc->max_size = 0;
+}
+
+static struct ggml_dyn_tallocr * ggml_dyn_tallocr_new(size_t alignment) {
+ struct ggml_dyn_tallocr * alloc = (struct ggml_dyn_tallocr *)malloc(sizeof(struct ggml_dyn_tallocr));
+
+ *alloc = (struct ggml_dyn_tallocr) {
+ /*.alignment = */ alignment,
+ /*.n_free_blocks = */ 0,
+ /*.free_blocks = */ {{0}},
+ /*.max_size = */ 0,
+#ifdef GGML_ALLOCATOR_DEBUG
+ /*.allocated_tensors = */ {{0}},
+#endif
+ };
+
+ ggml_dyn_tallocr_reset(alloc);
+
+ return alloc;
+}
+
+static void ggml_dyn_tallocr_free(struct ggml_dyn_tallocr * alloc) {
+ free(alloc);
+}
+
+static size_t ggml_dyn_tallocr_max_size(struct ggml_dyn_tallocr * alloc) {
+ return alloc->max_size;
+}
+
+
+/////////////////////////////////////
+
+// graph allocator
+
+struct hash_node {
+ int n_children;
+ int n_views;
+ int buffer_id;
+ size_t offset; // offset within the buffer
+ bool allocated;
+};
+
+struct tensor_alloc {
+ int buffer_id;
+ size_t offset;
+ size_t size_max; // 0 = pre-allocated, unused, or view
+};
+
+struct leaf_alloc {
+ int buffer_id;
+ struct tensor_alloc leaf;
+};
+
+struct node_alloc {
+ struct tensor_alloc dst;
+ struct tensor_alloc src[GGML_MAX_SRC];
+};
+
+struct ggml_gallocr {
+ ggml_backend_buffer_type_t * bufts; // [n_buffers]
+ ggml_backend_buffer_t * buffers; // [n_buffers]
+ struct ggml_dyn_tallocr ** buf_tallocs; // [n_buffers]
+ int n_buffers;
+
+ struct ggml_hash_set hash_set;
+ struct hash_node * hash_values; // [hash_set.size]
+
+ struct node_alloc * node_allocs; // [n_nodes]
+ int n_nodes;
+
+ struct leaf_alloc * leaf_allocs; // [n_leafs]
+ int n_leafs;
+};
+
+ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs) {
+ ggml_gallocr_t galloc = (ggml_gallocr_t)calloc(1, sizeof(struct ggml_gallocr));
+ GGML_ASSERT(galloc != NULL);
+
+ galloc->bufts = calloc(n_bufs, sizeof(ggml_backend_buffer_type_t));
+ GGML_ASSERT(galloc->bufts != NULL);
+
+ galloc->buffers = calloc(n_bufs, sizeof(ggml_backend_buffer_t));
+ GGML_ASSERT(galloc->buffers != NULL);
+
+ galloc->buf_tallocs = calloc(n_bufs, sizeof(struct ggml_dyn_tallocr *));
+ GGML_ASSERT(galloc->buf_tallocs != NULL);
+
+ for (int i = 0; i < n_bufs; i++) {
+ galloc->bufts[i] = bufts[i];
+ galloc->buffers[i] = NULL;
+
+ // check if the same buffer type is used multiple times and reuse the same allocator
+ for (int j = 0; j < i; j++) {
+ if (bufts[i] == bufts[j]) {
+ galloc->buf_tallocs[i] = galloc->buf_tallocs[j];
+ break;
+ }
+ }
+
+ if (galloc->buf_tallocs[i] == NULL) {
+ size_t alignment = ggml_backend_buft_get_alignment(bufts[i]);
+ galloc->buf_tallocs[i] = ggml_dyn_tallocr_new(alignment);
+ }
+ }
+ galloc->n_buffers = n_bufs;
+
+ return galloc;
+}
+
+ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft) {
+ return ggml_gallocr_new_n(&buft, 1);
+}
+
+void ggml_gallocr_free(ggml_gallocr_t galloc) {
+ if (galloc == NULL) {
+ return;
+ }
+
+ for (int i = 0; i < galloc->n_buffers; i++) {
+ if (galloc->buffers != NULL) {
+ // skip if already freed
+ bool freed = false;
+ for (int j = 0; j < i; j++) {
+ if (galloc->buffers[j] == galloc->buffers[i]) {
+ freed = true;
+ break;
+ }
+ }
+ if (!freed) {
+ ggml_backend_buffer_free(galloc->buffers[i]);
+ }
+ }
+ if (galloc->buf_tallocs != NULL) {
+ // skip if already freed
+ bool freed = false;
+ for (int j = 0; j < i; j++) {
+ if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) {
+ freed = true;
+ break;
+ }
+ }
+ if (!freed) {
+ ggml_dyn_tallocr_free(galloc->buf_tallocs[i]);
+ }
+ }
+ }
+
+ ggml_hash_set_free(&galloc->hash_set);
+ free(galloc->hash_values);
+ free(galloc->bufts);
+ free(galloc->buffers);
+ free(galloc->buf_tallocs);
+ free(galloc->node_allocs);
+ free(galloc->leaf_allocs);
+ free(galloc);
+}
+
+typedef struct ggml_gallocr * ggml_gallocr_t;
+
+static struct hash_node * ggml_gallocr_hash_get(ggml_gallocr_t galloc, struct ggml_tensor * t) {
+ size_t i = ggml_hash_find_or_insert(&galloc->hash_set, t);
+ return &galloc->hash_values[i];
+}
+
+static bool ggml_gallocr_is_own(ggml_gallocr_t galloc, struct ggml_tensor * t) {
+ return ggml_gallocr_hash_get(galloc, t)->allocated;
+}
+
+static void ggml_gallocr_set_node_offset(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id, size_t offset) {
+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);
+ hn->buffer_id = buffer_id;
+ hn->offset = offset;
+ hn->allocated = true;
+}
+
+static bool ggml_gallocr_is_allocated(ggml_gallocr_t galloc, struct ggml_tensor * t) {
+ return t->data != NULL || ggml_gallocr_hash_get(galloc, t)->allocated;
+}
+
+static void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id) {
+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);
+
+ if (!ggml_gallocr_is_allocated(galloc, node) && !ggml_is_view(node)) {
+ hn->allocated = true;
+ assert(hn->offset == 0);
+
+ // try to reuse a parent's buffer (inplace)
+ if (ggml_op_can_inplace(node->op)) {
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ struct ggml_tensor * parent = node->src[i];
+ if (parent == NULL) {
+ continue;
+ }
+
+ // if the node's data is external, then we cannot re-use it
+ if (!ggml_gallocr_is_own(galloc, parent)) {
+ AT_PRINTF("not reusing parent %s for %s as %p is external\n", parent->name, node->name, parent->data);
+ continue;
+ }
+
+ // outputs cannot be reused
+ if (parent->flags & GGML_TENSOR_FLAG_OUTPUT || (parent->view_src != NULL && parent->view_src->flags & GGML_TENSOR_FLAG_OUTPUT)) {
+ AT_PRINTF("not reusing parent %s for %s as it is an output\n", parent->name, node->name);
+ continue;
+ }
+
+ if (!ggml_are_same_layout(node, parent)) {
+ AT_PRINTF("not reusing parent %s for %s as layouts are different\n", parent->name, node->name);
+ continue;
+ }
+
+ struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);
+ if (p_hn->n_children == 1 && p_hn->n_views == 0) {
+ if (ggml_is_view(parent)) {
+ struct ggml_tensor * view_src = parent->view_src;
+ struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);
+ if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) {
+ AT_PRINTF("reusing view parent %s (%s) for %s\n", parent->name, view_src->name, node->name);
+ assert(view_src_hn->offset == p_hn->offset);
+ hn->buffer_id = p_hn->buffer_id;
+ hn->offset = p_hn->offset;
+ p_hn->allocated = false; // avoid freeing the parent
+ view_src_hn->allocated = false;
+ return;
+ }
+ } else {
+ AT_PRINTF("reusing parent %s for %s\n", parent->name, node->name);
+ hn->buffer_id = p_hn->buffer_id;
+ hn->offset = p_hn->offset;
+ p_hn->allocated = false; // avoid freeing the parent
+ return;
+ }
+ }
+ }
+ }
+ // allocate tensor from the buffer
+ struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];
+ ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];
+ size_t size = ggml_backend_buft_get_alloc_size(buft, node);
+ size_t offset = ggml_dyn_tallocr_alloc(alloc, size, node);
+ hn->buffer_id = buffer_id;
+ hn->offset = offset;
+ return;
+ }
+}
+
+static void ggml_gallocr_free_node(ggml_gallocr_t galloc, struct ggml_tensor * node) {
+ // graph outputs are never freed
+ if (node->flags & GGML_TENSOR_FLAG_OUTPUT) {
+ AT_PRINTF("not freeing output %s\n", node->name);
+ return;
+ }
+
+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);
+ size_t offset = hn->offset;
+ int buffer_id = hn->buffer_id;
+ struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];
+ ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];
+ size_t size = ggml_backend_buft_get_alloc_size(buft, node);
+ ggml_dyn_tallocr_free_tensor(alloc, offset, size, node);
+ hn->allocated = false;
+}
+
+static int get_node_buffer_id(const int * node_buffer_ids, int i) {
+ return node_buffer_ids ? node_buffer_ids[i] : 0;
+}
+
+static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {
+ // clear hash tables
+ ggml_hash_set_reset(&galloc->hash_set);
+ memset(galloc->hash_values, 0, sizeof(struct hash_node) * galloc->hash_set.size);
+
+ // allocate leafs
+ // these may be tensors that the application is not using in the graph, but may still want to allocate for other purposes
+ for (int i = 0; i < graph->n_leafs; i++) {
+ struct ggml_tensor * leaf = graph->leafs[i];
+ ggml_gallocr_allocate_node(galloc, leaf, get_node_buffer_id(leaf_buffer_ids, i));
+ }
+
+ // count number of children and views
+ // allocate other graph inputs and leafs first to avoid overwriting them
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+
+ // TODO: better way to add external dependencies
+ // GGML_OP_NONE does not appear normally in the graph nodes, but is used by ggml-backend to add dependencies to
+ // control when some tensors are allocated and freed. in this case, the dependencies are in `src`, but the node
+ // itself is never used and should not be considered a dependency
+ if (ggml_is_view(node) && node->op != GGML_OP_NONE) {
+ struct ggml_tensor * view_src = node->view_src;
+ ggml_gallocr_hash_get(galloc, view_src)->n_views += 1;
+ }
+
+ if (node->flags & GGML_TENSOR_FLAG_INPUT) {
+ ggml_gallocr_allocate_node(galloc, graph->nodes[i], get_node_buffer_id(node_buffer_ids, i));
+ }
+
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * src = node->src[j];
+ if (src == NULL) {
+ continue;
+ }
+
+ ggml_gallocr_hash_get(galloc, src)->n_children += 1;
+
+ // allocate explicit inputs
+ if (src->flags & GGML_TENSOR_FLAG_INPUT) {
+ ggml_gallocr_allocate_node(galloc, src, get_node_buffer_id(node_buffer_ids, i));
+ }
+ }
+ }
+
+ // allocate tensors
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ int buffer_id = get_node_buffer_id(node_buffer_ids, i);
+
+ // allocate parents (only leafs need to be allocated at this point)
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * parent = node->src[j];
+ if (parent == NULL) {
+ continue;
+ }
+ ggml_gallocr_allocate_node(galloc, parent, buffer_id);
+ }
+
+ // allocate node
+ ggml_gallocr_allocate_node(galloc, node, buffer_id);
+
+ AT_PRINTF("exec: %s (%s) <= ", ggml_op_desc(node), node->name);
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * parent = node->src[j];
+ if (parent == NULL) {
+ continue;
+ }
+ AT_PRINTF("%s", parent->name);
+ if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {
+ AT_PRINTF(", ");
+ }
+ }
+ AT_PRINTF("\n");
+
+ // update parents
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * parent = node->src[j];
+ if (parent == NULL) {
+ continue;
+ }
+ struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);
+ p_hn->n_children -= 1;
+
+ AT_PRINTF("parent %s: %d children, %d views, allocated: %d\n",
+ parent->name, p_hn->n_children, p_hn->n_views, p_hn->allocated);
+
+ if (p_hn->n_children == 0 && p_hn->n_views == 0) {
+ if (ggml_is_view(parent)) {
+ struct ggml_tensor * view_src = parent->view_src;
+ struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);
+ view_src_hn->n_views -= 1;
+ AT_PRINTF("view_src %s: %d children, %d views\n",
+ view_src->name, view_src_hn->n_children, view_src_hn->n_views);
+ if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src_hn->allocated) {
+ ggml_gallocr_free_node(galloc, view_src);
+ }
+ }
+ else if (p_hn->allocated) {
+ ggml_gallocr_free_node(galloc, parent);
+ }
+ }
+ AT_PRINTF("\n");
+ }
+ }
+}
+
+bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {
+ size_t min_hash_size = graph->n_nodes + graph->n_leafs;
+ // add 25% margin to avoid hash collisions
+ min_hash_size += min_hash_size / 4;
+
+ // initialize hash table
+ if (galloc->hash_set.size < min_hash_size) {
+ ggml_hash_set_free(&galloc->hash_set);
+ galloc->hash_set = ggml_hash_set_new(min_hash_size);
+ GGML_ASSERT(galloc->hash_set.keys != NULL);
+
+ free(galloc->hash_values);
+ galloc->hash_values = malloc(sizeof(struct hash_node) * galloc->hash_set.size);
+ GGML_ASSERT(galloc->hash_values != NULL);
+ }
+
+ // reset allocators
+ for (int i = 0; i < galloc->n_buffers; i++) {
+ ggml_dyn_tallocr_reset(galloc->buf_tallocs[i]);
+ }
+
+ // allocate in hash table
+ ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids);
+
+ // set the node_allocs from the hash table
+ if (galloc->n_nodes < graph->n_nodes) {
+ free(galloc->node_allocs);
+ galloc->node_allocs = calloc(graph->n_nodes, sizeof(struct node_alloc));
+ GGML_ASSERT(galloc->node_allocs != NULL);
+ }
+ galloc->n_nodes = graph->n_nodes;
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ struct node_alloc * node_alloc = &galloc->node_allocs[i];
+ if (node->view_src || node->data) {
+ node_alloc->dst.buffer_id = -1;
+ node_alloc->dst.offset = SIZE_MAX;
+ node_alloc->dst.size_max = 0;
+ } else {
+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);
+ node_alloc->dst.buffer_id = hn->buffer_id;
+ node_alloc->dst.offset = hn->offset;
+ node_alloc->dst.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], node);
+ }
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * src = node->src[j];
+ if (!src || src->view_src || src->data) {
+ node_alloc->src[j].buffer_id = -1;
+ node_alloc->src[j].offset = SIZE_MAX;
+ node_alloc->src[j].size_max = 0;
+ } else {
+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, src);
+ node_alloc->src[j].buffer_id = hn->buffer_id;
+ node_alloc->src[j].offset = hn->offset;
+ node_alloc->src[j].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], src);
+ }
+ }
+ }
+ if (galloc->n_leafs < graph->n_leafs) {
+ free(galloc->leaf_allocs);
+ galloc->leaf_allocs = calloc(graph->n_leafs, sizeof(galloc->leaf_allocs[0]));
+ GGML_ASSERT(galloc->leaf_allocs != NULL);
+ }
+ galloc->n_leafs = graph->n_leafs;
+ for (int i = 0; i < graph->n_leafs; i++) {
+ struct ggml_tensor * leaf = graph->leafs[i];
+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf);
+ galloc->leaf_allocs[i].buffer_id = hn->buffer_id;
+ if (leaf->view_src || leaf->data) {
+ galloc->leaf_allocs[i].leaf.buffer_id = -1;
+ galloc->leaf_allocs[i].leaf.offset = SIZE_MAX;
+ galloc->leaf_allocs[i].leaf.size_max = 0;
+ } else {
+ galloc->leaf_allocs[i].leaf.buffer_id = hn->buffer_id;
+ galloc->leaf_allocs[i].leaf.offset = hn->offset;
+ galloc->leaf_allocs[i].leaf.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf);
+ }
+ }
+
+ // reallocate buffers if needed
+ for (int i = 0; i < galloc->n_buffers; i++) {
+ // if the buffer type is used multiple times, we reuse the same buffer
+ for (int j = 0; j < i; j++) {
+ if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) {
+ galloc->buffers[i] = galloc->buffers[j];
+ break;
+ }
+ }
+
+ size_t cur_size = galloc->buffers[i] ? ggml_backend_buffer_get_size(galloc->buffers[i]) : 0;
+ size_t new_size = ggml_dyn_tallocr_max_size(galloc->buf_tallocs[i]);
+
+ // even if there are no tensors allocated in this buffer, we still need to allocate it to initialize views
+ if (new_size > cur_size || galloc->buffers[i] == NULL) {
+#ifndef NDEBUG
+ fprintf(stderr, "%s: reallocating %s buffer from size %.02f MiB to %.02f MiB\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), cur_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);
+#endif
+
+ ggml_backend_buffer_free(galloc->buffers[i]);
+ galloc->buffers[i] = ggml_backend_buft_alloc_buffer(galloc->bufts[i], new_size);
+ if (galloc->buffers[i] == NULL) {
+ fprintf(stderr, "%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), new_size);
+ return false;
+ }
+ ggml_backend_buffer_set_usage(galloc->buffers[i], GGML_BACKEND_BUFFER_USAGE_COMPUTE);
+ }
+ }
+
+ return true;
+}
+
+bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) {
+ return ggml_gallocr_reserve_n(galloc, graph, NULL, NULL);
+}
+
+static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * tensor, struct tensor_alloc * tensor_alloc) {
+ int buffer_id = tensor_alloc->buffer_id;
+ assert(tensor->data || tensor->view_src || ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], tensor) <= tensor_alloc->size_max);
+
+ if (tensor->view_src != NULL) {
+ if (tensor->buffer == NULL) {
+ assert(tensor_alloc->offset == SIZE_MAX);
+ if (tensor->view_src->buffer == NULL) {
+ // this tensor was allocated without ggml-backend
+ return;
+ }
+ ggml_backend_view_init(tensor);
+ }
+ } else {
+ if (tensor->data == NULL) {
+ assert(tensor_alloc->offset != SIZE_MAX);
+ assert(ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], tensor) <= tensor_alloc->size_max);
+ void * base = ggml_backend_buffer_get_base(galloc->buffers[buffer_id]);
+ void * addr = (char *)base + tensor_alloc->offset;
+ ggml_backend_tensor_alloc(galloc->buffers[buffer_id], tensor, addr);
+ } else {
+ if (tensor->buffer == NULL) {
+ // this tensor was allocated without ggml-backend
+ return;
+ }
+ }
+ }
+}
+
+static bool ggml_gallocr_node_needs_realloc(ggml_gallocr_t galloc, struct ggml_tensor * node, struct tensor_alloc * talloc) {
+ size_t node_size = (node->data || node->view_src) ? 0 : ggml_backend_buft_get_alloc_size(galloc->bufts[talloc->buffer_id], node);
+ return talloc->size_max >= node_size;
+}
+
+static bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {
+ if (galloc->n_nodes != graph->n_nodes) {
+#ifndef NDEBUG
+ fprintf(stderr, "%s: graph has different number of nodes\n", __func__);
+#endif
+ return true;
+ }
+
+ if (galloc->n_leafs != graph->n_leafs) {
+#ifndef NDEBUG
+ fprintf(stderr, "%s: graph has different number of leafs\n", __func__);
+#endif
+ return true;
+ }
+
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ struct node_alloc * node_alloc = &galloc->node_allocs[i];
+
+ if (!ggml_gallocr_node_needs_realloc(galloc, node, &node_alloc->dst)) {
+#ifndef NDEBUG
+ fprintf(stderr, "%s: node %s is not valid\n", __func__, node->name);
+#endif
+ return true;
+ }
+
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * src = node->src[j];
+ if (src == NULL) {
+ continue;
+ }
+ if (!ggml_gallocr_node_needs_realloc(galloc, src, &node_alloc->src[j])) {
+#ifndef NDEBUG
+ fprintf(stderr, "%s: src %d (%s) of node %s is not valid\n", __func__, j, src->name, node->name);
+#endif
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
+bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {
+ if (ggml_gallocr_needs_realloc(galloc, graph)) {
+ if (galloc->n_buffers == 1) {
+#ifndef NDEBUG
+ fprintf(stderr, "%s: reallocating buffers automatically\n", __func__);
+#endif
+ if (!ggml_gallocr_reserve(galloc, graph)) {
+ return false;
+ }
+ } else {
+#ifndef NDEBUG
+ fprintf(stderr, "%s: cannot reallocate multi buffer graph automatically, call reserve\n", __func__);
+#endif
+ return false;
+ }
+ }
+
+ // reset buffers
+ for (int i = 0; i < galloc->n_buffers; i++) {
+ if (galloc->buffers[i] != NULL) {
+ ggml_backend_buffer_reset(galloc->buffers[i]);
+ }
+ }
+
+ // allocate the graph tensors from the previous assignments
+ // leafs
+ for (int i = 0; i < graph->n_leafs; i++) {
+ struct ggml_tensor * leaf = graph->leafs[i];
+ struct leaf_alloc * leaf_alloc = &galloc->leaf_allocs[i];
+ ggml_gallocr_init_tensor(galloc, leaf, &leaf_alloc->leaf);
+ }
+ // nodes
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ struct node_alloc * node_alloc = &galloc->node_allocs[i];
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * src = node->src[j];
+ if (src == NULL) {
+ continue;
+ }
+ ggml_gallocr_init_tensor(galloc, src, &node_alloc->src[j]);
+ }
+ ggml_gallocr_init_tensor(galloc, node, &node_alloc->dst);
+ }
+
+ return true;
+}
+
+size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) {
+ GGML_ASSERT(buffer_id >= 0 && buffer_id < galloc->n_buffers);
+
+ if (galloc->buffers[buffer_id] == NULL) {
+ return 0;
+ }
+
+ for (int i = 0; i < buffer_id; i++) {
+ if (galloc->buffers[i] == galloc->buffers[buffer_id]) {
+ // this buffer is the same as a previous one due to the same buffer type being used multiple times
+ // only return the buffer size the first time it appears to avoid double counting
+ return 0;
+ }
+ }
+
+ return ggml_backend_buffer_get_size(galloc->buffers[buffer_id]);
+}
+
+// utils
+
+static bool alloc_tensor_range(struct ggml_context * ctx,
+ struct ggml_tensor * first, struct ggml_tensor * last,
+ ggml_backend_buffer_type_t buft, size_t size,
+ ggml_backend_buffer_t ** buffers, size_t * n_buffers) {
+ ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size);
+ if (buffer == NULL) {
+#ifndef NDEBUG
+ fprintf(stderr, "%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), size);
+#endif
+ for (size_t i = 0; i < *n_buffers; i++) {
+ ggml_backend_buffer_free((*buffers)[i]);
+ }
+ free(*buffers);
+ return false;
+ }
+
+ struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);
+
+ for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) {
+ if (t->data == NULL) {
+ if (t->view_src == NULL) {
+ ggml_tallocr_alloc(&tallocr, t);
+ } else if (t->buffer == NULL) {
+ ggml_backend_view_init(t);
+ }
+ } else {
+ if (t->view_src != NULL && t->buffer == NULL) {
+ // view of a pre-allocated tensor
+ ggml_backend_view_init(t);
+ }
+ }
+ }
+
+ *buffers = realloc(*buffers, sizeof(ggml_backend_buffer_t) * (*n_buffers + 1));
+ (*buffers)[(*n_buffers)++] = buffer;
+
+ return true;
+}
+
+ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) {
+ GGML_ASSERT(ggml_get_no_alloc(ctx) == true);
+
+ size_t alignment = ggml_backend_buft_get_alignment(buft);
+ size_t max_size = ggml_backend_buft_get_max_size(buft);
+
+ ggml_backend_buffer_t * buffers = NULL;
+ size_t n_buffers = 0;
+
+ size_t cur_buf_size = 0;
+ struct ggml_tensor * first = ggml_get_first_tensor(ctx);
+ for (struct ggml_tensor * t = first; t != NULL; t = ggml_get_next_tensor(ctx, t)) {
+ size_t this_size = 0;
+ if (t->data == NULL && t->view_src == NULL) {
+ this_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);
+ }
+
+ if (this_size > max_size) {
+ fprintf(stderr, "%s: tensor %s is too large to fit in a %s buffer (tensor size: %zu, max buffer size: %zu)\n",
+ __func__, t->name,
+ ggml_backend_buft_name(buft),
+ this_size, max_size);
+ for (size_t i = 0; i < n_buffers; i++) {
+ ggml_backend_buffer_free(buffers[i]);
+ }
+ free(buffers);
+ return NULL;
+ }
+
+ if ((cur_buf_size + this_size) > max_size) {
+ // allocate tensors in the current buffer
+ if (!alloc_tensor_range(ctx, first, t, buft, cur_buf_size, &buffers, &n_buffers)) {
+ return NULL;
+ }
+ first = t;
+ cur_buf_size = this_size;
+ } else {
+ cur_buf_size += this_size;
+ }
+ }
+
+ // allocate remaining tensors
+ if (cur_buf_size > 0) {
+ if (!alloc_tensor_range(ctx, first, NULL, buft, cur_buf_size, &buffers, &n_buffers)) {
+ return NULL;
+ }
+ }
+
+ if (n_buffers == 0) {
+#ifndef NDEBUG
+ fprintf(stderr, "%s: all tensors in the context are already allocated\n", __func__);
+#endif
+ return NULL;
+ }
+
+ ggml_backend_buffer_t buffer;
+ if (n_buffers == 1) {
+ buffer = buffers[0];
+ } else {
+ buffer = ggml_backend_multi_buffer_alloc_buffer(buffers, n_buffers);
+ }
+ free(buffers);
+ return buffer;
+}
+
+ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend) {
+ return ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_get_default_buffer_type(backend));
+}
diff --git a/llama/ggml-alloc.h b/llama/ggml-alloc.h
new file mode 100644
index 00000000..017d5298
--- /dev/null
+++ b/llama/ggml-alloc.h
@@ -0,0 +1,102 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "ggml.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;
+typedef struct ggml_backend_buffer * ggml_backend_buffer_t;
+typedef struct ggml_backend * ggml_backend_t;
+
+// Tensor allocator
+struct ggml_tallocr {
+ ggml_backend_buffer_t buffer;
+ void * base;
+ size_t alignment;
+ size_t offset;
+};
+
+GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer);
+GGML_API void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor);
+
+// Graph allocator
+/*
+ Example usage:
+ ggml_gallocr_t galloc = ggml_gallocr_new(ggml_bacckend_cpu_buffer_type());
+
+ // optional: create a worst-case graph and reserve the buffers to avoid reallocations
+ ggml_gallocr_reserve(galloc, build_graph(max_batch));
+
+ // allocate the graph
+ struct ggml_cgraph * graph = build_graph(batch);
+ ggml_gallocr_alloc_graph(galloc, graph);
+
+ printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0));
+
+ // evaluate the graph
+ ggml_backend_graph_compute(backend, graph);
+*/
+
+// special tensor flags for use with the graph allocator:
+// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses
+// ggml_set_output(): output tensors are never freed and never overwritten
+
+typedef struct ggml_gallocr * ggml_gallocr_t;
+
+GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft);
+GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs);
+GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc);
+
+// pre-allocate buffers from a measure graph - does not allocate or modify the graph
+// call with a worst-case graph to avoid buffer reallocations
+// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed
+// returns false if the buffer allocation failed
+GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph);
+GGML_API bool ggml_gallocr_reserve_n(
+ ggml_gallocr_t galloc,
+ struct ggml_cgraph * graph,
+ const int * node_buffer_ids,
+ const int * leaf_buffer_ids);
+
+// automatic reallocation if the topology changes when using a single buffer
+// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers)
+GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph);
+
+GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id);
+
+// Utils
+// Create a buffer and allocate all the tensors in a ggml_context
+GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);
+GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama/ggml-backend-impl.h b/llama/ggml-backend-impl.h
new file mode 100644
index 00000000..aeafdbd5
--- /dev/null
+++ b/llama/ggml-backend-impl.h
@@ -0,0 +1,179 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+// ggml-backend internal header
+
+#include "ggml-backend.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ //
+ // Backend buffer
+ //
+
+ // buffer type
+ typedef void * ggml_backend_buffer_type_context_t;
+
+ struct ggml_backend_buffer_type_i {
+ const char * (*GGML_CALL get_name) (ggml_backend_buffer_type_t buft);
+ // allocate a buffer of this type
+ ggml_backend_buffer_t (*GGML_CALL alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);
+ // tensor alignment
+ size_t (*GGML_CALL get_alignment) (ggml_backend_buffer_type_t buft);
+ // max buffer size that can be allocated
+ size_t (*GGML_CALL get_max_size) (ggml_backend_buffer_type_t buft);
+ // data size needed to allocate the tensor, including padding
+ size_t (*GGML_CALL get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);
+ // check if tensor data is in host memory
+ bool (*GGML_CALL is_host) (ggml_backend_buffer_type_t buft);
+ };
+
+ struct ggml_backend_buffer_type {
+ struct ggml_backend_buffer_type_i iface;
+ ggml_backend_buffer_type_context_t context;
+ };
+
+ // buffer
+ typedef void * ggml_backend_buffer_context_t;
+
+ struct ggml_backend_buffer_i {
+ const char * (*GGML_CALL get_name) (ggml_backend_buffer_t buffer);
+ void (*GGML_CALL free_buffer)(ggml_backend_buffer_t buffer);
+ void * (*GGML_CALL get_base) (ggml_backend_buffer_t buffer);
+ void (*GGML_CALL init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
+ void (*GGML_CALL set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+ void (*GGML_CALL get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
+ bool (*GGML_CALL cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst); // dst is in the buffer, src may be in any buffer
+ void (*GGML_CALL clear) (ggml_backend_buffer_t buffer, uint8_t value);
+ void (*GGML_CALL reset) (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras
+ };
+
+ struct ggml_backend_buffer {
+ struct ggml_backend_buffer_i iface;
+ ggml_backend_buffer_type_t buft;
+ ggml_backend_buffer_context_t context;
+ size_t size;
+ enum ggml_backend_buffer_usage usage;
+ };
+
+ GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init(
+ ggml_backend_buffer_type_t buft,
+ struct ggml_backend_buffer_i iface,
+ ggml_backend_buffer_context_t context,
+ size_t size);
+
+ // do not use directly, use ggml_backend_tensor_copy instead
+ bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
+
+ // buffer that contains a collection of buffers
+ GGML_CALL ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
+ GGML_CALL bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
+ GGML_CALL void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
+
+ //
+ // Backend
+ //
+
+ typedef void * ggml_backend_context_t;
+
+ struct ggml_backend_i {
+ const char * (*GGML_CALL get_name)(ggml_backend_t backend);
+
+ void (*GGML_CALL free)(ggml_backend_t backend);
+
+ // buffer allocation
+ ggml_backend_buffer_type_t (*GGML_CALL get_default_buffer_type)(ggml_backend_t backend);
+
+ // (optional) asynchronous tensor data access
+ void (*GGML_CALL set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+ void (*GGML_CALL get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
+ bool (*GGML_CALL cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);
+
+ // (optional) complete all pending operations
+ void (*GGML_CALL synchronize)(ggml_backend_t backend);
+
+ // compute graph with a plan (not used currently)
+ // create a new plan for a graph
+ ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph);
+ void (*GGML_CALL graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
+ // update the plan with a new graph - this should be faster than creating a new plan when the graph has the same topology
+ void (*GGML_CALL graph_plan_update) (ggml_backend_t backend, ggml_backend_graph_plan_t plan, const struct ggml_cgraph * cgraph);
+ // compute the graph with the plan
+ enum ggml_status (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);
+
+ // compute graph without a plan (async)
+ enum ggml_status (*GGML_CALL graph_compute) (ggml_backend_t backend, struct ggml_cgraph * cgraph);
+
+ // check if the backend can compute an operation
+ bool (*GGML_CALL supports_op)(ggml_backend_t backend, const struct ggml_tensor * op);
+
+ // check if the backend can use tensors allocated in a buffer type
+ bool (*GGML_CALL supports_buft)(ggml_backend_t backend, ggml_backend_buffer_type_t buft);
+
+ // check if the backend wants to run an operation, even if the weights are allocated in a CPU buffer
+ // these should be expensive operations with large batch sizes that may benefit from running on this backend
+ // even if the weight has to be copied from the CPU temporarily
+ bool (*GGML_CALL offload_op)(ggml_backend_t backend, const struct ggml_tensor * op);
+
+ // (optional) event synchronization
+ // create a new event that can record events on this backend instance
+ ggml_backend_event_t (*GGML_CALL event_new) (ggml_backend_t backend);
+ void (*GGML_CALL event_free) (ggml_backend_event_t event);
+ // record an event on the backend instance that created it
+ void (*GGML_CALL event_record) (ggml_backend_event_t event);
+ // wait for an event on on a different backend instance
+ void (*GGML_CALL event_wait) (ggml_backend_t backend, ggml_backend_event_t event);
+ // block until an event is recorded
+ void (*GGML_CALL event_synchronize) (ggml_backend_event_t event);
+ };
+
+ struct ggml_backend {
+ ggml_guid_t guid;
+
+ struct ggml_backend_i iface;
+ ggml_backend_context_t context;
+ };
+
+ struct ggml_backend_event {
+ ggml_backend_t backend;
+ void * context;
+ };
+
+ //
+ // Backend registry
+ //
+
+ typedef ggml_backend_t (*GGML_CALL ggml_backend_init_fn)(const char * params, void * user_data);
+
+ GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama/ggml-backend.c b/llama/ggml-backend.c
new file mode 100644
index 00000000..08af0149
--- /dev/null
+++ b/llama/ggml-backend.c
@@ -0,0 +1,2288 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "ggml-backend-impl.h"
+#include "ggml-alloc.h"
+#include "ggml-impl.h"
+
+#include <assert.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+// backend buffer type
+
+const char * ggml_backend_buft_name(ggml_backend_buffer_type_t buft) {
+ return buft->iface.get_name(buft);
+}
+
+GGML_CALL ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+ return buft->iface.alloc_buffer(buft, size);
+}
+
+size_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) {
+ return buft->iface.get_alignment(buft);
+}
+
+size_t ggml_backend_buft_get_max_size(ggml_backend_buffer_type_t buft) {
+ // get_max_size is optional, defaults to SIZE_MAX
+ if (buft->iface.get_max_size) {
+ return buft->iface.get_max_size(buft);
+ }
+ return SIZE_MAX;
+}
+
+GGML_CALL size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor) {
+ // get_alloc_size is optional, defaults to ggml_nbytes
+ if (buft->iface.get_alloc_size) {
+ size_t size = buft->iface.get_alloc_size(buft, tensor);
+ assert(size >= ggml_nbytes(tensor));
+ return size;
+ }
+ return ggml_nbytes(tensor);
+}
+
+bool ggml_backend_buft_is_host(ggml_backend_buffer_type_t buft) {
+ if (buft->iface.is_host) {
+ return buft->iface.is_host(buft);
+ }
+ return false;
+}
+
+// backend buffer
+
+GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init(
+ ggml_backend_buffer_type_t buft,
+ struct ggml_backend_buffer_i iface,
+ ggml_backend_buffer_context_t context,
+ size_t size) {
+ ggml_backend_buffer_t buffer = malloc(sizeof(struct ggml_backend_buffer));
+
+ (*buffer) = (struct ggml_backend_buffer) {
+ /* .interface = */ iface,
+ /* .buft = */ buft,
+ /* .context = */ context,
+ /* .size = */ size,
+ /* .usage = */ GGML_BACKEND_BUFFER_USAGE_ANY
+ };
+
+ return buffer;
+}
+
+const char * ggml_backend_buffer_name(ggml_backend_buffer_t buffer) {
+ return buffer->iface.get_name(buffer);
+}
+
+void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {
+ if (buffer == NULL) {
+ return;
+ }
+
+ if (buffer->iface.free_buffer != NULL) {
+ buffer->iface.free_buffer(buffer);
+ }
+
+// TODO: this needs to be freed in cuda and hipblas backends because
+// the cuda backend implementation compiled with msvc
+#if !defined(GGML_USE_CUDA) && !defined(GGML_USE_HIPBLAS)
+ free(buffer);
+#endif
+}
+
+size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {
+ return buffer->size;
+}
+
+void * ggml_backend_buffer_get_base(ggml_backend_buffer_t buffer) {
+ void * base = buffer->iface.get_base(buffer);
+
+ GGML_ASSERT(base != NULL && "backend buffer base cannot be NULL");
+
+ return base;
+}
+
+GGML_CALL void ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {
+ // init_tensor is optional
+ if (buffer->iface.init_tensor) {
+ buffer->iface.init_tensor(buffer, tensor);
+ }
+}
+
+size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer) {
+ return ggml_backend_buft_get_alignment(ggml_backend_buffer_get_type(buffer));
+}
+
+size_t ggml_backend_buffer_get_max_size(ggml_backend_buffer_t buffer) {
+ return ggml_backend_buft_get_max_size(ggml_backend_buffer_get_type(buffer));
+}
+
+size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {
+ return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_get_type(buffer), tensor);
+}
+
+void ggml_backend_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+ buffer->iface.clear(buffer, value);
+}
+
+bool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) {
+ return ggml_backend_buft_is_host(ggml_backend_buffer_get_type(buffer));
+}
+
+void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {
+ buffer->usage = usage;
+
+ // FIXME: add a generic callback to the buffer interface
+ if (ggml_backend_buffer_is_multi_buffer(buffer)) {
+ ggml_backend_multi_buffer_set_usage(buffer, usage);
+ }
+}
+
+enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage(ggml_backend_buffer_t buffer) {
+ return buffer->usage;
+}
+
+ggml_backend_buffer_type_t ggml_backend_buffer_get_type(ggml_backend_buffer_t buffer) {
+ return buffer->buft;
+}
+
+void ggml_backend_buffer_reset(ggml_backend_buffer_t buffer) {
+ if (buffer->iface.reset) {
+ buffer->iface.reset(buffer);
+ }
+}
+
+bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) {
+ ggml_backend_buffer_t dst_buf = dst->view_src ? dst->view_src->buffer : dst->buffer;
+ if (dst_buf->iface.cpy_tensor) {
+ return dst_buf->iface.cpy_tensor(dst_buf, src, dst);
+ }
+ return false;
+}
+
+// backend
+
+ggml_guid_t ggml_backend_guid(ggml_backend_t backend) {
+ if (backend == NULL) {
+ return NULL;
+ }
+ return backend->guid;
+}
+
+const char * ggml_backend_name(ggml_backend_t backend) {
+ if (backend == NULL) {
+ return "NULL";
+ }
+ return backend->iface.get_name(backend);
+}
+
+void ggml_backend_free(ggml_backend_t backend) {
+ if (backend == NULL) {
+ return;
+ }
+
+ backend->iface.free(backend);
+}
+
+ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend) {
+ return backend->iface.get_default_buffer_type(backend);
+}
+
+ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size) {
+ return ggml_backend_buft_alloc_buffer(ggml_backend_get_default_buffer_type(backend), size);
+}
+
+size_t ggml_backend_get_alignment(ggml_backend_t backend) {
+ return ggml_backend_buft_get_alignment(ggml_backend_get_default_buffer_type(backend));
+}
+
+size_t ggml_backend_get_max_size(ggml_backend_t backend) {
+ return ggml_backend_buft_get_max_size(ggml_backend_get_default_buffer_type(backend));
+}
+
+void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");
+
+ if (backend->iface.set_tensor_async == NULL) {
+ ggml_backend_tensor_set(tensor, data, offset, size);
+ } else {
+ backend->iface.set_tensor_async(backend, tensor, data, offset, size);
+ }
+}
+
+void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");
+
+ if (backend->iface.get_tensor_async == NULL) {
+ ggml_backend_tensor_get(tensor, data, offset, size);
+ } else {
+ backend->iface.get_tensor_async(backend, tensor, data, offset, size);
+ }
+}
+
+GGML_CALL void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
+
+ GGML_ASSERT(buf != NULL && "tensor buffer not set");
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");
+
+ if (!size) {
+ return;
+ }
+
+ buf->iface.set_tensor(buf, tensor, data, offset, size);
+}
+
+GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
+
+ GGML_ASSERT(buf != NULL && "tensor buffer not set");
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");
+
+ if (!size) {
+ return;
+ }
+
+ buf->iface.get_tensor(buf, tensor, data, offset, size);
+}
+
+void ggml_backend_synchronize(ggml_backend_t backend) {
+ if (backend->iface.synchronize == NULL) {
+ return;
+ }
+
+ backend->iface.synchronize(backend);
+}
+
+ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
+ GGML_ASSERT(backend->iface.graph_plan_create != NULL);
+
+ return backend->iface.graph_plan_create(backend, cgraph);
+}
+
+void ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {
+ GGML_ASSERT(backend->iface.graph_plan_free != NULL);
+
+ backend->iface.graph_plan_free(backend, plan);
+}
+
+enum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {
+ GGML_ASSERT(backend->iface.graph_plan_compute != NULL);
+
+ return backend->iface.graph_plan_compute(backend, plan);
+}
+
+enum ggml_status ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
+ enum ggml_status err = ggml_backend_graph_compute_async(backend, cgraph);
+ ggml_backend_synchronize(backend);
+ return err;
+}
+
+enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
+ return backend->iface.graph_compute(backend, cgraph);
+}
+
+bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
+ return backend->iface.supports_op(backend, op);
+}
+
+bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
+ return backend->iface.supports_buft(backend, buft);
+}
+
+bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op) {
+ if (backend->iface.offload_op != NULL) {
+ return backend->iface.offload_op(backend, op);
+ }
+ return false;
+}
+
+// backend copy
+
+static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {
+ if (a->type != b->type) {
+ return false;
+ }
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
+ if (a->ne[i] != b->ne[i]) {
+ return false;
+ }
+ if (a->nb[i] != b->nb[i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst) {
+ GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");
+
+ if (src == dst) {
+ return;
+ }
+
+ if (ggml_backend_buffer_is_host(src->buffer)) {
+ ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));
+ } else if (ggml_backend_buffer_is_host(dst->buffer)) {
+ ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));
+ } else if (!ggml_backend_buffer_copy_tensor(src, dst)) {
+#ifndef NDEBUG
+ fprintf(stderr, "%s: warning: slow copy from %s to %s\n", __func__, ggml_backend_buffer_name(src->buffer), ggml_backend_buffer_name(dst->buffer));
+#endif
+ size_t nbytes = ggml_nbytes(src);
+ void * data = malloc(nbytes);
+ ggml_backend_tensor_get(src, data, 0, nbytes);
+ ggml_backend_tensor_set(dst, data, 0, nbytes);
+ free(data);
+ }
+}
+
+void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst) {
+ GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");
+
+ if (src == dst) {
+ return;
+ }
+
+ if (backend_dst->iface.cpy_tensor_async != NULL) {
+ if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {
+ return;
+ }
+ }
+
+ // an async copy would normally happen after all the queued operations on both backends are completed
+ // to simulate the same behavior, we need to synchronize both backends first, and do a blocking copy
+ ggml_backend_synchronize(backend_src);
+ ggml_backend_synchronize(backend_dst);
+ ggml_backend_tensor_copy(src, dst);
+}
+
+// events
+
+ggml_backend_event_t ggml_backend_event_new(ggml_backend_t backend) {
+ if (backend->iface.event_new == NULL) {
+ return NULL;
+ }
+ return backend->iface.event_new(backend);
+}
+
+void ggml_backend_event_free(ggml_backend_event_t event) {
+ if (event == NULL) {
+ return;
+ }
+ event->backend->iface.event_free(event);
+}
+
+void ggml_backend_event_record(ggml_backend_event_t event) {
+ GGML_ASSERT(event->backend->iface.event_record != NULL);
+
+ event->backend->iface.event_record(event);
+}
+
+void ggml_backend_event_synchronize(ggml_backend_event_t event) {
+ GGML_ASSERT(event->backend->iface.event_synchronize != NULL);
+
+ event->backend->iface.event_synchronize(event);
+}
+
+void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {
+ GGML_ASSERT(backend->iface.event_wait != NULL);
+
+ backend->iface.event_wait(backend, event);
+}
+
+// backend registry
+
+#define GGML_REG_MAX_BACKENDS 64
+
+struct ggml_backend_reg {
+ char name[128];
+ ggml_backend_init_fn init_fn;
+ ggml_backend_buffer_type_t default_buffer_type;
+ void * user_data;
+};
+
+static struct ggml_backend_reg ggml_backend_registry[GGML_REG_MAX_BACKENDS];
+static size_t ggml_backend_registry_count = 0;
+
+GGML_CALL static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data);
+
+GGML_CALL static void ggml_backend_registry_init(void) {
+ static bool initialized = false;
+
+ if (initialized) {
+ return;
+ }
+
+ initialized = true;
+
+ ggml_backend_register("CPU", ggml_backend_reg_cpu_init, ggml_backend_cpu_buffer_type(), NULL);
+
+ // add forward decls here to avoid including the backend headers
+#ifdef GGML_USE_CUDA
+ extern GGML_CALL void ggml_backend_cuda_reg_devices(void);
+ ggml_backend_cuda_reg_devices();
+#endif
+
+#ifdef GGML_USE_SYCL
+ extern void ggml_backend_sycl_reg_devices(void);
+ ggml_backend_sycl_reg_devices();
+#endif
+
+#ifdef GGML_USE_METAL
+ extern GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data);
+ extern GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);
+ ggml_backend_register("Metal", ggml_backend_reg_metal_init, ggml_backend_metal_buffer_type(), NULL);
+#endif
+
+#ifdef GGML_USE_VULKAN
+ extern GGML_CALL int ggml_backend_vk_reg_devices(void);
+ ggml_backend_vk_reg_devices();
+#endif
+
+#ifdef GGML_USE_KOMPUTE
+ extern GGML_CALL void ggml_backend_kompute_reg_devices(void);
+ ggml_backend_kompute_reg_devices();
+#endif
+
+#ifdef GGML_USE_CANN
+ extern GGML_CALL int ggml_backend_cann_reg_devices(void);
+ ggml_backend_cann_reg_devices();
+#endif
+}
+
+GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) {
+ GGML_ASSERT(ggml_backend_registry_count < GGML_REG_MAX_BACKENDS);
+
+ size_t id = ggml_backend_registry_count;
+
+ ggml_backend_registry[id] = (struct ggml_backend_reg) {
+ /* .name = */ {0},
+ /* .fn = */ init_fn,
+ /* .default_buffer_type = */ default_buffer_type,
+ /* .user_data = */ user_data,
+ };
+
+ snprintf(ggml_backend_registry[id].name, sizeof(ggml_backend_registry[id].name), "%s", name);
+
+#ifndef NDEBUG
+ fprintf(stderr, "%s: registered backend %s\n", __func__, name);
+#endif
+
+ ggml_backend_registry_count++;
+}
+
+size_t ggml_backend_reg_get_count(void) {
+ ggml_backend_registry_init();
+
+ return ggml_backend_registry_count;
+}
+
+size_t ggml_backend_reg_find_by_name(const char * name) {
+ ggml_backend_registry_init();
+
+ for (size_t i = 0; i < ggml_backend_registry_count; i++) {
+ // TODO: case insensitive in a portable way
+ if (strcmp(ggml_backend_registry[i].name, name) == 0) {
+ return i;
+ }
+ }
+
+ // not found
+ return SIZE_MAX;
+}
+
+// init from backend:params string
+ggml_backend_t ggml_backend_reg_init_backend_from_str(const char * backend_str) {
+ ggml_backend_registry_init();
+
+ const char * params = strchr(backend_str, ':');
+ char backend_name[128];
+ if (params == NULL) {
+ snprintf(backend_name, sizeof(backend_name), "%s", backend_str);
+ params = "";
+ } else {
+ snprintf(backend_name, sizeof(backend_name), "%.*s", (int)(params - backend_str), backend_str);
+ params++;
+ }
+
+ size_t backend_i = ggml_backend_reg_find_by_name(backend_name);
+
+ if (backend_i == SIZE_MAX) {
+ fprintf(stderr, "%s: backend %s not found\n", __func__, backend_name);
+ return NULL;
+ }
+
+ return ggml_backend_reg_init_backend(backend_i, params);
+}
+
+const char * ggml_backend_reg_get_name(size_t i) {
+ ggml_backend_registry_init();
+
+ GGML_ASSERT(i < ggml_backend_registry_count);
+ return ggml_backend_registry[i].name;
+}
+
+ggml_backend_t ggml_backend_reg_init_backend(size_t i, const char * params) {
+ ggml_backend_registry_init();
+
+ GGML_ASSERT(i < ggml_backend_registry_count);
+ return ggml_backend_registry[i].init_fn(params, ggml_backend_registry[i].user_data);
+}
+
+ggml_backend_buffer_type_t ggml_backend_reg_get_default_buffer_type(size_t i) {
+ ggml_backend_registry_init();
+
+ GGML_ASSERT(i < ggml_backend_registry_count);
+ return ggml_backend_registry[i].default_buffer_type;
+}
+
+ggml_backend_buffer_t ggml_backend_reg_alloc_buffer(size_t i, size_t size) {
+ ggml_backend_registry_init();
+
+ GGML_ASSERT(i < ggml_backend_registry_count);
+ return ggml_backend_buft_alloc_buffer(ggml_backend_registry[i].default_buffer_type, size);
+}
+
+// backend CPU
+
+static const size_t TENSOR_ALIGNMENT = 32; // required for mmap as gguf only guarantees 32-byte alignment
+
+GGML_CALL static const char * ggml_backend_cpu_buffer_name(ggml_backend_buffer_t buffer) {
+ return "CPU";
+
+ GGML_UNUSED(buffer);
+}
+
+GGML_CALL static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
+ uintptr_t data = (uintptr_t)buffer->context;
+
+ // align the buffer
+ if (data % TENSOR_ALIGNMENT != 0) {
+ data = GGML_PAD(data, TENSOR_ALIGNMENT);
+ }
+
+ return (void *)data;
+}
+
+GGML_CALL static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+ free(buffer->context);
+}
+
+GGML_CALL static void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+ memcpy((char *)tensor->data + offset, data, size);
+
+ GGML_UNUSED(buffer);
+}
+
+GGML_CALL static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+ memcpy(data, (const char *)tensor->data + offset, size);
+
+ GGML_UNUSED(buffer);
+}
+
+GGML_CALL static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {
+ if (ggml_backend_buffer_is_host(src->buffer)) {
+ memcpy(dst->data, src->data, ggml_nbytes(src));
+ return true;
+ }
+ return false;
+
+ GGML_UNUSED(buffer);
+}
+
+GGML_CALL static void ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+ memset(buffer->context, value, buffer->size);
+}
+
+static struct ggml_backend_buffer_i cpu_backend_buffer_i = {
+ /* .get_name = */ ggml_backend_cpu_buffer_name,
+ /* .free_buffer = */ ggml_backend_cpu_buffer_free_buffer,
+ /* .get_base = */ ggml_backend_cpu_buffer_get_base,
+ /* .init_tensor = */ NULL, // no initialization required
+ /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,
+ /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,
+ /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
+ /* .clear = */ ggml_backend_cpu_buffer_clear,
+ /* .reset = */ NULL,
+};
+
+// for buffers from ptr, free is not called
+static struct ggml_backend_buffer_i cpu_backend_buffer_i_from_ptr = {
+ /* .get_name = */ ggml_backend_cpu_buffer_name,
+ /* .free_buffer = */ NULL, // ptr is not owned by the buffer, so it does not need to be freed
+ /* .get_base = */ ggml_backend_cpu_buffer_get_base,
+ /* .init_tensor = */ NULL, // no initialization required
+ /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,
+ /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,
+ /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
+ /* .clear = */ ggml_backend_cpu_buffer_clear,
+ /* .reset = */ NULL,
+};
+
+GGML_CALL static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
+ return "CPU";
+
+ GGML_UNUSED(buft);
+}
+
+GGML_CALL static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+ size += TENSOR_ALIGNMENT; // malloc may return an address that is not aligned
+ void * data = malloc(size); // TODO: use GGML_ALIGNED_MALLOC (move to ggml-impl.h)
+ if (data == NULL) {
+ fprintf(stderr, "%s: failed to allocate buffer of size %zu\n", __func__, size);
+ return NULL;
+ }
+
+ return ggml_backend_buffer_init(buft, cpu_backend_buffer_i, data, size);
+}
+
+GGML_CALL static size_t ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
+ return TENSOR_ALIGNMENT;
+
+ GGML_UNUSED(buft);
+}
+
+GGML_CALL static bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
+ return true;
+
+ GGML_UNUSED(buft);
+}
+
+GGML_CALL ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {
+ static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = {
+ /* .iface = */ {
+ /* .get_name = */ ggml_backend_cpu_buffer_type_get_name,
+ /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,
+ /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,
+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX
+ /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
+ /* .is_host = */ ggml_backend_cpu_buffer_type_is_host,
+ },
+ /* .context = */ NULL,
+ };
+
+ return &ggml_backend_cpu_buffer_type;
+}
+
+#ifdef GGML_USE_CPU_HBM
+
+// buffer type HBM
+
+#include <hbwmalloc.h>
+
+GGML_CALL static const char * ggml_backend_cpu_hbm_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
+ return "CPU_HBM";
+
+ GGML_UNUSED(buft);
+}
+
+GGML_CALL static const char * ggml_backend_cpu_hbm_buffer_get_name(ggml_backend_buffer_t buf) {
+ return "CPU_HBM";
+
+ GGML_UNUSED(buf);
+}
+
+GGML_CALL static void ggml_backend_cpu_hbm_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+ hbw_free(buffer->context);
+}
+
+GGML_CALL static ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+ //void * ptr = hbw_malloc(size);
+ void * ptr;
+ int result = hbw_posix_memalign(&ptr, ggml_backend_cpu_buffer_type_get_alignment(buft), size);
+ if (result != 0) {
+ fprintf(stderr, "failed to allocate HBM buffer of size %zu\n", size);
+ return NULL;
+ }
+
+ ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
+ buffer->buft = buft;
+ buffer->iface.get_name = ggml_backend_cpu_hbm_buffer_get_name;
+ buffer->iface.free_buffer = ggml_backend_cpu_hbm_buffer_free_buffer;
+
+ return buffer;
+}
+
+ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void) {
+ static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_hbm = {
+ /* .iface = */ {
+ /* .get_name = */ ggml_backend_cpu_hbm_buffer_type_get_name,
+ /* .alloc_buffer = */ ggml_backend_cpu_hbm_buffer_type_alloc_buffer,
+ /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,
+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX
+ /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
+ /* .is_host = */ ggml_backend_cpu_buffer_type_is_host,
+ },
+ /* .context = */ NULL,
+ };
+
+ return &ggml_backend_cpu_buffer_type_hbm;
+}
+#endif
+
+struct ggml_backend_cpu_context {
+ int n_threads;
+ ggml_threadpool_t threadpool;
+
+ void * work_data;
+ size_t work_size;
+
+ ggml_abort_callback abort_callback;
+ void * abort_callback_data;
+};
+
+GGML_CALL static const char * ggml_backend_cpu_name(ggml_backend_t backend) {
+ return "CPU";
+
+ GGML_UNUSED(backend);
+}
+
+GGML_CALL static void ggml_backend_cpu_free(ggml_backend_t backend) {
+ struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;
+ free(cpu_ctx->work_data);
+ free(cpu_ctx);
+ free(backend);
+}
+
+GGML_CALL static ggml_backend_buffer_type_t ggml_backend_cpu_get_default_buffer_type(ggml_backend_t backend) {
+ return ggml_backend_cpu_buffer_type();
+
+ GGML_UNUSED(backend);
+}
+
+struct ggml_backend_plan_cpu {
+ struct ggml_cplan cplan;
+ struct ggml_cgraph cgraph;
+};
+
+GGML_CALL static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backend_t backend, const struct ggml_cgraph * cgraph) {
+ struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;
+
+ struct ggml_backend_plan_cpu * cpu_plan = malloc(sizeof(struct ggml_backend_plan_cpu));
+
+ cpu_plan->cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);
+ cpu_plan->cgraph = *cgraph; // FIXME: deep copy
+
+ if (cpu_plan->cplan.work_size > 0) {
+ cpu_plan->cplan.work_data = malloc(cpu_plan->cplan.work_size);
+ if (cpu_plan->cplan.work_data == NULL) {
+ free(cpu_plan);
+ return NULL;
+ }
+ }
+
+ cpu_plan->cplan.abort_callback = cpu_ctx->abort_callback;
+ cpu_plan->cplan.abort_callback_data = cpu_ctx->abort_callback_data;
+
+ return cpu_plan;
+}
+
+GGML_CALL static void ggml_backend_cpu_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {
+ struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan;
+
+ free(cpu_plan->cplan.work_data);
+ free(cpu_plan);
+
+ GGML_UNUSED(backend);
+}
+
+GGML_CALL static enum ggml_status ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {
+ struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan;
+
+ return ggml_graph_compute(&cpu_plan->cgraph, &cpu_plan->cplan);
+
+ GGML_UNUSED(backend);
+}
+
+GGML_CALL static enum ggml_status ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
+ struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;
+
+ struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);
+
+ if (cpu_ctx->work_size < cplan.work_size) {
+ free(cpu_ctx->work_data);
+ cpu_ctx->work_data = malloc(cplan.work_size);
+ if (cpu_ctx->work_data == NULL) {
+ cpu_ctx->work_size = 0;
+ return GGML_STATUS_ALLOC_FAILED;
+ }
+ cpu_ctx->work_size = cplan.work_size;
+ }
+ cplan.work_data = cpu_ctx->work_data;
+
+ cplan.abort_callback = cpu_ctx->abort_callback;
+ cplan.abort_callback_data = cpu_ctx->abort_callback_data;
+
+ return ggml_graph_compute(cgraph, &cplan);
+}
+
+GGML_CALL static bool ggml_backend_cpu_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
+ switch (op->op) {
+ case GGML_OP_CPY:
+ return
+ op->type != GGML_TYPE_IQ2_XXS &&
+ op->type != GGML_TYPE_IQ2_XS &&
+ op->type != GGML_TYPE_IQ1_S &&
+ op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float
+ case GGML_OP_MUL_MAT:
+ return op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == ggml_internal_get_type_traits(op->src[0]->type).vec_dot_type;
+ default:
+ return true;
+ }
+
+ GGML_UNUSED(backend);
+}
+
+GGML_CALL static bool ggml_backend_cpu_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
+ return ggml_backend_buft_is_host(buft);
+
+ GGML_UNUSED(backend);
+}
+
+static struct ggml_backend_i cpu_backend_i = {
+ /* .get_name = */ ggml_backend_cpu_name,
+ /* .free = */ ggml_backend_cpu_free,
+ /* .get_default_buffer_type = */ ggml_backend_cpu_get_default_buffer_type,
+ /* .set_tensor_async = */ NULL,
+ /* .get_tensor_async = */ NULL,
+ /* .cpy_tensor_async = */ NULL,
+ /* .synchronize = */ NULL,
+ /* .graph_plan_create = */ ggml_backend_cpu_graph_plan_create,
+ /* .graph_plan_free = */ ggml_backend_cpu_graph_plan_free,
+ /* .graph_plan_update = */ NULL,
+ /* .graph_plan_compute = */ ggml_backend_cpu_graph_plan_compute,
+ /* .graph_compute = */ ggml_backend_cpu_graph_compute,
+ /* .supports_op = */ ggml_backend_cpu_supports_op,
+ /* .supports_buft = */ ggml_backend_cpu_supports_buft,
+ /* .offload_op = */ NULL,
+ /* .event_new = */ NULL,
+ /* .event_free = */ NULL,
+ /* .event_record = */ NULL,
+ /* .event_wait = */ NULL,
+ /* .event_synchronize = */ NULL,
+};
+
+static ggml_guid_t ggml_backend_cpu_guid(void) {
+ static ggml_guid guid = { 0xaa, 0x67, 0xc7, 0x43, 0x96, 0xe6, 0xa3, 0x8a, 0xe3, 0xaf, 0xea, 0x92, 0x36, 0xbc, 0xfc, 0x89 };
+ return &guid;
+}
+
+ggml_backend_t ggml_backend_cpu_init(void) {
+ struct ggml_backend_cpu_context * ctx = malloc(sizeof(struct ggml_backend_cpu_context));
+ if (ctx == NULL) {
+ return NULL;
+ }
+
+ ctx->n_threads = GGML_DEFAULT_N_THREADS;
+ ctx->threadpool = NULL;
+ ctx->work_data = NULL;
+ ctx->work_size = 0;
+ ctx->abort_callback = NULL;
+ ctx->abort_callback_data = NULL;
+
+ ggml_backend_t cpu_backend = malloc(sizeof(struct ggml_backend));
+ if (cpu_backend == NULL) {
+ free(ctx);
+ return NULL;
+ }
+
+ *cpu_backend = (struct ggml_backend) {
+ /* .guid = */ ggml_backend_cpu_guid(),
+ /* .interface = */ cpu_backend_i,
+ /* .context = */ ctx
+ };
+ return cpu_backend;
+}
+
+GGML_CALL bool ggml_backend_is_cpu(ggml_backend_t backend) {
+ return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cpu_guid());
+}
+
+void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {
+ GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));
+
+ struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;
+ ctx->n_threads = n_threads;
+}
+
+void ggml_backend_cpu_set_threadpool(ggml_backend_t backend_cpu, ggml_threadpool_t threadpool) {
+ GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));
+
+ struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;
+
+ if (ctx->threadpool && ctx->threadpool != threadpool) {
+ // already had a different threadpool, pause/suspend it before switching
+ ggml_threadpool_pause(ctx->threadpool);
+ }
+ ctx->threadpool = threadpool;
+}
+
+void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data) {
+ GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));
+
+ struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;
+ ctx->abort_callback = abort_callback;
+ ctx->abort_callback_data = abort_callback_data;
+}
+
+GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) {
+ GGML_ASSERT((uintptr_t)ptr % TENSOR_ALIGNMENT == 0 && "buffer pointer must be aligned");
+ return ggml_backend_buffer_init(ggml_backend_cpu_buffer_type(), cpu_backend_buffer_i_from_ptr, ptr, size);
+}
+
+GGML_CALL static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data) {
+ return ggml_backend_cpu_init();
+
+ GGML_UNUSED(params);
+ GGML_UNUSED(user_data);
+}
+
+// multi-buffer buffer
+
+struct ggml_backend_multi_buffer_context {
+ ggml_backend_buffer_t * buffers;
+ size_t n_buffers;
+};
+
+typedef struct ggml_backend_multi_buffer_context * ggml_backend_multi_buffer_context_t;
+
+GGML_CALL static const char * ggml_backend_multi_buffer_get_name(ggml_backend_buffer_t buffer) {
+ ggml_backend_multi_buffer_context_t ctx = (ggml_backend_multi_buffer_context_t) buffer->context;
+
+ return ctx->buffers[0]->iface.get_name(ctx->buffers[0]);
+}
+
+GGML_CALL static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+ ggml_backend_multi_buffer_context_t ctx = (ggml_backend_multi_buffer_context_t) buffer->context;
+ for (size_t i = 0; i < ctx->n_buffers; i++) {
+ ggml_backend_buffer_free(ctx->buffers[i]);
+ }
+
+ free(ctx->buffers);
+ free(ctx);
+}
+
+GGML_CALL static void ggml_backend_multi_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+ ggml_backend_multi_buffer_context_t ctx = (ggml_backend_multi_buffer_context_t) buffer->context;
+ for (size_t i = 0; i < ctx->n_buffers; i++) {
+ ggml_backend_buffer_clear(ctx->buffers[i], value);
+ }
+}
+
+static struct ggml_backend_buffer_i ggml_backend_multi_buffer_context_interface(void) {
+ static struct ggml_backend_buffer_i multi_backend_buffer_i = {
+ /* .get_name = */ ggml_backend_multi_buffer_get_name,
+ /* .free_buffer = */ ggml_backend_multi_buffer_free_buffer,
+ /* .get_base = */ NULL,
+ /* .init_tensor = */ NULL,
+ /* .set_tensor = */ NULL,
+ /* .get_tensor = */ NULL,
+ /* .cpy_tensor = */ NULL,
+ /* .clear = */ ggml_backend_multi_buffer_clear,
+ /* .reset = */ NULL,
+ };
+
+ return multi_backend_buffer_i;
+}
+
+GGML_CALL ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {
+ ggml_backend_multi_buffer_context_t ctx = (ggml_backend_multi_buffer_context_t) malloc(sizeof(struct ggml_backend_multi_buffer_context));
+ ctx->n_buffers = n_buffers;
+ ctx->buffers = (ggml_backend_buffer_t *) malloc(n_buffers * sizeof(ggml_backend_buffer_t));
+
+ GGML_ASSERT(ctx->buffers != NULL);
+
+ size_t total_size = 0;
+ for (size_t i = 0; i < n_buffers; i++) {
+ ctx->buffers[i] = buffers[i];
+ total_size += ggml_backend_buffer_get_size(buffers[i]);
+ }
+
+ return ggml_backend_buffer_init(buffers[0]->buft, ggml_backend_multi_buffer_context_interface(), ctx, total_size);
+}
+
+GGML_CALL bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer) {
+ return buffer->iface.get_name == ggml_backend_multi_buffer_get_name;
+}
+
+GGML_CALL void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {
+ GGML_ASSERT(ggml_backend_buffer_is_multi_buffer(buffer));
+ ggml_backend_multi_buffer_context_t ctx = (ggml_backend_multi_buffer_context_t) buffer->context;
+ for (size_t i = 0; i < ctx->n_buffers; i++) {
+ ggml_backend_buffer_set_usage(ctx->buffers[i], usage);
+ }
+}
+
+// creates a copy of the tensor with the same memory layout
+static struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, const struct ggml_tensor * tensor) {
+ struct ggml_tensor * dup = ggml_dup_tensor(ctx, tensor);
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
+ dup->nb[i] = tensor->nb[i];
+ }
+ return dup;
+}
+
+static bool ggml_is_view_op(enum ggml_op op) {
+ return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;
+}
+
+// scheduler
+
+#ifndef GGML_SCHED_MAX_BACKENDS
+#define GGML_SCHED_MAX_BACKENDS 16
+#endif
+
+#ifndef GGML_SCHED_MAX_SPLIT_INPUTS
+#define GGML_SCHED_MAX_SPLIT_INPUTS GGML_MAX_SRC
+#endif
+
+#ifndef GGML_SCHED_MAX_COPIES
+#define GGML_SCHED_MAX_COPIES 4
+#endif
+
+struct ggml_backend_sched_split {
+ int backend_id;
+ int i_start;
+ int i_end;
+ struct ggml_tensor * inputs[GGML_SCHED_MAX_SPLIT_INPUTS];
+ int n_inputs;
+ // graph view of this split
+ struct ggml_cgraph graph;
+};
+
+struct ggml_backend_sched {
+ bool is_reset; // true if the scheduler has been reset since the last graph split
+ bool is_alloc;
+
+ int n_backends;
+
+ ggml_backend_t backends[GGML_SCHED_MAX_BACKENDS];
+ ggml_backend_buffer_type_t bufts[GGML_SCHED_MAX_BACKENDS];
+ ggml_gallocr_t galloc;
+
+ // hash map of the nodes in the graph
+ struct ggml_hash_set hash_set;
+ int * hv_tensor_backend_ids; // [hash_set.size]
+ struct ggml_tensor ** hv_tensor_copies; // [hash_set.size][n_backends][n_copies]
+
+ int * node_backend_ids; // [graph_size]
+ int * leaf_backend_ids; // [graph_size]
+
+ int * prev_node_backend_ids; // [graph_size]
+ int * prev_leaf_backend_ids; // [graph_size]
+
+ // copy of the graph with modified inputs
+ struct ggml_cgraph graph;
+
+ // graph splits
+ struct ggml_backend_sched_split * splits;
+ int n_splits;
+ int splits_capacity;
+
+ // pipeline parallelism support
+ int n_copies;
+ int cur_copy;
+ ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES];
+ struct ggml_tensor * graph_inputs[GGML_SCHED_MAX_SPLIT_INPUTS];
+ int n_graph_inputs;
+
+ struct ggml_context * ctx;
+
+ ggml_backend_sched_eval_callback callback_eval;
+ void * callback_eval_user_data;
+
+ char * context_buffer;
+ size_t context_buffer_size;
+
+ bool debug;
+};
+
+#define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor)
+#define tensor_backend_id(tensor) sched->hv_tensor_backend_ids[hash_id(tensor)]
+#define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)]
+#define tensor_copy(tensor, backend_id, copy_id) tensor_id_copy(hash_id(tensor), backend_id, copy_id)
+
+// returns the priority of the backend, lower id is higher priority
+static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) {
+ for (int i = 0; i < sched->n_backends; i++) {
+ if (sched->backends[i] == backend) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, const struct ggml_tensor * tensor, const struct ggml_tensor * op) {
+ ggml_backend_buffer_t buffer = tensor->buffer;
+ if (buffer == NULL) {
+ return -1;
+ }
+
+ // find highest prio backend that supports the buffer type and the op
+ for (int i = 0; i < sched->n_backends; i++) {
+ if (ggml_backend_supports_buft(sched->backends[i], buffer->buft) &&
+ ggml_backend_supports_op(sched->backends[i], op)) {
+ return i;
+ }
+ }
+
+#ifndef NDEBUG
+ fprintf(stderr, "%s: warning: no backend supports op %s with a weight with buffer type %s used in tensor %s, the weight will need to be copied\n",
+ __func__, ggml_op_desc(tensor), ggml_backend_buffer_name(buffer), tensor->name);
+#endif
+
+ return -1;
+}
+
+#if 0
+#define GGML_SCHED_MAX_SPLITS_DEBUG 4096
+static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS_DEBUG*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only
+#define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__)
+#define GET_CAUSE(node) causes[hash_id(node)]
+#else
+#define SET_CAUSE(node, ...)
+#define GET_CAUSE(node) ""
+#endif
+
+// returns the backend that should be used for the node based on the current locations
+static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * tensor) {
+ // TODO: use supports_op to check if the backend supports the op
+
+ // assign pre-allocated nodes to their backend
+ int cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor, tensor);
+ if (cur_backend_id != -1) {
+ SET_CAUSE(tensor, "1.dst");
+ return cur_backend_id;
+ }
+
+ // view_src
+ if (tensor->view_src != NULL) {
+ cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src, tensor);
+ if (cur_backend_id != -1) {
+ SET_CAUSE(tensor, "1.vsrc");
+ return cur_backend_id;
+ }
+ }
+
+ // graph input
+ if (tensor->flags & GGML_TENSOR_FLAG_INPUT) {
+ cur_backend_id = sched->n_backends - 1; // last backend (assumed CPU)
+ SET_CAUSE(tensor, "1.inp");
+ return cur_backend_id;
+ }
+
+ // operations with weights are preferably run on the same backend as the weights
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ const struct ggml_tensor * src = tensor->src[i];
+ if (src == NULL) {
+ continue;
+ }
+ if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
+ int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src, tensor);
+ // check if a backend with higher prio wants to offload the op
+ if (src_backend_id == sched->n_backends - 1) {
+ for (int b = 0; b < src_backend_id; b++) {
+ if (ggml_backend_supports_op(sched->backends[b], tensor) && ggml_backend_offload_op(sched->backends[b], tensor)) {
+ SET_CAUSE(tensor, "1.off");
+ return b;
+ }
+ }
+ }
+ SET_CAUSE(tensor, "1.wgt%d", i);
+ return src_backend_id;
+ }
+ }
+
+ return -1;
+}
+
+static char * fmt_size(size_t size) {
+ static char buffer[128];
+ if (size >= 1024*1024) {
+ snprintf(buffer, sizeof(buffer), "%zuM", size/1024/1024);
+ } else {
+ snprintf(buffer, sizeof(buffer), "%zuK", size/1024);
+ }
+ return buffer;
+}
+
+static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
+ int cur_split = 0;
+ for (int i = 0; i < graph->n_nodes; i++) {
+ if (cur_split < sched->n_splits && i == sched->splits[cur_split].i_start) {
+ ggml_backend_t split_backend = sched->backends[sched->splits[cur_split].backend_id];
+ fprintf(stderr, "\n## SPLIT #%d: %s # %d inputs: ", cur_split, ggml_backend_name(split_backend),
+ sched->splits[cur_split].n_inputs);
+ for (int j = 0; j < sched->splits[cur_split].n_inputs; j++) {
+ fprintf(stderr, "[%s (%5.5s)] ", sched->splits[cur_split].inputs[j]->name,
+ fmt_size(ggml_nbytes(sched->splits[cur_split].inputs[j])));
+ }
+ fprintf(stderr, "\n");
+ cur_split++;
+ }
+ struct ggml_tensor * node = graph->nodes[i];
+ if (ggml_is_view_op(node->op)) {
+ continue;
+ }
+ ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node);
+ fprintf(stderr, "node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s]:", i, ggml_op_name(node->op), node->name,
+ fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node));
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * src = node->src[j];
+ if (src == NULL) {
+ continue;
+ }
+ ggml_backend_t src_backend = ggml_backend_sched_get_tensor_backend(sched, src);
+ fprintf(stderr, " %20.20s (%5.5s) [%5.5s %8.8s]", src->name,
+ fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src));
+ }
+ fprintf(stderr, "\n");
+ }
+}
+
+static bool ggml_backend_sched_buffer_supported(ggml_backend_sched_t sched, struct ggml_tensor * t, int backend_id) {
+ ggml_backend_buffer_t buf = t->view_src ? t->view_src->buffer : t->buffer;
+ ggml_backend_buffer_type_t buft = NULL;
+
+ if (buf) {
+ // the tensor is already allocated
+ buft = buf->buft;
+ } else {
+ // see if the tensor already has a backend assigned, and use the buffer type of that backend
+ int tensor_backend_id = tensor_backend_id(t);
+ if (tensor_backend_id == -1 && t->view_src) {
+ tensor_backend_id = tensor_backend_id(t->view_src);
+ }
+ if (tensor_backend_id != -1) {
+ buft = sched->bufts[tensor_backend_id];
+ }
+ }
+
+ return buft != NULL && ggml_backend_supports_buft(sched->backends[backend_id], buft);
+}
+
+static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, struct ggml_tensor * node, int cur_backend_id, int * node_backend_id) {
+ if (ggml_backend_supports_op(sched->backends[cur_backend_id], node)) {
+ *node_backend_id = cur_backend_id;
+ SET_CAUSE(node, "2.sup");
+ }
+}
+
+// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend
+static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
+ // reset splits
+ sched->n_splits = 0;
+ sched->n_graph_inputs = 0;
+ sched->is_reset = false;
+
+ struct ggml_init_params params = {
+ /* .mem_size = */ sched->context_buffer_size,
+ /* .mem_buffer = */ sched->context_buffer,
+ /* .no_alloc = */ true
+ };
+
+ ggml_free(sched->ctx);
+
+ sched->ctx = ggml_init(params);
+ if (sched->ctx == NULL) {
+ GGML_ABORT("%s: failed to initialize context\n", __func__);
+ }
+
+ // pass 1: assign backends to ops with pre-allocated inputs
+ for (int i = 0; i < graph->n_leafs; i++) {
+ struct ggml_tensor * leaf = graph->leafs[i];
+ int * leaf_backend_id = &tensor_backend_id(leaf);
+ // do not overwrite user assignments
+ if (*leaf_backend_id == -1) {
+ *leaf_backend_id = ggml_backend_sched_backend_id_from_cur(sched, leaf);
+ }
+ }
+
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ int * node_backend_id = &tensor_backend_id(node);
+ // do not overwrite user assignments
+ if (*node_backend_id == -1) {
+ *node_backend_id = ggml_backend_sched_backend_id_from_cur(sched, node);
+
+#if 0
+ // src
+ if (node->op == GGML_OP_NONE) {
+ continue;
+ }
+
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * src = node->src[j];
+ if (src == NULL) {
+ continue;
+ }
+ int * src_backend_id = &tensor_backend_id(src);
+ if (*src_backend_id == -1) {
+ *src_backend_id = ggml_backend_sched_backend_id_from_cur(sched, src);
+ }
+ }
+#endif
+ }
+ }
+
+ // pass 2: expand current backend assignments
+ // assign the same backend to adjacent nodes
+ // expand gpu backends (i.e. non last prio) up and down, ignoring cpu (the lowest priority backend)
+ // thus, cpu will never be used unless weights are on cpu, or there are no gpu ops between cpu ops
+ // ops unsupported by the backend being expanded will be left unassigned so that they can be assigned later when the locations of its inputs are known
+ // expand gpu down
+ {
+ int cur_backend_id = -1;
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ if (ggml_is_view_op(node->op)) {
+ continue;
+ }
+ int * node_backend_id = &tensor_backend_id(node);
+ if (*node_backend_id != -1) {
+ if (*node_backend_id == sched->n_backends - 1) {
+ // skip cpu (lowest prio backend)
+ cur_backend_id = -1;
+ } else {
+ cur_backend_id = *node_backend_id;
+ }
+ } else if (cur_backend_id != -1) {
+ ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);
+ }
+ }
+ }
+ // expand gpu up
+ {
+ int cur_backend_id = -1;
+ for (int i = graph->n_nodes - 1; i >= 0; i--) {
+ struct ggml_tensor * node = graph->nodes[i];
+ if (ggml_is_view_op(node->op)) {
+ continue;
+ }
+ int * node_backend_id = &tensor_backend_id(node);
+ if (*node_backend_id != -1) {
+ if (*node_backend_id == sched->n_backends - 1) {
+ // skip cpu (lowest prio backend)
+ cur_backend_id = -1;
+ } else {
+ cur_backend_id = *node_backend_id;
+ }
+ } else if (cur_backend_id != -1) {
+ ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);
+ }
+ }
+ }
+ // expand rest down
+ {
+ int cur_backend_id = -1;
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ if (ggml_is_view_op(node->op)) {
+ continue;
+ }
+ int * node_backend_id = &tensor_backend_id(node);
+ if (*node_backend_id != -1) {
+ cur_backend_id = *node_backend_id;
+ } else if (cur_backend_id != -1) {
+ ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);
+ }
+ }
+ }
+ // expand rest up
+ {
+ int cur_backend_id = -1;
+ for (int i = graph->n_nodes - 1; i >= 0; i--) {
+ struct ggml_tensor * node = graph->nodes[i];
+ if (ggml_is_view_op(node->op)) {
+ continue;
+ }
+ int * node_backend_id = &tensor_backend_id(node);
+ if (*node_backend_id != -1) {
+ cur_backend_id = *node_backend_id;
+ } else if (cur_backend_id != -1) {
+ ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);
+ }
+ }
+ }
+
+ // pass 3: upgrade nodes to higher prio backends with compatible buffer types
+ // if the tensor is already in the same buffer type (*) as another higher priority backend, we should move it there
+ // however, we also need to verify that the sources are in compatible buffer types
+ // (*) the actual requirement is more relaxed, the buffer type of the backend should be supported by all the users of this tensor further down the graph
+ // however, this is slow to verify, so we have a more strict requirement that the buffer type is the same
+ // this is not uncommon since multiple backends can use host memory, with the same buffer type (eg. BLAS and CPU)
+ // additionally, set remaining unassigned nodes to the backend with the most supported inputs
+ // only nodes that could not be assigned during expansion due to the backend not supporting the op should be unassigned at this point
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ if (ggml_is_view_op(node->op)) {
+ continue;
+ }
+ int * node_backend_id = &tensor_backend_id(node);
+ if (*node_backend_id == -1) {
+ // unassigned node: find the backend with the most supported inputs
+ int n_supported_best = -1;
+ for (int b = 0; b < sched->n_backends; b++) {
+ if (ggml_backend_supports_op(sched->backends[b], node)) {
+ int n_supported = 0;
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * src = node->src[j];
+ if (src == NULL) {
+ continue;
+ }
+ if ((tensor_backend_id(src) != -1 || tensor_backend_id(src->view_src) != -1) && ggml_backend_sched_buffer_supported(sched, src, b)) {
+ n_supported++;
+ }
+ }
+ if (n_supported > n_supported_best) {
+ n_supported_best = n_supported;
+ *node_backend_id = b;
+ SET_CAUSE(node, "3.best");
+ }
+ }
+ }
+ } else {
+ // assigned node: upgrade to higher prio backend if possible
+ for (int b = 0; b < *node_backend_id; b++) {
+ if (sched->bufts[b] == sched->bufts[*node_backend_id] && ggml_backend_supports_op(sched->backends[b], node)) {
+ bool supported = true;
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * src = node->src[j];
+ if (src == NULL) {
+ continue;
+ }
+ if (!ggml_backend_sched_buffer_supported(sched, src, b)) {
+ supported = false;
+ break;
+ }
+ }
+ if (supported) {
+ *node_backend_id = b;
+ SET_CAUSE(node, "3.upg");
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ // pass 4: assign backends to remaining src from dst and view_src
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ int * cur_backend_id = &tensor_backend_id(node);
+ if (node->view_src != NULL && *cur_backend_id == -1) {
+ *cur_backend_id = tensor_backend_id(node->view_src);
+ SET_CAUSE(node, "4.vsrc");
+ }
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * src = node->src[j];
+ if (src == NULL) {
+ continue;
+ }
+ int * src_backend_id = &tensor_backend_id(src);
+ if (*src_backend_id == -1) {
+ if (src->view_src != NULL) {
+ // views are always on the same backend as the source
+ *src_backend_id = tensor_backend_id(src->view_src);
+ SET_CAUSE(src, "4.vsrc");
+ } else {
+ *src_backend_id = *cur_backend_id;
+ SET_CAUSE(src, "4.cur");
+ }
+ }
+ }
+ }
+
+ // pass 5: split graph, find tensors that need to be copied
+ {
+ int i_split = 0;
+ struct ggml_backend_sched_split * split = &sched->splits[0];
+ // find the backend of the first split, skipping view ops
+ int i = 0;
+ for (; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ if (!ggml_is_view_op(node->op)) {
+ split->backend_id = tensor_backend_id(node);
+ break;
+ }
+ }
+ split->i_start = 0;
+ split->n_inputs = 0;
+ int cur_backend_id = split->backend_id;
+ for (; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+
+ if (ggml_is_view_op(node->op)) {
+ continue;
+ }
+
+ const int node_backend_id = tensor_backend_id(node);
+
+ assert(node_backend_id != -1); // all nodes should be assigned by now
+
+ // check if we should start a new split based on the sources of the current node
+ bool need_new_split = false;
+ if (node_backend_id == cur_backend_id && split->n_inputs > 0) {
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * src = node->src[j];
+ if (src == NULL) {
+ continue;
+ }
+ // check if a weight is on a different backend
+ // by starting a new split, the memory of the previously offloaded weights can be reused
+ if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
+ int src_backend_id = tensor_backend_id(src);
+ if (src_backend_id != cur_backend_id) {
+ need_new_split = true;
+ break;
+ }
+ }
+ // check if the split has too many inputs
+ // FIXME: count the number of inputs instead of only checking when full
+ if (split->n_inputs == GGML_SCHED_MAX_SPLIT_INPUTS) {
+ const size_t id = hash_id(src);
+ int src_backend_id = sched->hv_tensor_backend_ids[id];
+ bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id);
+ if (src_backend_id != cur_backend_id && tensor_id_copy(id, cur_backend_id, 0) == NULL && !supported) {
+ //printf("starting new split because of too many inputs: node %s, input %s\n", node->name, src->name);
+ need_new_split = true;
+ break;
+ }
+ }
+ }
+ }
+
+ if (node_backend_id != cur_backend_id || need_new_split) {
+ split->i_end = i;
+ i_split++;
+ if (i_split >= sched->splits_capacity) {
+ sched->splits_capacity *= 2;
+ sched->splits = realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split));
+ GGML_ASSERT(sched->splits != NULL);
+ }
+ split = &sched->splits[i_split];
+ split->backend_id = node_backend_id;
+ split->i_start = i;
+ split->n_inputs = 0;
+ cur_backend_id = node_backend_id;
+ }
+
+ // find inputs that are not on the same backend
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ struct ggml_tensor * src = node->src[j];
+ if (src == NULL) {
+ continue;
+ }
+
+ size_t src_id = hash_id(src);
+ const int src_backend_id = sched->hv_tensor_backend_ids[src_id];
+ assert(src_backend_id != -1); // all inputs should be assigned by now
+
+ if (src->flags & GGML_TENSOR_FLAG_INPUT && sched->n_copies > 1) {
+ if (tensor_id_copy(src_id, src_backend_id, 0) == NULL) {
+ ggml_backend_t backend = sched->backends[src_backend_id];
+ for (int c = 0; c < sched->n_copies; c++) {
+ struct ggml_tensor * tensor_copy;
+ if (c == sched->cur_copy) {
+ tensor_copy = src; // use the original tensor as the current copy
+ } else {
+ tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);
+ ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c);
+ }
+ if (sched->n_copies > 1) {
+ ggml_set_input(tensor_copy);
+ ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor
+ }
+ tensor_id_copy(src_id, src_backend_id, c) = tensor_copy;
+ SET_CAUSE(tensor_copy, "4.cpy");
+ }
+ int n_graph_inputs = sched->n_graph_inputs++;
+ GGML_ASSERT(n_graph_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);
+ sched->graph_inputs[n_graph_inputs] = src;
+ }
+ }
+
+ if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) {
+ // create a copy of the input in the split's backend
+ if (tensor_id_copy(src_id, cur_backend_id, 0) == NULL) {
+ ggml_backend_t backend = sched->backends[cur_backend_id];
+ for (int c = 0; c < sched->n_copies; c++) {
+ struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);
+ ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c);
+ if (sched->n_copies > 1) {
+ ggml_set_input(tensor_copy);
+ ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor
+ }
+ tensor_id_copy(src_id, cur_backend_id, c) = tensor_copy;
+ SET_CAUSE(tensor_copy, "4.cpy");
+ }
+ int n_inputs = split->n_inputs++;
+ GGML_ASSERT(n_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);
+ split->inputs[n_inputs] = src;
+ }
+ node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy);
+ }
+ }
+ }
+ split->i_end = graph->n_nodes;
+ sched->n_splits = i_split + 1;
+ }
+
+ if (sched->debug) {
+ ggml_backend_sched_print_assignments(sched, graph);
+ }
+
+ // swap node_backend_ids and leaf _backend_ids with prevs
+ {
+ int * tmp = sched->node_backend_ids;
+ sched->node_backend_ids = sched->prev_node_backend_ids;
+ sched->prev_node_backend_ids = tmp;
+
+ tmp = sched->leaf_backend_ids;
+ sched->leaf_backend_ids = sched->prev_leaf_backend_ids;
+ sched->prev_leaf_backend_ids = tmp;
+ }
+
+ int graph_size = graph->n_nodes + sched->n_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2;
+ if (sched->graph.size < graph_size) {
+ sched->graph.size = graph_size;
+ sched->graph.nodes = realloc(sched->graph.nodes, graph_size * sizeof(struct ggml_tensor *));
+ sched->graph.leafs = realloc(sched->graph.leafs, graph_size * sizeof(struct ggml_tensor *));
+ GGML_ASSERT(sched->graph.nodes != NULL);
+ GGML_ASSERT(sched->graph.leafs != NULL);
+ }
+ sched->graph.n_nodes = 0;
+ sched->graph.n_leafs = 0;
+
+ struct ggml_cgraph * graph_copy = &sched->graph;
+
+ for (int i = 0; i < sched->n_splits; i++) {
+ struct ggml_backend_sched_split * split = &sched->splits[i];
+ split->graph = ggml_graph_view(graph, split->i_start, split->i_end);
+
+ // add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the split
+ for (int j = 0; j < split->n_inputs; j++) {
+ assert(graph_copy->size > (graph_copy->n_nodes + 1));
+
+ struct ggml_tensor * input = split->inputs[j];
+ const size_t input_id = hash_id(input);
+ struct ggml_tensor * input_cpy = tensor_id_copy(input_id, split->backend_id, sched->cur_copy);
+
+ // add a dependency to the input source so that it is not freed before the copy is done
+ struct ggml_tensor * input_dep = ggml_view_tensor(sched->ctx, input);
+ input_dep->src[0] = input;
+ sched->node_backend_ids[graph_copy->n_nodes] = sched->hv_tensor_backend_ids[input_id];
+ graph_copy->nodes[graph_copy->n_nodes++] = input_dep;
+
+ // add a dependency to the input copy so that it is allocated at the start of the split
+ sched->node_backend_ids[graph_copy->n_nodes] = split->backend_id;
+ graph_copy->nodes[graph_copy->n_nodes++] = input_cpy;
+ }
+
+ for (int j = split->i_start; j < split->i_end; j++) {
+ assert(graph_copy->size > graph_copy->n_nodes);
+ sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]);
+ graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j];
+ }
+ }
+
+ if (sched->n_copies > 1) {
+ // add input copies as leafs so that they are allocated first
+ for (int i = 0; i < sched->n_graph_inputs; i++) {
+ struct ggml_tensor * input = sched->graph_inputs[i];
+ size_t id = hash_id(input);
+ int backend_id = tensor_backend_id(input);
+ for (int c = 0; c < sched->n_copies; c++) {
+ struct ggml_tensor * input_cpy = tensor_id_copy(id, backend_id, c);
+ sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id;
+ graph_copy->leafs[graph_copy->n_leafs++] = input_cpy;
+ }
+ }
+
+ for (int i = 0; i < sched->n_splits; i++) {
+ struct ggml_backend_sched_split * split = &sched->splits[i];
+ int backend_id = split->backend_id;
+ for (int j = 0; j < split->n_inputs; j++) {
+ struct ggml_tensor * input = split->inputs[j];
+ size_t id = hash_id(input);
+ for (int c = 0; c < sched->n_copies; c++) {
+ struct ggml_tensor * input_cpy = tensor_id_copy(id, backend_id, c);
+ sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id;
+ graph_copy->leafs[graph_copy->n_leafs++] = input_cpy;
+ }
+ }
+ }
+ }
+
+ // add leafs from the original graph
+ for (int i = 0; i < graph->n_leafs; i++) {
+ struct ggml_tensor * leaf = graph->leafs[i];
+ sched->leaf_backend_ids[graph_copy->n_leafs] = tensor_backend_id(leaf);
+ graph_copy->leafs[graph_copy->n_leafs++] = leaf;
+ }
+}
+
+static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) {
+ bool backend_ids_changed = false;
+ for (int i = 0; i < sched->graph.n_nodes; i++) {
+ if (sched->node_backend_ids[i] != sched->prev_node_backend_ids[i] &&
+ sched->bufts[sched->node_backend_ids[i]] != sched->bufts[sched->prev_node_backend_ids[i]]) {
+ backend_ids_changed = true;
+ break;
+ }
+ }
+ if (!backend_ids_changed) {
+ for (int i = 0; i < sched->graph.n_leafs; i++) {
+ if (sched->leaf_backend_ids[i] != sched->prev_leaf_backend_ids[i] &&
+ sched->bufts[sched->leaf_backend_ids[i]] != sched->bufts[sched->prev_leaf_backend_ids[i]]) {
+ backend_ids_changed = true;
+ break;
+ }
+ }
+ }
+
+ // allocate graph
+ if (backend_ids_changed || !ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) {
+ // the re-allocation may cause the split inputs to be moved to a different address
+ ggml_backend_sched_synchronize(sched);
+#ifndef NDEBUG
+ fprintf(stderr, "%s: failed to allocate graph, reserving (backend_ids_changed = %d)\n", __func__, backend_ids_changed);
+#endif
+ ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids);
+ if (!ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) {
+ fprintf(stderr, "%s: failed to allocate graph\n", __func__);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {
+ struct ggml_backend_sched_split * splits = sched->splits;
+
+ for (int i = 0; i < sched->n_splits; i++) {
+ struct ggml_backend_sched_split * split = &splits[i];
+ int split_backend_id = split->backend_id;
+ ggml_backend_t split_backend = sched->backends[split_backend_id];
+
+ // copy the input tensors to the split backend
+ for (int j = 0; j < split->n_inputs; j++) {
+ ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[j]);
+ struct ggml_tensor * input = split->inputs[j];
+ struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy);
+
+ if (input->flags & GGML_TENSOR_FLAG_INPUT) {
+ // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done
+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
+ ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
+ } else {
+ ggml_backend_synchronize(split_backend);
+ }
+ ggml_backend_tensor_copy(input, input_cpy);
+ } else {
+ // wait for the split backend to finish using the input before overwriting it
+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
+ ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);
+ } else {
+ ggml_backend_synchronize(split_backend);
+ }
+ // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events
+ // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface
+ if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) {
+ ggml_backend_synchronize(input_backend);
+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
+ ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
+ } else {
+ ggml_backend_synchronize(split_backend);
+ }
+ ggml_backend_tensor_copy(input, input_cpy);
+ }
+ }
+ }
+
+ if (!sched->callback_eval) {
+ enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph);
+ if (ec != GGML_STATUS_SUCCESS) {
+ return ec;
+ }
+ } else {
+ // similar to ggml_backend_compare_graph_backend
+ for (int j0 = 0; j0 < split->graph.n_nodes; j0++) {
+ struct ggml_tensor * t = split->graph.nodes[j0];
+
+ // check if the user needs data from this node
+ bool need = sched->callback_eval(t, true, sched->callback_eval_user_data);
+
+ int j1 = j0;
+
+ // determine the range [j0, j1] of nodes that can be computed together
+ while (!need && j1 < split->graph.n_nodes - 1) {
+ t = split->graph.nodes[++j1];
+ need = sched->callback_eval(t, true, sched->callback_eval_user_data);
+ }
+
+ struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1);
+
+ enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &gv);
+ if (ec != GGML_STATUS_SUCCESS) {
+ return ec;
+ }
+
+ // TODO: pass backend to the callback, then the user can decide if they want to synchronize
+ ggml_backend_synchronize(split_backend);
+
+ if (need && !sched->callback_eval(t, false, sched->callback_eval_user_data)) {
+ break;
+ }
+
+ j0 = j1;
+ }
+ }
+
+ // record the event of this copy
+ if (split->n_inputs > 0) {
+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
+ ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy]);
+ }
+ }
+ }
+
+ sched->cur_copy = (sched->cur_copy + 1) % sched->n_copies;
+
+ return GGML_STATUS_SUCCESS;
+}
+
+ggml_backend_sched_t ggml_backend_sched_new(
+ ggml_backend_t * backends,
+ ggml_backend_buffer_type_t * bufts,
+ int n_backends,
+ size_t graph_size,
+ bool parallel) {
+ GGML_ASSERT(n_backends > 0);
+ GGML_ASSERT(n_backends <= GGML_SCHED_MAX_BACKENDS);
+ GGML_ASSERT(ggml_backend_is_cpu(backends[n_backends - 1])); // last backend must be CPU
+
+ struct ggml_backend_sched * sched = calloc(1, sizeof(struct ggml_backend_sched));
+
+ sched->debug = getenv("GGML_SCHED_DEBUG") != NULL;
+ sched->n_backends = n_backends;
+ sched->n_copies = parallel ? GGML_SCHED_MAX_COPIES : 1;
+
+ // initialize hash table
+ // FIXME: needs to be size*2 to account for leafs (do it in graph_split instead)
+ sched->hash_set = ggml_hash_set_new(graph_size);
+ sched->hv_tensor_backend_ids = malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));
+ sched->hv_tensor_copies = malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));
+
+ const size_t ggml_sched_max_splits = graph_size; // at most there is one split for each node in the graph
+ const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2;
+ sched->node_backend_ids = calloc(nodes_size, sizeof(sched->node_backend_ids[0]));
+ sched->leaf_backend_ids = calloc(nodes_size, sizeof(sched->leaf_backend_ids[0]));
+ sched->prev_node_backend_ids = calloc(nodes_size, sizeof(sched->prev_node_backend_ids[0]));
+ sched->prev_leaf_backend_ids = calloc(nodes_size, sizeof(sched->prev_leaf_backend_ids[0]));
+
+ sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false);
+ sched->context_buffer = malloc(sched->context_buffer_size);
+
+ const int initial_splits_capacity = 16;
+ sched->splits = calloc(initial_splits_capacity, sizeof(sched->splits[0]));
+ sched->splits_capacity = initial_splits_capacity;
+
+ for (int b = 0; b < n_backends; b++) {
+ sched->backends[b] = backends[b];
+ sched->bufts[b] = bufts ? bufts[b] : ggml_backend_get_default_buffer_type(backends[b]);
+ GGML_ASSERT(ggml_backend_supports_buft(backends[b], sched->bufts[b]));
+ if (sched->n_copies > 1) {
+ for (int c = 0; c < sched->n_copies; c++) {
+ sched->events[b][c] = ggml_backend_event_new(backends[b]);
+ }
+ }
+ }
+
+ sched->galloc = ggml_gallocr_new_n(sched->bufts, n_backends);
+
+ ggml_backend_sched_reset(sched);
+
+ return sched;
+}
+
+void ggml_backend_sched_free(ggml_backend_sched_t sched) {
+ if (sched == NULL) {
+ return;
+ }
+ for (int b = 0; b < sched->n_backends; b++) {
+ for (int c = 0; c < sched->n_copies; c++) {
+ ggml_backend_event_free(sched->events[b][c]);
+ }
+ }
+ ggml_gallocr_free(sched->galloc);
+ ggml_free(sched->ctx);
+ ggml_hash_set_free(&sched->hash_set);
+ free(sched->splits);
+ free(sched->hv_tensor_backend_ids);
+ free(sched->hv_tensor_copies);
+ free(sched->node_backend_ids);
+ free(sched->leaf_backend_ids);
+ free(sched->prev_node_backend_ids);
+ free(sched->prev_leaf_backend_ids);
+ free(sched->context_buffer);
+ free(sched->graph.nodes);
+ free(sched->graph.leafs);
+ free(sched);
+}
+
+void ggml_backend_sched_reset(ggml_backend_sched_t sched) {
+ // reset state for the next run
+ if (!sched->is_reset) {
+ ggml_hash_set_reset(&sched->hash_set);
+ memset(sched->hv_tensor_backend_ids, -1, sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));
+ memset(sched->hv_tensor_copies, 0, sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));
+ sched->is_reset = true;
+ }
+ sched->is_alloc = false;
+}
+
+bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) {
+ GGML_ASSERT((int)sched->hash_set.size >= measure_graph->n_nodes + measure_graph->n_leafs);
+
+ ggml_backend_sched_split_graph(sched, measure_graph);
+
+ if (!ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids)) {
+ return false;
+ }
+
+ ggml_backend_sched_reset(sched);
+ ggml_backend_sched_synchronize(sched);
+
+ return true;
+}
+
+bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
+ GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + graph->n_leafs);
+
+ ggml_backend_sched_split_graph(sched, graph);
+
+
+ if (!ggml_backend_sched_alloc_splits(sched)) {
+ return false;
+ }
+
+ sched->is_alloc = true;
+
+ return true;
+}
+
+enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
+ enum ggml_status err = ggml_backend_sched_graph_compute_async(sched, graph);
+ ggml_backend_sched_synchronize(sched);
+ return err;
+}
+
+enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
+ if (!sched->is_reset && !sched->is_alloc) {
+ ggml_backend_sched_reset(sched);
+ }
+
+ if (!sched->is_alloc) {
+ if (!ggml_backend_sched_alloc_graph(sched, graph)) {
+ return GGML_STATUS_ALLOC_FAILED;
+ }
+ }
+
+ return ggml_backend_sched_compute_splits(sched);
+}
+
+void ggml_backend_sched_synchronize(ggml_backend_sched_t sched) {
+ for (int i = 0; i < sched->n_backends; i++) {
+ ggml_backend_synchronize(sched->backends[i]);
+ }
+}
+
+void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data) {
+ sched->callback_eval = callback;
+ sched->callback_eval_user_data = user_data;
+}
+
+int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) {
+ return sched->n_splits;
+}
+
+int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched) {
+ return sched->n_copies;
+}
+
+int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched) {
+ return sched->n_backends;
+}
+
+ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i) {
+ GGML_ASSERT(i >= 0 && i < sched->n_backends);
+ return sched->backends[i];
+}
+
+size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend) {
+ int backend_index = ggml_backend_sched_backend_id(sched, backend);
+ GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);
+
+ return ggml_gallocr_get_buffer_size(sched->galloc, backend_index);
+}
+
+void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) {
+ int backend_index = ggml_backend_sched_backend_id(sched, backend);
+ GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);
+ tensor_backend_id(node) = backend_index;
+ SET_CAUSE(node, "usr");
+ sched->is_reset = false;
+}
+
+ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) {
+ int backend_index = tensor_backend_id(node);
+ if (backend_index == -1) {
+ return NULL;
+ }
+ return sched->backends[backend_index];
+}
+
+// utils
+
+void ggml_backend_view_init(struct ggml_tensor * tensor) {
+ GGML_ASSERT(tensor->buffer == NULL);
+ GGML_ASSERT(tensor->view_src != NULL);
+ GGML_ASSERT(tensor->view_src->buffer != NULL);
+ GGML_ASSERT(tensor->view_src->data != NULL);
+
+ tensor->buffer = tensor->view_src->buffer;
+ tensor->data = (char *)tensor->view_src->data + tensor->view_offs;
+ ggml_backend_buffer_init_tensor(tensor->buffer, tensor);
+}
+
+void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr) {
+ GGML_ASSERT(tensor->buffer == NULL);
+ GGML_ASSERT(tensor->data == NULL);
+ GGML_ASSERT(tensor->view_src == NULL);
+ GGML_ASSERT(addr >= ggml_backend_buffer_get_base(buffer));
+ GGML_ASSERT((char *)addr + ggml_backend_buffer_get_alloc_size(buffer, tensor) <=
+ (char *)ggml_backend_buffer_get_base(buffer) + ggml_backend_buffer_get_size(buffer));
+
+ tensor->buffer = buffer;
+ tensor->data = addr;
+ ggml_backend_buffer_init_tensor(buffer, tensor);
+}
+
+static struct ggml_tensor * graph_copy_dup_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies,
+ struct ggml_context * ctx_allocated, struct ggml_context * ctx_unallocated, struct ggml_tensor * src) {
+
+ GGML_ASSERT(src != NULL);
+ GGML_ASSERT(src->data && "graph must be allocated");
+
+ size_t id = ggml_hash_insert(&hash_set, src);
+ if (id == GGML_HASHSET_ALREADY_EXISTS) {
+ return node_copies[ggml_hash_find(&hash_set, src)];
+ }
+
+ struct ggml_tensor * dst = ggml_dup_tensor_layout(src->data && !src->view_src ? ctx_allocated : ctx_unallocated, src);
+ if (src->view_src != NULL) {
+ dst->view_src = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, src->view_src);
+ dst->view_offs = src->view_offs;
+ }
+ dst->op = src->op;
+ memcpy(dst->op_params, src->op_params, sizeof(dst->op_params));
+ ggml_set_name(dst, src->name);
+
+ // copy src
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ struct ggml_tensor * s = src->src[i];
+ if (s == NULL) {
+ continue;
+ }
+ dst->src[i] = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, s);
+ }
+
+ node_copies[id] = dst;
+ return dst;
+}
+
+static void graph_copy_init_tensor(struct ggml_hash_set * hash_set, struct ggml_tensor ** node_copies, bool * node_init, struct ggml_tensor * src) {
+ size_t id = ggml_hash_find(hash_set, src);
+ if (node_init[id]) {
+ return;
+ }
+ node_init[id] = true;
+
+ struct ggml_tensor * dst = node_copies[id];
+ if (dst->view_src != NULL) {
+ graph_copy_init_tensor(hash_set, node_copies, node_init, src->view_src);
+ ggml_backend_view_init(dst);
+ }
+ else {
+ ggml_backend_tensor_copy(src, dst);
+ }
+
+ // init src
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ struct ggml_tensor * s = src->src[i];
+ if (s == NULL) {
+ continue;
+ }
+ graph_copy_init_tensor(hash_set, node_copies, node_init, s);
+ }
+}
+
+struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph) {
+ struct ggml_hash_set hash_set = ggml_hash_set_new(graph->visited_hash_set.size);
+ struct ggml_tensor ** node_copies = calloc(hash_set.size, sizeof(node_copies[0])); // NOLINT
+ bool * node_init = calloc(hash_set.size, sizeof(node_init[0]));
+
+ struct ggml_init_params params = {
+ /* .mem_size = */ ggml_tensor_overhead()*hash_set.size + ggml_graph_overhead_custom(graph->size, false),
+ /* .mem_buffer = */ NULL,
+ /* .no_alloc = */ true
+ };
+
+ struct ggml_context * ctx_allocated = ggml_init(params);
+ struct ggml_context * ctx_unallocated = ggml_init(params);
+
+ if (ctx_allocated == NULL || ctx_unallocated == NULL) {
+ fprintf(stderr, "failed to allocate context for graph copy\n");
+ ggml_hash_set_free(&hash_set);
+ free(node_copies);
+ free(node_init);
+ ggml_free(ctx_allocated);
+ ggml_free(ctx_unallocated);
+ return (struct ggml_backend_graph_copy) {
+ /* .buffer = */ NULL,
+ /* .ctx_allocated = */ NULL,
+ /* .ctx_unallocated = */ NULL,
+ /* .graph = */ NULL,
+ };
+ }
+
+ // dup nodes
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, node);
+ }
+
+ // allocate nodes
+ ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx_allocated, backend);
+ if (buffer == NULL) {
+ fprintf(stderr, "failed to allocate buffer for graph copy\n");
+ ggml_hash_set_free(&hash_set);
+ free(node_copies);
+ free(node_init);
+ ggml_free(ctx_allocated);
+ ggml_free(ctx_unallocated);
+ return (struct ggml_backend_graph_copy) {
+ /* .buffer = */ NULL,
+ /* .ctx_allocated = */ NULL,
+ /* .ctx_unallocated = */ NULL,
+ /* .graph = */ NULL,
+ };
+ }
+
+ //printf("copy buffer size: %zu MB\n", ggml_backend_buffer_get_size(buffer) / 1024 / 1024);
+
+ // copy data and init views
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ graph_copy_init_tensor(&hash_set, node_copies, node_init, node);
+ }
+
+ // build graph copy
+ struct ggml_cgraph * graph_copy = ggml_new_graph_custom(ctx_allocated, graph->size, false);
+ for (int i = 0; i < graph->n_nodes; i++) {
+ struct ggml_tensor * node = graph->nodes[i];
+ struct ggml_tensor * node_copy = node_copies[ggml_hash_find(&hash_set, node)];
+ graph_copy->nodes[i] = node_copy;
+ }
+ graph_copy->n_nodes = graph->n_nodes;
+
+ ggml_hash_set_free(&hash_set);
+ free(node_copies);
+ free(node_init);
+
+ return (struct ggml_backend_graph_copy) {
+ /* .buffer = */ buffer,
+ /* .ctx_allocated = */ ctx_allocated,
+ /* .ctx_unallocated = */ ctx_unallocated,
+ /* .graph = */ graph_copy,
+ };
+}
+
+void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy) {
+ ggml_backend_buffer_free(copy.buffer);
+ ggml_free(copy.ctx_allocated);
+ ggml_free(copy.ctx_unallocated);
+}
+
+bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data) {
+ struct ggml_backend_graph_copy copy = ggml_backend_graph_copy(backend2, graph);
+ if (copy.buffer == NULL) {
+ return false;
+ }
+
+ struct ggml_cgraph * g1 = graph;
+ struct ggml_cgraph * g2 = copy.graph;
+
+ assert(g1->n_nodes == g2->n_nodes);
+
+ for (int i = 0; i < g1->n_nodes; i++) {
+ //printf("eval %d/%d\n", i, g1->n_nodes);
+ struct ggml_tensor * t1 = g1->nodes[i];
+ struct ggml_tensor * t2 = g2->nodes[i];
+
+ assert(t1->op == t2->op && ggml_are_same_layout(t1, t2));
+
+ struct ggml_cgraph g1v = ggml_graph_view(g1, i, i + 1);
+ struct ggml_cgraph g2v = ggml_graph_view(g2, i, i + 1);
+
+ ggml_backend_graph_compute(backend1, &g1v);
+ ggml_backend_graph_compute(backend2, &g2v);
+
+ if (ggml_is_view_op(t1->op)) {
+ continue;
+ }
+
+ // compare results, calculate rms etc
+ if (!callback(i, t1, t2, user_data)) {
+ break;
+ }
+ }
+
+ ggml_backend_graph_copy_free(copy);
+
+ return true;
+}
diff --git a/llama/ggml-backend.h b/llama/ggml-backend.h
new file mode 100644
index 00000000..48385bc4
--- /dev/null
+++ b/llama/ggml-backend.h
@@ -0,0 +1,266 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "ggml.h"
+#include "ggml-alloc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;
+ typedef struct ggml_backend_buffer * ggml_backend_buffer_t;
+ typedef struct ggml_backend_event * ggml_backend_event_t;
+ typedef struct ggml_backend * ggml_backend_t;
+ typedef void * ggml_backend_graph_plan_t;
+
+ //
+ // Backend buffer
+ //
+
+ // buffer type
+ GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft);
+ GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size);
+ GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);
+ GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft);
+ GGML_API GGML_CALL size_t ggml_backend_buft_get_alloc_size (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor);
+ GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);
+
+ // buffer
+ enum ggml_backend_buffer_usage {
+ GGML_BACKEND_BUFFER_USAGE_ANY = 0,
+ GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,
+ GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2,
+ };
+
+ GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer);
+ GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);
+ GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);
+ GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);
+ GGML_API GGML_CALL void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
+ GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);
+ GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer);
+ GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
+ GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);
+ GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);
+ GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
+ GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer);
+ GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer);
+ GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);
+
+ //
+ // Backend
+ //
+
+ GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend);
+ GGML_API const char * ggml_backend_name(ggml_backend_t backend);
+ GGML_API void ggml_backend_free(ggml_backend_t backend);
+
+ GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);
+ GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);
+ GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);
+ GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend);
+
+ GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+ GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
+
+ // "offset" refers to the offset of the tensor data for setting/getting data
+ GGML_API GGML_CALL void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+ GGML_API GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
+
+ GGML_API void ggml_backend_synchronize(ggml_backend_t backend);
+
+ GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);
+ GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
+
+ GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
+ GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);
+ GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);
+ GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
+ GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);
+ GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);
+
+ // tensor copy between different backends
+ GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);
+
+ // asynchronous copy
+ // the copy is performed after all the currently queued operations in backend_src
+ // backend_dst will wait for the copy to complete before performing other operations
+ // automatic fallback to sync copy if async is not supported
+ GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst);
+
+ // events
+ GGML_API ggml_backend_event_t ggml_backend_event_new (ggml_backend_t backend);
+ GGML_API void ggml_backend_event_free (ggml_backend_event_t event);
+ GGML_API void ggml_backend_event_record (ggml_backend_event_t event);
+ GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event);
+ GGML_API void ggml_backend_event_wait (ggml_backend_t backend, ggml_backend_event_t event);
+
+ //
+ // CPU backend
+ //
+
+ GGML_API ggml_backend_t ggml_backend_cpu_init(void);
+
+ GGML_API GGML_CALL bool ggml_backend_is_cpu (ggml_backend_t backend);
+ GGML_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads);
+ GGML_API void ggml_backend_cpu_set_threadpool (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool);
+ GGML_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);
+
+ // Create a backend buffer from an existing pointer
+ GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
+
+ GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);
+
+#ifdef GGML_USE_CPU_HBM
+ GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void);
+#endif
+
+ //
+ // Backend registry
+ //
+
+ // The backend registry is a registry of all the available backends, and allows initializing backends in a generic way
+
+ GGML_API size_t ggml_backend_reg_get_count(void);
+ GGML_API size_t ggml_backend_reg_find_by_name(const char * name);
+ GGML_API ggml_backend_t ggml_backend_reg_init_backend_from_str(const char * backend_str); // str is backend_name:params (params is optional)
+ GGML_API const char * ggml_backend_reg_get_name(size_t i);
+ GGML_API ggml_backend_t ggml_backend_reg_init_backend(size_t i, const char * params); // params is backend-specific
+ GGML_API ggml_backend_buffer_type_t ggml_backend_reg_get_default_buffer_type(size_t i);
+ GGML_API ggml_backend_buffer_t ggml_backend_reg_alloc_buffer(size_t i, size_t size);
+
+ //
+ // Backend scheduler
+ //
+
+ // The backend scheduler allows for multiple backends to be used together
+ // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends
+ // The backends are selected based on:
+ // - the backend that supports the operation
+ // - the location of the pre-allocated tensors (e.g. the weights)
+ /*
+ Example usage:
+
+ // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned
+ // preferrably to run on the same backend as the buffer
+ ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+
+ sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false);
+
+ // initialize buffers from a max size graph (optional)
+ reserve_graph = build_graph(sched, max_batch_size);
+
+ // manually assign nodes to a backend (optional, should not be needed in most cases)
+ struct ggml_tensor * node = ggml_mul_mat(ctx, ...);
+ ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);
+
+ ggml_backend_sched_reserve(sched, reserve_graph);
+
+ // compute
+ graph = build_graph(sched);
+ ggml_backend_sched_graph_compute(sched, graph);
+
+ // if there are graph inputs:
+ ggml_backend_sched_reset(sched);
+ ggml_backend_sched_alloc_graph(sched, graph);
+ ggml_backend_tensor_set(input_tensor, ...);
+ ggml_backend_sched_graph_compute(sched, graph);
+ }
+ */
+
+ struct ggml_backend_sched;
+ typedef struct ggml_backend_sched * ggml_backend_sched_t;
+
+ // when ask == true, the scheduler wants to know if the user wants to observe this node
+ // this allows the scheduler to batch nodes together in order to evaluate them in a single call
+ //
+ // when ask == false, the scheduler is passing the node tensor to the user for observation
+ // if the user returns false, the scheduler will cancel the graph compute
+ //
+ typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);
+
+ // Initialize a backend scheduler
+ GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);
+ GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);
+
+ // Initialize backend buffers from a measure graph
+ GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph);
+
+ GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched);
+ GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i);
+
+ // Get the number of splits of the last graph
+ GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);
+ GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);
+
+ GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);
+
+ GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);
+ GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);
+
+ // Allocate and compute graph on the backend scheduler
+ GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
+ GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
+ GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
+ GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched);
+
+ // Reset all assignments and allocators - must be called before changing the node backends
+ GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);
+
+ // Set a callback to be called for each resulting node during graph compute
+ GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);
+
+ //
+ // Utils
+ //
+
+ struct ggml_backend_graph_copy {
+ ggml_backend_buffer_t buffer;
+ struct ggml_context * ctx_allocated;
+ struct ggml_context * ctx_unallocated;
+ struct ggml_cgraph * graph;
+ };
+
+ // Copy a graph to a different backend
+ GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);
+ GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);
+
+ typedef bool (*GGML_CALL ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);
+
+ // Compare the output of two backends
+ GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data);
+
+ // Tensor initialization
+ GGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);
+ GGML_API void ggml_backend_view_init(struct ggml_tensor * tensor);
+
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama/ggml-blas.cpp b/llama/ggml-blas.cpp
new file mode 100644
index 00000000..ef7b0415
--- /dev/null
+++ b/llama/ggml-blas.cpp
@@ -0,0 +1,397 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifdef GGML_USE_BLAS
+
+#include "ggml-blas.h"
+#include "ggml-backend-impl.h"
+
+#include <future>
+#include <vector>
+
+#if defined(GGML_USE_ACCELERATE)
+# include <Accelerate/Accelerate.h>
+#elif defined(GGML_BLAS_USE_MKL)
+# include <mkl.h>
+#elif defined(GGML_BLAS_USE_BLIS)
+# include <blis.h>
+#elif defined(GGML_BLAS_USE_NVPL)
+# include <nvpl_blas.h>
+#else
+# include <cblas.h>
+#endif
+
+struct ggml_backend_blas_context {
+ int n_threads = GGML_DEFAULT_N_THREADS;
+ std::unique_ptr<char[]> work_data;
+ size_t work_size = 0;
+#ifndef GGML_USE_OPENMP
+ std::vector<std::future<void>> tasks;
+#endif
+};
+
+// helper function to determine if it is better to use BLAS or not
+// for large matrices, BLAS is faster
+static bool ggml_backend_blas_use_blas(const struct ggml_tensor * dst) {
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ const int64_t ne10 = src1->ne[0];
+
+ const int64_t ne0 = dst->ne[0];
+ const int64_t ne1 = dst->ne[1];
+
+ // TODO: find the optimal values for these
+ if (ggml_is_contiguous(src0) &&
+ ggml_is_contiguous(src1) &&
+ src1->type == GGML_TYPE_F32 &&
+ (ne0 >= 32 && ne1 >= 32 && ne10 >= 32)) {
+
+ /*printf("BLAS: %d %d %d %d %d\n", ne0, ne1, ne10, ne00, ne01);*/
+ return true;
+ }
+
+ return false;
+}
+
+static void ggml_backend_blas_mul_mat(ggml_backend_blas_context * ctx, struct ggml_tensor * dst) {
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const enum ggml_type type = src0->type;
+
+ GGML_ASSERT(ne0 == ne01);
+ GGML_ASSERT(ne1 == ne11);
+ GGML_ASSERT(ne2 == ne12);
+ GGML_ASSERT(ne3 == ne13);
+
+ // we don't support permuted src0 or src1
+ GGML_ASSERT(nb00 == ggml_type_size(type));
+ GGML_ASSERT(nb10 == ggml_type_size(src1->type));
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ GGML_ASSERT(nb0 <= nb1);
+ GGML_ASSERT(nb1 <= nb2);
+ GGML_ASSERT(nb2 <= nb3);
+
+ // broadcast factors
+ const int64_t r2 = ne12/ne02;
+ const int64_t r3 = ne13/ne03;
+
+ const int64_t ne_plane = ne01*ne00;
+ const size_t desired_wsize = type == GGML_TYPE_F32 ? 0 : ne03*ne02*ne_plane*sizeof(float);
+
+ if (ctx->work_size < desired_wsize) {
+ ctx->work_data.reset(new char[desired_wsize]);
+ ctx->work_size = desired_wsize;
+ }
+ void * wdata = ctx->work_data.get();
+
+ // convert src0 to float
+ if (type != GGML_TYPE_F32) {
+ ggml_type_traits_t type_traits = ggml_internal_get_type_traits(type);
+ ggml_to_float_t const to_float = type_traits.to_float;
+
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ const void * x = (char *) src0->data + i02*nb02 + i03*nb03;
+ float * const wplane = (float *) wdata + i02*ne_plane + i03*ne02*ne_plane;
+
+ const int min_cols_per_thread = 4096;
+ const int min_rows_per_thread = std::max((int)(min_cols_per_thread/ne00), 1);
+ const int n_threads = std::max(std::min(ctx->n_threads, (int)(ne01/min_rows_per_thread)), 1);
+
+#ifdef GGML_USE_OPENMP
+ #pragma omp parallel for num_threads(n_threads)
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00);
+ }
+#else
+ for (int i = 1; i < n_threads; i++) {
+ const int64_t start = i*ne01/n_threads;
+ const int64_t end = (i + 1)*ne01/n_threads;
+ if (start < end) {
+ ctx->tasks.push_back(std::async(std::launch::async, [=]() {
+ for (int64_t i01 = start; i01 < end; i01++) {
+ to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00);
+ }
+ }));
+ }
+ }
+ {
+ // reuse the current thread for the first task
+ const int64_t start = 0;
+ const int64_t end = ne01/n_threads;
+ for (int64_t i01 = start; i01 < end; i01++) {
+ to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00);
+ }
+ }
+#endif
+ }
+ }
+
+#ifndef GGML_USE_OPENMP
+ // wait for all tasks to finish
+ for (auto & task : ctx->tasks) {
+ task.get();
+ }
+ ctx->tasks.clear();
+#endif
+ }
+
+#if defined(OPENBLAS_VERSION)
+ openblas_set_num_threads(ctx->n_threads);
+#endif
+
+#if defined(GGML_BLAS_USE_BLIS)
+ bli_thread_set_num_threads(ctx->n_threads);
+#endif
+
+#if defined(GGML_BLAS_USE_NVPL)
+ nvpl_blas_set_num_threads(ctx->n_threads);
+#endif
+
+ for (int64_t i13 = 0; i13 < ne13; i13++) {
+ for (int64_t i12 = 0; i12 < ne12; i12++) {
+ const int64_t i03 = i13/r3;
+ const int64_t i02 = i12/r2;
+
+ const float * x = (float *) ((char *) src0->data + i02*nb02 + i03*nb03);
+ const float * y = (float *) ((char *) src1->data + i12*nb12 + i13*nb13);
+ float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3);
+
+ if (type != GGML_TYPE_F32) {
+ x = (float *) wdata + i02*ne_plane + i03*ne02*ne_plane;
+ }
+
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,
+ ne1, ne01, ne10,
+ 1.0f, y, ne10,
+ x, ne00,
+ 0.0f, d, ne01);
+ }
+ }
+}
+
+static void ggml_backend_blas_out_prod(ggml_backend_blas_context * ctx, struct ggml_tensor * dst) {
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT(ne0 == ne00);
+ GGML_ASSERT(ne1 == ne10);
+ GGML_ASSERT(ne2 == ne02);
+ GGML_ASSERT(ne02 == ne12);
+ GGML_ASSERT(ne3 == ne13);
+ GGML_ASSERT(ne03 == ne13);
+
+ // we don't support permuted src0 or src1
+ GGML_ASSERT(nb00 == sizeof(float));
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ // GGML_ASSERT(nb0 <= nb1);
+ // GGML_ASSERT(nb1 <= nb2);
+ // GGML_ASSERT(nb2 <= nb3);
+
+ // Arguments to ggml_compute_forward_out_prod (expressed as major,minor)
+ // src0: (k,n)
+ // src1: (k,m)
+ // dst: (m,n)
+ //
+ // Arguments to sgemm (see https://github.com/Reference-LAPACK/lapack/blob/master/BLAS/SRC/sgemm.f)
+ // Also expressed as (major,minor)
+ // a: (m,k): so src1 transposed
+ // b: (k,n): so src0
+ // c: (m,n)
+ //
+ // However, if ggml_is_transposed(src1) is true, then
+ // src1->data already contains a transposed version, so sgemm mustn't
+ // transpose it further.
+
+ int n = src0->ne[0];
+ int k = src0->ne[1];
+ int m = src1->ne[0];
+
+ CBLAS_TRANSPOSE transposeA;
+ int lda;
+
+ if (!ggml_is_transposed(src1)) {
+ transposeA = CblasTrans;
+ lda = m;
+ } else {
+ transposeA = CblasNoTrans;
+ lda = k;
+ }
+
+ float * a = (float *) ((char *) src1->data);
+ float * b = (float *) ((char *) src0->data);
+ float * c = (float *) ((char *) dst->data);
+
+ cblas_sgemm(CblasRowMajor, transposeA, CblasNoTrans, m, n, k, 1.0, a, lda, b, n, 0.0, c, n);
+
+ GGML_UNUSED(ctx);
+}
+
+// backend interface
+
+GGML_CALL static const char * ggml_backend_blas_name(ggml_backend_t backend) {
+ return "BLAS";
+
+ GGML_UNUSED(backend);
+}
+
+GGML_CALL static void ggml_backend_blas_free(ggml_backend_t backend) {
+ ggml_backend_blas_context * ctx = (ggml_backend_blas_context *)backend->context;
+ delete ctx;
+ delete backend;
+}
+
+GGML_CALL static ggml_backend_buffer_type_t ggml_backend_blas_get_default_buffer_type(ggml_backend_t backend) {
+ return ggml_backend_cpu_buffer_type();
+
+ GGML_UNUSED(backend);
+}
+
+GGML_CALL static enum ggml_status ggml_backend_blas_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
+ ggml_backend_blas_context * ctx = (ggml_backend_blas_context *)backend->context;
+
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ struct ggml_tensor * node = cgraph->nodes[i];
+
+ switch (node->op) {
+ case GGML_OP_MUL_MAT:
+ ggml_backend_blas_mul_mat(ctx, node);
+ break;
+
+ case GGML_OP_OUT_PROD:
+ ggml_backend_blas_out_prod(ctx, node);
+ break;
+
+ case GGML_OP_NONE:
+ case GGML_OP_RESHAPE:
+ case GGML_OP_VIEW:
+ case GGML_OP_PERMUTE:
+ case GGML_OP_TRANSPOSE:
+ break;
+
+ default:
+ GGML_ABORT("%s: unsupported op %s\n", __func__, ggml_op_desc(node));
+ }
+ }
+
+ return GGML_STATUS_SUCCESS;
+
+ GGML_UNUSED(backend);
+}
+
+GGML_CALL static bool ggml_backend_blas_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+
+ return (op->op == GGML_OP_MUL_MAT && ggml_backend_blas_use_blas(op)) ||
+ (op->op == GGML_OP_OUT_PROD && op->src[0]->type == GGML_TYPE_F32 &&
+ op->src[1]->type == GGML_TYPE_F32 &&
+ ggml_is_matrix(src0) &&
+ ggml_is_matrix(src1) &&
+ ggml_is_contiguous(src0) &&
+ (ggml_is_contiguous(src1) || ggml_is_transposed(src1)));
+
+ GGML_UNUSED(backend);
+}
+
+GGML_CALL static bool ggml_backend_blas_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
+ return ggml_backend_buft_is_host(buft);
+
+ GGML_UNUSED(backend);
+}
+
+static struct ggml_backend_i blas_backend_i = {
+ /* .get_name = */ ggml_backend_blas_name,
+ /* .free = */ ggml_backend_blas_free,
+ /* .get_default_buffer_type = */ ggml_backend_blas_get_default_buffer_type,
+ /* .set_tensor_async = */ NULL,
+ /* .get_tensor_async = */ NULL,
+ /* .cpy_tensor_async = */ NULL,
+ /* .synchronize = */ NULL,
+ /* .graph_plan_create = */ NULL,
+ /* .graph_plan_free = */ NULL,
+ /* .graph_plan_update = */ NULL,
+ /* .graph_plan_compute = */ NULL,
+ /* .graph_compute = */ ggml_backend_blas_graph_compute,
+ /* .supports_op = */ ggml_backend_blas_supports_op,
+ /* .supports_buft = */ ggml_backend_blas_supports_buft,
+ /* .offload_op = */ NULL,
+ /* .event_new = */ NULL,
+ /* .event_free = */ NULL,
+ /* .event_record = */ NULL,
+ /* .event_wait = */ NULL,
+ /* .event_synchronize = */ NULL,
+};
+
+static ggml_guid_t ggml_backend_blas_guid(void) {
+ static ggml_guid guid = { 0x12, 0xa8, 0xae, 0xf4, 0xc0, 0x1e, 0x61, 0x97, 0x8f, 0xeb, 0x33, 0x04, 0xa1, 0x33, 0x51, 0x2d };
+ return &guid;
+}
+
+ggml_backend_t ggml_backend_blas_init(void) {
+ ggml_backend_blas_context * ctx = new ggml_backend_blas_context;
+
+ ggml_backend_t backend = new ggml_backend {
+ /* .guid = */ ggml_backend_blas_guid(),
+ /* .interface = */ blas_backend_i,
+ /* .context = */ ctx,
+ };
+
+#if !defined(NDEBUG) && defined(OPENBLAS_VERSION) && defined(GGML_USE_OPENMP)
+ if (openblas_get_parallel() != OPENBLAS_OPENMP) {
+ fprintf(stderr, "%s: warning: ggml is using OpenMP, but OpenBLAS was compiled without OpenMP support\n", __func__);
+ }
+#endif
+
+#if !defined(NDEBUG) && defined(BLIS_ENABLE_CBLAS) && defined(GGML_USE_OPENMP) && !defined(BLIS_ENABLE_OPENMP)
+ fprintf(stderr, "%s: warning: ggml is using OpenMP, but BLIS was compiled without OpenMP support\n", __func__);
+#endif
+
+ return backend;
+}
+
+GGML_CALL bool ggml_backend_is_blas(ggml_backend_t backend) {
+ return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_blas_guid());
+}
+
+void ggml_backend_blas_set_n_threads(ggml_backend_t backend_blas, int n_threads) {
+ GGML_ASSERT(ggml_backend_is_blas(backend_blas));
+
+ ggml_backend_blas_context * ctx = (ggml_backend_blas_context *)backend_blas->context;
+ ctx->n_threads = n_threads;
+}
+
+#endif
diff --git a/llama/ggml-blas.h b/llama/ggml-blas.h
new file mode 100644
index 00000000..6e5d2556
--- /dev/null
+++ b/llama/ggml-blas.h
@@ -0,0 +1,49 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// backend API
+GGML_API GGML_CALL ggml_backend_t ggml_backend_blas_init(void);
+
+GGML_API GGML_CALL bool ggml_backend_is_blas(ggml_backend_t backend);
+
+// number of threads used for conversion to float
+// for openblas and blis, this will also set the number of threads used for blas operations
+GGML_API GGML_CALL void ggml_backend_blas_set_n_threads(ggml_backend_t backend_blas, int n_threads);
+
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama/ggml-common.h b/llama/ggml-common.h
new file mode 100644
index 00000000..b449d362
--- /dev/null
+++ b/llama/ggml-common.h
@@ -0,0 +1,1859 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef GGML_COMMON_DECL
+
+#if defined(GGML_COMMON_DECL_C)
+#include <stdint.h>
+
+typedef uint16_t ggml_half;
+typedef uint32_t ggml_half2;
+
+#define GGML_COMMON_AGGR
+
+#define GGML_COMMON_DECL
+#elif defined(GGML_COMMON_DECL_METAL)
+#include <metal_stdlib>
+
+typedef half ggml_half;
+typedef half2 ggml_half2;
+
+#define GGML_COMMON_AGGR
+
+#define GGML_COMMON_DECL
+#elif defined(GGML_COMMON_DECL_CUDA)
+#if defined(GGML_COMMON_DECL_MUSA)
+#include <musa_fp16.h>
+#else
+#include <cuda_fp16.h>
+#endif
+#include <cstdint>
+
+typedef half ggml_half;
+typedef half2 ggml_half2;
+
+#define GGML_COMMON_AGGR data
+
+#define GGML_COMMON_DECL
+#elif defined(GGML_COMMON_DECL_HIP)
+#include <hip/hip_fp16.h>
+#include <cstdint>
+
+typedef half ggml_half;
+typedef half2 ggml_half2;
+
+#define GGML_COMMON_AGGR data
+
+#define GGML_COMMON_DECL
+#elif defined(GGML_COMMON_DECL_SYCL)
+#include <sycl/half_type.hpp>
+#include <cstdint>
+
+typedef sycl::half ggml_half;
+typedef sycl::half2 ggml_half2;
+
+#define GGML_COMMON_AGGR data
+
+#define GGML_COMMON_DECL
+#endif
+
+#if defined(GGML_COMMON_DECL)
+
+#ifndef __cplusplus
+#ifndef static_assert
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)
+#define static_assert(cond, msg) _Static_assert(cond, msg)
+#else
+#define static_assert(cond, msg) struct global_scope_noop_trick
+#endif
+#endif
+#endif // __cplusplus
+
+// QK = number of values after dequantization
+// QK_K = super-block size
+
+#define QK_K 256
+#define K_SCALE_SIZE 12
+
+#if defined(GGML_COMMON_DECL_CUDA) || defined(GGML_COMMON_DECL_HIP) || defined(GGML_COMMON_DECL_SYCL)
+// QR = QK / number of values before dequantization
+// QI = number of 32 bit integers before dequantization
+
+#define QI4_0 (QK4_0 / (4 * QR4_0))
+#define QR4_0 2
+
+#define QI4_1 (QK4_1 / (4 * QR4_1))
+#define QR4_1 2
+
+#define QI5_0 (QK5_0 / (4 * QR5_0))
+#define QR5_0 2
+
+#define QI5_1 (QK5_1 / (4 * QR5_1))
+#define QR5_1 2
+
+#define QI8_0 (QK8_0 / (4 * QR8_0))
+#define QR8_0 1
+
+#define QI8_1 (QK8_1 / (4 * QR8_1))
+#define QR8_1 1
+
+#define QI2_K (QK_K / (4*QR2_K))
+#define QR2_K 4
+
+#define QI3_K (QK_K / (4*QR3_K))
+#define QR3_K 4
+
+#define QI4_K (QK_K / (4*QR4_K))
+#define QR4_K 2
+
+#define QI5_K (QK_K / (4*QR5_K))
+#define QR5_K 2
+
+#define QI6_K (QK_K / (4*QR6_K))
+#define QR6_K 2
+
+#define QI2_XXS (QK_K / (4*QR2_XXS))
+#define QR2_XXS 4
+
+#define QI2_XS (QK_K / (4*QR2_XS))
+#define QR2_XS 4
+
+#define QI2_S (QK_K / (4*QR2_S))
+#define QR2_S 4
+
+#define QI3_XXS (QK_K / (4*QR3_XXS))
+#define QR3_XXS 4
+
+#define QI3_XS (QK_K / (4*QR3_XS))
+#define QR3_XS 4
+
+#define QI1_S (QK_K / (4*QR1_S))
+#define QR1_S 8
+
+#define QI1_M (QK_K / (4*QR1_M))
+#define QR1_M 8
+
+#define QI4_NL (QK4_NL / (4*QR4_NL))
+#define QR4_NL 2
+
+#define QI4_XS (QK_K / (4*QR4_XS))
+#define QR4_XS 2
+
+#define QI3_S (QK_K / (4*QR3_S))
+#define QR3_S 4
+
+#endif // GGML_COMMON_DECL_CUDA || GGML_COMMON_DECL_HIP
+
+#define QK4_0 32
+typedef struct {
+ ggml_half d; // delta
+ uint8_t qs[QK4_0 / 2]; // nibbles / quants
+} block_q4_0;
+static_assert(sizeof(block_q4_0) == sizeof(ggml_half) + QK4_0 / 2, "wrong q4_0 block size/padding");
+
+#define QK4_1 32
+typedef struct {
+ union {
+ struct {
+ ggml_half d; // delta
+ ggml_half m; // min
+ } GGML_COMMON_AGGR;
+ ggml_half2 dm;
+ };
+ uint8_t qs[QK4_1 / 2]; // nibbles / quants
+} block_q4_1;
+static_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_half) + QK4_1 / 2, "wrong q4_1 block size/padding");
+
+#define QK5_0 32
+typedef struct {
+ ggml_half d; // delta
+ uint8_t qh[4]; // 5-th bit of quants
+ uint8_t qs[QK5_0 / 2]; // nibbles / quants
+} block_q5_0;
+static_assert(sizeof(block_q5_0) == sizeof(ggml_half) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding");
+
+#define QK5_1 32
+typedef struct {
+ union {
+ struct {
+ ggml_half d; // delta
+ ggml_half m; // min
+ } GGML_COMMON_AGGR;
+ ggml_half2 dm;
+ };
+ uint8_t qh[4]; // 5-th bit of quants
+ uint8_t qs[QK5_1 / 2]; // nibbles / quants
+} block_q5_1;
+static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_half) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding");
+
+#define QK8_0 32
+typedef struct {
+ ggml_half d; // delta
+ int8_t qs[QK8_0]; // quants
+} block_q8_0;
+static_assert(sizeof(block_q8_0) == sizeof(ggml_half) + QK8_0, "wrong q8_0 block size/padding");
+
+#define QK8_1 32
+typedef struct {
+ union {
+ struct {
+ ggml_half d; // delta
+ ggml_half s; // d * sum(qs[i])
+ } GGML_COMMON_AGGR;
+ ggml_half2 ds;
+ };
+ int8_t qs[QK8_1]; // quants
+} block_q8_1;
+static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_half) + QK8_1, "wrong q8_1 block size/padding");
+
+typedef struct {
+ ggml_half d[4]; // deltas for 4 q4_0 blocks
+ uint8_t qs[QK4_0 * 2]; // nibbles / quants for 4 q4_0 blocks
+} block_q4_0x4;
+static_assert(sizeof(block_q4_0x4) == 4 * sizeof(ggml_half) + QK4_0 * 2, "wrong q4_0x4 block size/padding");
+
+typedef struct {
+ ggml_half d[8]; // deltas for 8 q4_0 blocks
+ uint8_t qs[QK4_0 * 4]; // nibbles / quants for 8 q4_0 blocks
+} block_q4_0x8;
+static_assert(sizeof(block_q4_0x8) == 8 * sizeof(ggml_half) + QK4_0 * 4, "wrong q4_0x8 block size/padding");
+
+typedef struct {
+ ggml_half d[4]; // deltas for 4 q8_0 blocks
+ int8_t qs[QK8_0 * 4]; // quants for 4 q8_0 blocks
+} block_q8_0x4;
+static_assert(sizeof(block_q8_0x4) == 4 * sizeof(ggml_half) + QK8_0 * 4, "wrong q8_0x4 block size/padding");
+
+typedef struct {
+ ggml_half d[8]; // deltas for 8 q8_0 blocks
+ int8_t qs[QK8_0 * 8]; // quants for 8 q8_0 blocks
+} block_q8_0x8;
+static_assert(sizeof(block_q8_0x8) == 8 * sizeof(ggml_half) + QK8_0 * 8, "wrong q8_0x8 block size/padding");
+
+//
+// Super-block quantization structures
+//
+
+// 2-bit quantization
+// weight is represented as x = a * q + b
+// 16 blocks of 16 elements each
+// Effectively 2.625 bits per weight
+typedef struct {
+ uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits
+ uint8_t qs[QK_K/4]; // quants
+ union {
+ struct {
+ ggml_half d; // super-block scale for quantized scales
+ ggml_half dmin; // super-block scale for quantized mins
+ } GGML_COMMON_AGGR;
+ ggml_half2 dm;
+ };
+} block_q2_K;
+static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");
+
+// 3-bit quantization
+// weight is represented as x = a * q
+// 16 blocks of 16 elements each
+// Effectively 3.4375 bits per weight
+typedef struct {
+ uint8_t hmask[QK_K/8]; // quants - high bit
+ uint8_t qs[QK_K/4]; // quants - low 2 bits
+ uint8_t scales[12]; // scales, quantized with 6 bits
+ ggml_half d; // super-block scale
+} block_q3_K;
+static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding");
+
+// 4-bit quantization
+// 8 blocks of 32 elements each
+// weight is represented as x = a * q + b
+// Effectively 4.5 bits per weight
+typedef struct {
+ union {
+ struct {
+ ggml_half d; // super-block scale for quantized scales
+ ggml_half dmin; // super-block scale for quantized mins
+ } GGML_COMMON_AGGR;
+ ggml_half2 dm;
+ };
+ uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
+ uint8_t qs[QK_K/2]; // 4--bit quants
+} block_q4_K;
+static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding");
+
+// 5-bit quantization
+// 8 blocks of 32 elements each
+// weight is represented as x = a * q + b
+// Effectively 5.5 bits per weight
+typedef struct {
+ union {
+ struct {
+ ggml_half d; // super-block scale for quantized scales
+ ggml_half dmin; // super-block scale for quantized mins
+ } GGML_COMMON_AGGR;
+ ggml_half2 dm;
+ };
+ uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
+ uint8_t qh[QK_K/8]; // quants, high bit
+ uint8_t qs[QK_K/2]; // quants, low 4 bits
+} block_q5_K;
+static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding");
+
+// 6-bit quantization
+// weight is represented as x = a * q
+// 16 blocks of 16 elements each
+// Effectively 6.5625 bits per weight
+typedef struct {
+ uint8_t ql[QK_K/2]; // quants, lower 4 bits
+ uint8_t qh[QK_K/4]; // quants, upper 2 bits
+ int8_t scales[QK_K/16]; // scales, quantized with 8 bits
+ ggml_half d; // super-block scale
+} block_q6_K;
+static_assert(sizeof(block_q6_K) == sizeof(ggml_half) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding");
+
+// This is only used for intermediate quantization and dot products
+typedef struct {
+ float d; // delta
+ int8_t qs[QK_K]; // quants
+ int16_t bsums[QK_K/16]; // sum of quants in groups of 16
+} block_q8_K;
+static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding");
+
+// (Almost) "true" 2-bit quantization.
+// Due to the need to use blocks as per ggml design, it ends up using
+// 2.0625 bpw because of the 16-bit scale for each block of 256.
+typedef struct {
+ ggml_half d;
+ uint16_t qs[QK_K/8];
+} block_iq2_xxs;
+static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding");
+
+// 2.3125 bpw quants
+typedef struct {
+ ggml_half d;
+ uint16_t qs[QK_K/8];
+ uint8_t scales[QK_K/32];
+} block_iq2_xs;
+static_assert(sizeof(block_iq2_xs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");
+
+// 2.5625 bpw quants
+typedef struct {
+ ggml_half d;
+ uint8_t qs[QK_K/4];
+ uint8_t qh[QK_K/32];
+ uint8_t scales[QK_K/32];
+} block_iq2_s;
+static_assert(sizeof(block_iq2_s) == sizeof(ggml_half) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding");
+
+// (Almost) "true" 3-bit quantization.
+// Due to the need to use blocks as per ggml design, it ends up using
+// 3.0625 bpw because of the 16-bit scale for each block of 256.
+typedef struct {
+ ggml_half d;
+ uint8_t qs[3*QK_K/8];
+} block_iq3_xxs;
+static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_half) + 3*(QK_K/8), "wrong iq3_xxs block size/padding");
+
+// 3.4375 bpw
+#define IQ3S_N_SCALE QK_K/64
+typedef struct {
+ ggml_half d;
+ uint8_t qs[QK_K/4];
+ uint8_t qh[QK_K/32];
+ uint8_t signs[QK_K/8];
+ uint8_t scales[IQ3S_N_SCALE];
+} block_iq3_s;
+static_assert(sizeof(block_iq3_s) == sizeof(ggml_half) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding");
+
+typedef struct {
+ ggml_half d;
+ uint8_t qs[QK_K/8];
+ uint16_t qh[QK_K/32];
+} block_iq1_s;
+static_assert(sizeof(block_iq1_s) == sizeof(ggml_half) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding");
+
+// 1.75 bpw
+typedef struct {
+ uint8_t qs[QK_K/8]; // grid index, low 8 bits
+ uint8_t qh[QK_K/16]; // grid index, high 3 bits + grid shift bit (for two groups of 8)
+ uint8_t scales[QK_K/32]; // 3-bit block scales (4-bit if QK_K == 64)
+} block_iq1_m;
+static_assert(sizeof(block_iq1_m) == QK_K/8 + QK_K/16 + QK_K/32, "wrong iq1_m block size/padding");
+
+// Used by IQ1_M quants
+typedef union {
+ ggml_half f16;
+ uint16_t u16;
+} iq1m_scale_t;
+
+// Non-linear quants
+#define QK4_NL 32
+typedef struct {
+ ggml_half d;
+ uint8_t qs[QK4_NL/2];
+} block_iq4_nl;
+static_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, "wrong iq4_nl block size/padding");
+
+typedef struct {
+ ggml_half d;
+ uint16_t scales_h;
+ uint8_t scales_l[QK_K/64];
+ uint8_t qs[QK_K/2];
+} block_iq4_xs;
+static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");
+
+#endif // GGML_COMMON_DECL
+#endif // GGML_COMMON_DECL
+
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef GGML_COMMON_IMPL
+
+#if defined(GGML_COMMON_IMPL_C)
+#include <stdint.h>
+
+#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
+#define GGML_TABLE_END() };
+
+#define GGML_COMMON_IMPL
+#elif defined(GGML_COMMON_IMPL_METAL)
+#include <metal_stdlib>
+
+#define GGML_TABLE_BEGIN(type, name, size) static const constant type name[size] = {
+#define GGML_TABLE_END() };
+
+#define GGML_COMMON_IMPL
+#elif defined(GGML_COMMON_IMPL_CUDA) || defined(GGML_COMMON_IMPL_HIP) || defined(GGML_COMMON_IMPL_MUSA)
+#include <cstdint>
+
+#define GGML_TABLE_BEGIN(type, name, size) static const __device__ type name[size] = {
+#define GGML_TABLE_END() };
+
+#define GGML_COMMON_IMPL
+#elif defined(GGML_COMMON_IMPL_SYCL)
+
+#include <cstdint>
+
+#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
+#define GGML_TABLE_END() };
+
+#define GGML_COMMON_IMPL
+#endif
+
+#if defined(GGML_COMMON_IMPL)
+
+GGML_TABLE_BEGIN(uint8_t, kmask_iq2xs, 8)
+ 1, 2, 4, 8, 16, 32, 64, 128
+GGML_TABLE_END()
+
+GGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128)
+ 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,
+ 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,
+ 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,
+ 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,
+ 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,
+ 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,
+ 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,
+ 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,
+GGML_TABLE_END()
+
+//#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics
+GGML_TABLE_BEGIN(uint64_t, ksigns64, 128)
+ 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,
+ 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,
+ 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,
+ 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,
+ 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,
+ 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,
+ 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,
+ 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,
+ 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,
+ 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,
+ 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,
+ 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,
+ 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,
+ 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,
+ 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,
+ 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,
+ 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,
+ 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,
+ 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,
+ 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,
+ 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,
+ 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,
+ 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,
+ 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,
+ 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,
+ 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,
+ 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,
+ 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,
+ 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,
+ 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,
+ 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,
+ 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,
+GGML_TABLE_END()
+//#endif
+
+
+GGML_TABLE_BEGIN(uint64_t, iq2xxs_grid, 256)
+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,
+ 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,
+ 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,
+ 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,
+ 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,
+ 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,
+ 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,
+ 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,
+ 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,
+ 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,
+ 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,
+ 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,
+ 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,
+ 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,
+ 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,
+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,
+ 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,
+ 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,
+ 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,
+ 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,
+ 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,
+ 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,
+ 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,
+ 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,
+ 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,
+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,
+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,
+ 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,
+ 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,
+ 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,
+ 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,
+ 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,
+ 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,
+ 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,
+ 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,
+ 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,
+ 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,
+ 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,
+ 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,
+ 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,
+ 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,
+ 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,
+ 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,
+ 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,
+ 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,
+ 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,
+ 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,
+ 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,
+ 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,
+ 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,
+ 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,
+ 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,
+ 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,
+ 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,
+ 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,
+ 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,
+ 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,
+ 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,
+ 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,
+ 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,
+ 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,
+ 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,
+ 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,
+GGML_TABLE_END()
+
+GGML_TABLE_BEGIN(uint64_t, iq2xs_grid, 512)
+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,
+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,
+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,
+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,
+ 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,
+ 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,
+ 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,
+ 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,
+ 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,
+ 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,
+ 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,
+ 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,
+ 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,
+ 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,
+ 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,
+ 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,
+ 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,
+ 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,
+ 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,
+ 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,
+ 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,
+ 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,
+ 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,
+ 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,
+ 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,
+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,
+ 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,
+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,
+ 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,
+ 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,
+ 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,
+ 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,
+ 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,
+ 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,
+ 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,
+ 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,
+ 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,
+ 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,
+ 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,
+ 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,
+ 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,
+ 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,
+ 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,
+ 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,
+ 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,
+ 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,
+ 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,
+ 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,
+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,
+ 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,
+ 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,
+ 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,
+ 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,
+ 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,
+ 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,
+ 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,
+ 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,
+ 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,
+ 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,
+ 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,
+ 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,
+ 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,
+ 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,
+ 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,
+ 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,
+ 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,
+ 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,
+ 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,
+ 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,
+ 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,
+ 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,
+ 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,
+ 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,
+ 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,
+ 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,
+ 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,
+ 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,
+ 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,
+ 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,
+ 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,
+ 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,
+ 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,
+ 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,
+ 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,
+ 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,
+ 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,
+ 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,
+ 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,
+ 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,
+ 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,
+ 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,
+ 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,
+ 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,
+ 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,
+ 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,
+ 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,
+ 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,
+ 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,
+ 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,
+ 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,
+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,
+ 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,
+ 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,
+ 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,
+ 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,
+ 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,
+ 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,
+ 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,
+ 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,
+ 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,
+ 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,
+ 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,
+ 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,
+ 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,
+ 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,
+ 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,
+ 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,
+ 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,
+ 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,
+ 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,
+ 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,
+ 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,
+ 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,
+ 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,
+ 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,
+ 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,
+GGML_TABLE_END()
+
+GGML_TABLE_BEGIN(uint64_t, iq2s_grid, 1024)
+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,
+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,
+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,
+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,
+ 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,
+ 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,
+ 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,
+ 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,
+ 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,
+ 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,
+ 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,
+ 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,
+ 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,
+ 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,
+ 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,
+ 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,
+ 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,
+ 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,
+ 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,
+ 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,
+ 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,
+ 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,
+ 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,
+ 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,
+ 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,
+ 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,
+ 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,
+ 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,
+ 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,
+ 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,
+ 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,
+ 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,
+ 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,
+ 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,
+ 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,
+ 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,
+ 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,
+ 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,
+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,
+ 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,
+ 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,
+ 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,
+ 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,
+ 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,
+ 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,
+ 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,
+ 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,
+ 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,
+ 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,
+ 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,
+ 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,
+ 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,
+ 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,
+ 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,
+ 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,
+ 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,
+ 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,
+ 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,
+ 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,
+ 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,
+ 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,
+ 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,
+ 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,
+ 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,
+ 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,
+ 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,
+ 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,
+ 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,
+ 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,
+ 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,
+ 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,
+ 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,
+ 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,
+ 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,
+ 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,
+ 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,
+ 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,
+ 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,
+ 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,
+ 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,
+ 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,
+ 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,
+ 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,
+ 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,
+ 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,
+ 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,
+ 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,
+ 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,
+ 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,
+ 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,
+ 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,
+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,
+ 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,
+ 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,
+ 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,
+ 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,
+ 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,
+ 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,
+ 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,
+ 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,
+ 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,
+ 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,
+ 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,
+ 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,
+ 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,
+ 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,
+ 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,
+ 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,
+ 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,
+ 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,
+ 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,
+ 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,
+ 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,
+ 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,
+ 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,
+ 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,
+ 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,
+ 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,
+ 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,
+ 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,
+ 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,
+ 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,
+ 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,
+ 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,
+ 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,
+ 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,
+ 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,
+ 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,
+ 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,
+ 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,
+ 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,
+ 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,
+ 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,
+ 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,
+ 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,
+ 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,
+ 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,
+ 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,
+ 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,
+ 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,
+ 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,
+ 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,
+ 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,
+ 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,
+ 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,
+ 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,
+ 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,
+ 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,
+ 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,
+ 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,
+ 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,
+ 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,
+ 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,
+ 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,
+ 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,
+ 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,
+ 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,
+ 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,
+ 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,
+ 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,
+ 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,
+ 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,
+ 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,
+ 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,
+ 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,
+ 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,
+ 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,
+ 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,
+ 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,
+ 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,
+ 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,
+ 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,
+ 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,
+ 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,
+ 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,
+ 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,
+ 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,
+ 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,
+ 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,
+ 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,
+ 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,
+ 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,
+ 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,
+ 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,
+ 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,
+ 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,
+ 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,
+ 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,
+ 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,
+ 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,
+ 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,
+ 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,
+ 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,
+ 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,
+ 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,
+ 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,
+ 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,
+ 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,
+ 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,
+ 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,
+ 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,
+ 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,
+ 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,
+ 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,
+ 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,
+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,
+ 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,
+ 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,
+ 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,
+ 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,
+ 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,
+ 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,
+ 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,
+ 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,
+ 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,
+ 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,
+ 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,
+ 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,
+ 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,
+ 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,
+ 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,
+ 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,
+ 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,
+ 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,
+ 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,
+ 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,
+ 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,
+ 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,
+ 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,
+ 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,
+ 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,
+ 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,
+ 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,
+ 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,
+ 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,
+ 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,
+ 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,
+ 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,
+ 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,
+ 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,
+ 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,
+ 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,
+ 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,
+ 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,
+ 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,
+ 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,
+ 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,
+ 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,
+ 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,
+ 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,
+ 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,
+ 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,
+ 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,
+ 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,
+GGML_TABLE_END()
+
+GGML_TABLE_BEGIN(uint32_t, iq3xxs_grid, 256)
+ 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,
+ 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,
+ 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,
+ 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,
+ 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,
+ 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,
+ 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,
+ 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,
+ 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,
+ 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,
+ 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,
+ 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,
+ 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,
+ 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,
+ 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,
+ 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,
+ 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,
+ 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,
+ 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,
+ 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,
+ 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,
+ 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,
+ 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,
+ 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,
+ 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,
+ 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,
+ 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,
+ 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,
+ 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,
+ 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,
+ 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,
+ 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,
+GGML_TABLE_END()
+
+GGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512)
+ 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305,
+ 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905,
+ 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09,
+ 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b,
+ 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b,
+ 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d,
+ 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03,
+ 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505,
+ 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03,
+ 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901,
+ 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d,
+ 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303,
+ 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501,
+ 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105,
+ 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505,
+ 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101,
+ 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707,
+ 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b,
+ 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01,
+ 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f,
+ 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305,
+ 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103,
+ 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509,
+ 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503,
+ 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b,
+ 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f,
+ 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f,
+ 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f,
+ 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109,
+ 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f,
+ 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509,
+ 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501,
+ 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303,
+ 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f,
+ 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907,
+ 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703,
+ 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03,
+ 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01,
+ 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01,
+ 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903,
+ 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505,
+ 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b,
+ 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107,
+ 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509,
+ 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303,
+ 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103,
+ 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05,
+ 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b,
+ 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f,
+ 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701,
+ 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909,
+ 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305,
+ 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d,
+ 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b,
+ 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d,
+ 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307,
+ 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09,
+ 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309,
+ 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709,
+ 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f,
+ 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303,
+ 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503,
+ 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b,
+ 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101,
+GGML_TABLE_END()
+
+#define NGRID_IQ1S 2048
+#define IQ1S_DELTA 0.125f
+#define IQ1M_DELTA 0.125f
+#if defined(GGML_COMMON_IMPL_C)
+GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S)
+ 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff,
+ 0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff,
+ 0xffffffffff01ff01, 0xffffffffff0101ff, 0xffffffffff010101, 0xffffffff00ff0000,
+ 0xffffffff0000ff00, 0xffffffff000000ff, 0xffffffff00000001, 0xffffffff00010000,
+ 0xffffffff01ffffff, 0xffffffff01ffff01, 0xffffffff01ff01ff, 0xffffffff01ff0101,
+ 0xffffffff01000000, 0xffffffff0101ffff, 0xffffffff0101ff01, 0xffffffff010101ff,
+ 0xffffffff01010101, 0xffffff00ffff00ff, 0xffffff00ffff0000, 0xffffff00ff00ff00,
+ 0xffffff00ff0000ff, 0xffffff00ff000001, 0xffffff00ff000100, 0xffffff00ff000101,
+ 0xffffff00ff010000, 0xffffff0000ffff00, 0xffffff0000ff0001, 0xffffff0000ff0100,
+ 0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff000001ff00,
+ 0xffffff00000100ff, 0xffffff0000010001, 0xffffff00000101ff, 0xffffff0001ff0000,
+ 0xffffff000100ff00, 0xffffff00010000ff, 0xffffff0001000001, 0xffffff0001010000,
+ 0xffffff01ffffffff, 0xffffff01ffffff01, 0xffffff01ffff01ff, 0xffffff01ffff0101,
+ 0xffffff01ff000000, 0xffffff01ff01ffff, 0xffffff01ff01ff01, 0xffffff01ff0101ff,
+ 0xffffff01ff010101, 0xffffff0100ff0000, 0xffffff010000ff00, 0xffffff0100000100,
+ 0xffffff01000100ff, 0xffffff0100010100, 0xffffff0101ffffff, 0xffffff0101ffff01,
+ 0xffffff0101ff01ff, 0xffffff0101ff0101, 0xffffff010100ff00, 0xffffff0101000000,
+ 0xffffff0101000100, 0xffffff010101ffff, 0xffffff010101ff01, 0xffffff01010101ff,
+ 0xffffff0101010101, 0xffff00ffff00ff00, 0xffff00ffff0000ff, 0xffff00ffff000001,
+ 0xffff00ffff010000, 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000,
+ 0xffff00ff00000101, 0xffff00ff000100ff, 0xffff00ff00010000, 0xffff00ff0100ff00,
+ 0xffff00ff01000100, 0xffff00ff01010000, 0xffff0000ffffff00, 0xffff0000ffff00ff,
+ 0xffff0000ffff0000, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff0000ff0001ff,
+ 0xffff0000ff000101, 0xffff0000ff010100, 0xffff000000ffffff, 0xffff000000ff0000,
+ 0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff,
+ 0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff,
+ 0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff000000010101,
+ 0xffff000001ffff00, 0xffff00000100ff00, 0xffff000001000000, 0xffff0000010001ff,
+ 0xffff000001000101, 0xffff00000101ff00, 0xffff0000010100ff, 0xffff000001010000,
+ 0xffff000001010001, 0xffff000001010100, 0xffff0001ff0000ff, 0xffff0001ff000100,
+ 0xffff000100ffff00, 0xffff000100ff00ff, 0xffff00010000ffff, 0xffff00010000ff01,
+ 0xffff000100000000, 0xffff0001000001ff, 0xffff00010001ffff, 0xffff00010001ff00,
+ 0xffff000100010001, 0xffff000100010100, 0xffff000101ff0000, 0xffff00010100ff00,
+ 0xffff0001010000ff, 0xffff000101000100, 0xffff01ffffffffff, 0xffff01ffffffff01,
+ 0xffff01ffffff01ff, 0xffff01ffffff0101, 0xffff01ffff000000, 0xffff01ffff01ffff,
+ 0xffff01ffff01ff01, 0xffff01ffff0101ff, 0xffff01ffff010101, 0xffff01ff00ff0000,
+ 0xffff01ff0000ff00, 0xffff01ff00000001, 0xffff01ff00010000, 0xffff01ff01ffffff,
+ 0xffff01ff01ffff01, 0xffff01ff01ff01ff, 0xffff01ff01ff0101, 0xffff01ff01000000,
+ 0xffff01ff0101ffff, 0xffff01ff0101ff01, 0xffff01ff010101ff, 0xffff01ff01010101,
+ 0xffff0100ffff0000, 0xffff0100ff00ff00, 0xffff0100ff0000ff, 0xffff0100ff000100,
+ 0xffff0100ff0100ff, 0xffff0100ff010000, 0xffff010000ffff00, 0xffff01000000ffff,
+ 0xffff01000000ff00, 0xffff010000000000, 0xffff01000001ff00, 0xffff0100000100ff,
+ 0xffff010000010100, 0xffff01000100ff00, 0xffff0100010000ff, 0xffff010001000001,
+ 0xffff010001000100, 0xffff010001010000, 0xffff0101ffffffff, 0xffff0101ffffff01,
+ 0xffff0101ffff01ff, 0xffff0101ffff0101, 0xffff0101ff000000, 0xffff0101ff01ffff,
+ 0xffff0101ff01ff01, 0xffff0101ff0101ff, 0xffff0101ff010101, 0xffff010100ff0000,
+ 0xffff01010000ff00, 0xffff010100000100, 0xffff01010001ff00, 0xffff010100010000,
+ 0xffff010101ffffff, 0xffff010101ffff01, 0xffff010101ff0000, 0xffff010101ff01ff,
+ 0xffff010101ff0101, 0xffff010101000000, 0xffff01010101ffff, 0xffff01010101ff01,
+ 0xffff0101010101ff, 0xffff010101010101, 0xff00ffffff00ffff, 0xff00ffffff00ff00,
+ 0xff00ffffff0000ff, 0xff00ffffff000100, 0xff00ffffff0100ff, 0xff00ffffff010000,
+ 0xff00ffff00ffff00, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, 0xff00ffff00000000,
+ 0xff00ffff000001ff, 0xff00ffff0001ff00, 0xff00ffff000100ff, 0xff00ffff00010000,
+ 0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff010000ff, 0xff00ffff01000001,
+ 0xff00ffff0101ff00, 0xff00ffff01010000, 0xff00ff00ffffff00, 0xff00ff00ffff00ff,
+ 0xff00ff00ffff0001, 0xff00ff00ffff0100, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01,
+ 0xff00ff00ff000000, 0xff00ff00ff0001ff, 0xff00ff00ff01ff00, 0xff00ff00ff0100ff,
+ 0xff00ff00ff010100, 0xff00ff0000ff0000, 0xff00ff0000ff0101, 0xff00ff000000ffff,
+ 0xff00ff000000ff00, 0xff00ff000000ff01, 0xff00ff00000000ff, 0xff00ff0000000000,
+ 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, 0xff00ff0000010000,
+ 0xff00ff0001ff00ff, 0xff00ff000100ff01, 0xff00ff0001000000, 0xff00ff000101ff00,
+ 0xff00ff00010100ff, 0xff00ff01ff00ff00, 0xff00ff01ff0000ff, 0xff00ff01ff000001,
+ 0xff00ff01ff010000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, 0xff00ff0100ff0100,
+ 0xff00ff010000ff01, 0xff00ff0100000000, 0xff00ff01000001ff, 0xff00ff0100000101,
+ 0xff00ff01000100ff, 0xff00ff0100010001, 0xff00ff0101ff0000, 0xff00ff010100ff00,
+ 0xff00ff01010000ff, 0xff00ff0101000001, 0xff00ff0101010000, 0xff0000ffffffff00,
+ 0xff0000ffffff0001, 0xff0000ffffff0100, 0xff0000ffff0000ff, 0xff0000ffff000000,
+ 0xff0000ffff0001ff, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ffff010001,
+ 0xff0000ff00ffff00, 0xff0000ff00ff0000, 0xff0000ff00ff0001, 0xff0000ff00ff01ff,
+ 0xff0000ff00ff0101, 0xff0000ff0000ff00, 0xff0000ff000000ff, 0xff0000ff00000000,
+ 0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff0001ff01, 0xff0000ff00010000,
+ 0xff0000ff000101ff, 0xff0000ff01ff00ff, 0xff0000ff01ff0100, 0xff0000ff0100ffff,
+ 0xff0000ff010000ff, 0xff0000ff01000000, 0xff0000ff010001ff, 0xff0000ff01000100,
+ 0xff0000ff01000101, 0xff0000ff0101ff00, 0xff0000ff010100ff, 0xff0000ff01010000,
+ 0xff0000ff01010100, 0xff000000ffffff01, 0xff000000ffff0000, 0xff000000ffff0101,
+ 0xff000000ff00ff00, 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001,
+ 0xff000000ff000100, 0xff000000ff01ffff, 0xff000000ff01ff01, 0xff000000ff010000,
+ 0xff000000ff0101ff, 0xff000000ff010101, 0xff00000000ffff00, 0xff00000000ff00ff,
+ 0xff00000000ff0000, 0xff00000000ff0001, 0xff0000000000ff00, 0xff0000000000ff01,
+ 0xff000000000000ff, 0xff00000000000000, 0xff00000000000001, 0xff00000000000100,
+ 0xff00000000000101, 0xff0000000001ff00, 0xff000000000100ff, 0xff00000000010000,
+ 0xff00000000010001, 0xff00000000010100, 0xff00000001ffffff, 0xff00000001ffff01,
+ 0xff00000001ff00ff, 0xff00000001ff0000, 0xff00000001ff01ff, 0xff00000001ff0101,
+ 0xff0000000100ffff, 0xff0000000100ff00, 0xff000000010000ff, 0xff00000001000000,
+ 0xff00000001000001, 0xff00000001000100, 0xff00000001000101, 0xff0000000101ffff,
+ 0xff0000000101ff01, 0xff00000001010000, 0xff000001ffffff00, 0xff000001ffff00ff,
+ 0xff000001ffff0000, 0xff000001ffff0001, 0xff000001ff000000, 0xff000001ff000001,
+ 0xff000001ff0001ff, 0xff000001ff000101, 0xff000001ff01ff00, 0xff000001ff010001,
+ 0xff00000100ffffff, 0xff00000100ffff01, 0xff00000100ff00ff, 0xff00000100ff0000,
+ 0xff00000100ff01ff, 0xff00000100ff0101, 0xff0000010000ff00, 0xff00000100000000,
+ 0xff00000100000001, 0xff000001000001ff, 0xff00000100000100, 0xff0000010001ff00,
+ 0xff000001000100ff, 0xff00000100010000, 0xff000001000101ff, 0xff00000100010100,
+ 0xff00000100010101, 0xff00000101ff0001, 0xff00000101ff0101, 0xff0000010100ff01,
+ 0xff00000101000000, 0xff000001010100ff, 0xff00000101010100, 0xff0001ffff00ff00,
+ 0xff0001ffff000001, 0xff0001ffff010000, 0xff0001ff00ffff00, 0xff0001ff00ff00ff,
+ 0xff0001ff00ff0001, 0xff0001ff00ff0100, 0xff0001ff0000ffff, 0xff0001ff00000000,
+ 0xff0001ff000001ff, 0xff0001ff00000101, 0xff0001ff0001ffff, 0xff0001ff0001ff00,
+ 0xff0001ff000100ff, 0xff0001ff00010001, 0xff0001ff00010100, 0xff0001ff01ff0000,
+ 0xff0001ff0100ff00, 0xff0001ff010000ff, 0xff0001ff01010000, 0xff000100ff00ffff,
+ 0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff000101, 0xff000100ff01ff00,
+ 0xff000100ff010000, 0xff00010000ffff01, 0xff00010000ff00ff, 0xff00010000ff0000,
+ 0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000,
+ 0xff00010000000001, 0xff00010000000100, 0xff00010000000101, 0xff0001000001ffff,
+ 0xff00010000010000, 0xff00010000010101, 0xff00010001ff0100, 0xff0001000100ff00,
+ 0xff0001000100ff01, 0xff00010001000000, 0xff000100010001ff, 0xff0001000101ff00,
+ 0xff00010001010001, 0xff00010001010100, 0xff000101ffff0100, 0xff000101ff000001,
+ 0xff000101ff0100ff, 0xff000101ff010001, 0xff00010100ff00ff, 0xff00010100ff0001,
+ 0xff00010100ff0100, 0xff0001010000ffff, 0xff0001010000ff01, 0xff00010100000000,
+ 0xff000101000001ff, 0xff0001010001ff00, 0xff00010100010001, 0xff00010100010100,
+ 0xff00010101ff0000, 0xff0001010100ff00, 0xff00010101000001, 0xff00010101000101,
+ 0xff01ffffffffffff, 0xff01ffffffffff01, 0xff01ffffffff01ff, 0xff01ffffffff0101,
+ 0xff01ffffff000000, 0xff01ffffff01ffff, 0xff01ffffff01ff01, 0xff01ffffff010000,
+ 0xff01ffffff0101ff, 0xff01ffffff010101, 0xff01ffff00ff0000, 0xff01ffff0000ff00,
+ 0xff01ffff00000100, 0xff01ffff0001ff00, 0xff01ffff00010000, 0xff01ffff01ffffff,
+ 0xff01ffff01ffff01, 0xff01ffff01ff01ff, 0xff01ffff01ff0101, 0xff01ffff01000000,
+ 0xff01ffff0101ffff, 0xff01ffff0101ff01, 0xff01ffff01010000, 0xff01ffff010101ff,
+ 0xff01ffff01010101, 0xff01ff00ffff0000, 0xff01ff00ff00ff00, 0xff01ff00ff0000ff,
+ 0xff01ff00ff000100, 0xff01ff00ff010000, 0xff01ff0000ffff01, 0xff01ff0000ff00ff,
+ 0xff01ff0000ff0100, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101,
+ 0xff01ff000001ff00, 0xff01ff00000100ff, 0xff01ff0000010000, 0xff01ff0000010001,
+ 0xff01ff0001ff0000, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001000100,
+ 0xff01ff0001010000, 0xff01ff01ffffff00, 0xff01ff01ffff01ff, 0xff01ff01ffff0101,
+ 0xff01ff01ff00ff00, 0xff01ff01ff000000, 0xff01ff01ff01ffff, 0xff01ff01ff01ff01,
+ 0xff01ff01ff0101ff, 0xff01ff01ff010101, 0xff01ff0100ff0000, 0xff01ff010000ff00,
+ 0xff01ff0100000001, 0xff01ff0100000100, 0xff01ff0100010000, 0xff01ff0101ffff00,
+ 0xff01ff0101ff01ff, 0xff01ff0101ff0101, 0xff01ff010100ff00, 0xff01ff0101000000,
+ 0xff01ff010101ffff, 0xff01ff010101ff01, 0xff01ff01010101ff, 0xff01ff0101010101,
+ 0xff0100ffffff0000, 0xff0100ffff0000ff, 0xff0100ffff000001, 0xff0100ffff000100,
+ 0xff0100ffff010000, 0xff0100ff00ff00ff, 0xff0100ff00ff0000, 0xff0100ff00ff0001,
+ 0xff0100ff00ff0100, 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff,
+ 0xff0100ff00000101, 0xff0100ff00010001, 0xff0100ff01ff0000, 0xff0100ff0100ff00,
+ 0xff0100ff010000ff, 0xff0100ff01000100, 0xff0100ff0101ff00, 0xff0100ff01010000,
+ 0xff010000ffff0100, 0xff010000ff000000, 0xff010000ff01ff00, 0xff010000ff010100,
+ 0xff01000000ffffff, 0xff01000000ff0000, 0xff01000000ff01ff, 0xff0100000000ff00,
+ 0xff010000000000ff, 0xff01000000000000, 0xff01000000000100, 0xff0100000001ff01,
+ 0xff01000000010000, 0xff010000000101ff, 0xff01000001ff0100, 0xff0100000100ffff,
+ 0xff010000010000ff, 0xff01000001000000, 0xff010000010001ff, 0xff01000001000101,
+ 0xff0100000101ff00, 0xff010000010100ff, 0xff01000001010001, 0xff01000001010100,
+ 0xff010001ffff0000, 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100,
+ 0xff010001ff010000, 0xff01000100ffff00, 0xff01000100ff0100, 0xff01000100000000,
+ 0xff0100010001ffff, 0xff0100010001ff00, 0xff01000100010100, 0xff01000101ff00ff,
+ 0xff01000101ff0001, 0xff0100010100ffff, 0xff01000101000101, 0xff0101ffffffffff,
+ 0xff0101ffffffff01, 0xff0101ffffff01ff, 0xff0101ffffff0101, 0xff0101ffff000000,
+ 0xff0101ffff01ffff, 0xff0101ffff01ff01, 0xff0101ffff0101ff, 0xff0101ffff010101,
+ 0xff0101ff00ff0000, 0xff0101ff0000ff00, 0xff0101ff000000ff, 0xff0101ff00010000,
+ 0xff0101ff01ffffff, 0xff0101ff01ffff01, 0xff0101ff01ff01ff, 0xff0101ff01ff0101,
+ 0xff0101ff0101ffff, 0xff0101ff0101ff01, 0xff0101ff010101ff, 0xff0101ff01010101,
+ 0xff010100ffff0100, 0xff010100ff00ff00, 0xff010100ff0000ff, 0xff010100ff000100,
+ 0xff010100ff010000, 0xff01010000ff0001, 0xff01010000ff0100, 0xff0101000000ff01,
+ 0xff01010000000000, 0xff0101000001ff00, 0xff010100000100ff, 0xff01010000010001,
+ 0xff01010000010100, 0xff01010001ff0000, 0xff0101000100ffff, 0xff01010001000001,
+ 0xff01010001000100, 0xff010100010100ff, 0xff01010001010000, 0xff010101ffffffff,
+ 0xff010101ffffff01, 0xff010101ffff01ff, 0xff010101ffff0101, 0xff010101ff01ffff,
+ 0xff010101ff01ff01, 0xff010101ff0101ff, 0xff010101ff010101, 0xff01010100ff0000,
+ 0xff0101010000ff00, 0xff01010100000001, 0xff01010100000100, 0xff01010100010000,
+ 0xff01010101ffffff, 0xff01010101ffff01, 0xff01010101ff01ff, 0xff01010101ff0101,
+ 0xff01010101000000, 0xff0101010101ffff, 0xff0101010101ff01, 0xff010101010101ff,
+ 0xff01010101010101, 0x00ffffffffff0000, 0x00ffffffff00ff00, 0x00ffffffff000001,
+ 0x00ffffffff010000, 0x00ffffff00ff0100, 0x00ffffff0000ff01, 0x00ffffff00000000,
+ 0x00ffffff000001ff, 0x00ffffff00000101, 0x00ffffff0001ff00, 0x00ffffff000100ff,
+ 0x00ffffff00010001, 0x00ffffff010000ff, 0x00ffffff01000100, 0x00ffffff0101ff00,
+ 0x00ffffff01010001, 0x00ffff00ffffffff, 0x00ffff00ffffff00, 0x00ffff00ffff00ff,
+ 0x00ffff00ffff0001, 0x00ffff00ffff0100, 0x00ffff00ff00ff01, 0x00ffff00ff000000,
+ 0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff000101, 0x00ffff00ff01ff00,
+ 0x00ffff00ff010001, 0x00ffff00ff010100, 0x00ffff0000ff0000, 0x00ffff0000ff01ff,
+ 0x00ffff0000ff0101, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000,
+ 0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000,
+ 0x00ffff00000101ff, 0x00ffff0000010101, 0x00ffff0001ffff00, 0x00ffff0001ff00ff,
+ 0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff000100ff01, 0x00ffff0001000000,
+ 0x00ffff000101ffff, 0x00ffff000101ff00, 0x00ffff000101ff01, 0x00ffff01ffff0000,
+ 0x00ffff01ff00ff00, 0x00ffff01ff0000ff, 0x00ffff01ff000001, 0x00ffff01ff010000,
+ 0x00ffff0100ffff00, 0x00ffff010000ff01, 0x00ffff0100000000, 0x00ffff0100000101,
+ 0x00ffff01000100ff, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ffff01010000ff,
+ 0x00ffff0101010000, 0x00ff00ffffffff00, 0x00ff00ffff000000, 0x00ff00ffff000100,
+ 0x00ff00ffff010100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, 0x00ff00ff00ff0101,
+ 0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001,
+ 0x00ff00ff0001ff00, 0x00ff00ff0001ff01, 0x00ff00ff00010000, 0x00ff00ff000101ff,
+ 0x00ff00ff00010101, 0x00ff00ff01ffff00, 0x00ff00ff01ff0001, 0x00ff00ff01ff0100,
+ 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, 0x00ff00ff0101ffff,
+ 0x00ff00ff0101ff00, 0x00ff00ff01010100, 0x00ff0000ffffff00, 0x00ff0000ffffff01,
+ 0x00ff0000ffff0000, 0x00ff0000ffff0101, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff,
+ 0x00ff0000ff000000, 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff,
+ 0x00ff0000ff010000, 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff,
+ 0x00ff000000ff0000, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff,
+ 0x00ff00000000ff00, 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001,
+ 0x00ff0000000001ff, 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff,
+ 0x00ff000000010000, 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01,
+ 0x00ff000001ff00ff, 0x00ff000001ff0000, 0x00ff000001ff01ff, 0x00ff00000100ff00,
+ 0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, 0x00ff000001000100,
+ 0x00ff000001000101, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101,
+ 0x00ff0001ffffff00, 0x00ff0001ffff0000, 0x00ff0001ffff0100, 0x00ff0001ff0000ff,
+ 0x00ff0001ff000000, 0x00ff0001ff0001ff, 0x00ff0001ff000101, 0x00ff0001ff01ff00,
+ 0x00ff0001ff0100ff, 0x00ff0001ff010100, 0x00ff000100ffffff, 0x00ff000100ffff01,
+ 0x00ff000100ff0000, 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00,
+ 0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100,
+ 0x00ff00010001ff01, 0x00ff000100010000, 0x00ff0001000101ff, 0x00ff000101ffff00,
+ 0x00ff000101ff0000, 0x00ff000101ff0101, 0x00ff0001010000ff, 0x00ff000101000000,
+ 0x00ff00010101ff00, 0x00ff0001010100ff, 0x00ff000101010001, 0x00ff01ffffff0000,
+ 0x00ff01ffff00ff00, 0x00ff01ffff000000, 0x00ff01ffff000101, 0x00ff01ffff010000,
+ 0x00ff01ff00ffff01, 0x00ff01ff00ff0100, 0x00ff01ff0000ffff, 0x00ff01ff00000000,
+ 0x00ff01ff000001ff, 0x00ff01ff0001ff00, 0x00ff01ff000100ff, 0x00ff01ff00010001,
+ 0x00ff01ff00010100, 0x00ff01ff01ff0000, 0x00ff01ff0100ff00, 0x00ff01ff010000ff,
+ 0x00ff01ff01000001, 0x00ff01ff01000100, 0x00ff01ff01010000, 0x00ff0100ffffff00,
+ 0x00ff0100ffff0000, 0x00ff0100ffff0001, 0x00ff0100ffff0101, 0x00ff0100ff00ffff,
+ 0x00ff0100ff0000ff, 0x00ff0100ff000000, 0x00ff0100ff0001ff, 0x00ff0100ff01ff00,
+ 0x00ff0100ff0100ff, 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000,
+ 0x00ff010000ff0101, 0x00ff01000000ff00, 0x00ff01000000ff01, 0x00ff0100000000ff,
+ 0x00ff010000000000, 0x00ff010000000001, 0x00ff010000000100, 0x00ff01000001ffff,
+ 0x00ff01000001ff01, 0x00ff010000010000, 0x00ff010000010001, 0x00ff010000010101,
+ 0x00ff010001ff0001, 0x00ff010001ff0100, 0x00ff01000100ff01, 0x00ff010001000000,
+ 0x00ff010001000001, 0x00ff0100010001ff, 0x00ff01000101ff00, 0x00ff0100010100ff,
+ 0x00ff010001010001, 0x00ff010001010100, 0x00ff0101ff000001, 0x00ff010100ff00ff,
+ 0x00ff010100ff0001, 0x00ff010100ff0100, 0x00ff010100000000, 0x00ff0101000001ff,
+ 0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x00ff0101010000ff,
+ 0x00ff010101010000, 0x0000ffffffffff00, 0x0000ffffffff00ff, 0x0000ffffffff0000,
+ 0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000,
+ 0x0000ffffff000101, 0x0000ffffff01ff00, 0x0000ffffff0100ff, 0x0000ffffff010100,
+ 0x0000ffff00ffffff, 0x0000ffff00ff0000, 0x0000ffff00ff01ff, 0x0000ffff0000ff00,
+ 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100,
+ 0x0000ffff00010000, 0x0000ffff000101ff, 0x0000ffff01ff0001, 0x0000ffff01ff0100,
+ 0x0000ffff01000000, 0x0000ffff010001ff, 0x0000ffff0101ffff, 0x0000ffff0101ff00,
+ 0x0000ffff01010001, 0x0000ffff01010100, 0x0000ff00ffff0000, 0x0000ff00ffff01ff,
+ 0x0000ff00ffff0100, 0x0000ff00ffff0101, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff,
+ 0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff0001ff, 0x0000ff00ff000100,
+ 0x0000ff00ff01ffff, 0x0000ff00ff010000, 0x0000ff00ff010001, 0x0000ff00ff0101ff,
+ 0x0000ff00ff010101, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, 0x0000ff0000ff0000,
+ 0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, 0x0000ff000000ff00,
+ 0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,
+ 0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, 0x0000ff000001ff00,
+ 0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, 0x0000ff0000010100,
+ 0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff,
+ 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff000101ffff,
+ 0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ffffff00, 0x0000ff01ffff0001,
+ 0x0000ff01ff00ff01, 0x0000ff01ff000000, 0x0000ff01ff000101, 0x0000ff01ff01ff00,
+ 0x0000ff01ff0100ff, 0x0000ff0100ffff01, 0x0000ff0100ff0000, 0x0000ff0100ff0101,
+ 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001,
+ 0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, 0x0000ff0101ff0000,
+ 0x0000ff010100ffff, 0x0000ff010100ff01, 0x0000ff0101000000, 0x0000ff0101000100,
+ 0x0000ff0101000101, 0x0000ff01010100ff, 0x000000ffffff00ff, 0x000000ffffff0000,
+ 0x000000ffff00ff00, 0x000000ffff0000ff, 0x000000ffff000000, 0x000000ffff000001,
+ 0x000000ffff0001ff, 0x000000ffff000100, 0x000000ffff01ff00, 0x000000ffff010000,
+ 0x000000ffff0101ff, 0x000000ffff010101, 0x000000ff00ffff00, 0x000000ff00ff00ff,
+ 0x000000ff00ff0000, 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101,
+ 0x000000ff0000ffff, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,
+ 0x000000ff00000001, 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101,
+ 0x000000ff0001ff00, 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000,
+ 0x000000ff00010001, 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff01ff01ff,
+ 0x000000ff01ff0101, 0x000000ff0100ff00, 0x000000ff010000ff, 0x000000ff01000000,
+ 0x000000ff01000001, 0x000000ff01000100, 0x000000ff0101ff00, 0x000000ff010100ff,
+ 0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01,
+ 0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100,
+ 0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff,
+ 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101,
+ 0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001,
+ 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01,
+ 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff,
+ 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,
+ 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff,
+ 0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00,
+ 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff,
+ 0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff,
+ 0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff,
+ 0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001,
+ 0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff,
+ 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffffff,
+ 0x00000001ffffff00, 0x00000001ffffff01, 0x00000001ffff00ff, 0x00000001ffff0001,
+ 0x00000001ffff01ff, 0x00000001ffff0100, 0x00000001ff00ff00, 0x00000001ff0000ff,
+ 0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, 0x00000001ff01ffff,
+ 0x00000001ff01ff00, 0x00000001ff01ff01, 0x00000001ff0100ff, 0x00000001ff010000,
+ 0x00000001ff010001, 0x00000001ff0101ff, 0x00000001ff010100, 0x0000000100ffff00,
+ 0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff01ff, 0x0000000100ff0100,
+ 0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01,
+ 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff,
+ 0x0000000100000100, 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff,
+ 0x0000000100010000, 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000,
+ 0x0000000101ff0001, 0x0000000101ff01ff, 0x0000000101ff0100, 0x0000000101ff0101,
+ 0x000000010100ff00, 0x0000000101000000, 0x0000000101000101, 0x000000010101ff01,
+ 0x0000000101010000, 0x0000000101010001, 0x00000001010101ff, 0x0000000101010100,
+ 0x000001ffffff00ff, 0x000001ffffff0000, 0x000001ffffff0001, 0x000001ffffff0100,
+ 0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff0001ff, 0x000001ffff01ff00,
+ 0x000001ffff010101, 0x000001ff00ff0000, 0x000001ff00ff01ff, 0x000001ff00ff0101,
+ 0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, 0x000001ff00000001,
+ 0x000001ff000001ff, 0x000001ff00000100, 0x000001ff0001ffff, 0x000001ff0001ff01,
+ 0x000001ff000100ff, 0x000001ff00010000, 0x000001ff01ffff01, 0x000001ff01ff0100,
+ 0x000001ff0100ffff, 0x000001ff0100ff01, 0x000001ff01000000, 0x000001ff010001ff,
+ 0x000001ff0101ff00, 0x000001ff01010100, 0x00000100ffffff00, 0x00000100ffffff01,
+ 0x00000100ffff0000, 0x00000100ffff0101, 0x00000100ff00ff00, 0x00000100ff0000ff,
+ 0x00000100ff000000, 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000,
+ 0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001,
+ 0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01,
+ 0x00000100000000ff, 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff,
+ 0x0000010000000100, 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff,
+ 0x0000010000010000, 0x0000010000010001, 0x0000010000010100, 0x0000010001ffff00,
+ 0x0000010001ff0000, 0x0000010001ff0100, 0x000001000100ff00, 0x00000100010000ff,
+ 0x0000010001000000, 0x0000010001000001, 0x00000100010001ff, 0x0000010001000100,
+ 0x0000010001010000, 0x00000101ffff00ff, 0x00000101ffff01ff, 0x00000101ff000000,
+ 0x00000101ff000101, 0x00000101ff01ffff, 0x00000101ff010000, 0x00000101ff010001,
+ 0x00000101ff010100, 0x0000010100ff0000, 0x0000010100ff01ff, 0x0000010100ff0100,
+ 0x000001010000ff00, 0x0000010100000000, 0x0000010100000001, 0x00000101000001ff,
+ 0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, 0x00000101000101ff,
+ 0x0000010100010101, 0x0000010101ffff00, 0x0000010101ff0101, 0x000001010100ff01,
+ 0x0000010101000000, 0x0000010101000001, 0x00000101010001ff, 0x0000010101000101,
+ 0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, 0x0001ffffff000001,
+ 0x0001ffffff000100, 0x0001ffffff010000, 0x0001ffff00ff00ff, 0x0001ffff0000ffff,
+ 0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff00000101,
+ 0x0001ffff0001ff00, 0x0001ffff000100ff, 0x0001ffff00010001, 0x0001ffff00010100,
+ 0x0001ffff01ffff00, 0x0001ffff01000001, 0x0001ffff01010000, 0x0001ff00ffffff00,
+ 0x0001ff00ffff00ff, 0x0001ff00ffff0001, 0x0001ff00ffff0100, 0x0001ff00ff00ff01,
+ 0x0001ff00ff000000, 0x0001ff00ff01ff00, 0x0001ff00ff01ff01, 0x0001ff00ff010001,
+ 0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff0000ff0100, 0x0001ff000000ff00,
+ 0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000,
+ 0x0001ff0000010001, 0x0001ff0000010101, 0x0001ff0001ff00ff, 0x0001ff0001ff0101,
+ 0x0001ff000100ff01, 0x0001ff0001000000, 0x0001ff000101ff00, 0x0001ff0001010001,
+ 0x0001ff0001010100, 0x0001ff01ff00ff00, 0x0001ff01ff000001, 0x0001ff01ff000100,
+ 0x0001ff0100ffffff, 0x0001ff0100ffff00, 0x0001ff0100ff0001, 0x0001ff0100000000,
+ 0x0001ff0100000001, 0x0001ff01000001ff, 0x0001ff010001ffff, 0x0001ff0101ff0000,
+ 0x0001ff010100ff00, 0x0001ff0101000001, 0x0001ff0101010000, 0x000100ffff00ff00,
+ 0x000100ffff00ff01, 0x000100ffff000000, 0x000100ffff000001, 0x000100ffff000101,
+ 0x000100ffff01ff00, 0x000100ffff010001, 0x000100ffff010100, 0x000100ff00ffffff,
+ 0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff00ff0101,
+ 0x000100ff0000ff00, 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000001,
+ 0x000100ff00000100, 0x000100ff00000101, 0x000100ff0001ffff, 0x000100ff0001ff01,
+ 0x000100ff00010000, 0x000100ff01ff00ff, 0x000100ff01ff0000, 0x000100ff01ff0100,
+ 0x000100ff0100ffff, 0x000100ff0100ff01, 0x000100ff010000ff, 0x000100ff01000000,
+ 0x000100ff01000001, 0x000100ff010001ff, 0x000100ff01000101, 0x000100ff0101ff00,
+ 0x000100ff010100ff, 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff01ff,
+ 0x00010000ffff0101, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001,
+ 0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0000, 0x0001000000ff0001,
+ 0x0001000000ff0100, 0x000100000000ffff, 0x000100000000ff00, 0x00010000000000ff,
+ 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, 0x000100000001ff00,
+ 0x00010000000100ff, 0x0001000000010000, 0x0001000000010001, 0x0001000000010100,
+ 0x0001000001ff0001, 0x0001000001ff0100, 0x0001000001ff0101, 0x000100000100ff00,
+ 0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101,
+ 0x000100000101ff01, 0x0001000001010000, 0x0001000001010001, 0x00010000010101ff,
+ 0x00010001ffffff01, 0x00010001ffff0100, 0x00010001ff000000, 0x00010001ff01ffff,
+ 0x00010001ff010001, 0x00010001ff0101ff, 0x00010001ff010100, 0x0001000100ffffff,
+ 0x0001000100ff0000, 0x0001000100ff01ff, 0x0001000100ff0101, 0x000100010000ff00,
+ 0x00010001000000ff, 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff,
+ 0x0001000100000101, 0x000100010001ffff, 0x0001000100010000, 0x00010001000101ff,
+ 0x0001000101ffffff, 0x0001000101ffff01, 0x0001000101ff0000, 0x0001000101ff0101,
+ 0x00010001010000ff, 0x0001000101000001, 0x00010001010001ff, 0x0001000101000100,
+ 0x000100010101ffff, 0x00010001010100ff, 0x0001000101010001, 0x0001000101010101,
+ 0x000101ffff000001, 0x000101ffff000100, 0x000101ffff010000, 0x000101ff00ffff00,
+ 0x000101ff0000ff01, 0x000101ff00000000, 0x000101ff00000101, 0x000101ff0001ff00,
+ 0x000101ff00010100, 0x000101ff01ff0000, 0x000101ff0100ff00, 0x000101ff010001ff,
+ 0x000101ff01010001, 0x00010100ffffff00, 0x00010100ffff00ff, 0x00010100ff00ffff,
+ 0x00010100ff000000, 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001,
+ 0x00010100ff010100, 0x0001010000ffffff, 0x0001010000ffff00, 0x0001010000ff0000,
+ 0x0001010000ff0001, 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff,
+ 0x0001010000000000, 0x0001010000000001, 0x0001010000000100, 0x000101000001ffff,
+ 0x0001010000010000, 0x0001010000010101, 0x0001010001ffff01, 0x0001010001ff00ff,
+ 0x0001010001ff0101, 0x0001010001000000, 0x000101000101ff00, 0x00010100010100ff,
+ 0x0001010001010000, 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff000001,
+ 0x00010101ff0001ff, 0x0001010100ffff00, 0x0001010100ff00ff, 0x0001010100ff0100,
+ 0x000101010000ffff, 0x0001010100000000, 0x00010101000001ff, 0x0001010100000101,
+ 0x00010101000100ff, 0x0001010100010000, 0x0001010100010100, 0x0001010101ff0001,
+ 0x00010101010000ff, 0x00010101010001ff, 0x0001010101000101, 0x0001010101010001,
+ 0x01ffffffffffffff, 0x01ffffffffffff01, 0x01ffffffffff01ff, 0x01ffffffffff0101,
+ 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, 0x01ffffffff010101,
+ 0x01ffffff00ff0000, 0x01ffffff0000ffff, 0x01ffffff0000ff00, 0x01ffffff000000ff,
+ 0x01ffffff00000001, 0x01ffffff00000100, 0x01ffffff00010000, 0x01ffffff01ffffff,
+ 0x01ffffff01ffff01, 0x01ffffff01ff01ff, 0x01ffffff01ff0101, 0x01ffffff01000000,
+ 0x01ffffff0101ffff, 0x01ffffff0101ff01, 0x01ffffff010101ff, 0x01ffffff01010101,
+ 0x01ffff00ffff0000, 0x01ffff00ff00ff00, 0x01ffff00ff0000ff, 0x01ffff00ff000001,
+ 0x01ffff00ff000100, 0x01ffff00ff010000, 0x01ffff0000ffff00, 0x01ffff0000ff00ff,
+ 0x01ffff0000ff0100, 0x01ffff000000ffff, 0x01ffff000000ff01, 0x01ffff0000000000,
+ 0x01ffff0000000001, 0x01ffff00000001ff, 0x01ffff0000000100, 0x01ffff00000100ff,
+ 0x01ffff0000010001, 0x01ffff0000010100, 0x01ffff0001ff0000, 0x01ffff0001ff0100,
+ 0x01ffff00010000ff, 0x01ffff0001000001, 0x01ffff0001000100, 0x01ffff0001010000,
+ 0x01ffff01ffffffff, 0x01ffff01ffffff01, 0x01ffff01ffff01ff, 0x01ffff01ffff0101,
+ 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff01ff01, 0x01ffff01ff0101ff,
+ 0x01ffff01ff010101, 0x01ffff010000ff00, 0x01ffff01000000ff, 0x01ffff0100000100,
+ 0x01ffff0100010000, 0x01ffff0101ffffff, 0x01ffff0101ffff01, 0x01ffff0101ff01ff,
+ 0x01ffff0101ff0101, 0x01ffff0101000000, 0x01ffff010101ffff, 0x01ffff010101ff01,
+ 0x01ffff01010101ff, 0x01ffff0101010101, 0x01ff00ffff0000ff, 0x01ff00ffff000100,
+ 0x01ff00ff00ffff00, 0x01ff00ff00ff00ff, 0x01ff00ff0000ff00, 0x01ff00ff00000000,
+ 0x01ff00ff00000101, 0x01ff00ff0001ff00, 0x01ff00ff000100ff, 0x01ff00ff00010100,
+ 0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ffffff00, 0x01ff0000ffff0100,
+ 0x01ff0000ff00ff01, 0x01ff0000ff000000, 0x01ff0000ff000101, 0x01ff0000ff010001,
+ 0x01ff0000ff010100, 0x01ff000000ffffff, 0x01ff000000ffff00, 0x01ff000000ff0000,
+ 0x01ff000000ff01ff, 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000,
+ 0x01ff000000000001, 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000,
+ 0x01ff000000010001, 0x01ff0000000101ff, 0x01ff000000010101, 0x01ff000001ffff00,
+ 0x01ff000001ff00ff, 0x01ff000001ff0001, 0x01ff000001ff0100, 0x01ff00000100ffff,
+ 0x01ff00000100ff01, 0x01ff000001000000, 0x01ff0000010001ff, 0x01ff000001010001,
+ 0x01ff0001ff00ff00, 0x01ff0001ff000001, 0x01ff0001ff000100, 0x01ff0001ff010000,
+ 0x01ff000100ffff00, 0x01ff000100ff00ff, 0x01ff000100ff0100, 0x01ff000100ff0101,
+ 0x01ff00010000ffff, 0x01ff000100000000, 0x01ff000100000100, 0x01ff000100000101,
+ 0x01ff00010001ff00, 0x01ff000100010001, 0x01ff000100010101, 0x01ff000101ff0000,
+ 0x01ff00010100ff00, 0x01ff000101000101, 0x01ff0001010100ff, 0x01ff01ffffffffff,
+ 0x01ff01ffffffff01, 0x01ff01ffffff01ff, 0x01ff01ffffff0101, 0x01ff01ffff000000,
+ 0x01ff01ffff01ffff, 0x01ff01ffff01ff01, 0x01ff01ffff0101ff, 0x01ff01ffff010101,
+ 0x01ff01ff00ffff00, 0x01ff01ff00ff0000, 0x01ff01ff0000ff00, 0x01ff01ff000000ff,
+ 0x01ff01ff00000100, 0x01ff01ff00010000, 0x01ff01ff00010100, 0x01ff01ff01ffffff,
+ 0x01ff01ff01ffff01, 0x01ff01ff01ff01ff, 0x01ff01ff01ff0101, 0x01ff01ff01000000,
+ 0x01ff01ff0101ffff, 0x01ff01ff0101ff01, 0x01ff01ff010101ff, 0x01ff01ff01010101,
+ 0x01ff0100ffff0000, 0x01ff0100ffff0001, 0x01ff0100ff00ff00, 0x01ff0100ff0000ff,
+ 0x01ff0100ff000001, 0x01ff0100ff010000, 0x01ff010000ffff00, 0x01ff010000ff00ff,
+ 0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff01000000ffff, 0x01ff01000000ff01,
+ 0x01ff010000000000, 0x01ff010000000101, 0x01ff01000001ff00, 0x01ff0100000100ff,
+ 0x01ff010001ff0000, 0x01ff010001000001, 0x01ff010001000100, 0x01ff010001010000,
+ 0x01ff0101ffffffff, 0x01ff0101ffffff01, 0x01ff0101ffff01ff, 0x01ff0101ffff0101,
+ 0x01ff0101ff000000, 0x01ff0101ff01ffff, 0x01ff0101ff01ff01, 0x01ff0101ff0101ff,
+ 0x01ff0101ff010101, 0x01ff010100ff0000, 0x01ff01010000ff00, 0x01ff0101000000ff,
+ 0x01ff010100000001, 0x01ff010101ffffff, 0x01ff010101ffff01, 0x01ff010101ff01ff,
+ 0x01ff010101ff0101, 0x01ff010101000000, 0x01ff01010101ffff, 0x01ff01010101ff01,
+ 0x01ff0101010101ff, 0x01ff010101010101, 0x0100ffffffff0000, 0x0100ffffff00ff00,
+ 0x0100ffffff000001, 0x0100ffffff0001ff, 0x0100ffffff000100, 0x0100ffffff010000,
+ 0x0100ffff00ffff00, 0x0100ffff00ff0001, 0x0100ffff00ff0100, 0x0100ffff00000000,
+ 0x0100ffff000001ff, 0x0100ffff00000101, 0x0100ffff00010100, 0x0100ffff00010101,
+ 0x0100ffff01ff0000, 0x0100ffff0100ff00, 0x0100ffff010000ff, 0x0100ffff01000001,
+ 0x0100ffff01000100, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff00ff,
+ 0x0100ff00ffff0001, 0x0100ff00ffff0100, 0x0100ff00ff00ffff, 0x0100ff00ff000000,
+ 0x0100ff00ff0001ff, 0x0100ff00ff000101, 0x0100ff00ff01ff00, 0x0100ff00ff0100ff,
+ 0x0100ff00ff010001, 0x0100ff00ff010100, 0x0100ff0000ffffff, 0x0100ff0000ff0000,
+ 0x0100ff000000ffff, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000,
+ 0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000,
+ 0x0100ff0001ff00ff, 0x0100ff0001ff0001, 0x0100ff000100ff01, 0x0100ff0001000000,
+ 0x0100ff00010001ff, 0x0100ff000101ff00, 0x0100ff00010100ff, 0x0100ff0001010001,
+ 0x0100ff0001010100, 0x0100ff01ffff0000, 0x0100ff01ff00ff00, 0x0100ff01ff0000ff,
+ 0x0100ff01ff000100, 0x0100ff01ff010000, 0x0100ff0100ff00ff, 0x0100ff0100ff0001,
+ 0x0100ff0100ff0100, 0x0100ff010000ffff, 0x0100ff010000ff01, 0x0100ff0100000000,
+ 0x0100ff01000001ff, 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101ff0000,
+ 0x0100ff01010000ff, 0x0100ff0101000001, 0x0100ff0101010100, 0x010000ffffffff00,
+ 0x010000ffffff00ff, 0x010000ffffff0001, 0x010000ffff00ffff, 0x010000ffff000000,
+ 0x010000ffff0001ff, 0x010000ffff010001, 0x010000ff00ffffff, 0x010000ff00ff0101,
+ 0x010000ff0000ff00, 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001,
+ 0x010000ff000001ff, 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff0001ff00,
+ 0x010000ff0001ff01, 0x010000ff00010000, 0x010000ff01ff00ff, 0x010000ff01ff0001,
+ 0x010000ff0100ff01, 0x010000ff010000ff, 0x010000ff01000000, 0x010000ff010001ff,
+ 0x010000ff0101ff00, 0x010000ff01010100, 0x01000000ffffffff, 0x01000000ffff0000,
+ 0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ffff, 0x01000000ff00ff00,
+ 0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100,
+ 0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010100, 0x01000000ff010101,
+ 0x0100000000ffff00, 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001,
+ 0x0100000000ff0100, 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01,
+ 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff,
+ 0x0100000000000100, 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff,
+ 0x0100000000010000, 0x0100000000010001, 0x0100000000010100, 0x0100000001ffff00,
+ 0x0100000001ff0000, 0x0100000001ff01ff, 0x010000000100ff00, 0x010000000100ff01,
+ 0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, 0x0100000001000100,
+ 0x0100000001000101, 0x010000000101ffff, 0x010000000101ff01, 0x0100000001010000,
+ 0x01000000010101ff, 0x0100000001010101, 0x01000001ffffff00, 0x01000001ffff00ff,
+ 0x01000001ff00ffff, 0x01000001ff000000, 0x01000001ff000100, 0x01000001ff01ffff,
+ 0x01000001ff010001, 0x01000001ff010100, 0x0100000100ff0000, 0x0100000100ff01ff,
+ 0x0100000100ff0100, 0x010000010000ff00, 0x010000010000ff01, 0x0100000100000000,
+ 0x0100000100000001, 0x0100000100000100, 0x0100000100010000, 0x01000001000101ff,
+ 0x0100000101ffff01, 0x0100000101ff00ff, 0x0100000101ff0100, 0x0100000101ff0101,
+ 0x010000010100ff01, 0x01000001010000ff, 0x0100000101000000, 0x01000001010100ff,
+ 0x0100000101010001, 0x0100000101010100, 0x010001ffffff0000, 0x010001ffff000001,
+ 0x010001ffff000100, 0x010001ffff010000, 0x010001ff00ffff00, 0x010001ff00ff0001,
+ 0x010001ff0000ffff, 0x010001ff0000ff01, 0x010001ff00000000, 0x010001ff00000001,
+ 0x010001ff00000101, 0x010001ff000100ff, 0x010001ff00010000, 0x010001ff01ff0000,
+ 0x010001ff0100ff00, 0x010001ff01000001, 0x010001ff01000100, 0x010001ff01010000,
+ 0x01000100ffff00ff, 0x01000100ffff0001, 0x01000100ffff0100, 0x01000100ff00ffff,
+ 0x01000100ff00ff01, 0x01000100ff000000, 0x01000100ff0001ff, 0x01000100ff000101,
+ 0x01000100ff01ffff, 0x01000100ff01ff00, 0x01000100ff0100ff, 0x01000100ff010001,
+ 0x0100010000ffffff, 0x0100010000ffff01, 0x0100010000ff0000, 0x0100010000ff01ff,
+ 0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000,
+ 0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000,
+ 0x0100010000010001, 0x0100010000010101, 0x0100010001ffff00, 0x0100010001ff00ff,
+ 0x010001000100ffff, 0x010001000100ff01, 0x0100010001000000, 0x0100010001000101,
+ 0x010001000101ff00, 0x0100010001010001, 0x01000101ffff0000, 0x01000101ff000000,
+ 0x01000101ff010000, 0x0100010100ff00ff, 0x0100010100ff0001, 0x0100010100ff0100,
+ 0x010001010000ffff, 0x0100010100000000, 0x01000101000001ff, 0x010001010001ff00,
+ 0x0100010101ff0000, 0x010001010100ff00, 0x01000101010000ff, 0x0100010101000000,
+ 0x0100010101000001, 0x0101ffffffffffff, 0x0101ffffffffff01, 0x0101ffffffff01ff,
+ 0x0101ffffffff0101, 0x0101ffffff000000, 0x0101ffffff01ffff, 0x0101ffffff01ff01,
+ 0x0101ffffff0101ff, 0x0101ffffff010101, 0x0101ffff00ff0000, 0x0101ffff0000ff00,
+ 0x0101ffff000000ff, 0x0101ffff00000001, 0x0101ffff00000100, 0x0101ffff01ffffff,
+ 0x0101ffff01ffff01, 0x0101ffff01ff01ff, 0x0101ffff01ff0101, 0x0101ffff01000000,
+ 0x0101ffff0101ffff, 0x0101ffff0101ff01, 0x0101ffff010101ff, 0x0101ffff01010101,
+ 0x0101ff00ffff0000, 0x0101ff00ffff0100, 0x0101ff00ff00ff00, 0x0101ff00ff0000ff,
+ 0x0101ff00ff000001, 0x0101ff00ff000100, 0x0101ff00ff000101, 0x0101ff0000ff0001,
+ 0x0101ff0000ff0100, 0x0101ff000000ff00, 0x0101ff0000000000, 0x0101ff00000001ff,
+ 0x0101ff0000000101, 0x0101ff000001ff00, 0x0101ff00000100ff, 0x0101ff0001ff0000,
+ 0x0101ff000100ffff, 0x0101ff000100ff01, 0x0101ff0001000001, 0x0101ff0001000100,
+ 0x0101ff01ffffff01, 0x0101ff01ffff01ff, 0x0101ff01ffff0101, 0x0101ff01ff00ffff,
+ 0x0101ff01ff000100, 0x0101ff01ff01ff01, 0x0101ff01ff0101ff, 0x0101ff01ff010101,
+ 0x0101ff0100ff0000, 0x0101ff010000ff00, 0x0101ff0100000001, 0x0101ff0100000100,
+ 0x0101ff0100010000, 0x0101ff0101ffffff, 0x0101ff0101ffff01, 0x0101ff0101ff01ff,
+ 0x0101ff0101ff0101, 0x0101ff0101000000, 0x0101ff010101ffff, 0x0101ff010101ff01,
+ 0x0101ff01010101ff, 0x0101ff0101010101, 0x010100ffff000100, 0x010100ffff010000,
+ 0x010100ff00ffff00, 0x010100ff00ff00ff, 0x010100ff0000ffff, 0x010100ff000000ff,
+ 0x010100ff00000000, 0x010100ff000001ff, 0x010100ff00000101, 0x010100ff0001ff00,
+ 0x010100ff00010000, 0x010100ff00010001, 0x010100ff000101ff, 0x010100ff00010100,
+ 0x010100ff01ff0000, 0x01010000ffff0001, 0x01010000ffff0100, 0x01010000ff00ffff,
+ 0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, 0x01010000ff010001,
+ 0x01010000ff010100, 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00,
+ 0x01010000000000ff, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100,
+ 0x0101000000010000, 0x0101000000010101, 0x0101000001ffff00, 0x0101000001ff00ff,
+ 0x0101000001ff0000, 0x0101000001ff0001, 0x0101000001ff0100, 0x010100000100ff01,
+ 0x0101000001000000, 0x01010000010001ff, 0x01010001ffff0000, 0x01010001ff00ff00,
+ 0x01010001ff000001, 0x01010001ff000101, 0x01010001ff01ff00, 0x01010001ff010000,
+ 0x0101000100ff00ff, 0x0101000100ff0001, 0x0101000100ff0101, 0x010100010000ff01,
+ 0x0101000100000000, 0x0101000100000001, 0x01010001000001ff, 0x010100010001ffff,
+ 0x010100010001ff01, 0x0101000101ff0001, 0x010100010100ffff, 0x0101000101000000,
+ 0x0101000101000001, 0x0101000101000100, 0x010100010101ff00, 0x01010001010100ff,
+ 0x0101000101010001, 0x010101ffffffffff, 0x010101ffffffff01, 0x010101ffffff01ff,
+ 0x010101ffffff0101, 0x010101ffff01ffff, 0x010101ffff01ff01, 0x010101ffff0101ff,
+ 0x010101ffff010101, 0x010101ff0000ff00, 0x010101ff000000ff, 0x010101ff00000001,
+ 0x010101ff00000100, 0x010101ff01ffffff, 0x010101ff01ffff01, 0x010101ff01ff01ff,
+ 0x010101ff01ff0101, 0x010101ff01000000, 0x010101ff0101ffff, 0x010101ff0101ff01,
+ 0x010101ff010101ff, 0x010101ff01010101, 0x01010100ffff0000, 0x01010100ff0000ff,
+ 0x01010100ff000100, 0x01010100ff01ff00, 0x01010100ff010000, 0x0101010000ffff00,
+ 0x010101000000ffff, 0x0101010000000000, 0x0101010000000101, 0x010101000001ff00,
+ 0x0101010000010001, 0x0101010000010100, 0x010101000100ffff, 0x0101010001000001,
+ 0x01010101ffffffff, 0x01010101ffffff01, 0x01010101ffff01ff, 0x01010101ffff0101,
+ 0x01010101ff01ffff, 0x01010101ff01ff01, 0x01010101ff0101ff, 0x01010101ff010101,
+ 0x010101010000ff00, 0x01010101000000ff, 0x0101010100000001, 0x0101010101ffffff,
+ 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000,
+ 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101,
+GGML_TABLE_END()
+#else
+GGML_TABLE_BEGIN(uint32_t, iq1s_grid_gpu, NGRID_IQ1S)
+ 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000,
+ 0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101,
+ 0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200,
+ 0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212,
+ 0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011,
+ 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111,
+ 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220,
+ 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022,
+ 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,
+ 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101,
+ 0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110,
+ 0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111,
+ 0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010,
+ 0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210,
+ 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221,
+ 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021,
+ 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002,
+ 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,
+ 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101,
+ 0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211,
+ 0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110,
+ 0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022,
+ 0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121,
+ 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220,
+ 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001,
+ 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101,
+ 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,
+ 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012,
+ 0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010,
+ 0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111,
+ 0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122,
+ 0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222,
+ 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001,
+ 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102,
+ 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101,
+ 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,
+ 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101,
+ 0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112,
+ 0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110,
+ 0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211,
+ 0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012,
+ 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111,
+ 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120,
+ 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122,
+ 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,
+ 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221,
+ 0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001,
+ 0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101,
+ 0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101,
+ 0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011,
+ 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111,
+ 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011,
+ 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122,
+ 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,
+ 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222,
+ 0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101,
+ 0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000,
+ 0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200,
+ 0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110,
+ 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112,
+ 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222,
+ 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021,
+ 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,
+ 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201,
+ 0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200,
+ 0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101,
+ 0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011,
+ 0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010,
+ 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211,
+ 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121,
+ 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000,
+ 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,
+ 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202,
+ 0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211,
+ 0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112,
+ 0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020,
+ 0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121,
+ 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222,
+ 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102,
+ 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100,
+ 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,
+ 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011,
+ 0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111,
+ 0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110,
+ 0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121,
+ 0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222,
+ 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201,
+ 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102,
+ 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201,
+ 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,
+ 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010,
+ 0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010,
+ 0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110,
+ 0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011,
+ 0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212,
+ 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021,
+ 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021,
+ 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021,
+ 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,
+ 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101,
+ 0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100,
+ 0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010,
+ 0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111,
+ 0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010,
+ 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111,
+ 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120,
+ 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120,
+ 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,
+ 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001,
+ 0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201,
+ 0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210,
+ 0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211,
+ 0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111,
+ 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112,
+ 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211,
+ 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010,
+ 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,
+ 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122,
+ 0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221,
+ 0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102,
+ 0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100,
+ 0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101,
+ 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101,
+ 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101,
+ 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012,
+ 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,
+ 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112,
+ 0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210,
+ 0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210,
+ 0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210,
+ 0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010,
+ 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110,
+ 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122,
+ 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020,
+ 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,
+ 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022,
+ 0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120,
+ 0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222,
+ 0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221,
+ 0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001,
+ 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102,
+ 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201,
+ 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012,
+ 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,
+ 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012,
+ 0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110,
+ 0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110,
+ 0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121,
+ 0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221,
+ 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220,
+ 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222,
+ 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000,
+ 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,
+ 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012,
+ 0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011,
+ 0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212,
+ 0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221,
+ 0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121,
+ 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202,
+ 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202,
+ 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002,
+ 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,
+ 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210,
+ 0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112,
+ 0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011,
+ 0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011,
+ 0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210,
+ 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020,
+ 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220,
+ 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222,
+ 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,
+ 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001,
+ 0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010,
+ 0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111,
+ 0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010,
+ 0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110,
+ 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221,
+ 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122,
+ 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202,
+ 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,
+ 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101,
+ 0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112,
+ 0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111,
+ 0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211,
+ 0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222,
+ 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221,
+ 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022,
+ 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101,
+ 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,
+ 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111,
+ 0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111,
+ 0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010,
+ 0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121,
+ 0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222,
+ 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000,
+ 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202,
+ 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000,
+ 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,
+ 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110,
+ 0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110,
+ 0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222,
+ 0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120,
+ 0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022,
+ 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101,
+ 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202,
+ 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110,
+ 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,
+ 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111,
+ 0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111,
+ 0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120,
+ 0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121,
+ 0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001,
+ 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202,
+ 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001,
+ 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200,
+ 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,
+ 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212,
+ 0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012,
+ 0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110,
+ 0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012,
+ 0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111,
+ 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020,
+ 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121,
+ 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222,
+ 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,
+ 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102,
+ 0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101,
+ 0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212,
+ 0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210,
+ 0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111,
+ 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212,
+ 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221,
+ 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121,
+ 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,
+ 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000,
+ 0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202,
+ 0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112,
+ 0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111,
+ 0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020,
+ 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221,
+ 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022,
+ 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100,
+ 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,
+ 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112,
+ 0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211,
+ 0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012,
+ 0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121,
+ 0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020,
+ 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120,
+ 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200,
+ 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200,
+ 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,
+ 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011,
+ 0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222,
+ 0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020,
+ 0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222,
+GGML_TABLE_END()
+#endif
+
+#endif // GGML_COMMON_IMPL
+#endif // GGML_COMMON_IMPL
diff --git a/llama/ggml-cuda.cu b/llama/ggml-cuda.cu
new file mode 100644
index 00000000..b75de19a
--- /dev/null
+++ b/llama/ggml-cuda.cu
@@ -0,0 +1,3147 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "ggml-cuda.h"
+#include "ggml.h"
+#include "ggml-backend-impl.h"
+
+#include "ggml-cuda/common.cuh"
+#include "ggml-cuda/acc.cuh"
+#include "ggml-cuda/arange.cuh"
+#include "ggml-cuda/argsort.cuh"
+#include "ggml-cuda/binbcast.cuh"
+#include "ggml-cuda/clamp.cuh"
+#include "ggml-cuda/concat.cuh"
+#include "ggml-cuda/conv-transpose-1d.cuh"
+#include "ggml-cuda/convert.cuh"
+#include "ggml-cuda/cpy.cuh"
+#include "ggml-cuda/cross-entropy-loss.cuh"
+#include "ggml-cuda/diagmask.cuh"
+#include "ggml-cuda/dmmv.cuh"
+#include "ggml-cuda/fattn.cuh"
+#include "ggml-cuda/getrows.cuh"
+#include "ggml-cuda/im2col.cuh"
+#include "ggml-cuda/mmq.cuh"
+#include "ggml-cuda/mmvq.cuh"
+#include "ggml-cuda/norm.cuh"
+#include "ggml-cuda/pad.cuh"
+#include "ggml-cuda/pool2d.cuh"
+#include "ggml-cuda/quantize.cuh"
+#include "ggml-cuda/rope.cuh"
+#include "ggml-cuda/scale.cuh"
+#include "ggml-cuda/softmax.cuh"
+#include "ggml-cuda/sumrows.cuh"
+#include "ggml-cuda/tsembd.cuh"
+#include "ggml-cuda/unary.cuh"
+#include "ggml-cuda/upscale.cuh"
+
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <cinttypes>
+#include <cstddef>
+#include <cstdint>
+#include <float.h>
+#include <limits>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string>
+#include <vector>
+
+static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");
+
+static void ggml_cuda_default_log_callback(enum ggml_log_level level, const char * msg, void * user_data) {
+ GGML_UNUSED(level);
+ GGML_UNUSED(user_data);
+ fprintf(stderr, "%s", msg);
+}
+
+ggml_log_callback ggml_cuda_log_callback = ggml_cuda_default_log_callback;
+void * ggml_cuda_log_user_data = NULL;
+
+GGML_API void ggml_backend_cuda_log_set_callback(ggml_log_callback log_callback, void * user_data) {
+ ggml_cuda_log_callback = log_callback;
+ ggml_cuda_log_user_data = user_data;
+}
+
+#define GGML_CUDA_LOG_INFO(...) ggml_cuda_log(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
+#define GGML_CUDA_LOG_WARN(...) ggml_cuda_log(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
+#define GGML_CUDA_LOG_ERROR(...) ggml_cuda_log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
+
+GGML_ATTRIBUTE_FORMAT(2, 3)
+static void ggml_cuda_log(enum ggml_log_level level, const char * format, ...) {
+ if (ggml_cuda_log_callback != NULL) {
+ va_list args;
+ va_start(args, format);
+ char buffer[128];
+ int len = vsnprintf(buffer, 128, format, args);
+ if (len < 128) {
+ ggml_cuda_log_callback(level, buffer, ggml_cuda_log_user_data);
+ } else {
+ std::vector<char> buffer2(len + 1); // vsnprintf adds a null terminator
+ va_end(args);
+ va_start(args, format);
+ vsnprintf(&buffer2[0], buffer2.size(), format, args);
+ ggml_cuda_log_callback(level, buffer2.data(), ggml_cuda_log_user_data);
+ }
+ va_end(args);
+ }
+}
+
+[[noreturn]]
+void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) {
+ int id = -1; // in case cudaGetDevice fails
+ cudaGetDevice(&id);
+
+ GGML_CUDA_LOG_ERROR("CUDA error: %s\n", msg);
+ GGML_CUDA_LOG_ERROR(" current device: %d, in function %s at %s:%d\n", id, func, file, line);
+ GGML_CUDA_LOG_ERROR(" %s\n", stmt);
+ // abort with GGML_ASSERT to get a stack trace
+ GGML_ABORT("CUDA error");
+}
+
+// this is faster on Windows
+// probably because the Windows CUDA libraries forget to make this check before invoking the drivers
+void ggml_cuda_set_device(int device) {
+ int current_device;
+ CUDA_CHECK(cudaGetDevice(¤t_device));
+
+ if (device == current_device) {
+ return;
+ }
+
+ CUDA_CHECK(cudaSetDevice(device));
+}
+
+int ggml_cuda_get_device() {
+ int id;
+ CUDA_CHECK(cudaGetDevice(&id));
+ return id;
+}
+
+static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device) {
+ ggml_cuda_set_device(device);
+#if defined(GGML_USE_HIPBLAS) && defined(GGML_HIP_UMA)
+ auto res = hipMallocManaged(ptr, size);
+ if (res == hipSuccess) {
+ // if error we "need" to know why...
+ CUDA_CHECK(hipMemAdvise(*ptr, size, hipMemAdviseSetCoarseGrain, device));
+ }
+ return res;
+#else
+
+#if !defined(GGML_USE_HIPBLAS) && !defined(GGML_USE_MUSA)
+ cudaError_t err;
+ if (getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr)
+ {
+ err = cudaMallocManaged(ptr, size);
+ }
+ else
+ {
+ err = cudaMalloc(ptr, size);
+ }
+ return err;
+#else
+ return cudaMalloc(ptr, size);
+#endif // !defined(GGML_USE_HIPBLAS) && !defined(GGML_USE_MUSA)
+
+#endif
+}
+
+static ggml_cuda_device_info ggml_cuda_init() {
+#ifdef __HIP_PLATFORM_AMD__
+ // Workaround for a rocBLAS bug when using multiple graphics cards:
+ // https://github.com/ROCmSoftwarePlatform/rocBLAS/issues/1346
+ rocblas_initialize();
+ CUDA_CHECK(cudaDeviceSynchronize());
+#endif
+
+ ggml_cuda_device_info info = {};
+
+ cudaError_t err = cudaGetDeviceCount(&info.device_count);
+ if (err != cudaSuccess) {
+ GGML_CUDA_LOG_ERROR("%s: failed to initialize " GGML_CUDA_NAME ": %s\n", __func__, cudaGetErrorString(err));
+ return info;
+ }
+
+ GGML_ASSERT(info.device_count <= GGML_CUDA_MAX_DEVICES);
+
+ int64_t total_vram = 0;
+#ifdef GGML_CUDA_FORCE_MMQ
+ GGML_CUDA_LOG_INFO("%s: GGML_CUDA_FORCE_MMQ: yes\n", __func__);
+#else
+ GGML_CUDA_LOG_INFO("%s: GGML_CUDA_FORCE_MMQ: no\n", __func__);
+#endif // GGML_CUDA_FORCE_MMQ
+#ifdef GGML_CUDA_FORCE_CUBLAS
+ GGML_CUDA_LOG_INFO("%s: GGML_CUDA_FORCE_CUBLAS: yes\n", __func__);
+#else
+ GGML_CUDA_LOG_INFO("%s: GGML_CUDA_FORCE_CUBLAS: no\n", __func__);
+#endif // GGML_CUDA_FORCE_CUBLAS
+ GGML_CUDA_LOG_INFO("%s: found %d " GGML_CUDA_NAME " devices:\n", __func__, info.device_count);
+ for (int id = 0; id < info.device_count; ++id) {
+ int device_vmm = 0;
+
+#if !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM) && !defined(GGML_USE_MUSA)
+ CUdevice device;
+ CU_CHECK(cuDeviceGet(&device, id));
+ CU_CHECK(cuDeviceGetAttribute(&device_vmm, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, device));
+
+ if (device_vmm) {
+ CUmemAllocationProp alloc_prop = {};
+ alloc_prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
+ alloc_prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
+ alloc_prop.location.id = id;
+ CU_CHECK(cuMemGetAllocationGranularity(&info.devices[id].vmm_granularity, &alloc_prop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED));
+ }
+#endif // !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM) && !defined(GGML_USE_MUSA)
+ info.devices[id].vmm = !!device_vmm;
+
+ cudaDeviceProp prop;
+ CUDA_CHECK(cudaGetDeviceProperties(&prop, id));
+ GGML_CUDA_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no");
+
+ info.default_tensor_split[id] = total_vram;
+ total_vram += prop.totalGlobalMem;
+
+ info.devices[id].nsm = prop.multiProcessorCount;
+ info.devices[id].smpb = prop.sharedMemPerBlock;
+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+ info.devices[id].smpbo = prop.sharedMemPerBlock;
+ info.devices[id].cc = 100*prop.major + 10*prop.minor + CC_OFFSET_AMD;
+#else
+ info.devices[id].smpbo = prop.sharedMemPerBlockOptin;
+ info.devices[id].cc = 100*prop.major + 10*prop.minor;
+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+ }
+
+ for (int id = 0; id < info.device_count; ++id) {
+ info.default_tensor_split[id] /= total_vram;
+ }
+
+ // configure logging to stdout
+ // CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr));
+
+ return info;
+}
+
+const ggml_cuda_device_info & ggml_cuda_info() {
+ static ggml_cuda_device_info info = ggml_cuda_init();
+ return info;
+}
+
+// #define DEBUG_CUDA_MALLOC
+
+// buffer pool for cuda (legacy)
+struct ggml_cuda_pool_leg : public ggml_cuda_pool {
+ static const int MAX_BUFFERS = 256;
+
+ int device;
+ struct ggml_cuda_buffer {
+ void * ptr = nullptr;
+ size_t size = 0;
+ };
+
+ ggml_cuda_buffer buffer_pool[MAX_BUFFERS] = {};
+ size_t pool_size = 0;
+
+ explicit ggml_cuda_pool_leg(int device) :
+ device(device) {
+ }
+
+ ~ggml_cuda_pool_leg() {
+ ggml_cuda_set_device(device);
+ for (int i = 0; i < MAX_BUFFERS; ++i) {
+ ggml_cuda_buffer & b = buffer_pool[i];
+ if (b.ptr != nullptr) {
+ CUDA_CHECK(cudaFree(b.ptr));
+ pool_size -= b.size;
+ }
+ }
+ GGML_ASSERT(pool_size == 0);
+ }
+
+ void * alloc(size_t size, size_t * actual_size) override {
+#ifdef DEBUG_CUDA_MALLOC
+ int nnz = 0;
+ size_t max_size = 0;
+#endif
+ size_t best_diff = 1ull << 36;
+ int ibest = -1;
+ for (int i = 0; i < MAX_BUFFERS; ++i) {
+ ggml_cuda_buffer& b = buffer_pool[i];
+ if (b.ptr != nullptr) {
+#ifdef DEBUG_CUDA_MALLOC
+ ++nnz;
+ if (b.size > max_size) max_size = b.size;
+#endif
+ if (b.size >= size) {
+ size_t diff = b.size - size;
+ if (diff < best_diff) {
+ best_diff = diff;
+ ibest = i;
+ if (!best_diff) {
+ void * ptr = b.ptr;
+ *actual_size = b.size;
+ b.ptr = nullptr;
+ b.size = 0;
+ return ptr;
+ }
+ }
+ }
+ }
+ }
+ if (ibest >= 0) {
+ ggml_cuda_buffer& b = buffer_pool[ibest];
+ void * ptr = b.ptr;
+ *actual_size = b.size;
+ b.ptr = nullptr;
+ b.size = 0;
+ return ptr;
+ }
+ void * ptr;
+ size_t look_ahead_size = (size_t) (1.05 * size);
+ look_ahead_size = 256 * ((look_ahead_size + 255)/256);
+ ggml_cuda_set_device(device);
+ CUDA_CHECK(ggml_cuda_device_malloc(&ptr, look_ahead_size, device));
+ *actual_size = look_ahead_size;
+ pool_size += look_ahead_size;
+#ifdef DEBUG_CUDA_MALLOC
+ GGML_CUDA_LOG_INFO("%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, device, nnz,
+ (uint32_t)(max_size / 1024 / 1024), (uint32_t)(pool_size / 1024 / 1024), (uint32_t)(size / 1024 / 1024));
+#endif
+ return ptr;
+ }
+
+ void free(void * ptr, size_t size) override {
+ for (int i = 0; i < MAX_BUFFERS; ++i) {
+ ggml_cuda_buffer& b = buffer_pool[i];
+ if (b.ptr == nullptr) {
+ b.ptr = ptr;
+ b.size = size;
+ return;
+ }
+ }
+ GGML_CUDA_LOG_WARN("Cuda buffer pool full, increase MAX_CUDA_BUFFERS\n");
+ ggml_cuda_set_device(device);
+ CUDA_CHECK(cudaFree(ptr));
+ pool_size -= size;
+ }
+};
+
+// pool with virtual memory
+#if !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM) && !defined(GGML_USE_MUSA)
+struct ggml_cuda_pool_vmm : public ggml_cuda_pool {
+ static const size_t CUDA_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB
+
+ int device;
+ CUdeviceptr pool_addr = 0;
+ size_t pool_used = 0;
+ size_t pool_size = 0;
+ size_t granularity;
+
+ explicit ggml_cuda_pool_vmm(int device) :
+ device(device),
+ granularity(ggml_cuda_info().devices[device].vmm_granularity) {
+ }
+
+ ~ggml_cuda_pool_vmm() {
+ if (pool_addr != 0) {
+ CU_CHECK(cuMemUnmap(pool_addr, pool_size));
+ CU_CHECK(cuMemAddressFree(pool_addr, CUDA_POOL_VMM_MAX_SIZE));
+ }
+ }
+
+ void * alloc(size_t size, size_t * actual_size) override {
+ // round up the allocation size to the alignment to ensure that all allocations are aligned for all data types
+ const size_t alignment = 128;
+ size = alignment * ((size + alignment - 1) / alignment);
+
+ size_t avail = pool_size - pool_used;
+
+ if (size > avail) {
+ // round up to the next multiple of the granularity
+ size_t reserve_size = size - avail;
+ reserve_size = granularity * ((reserve_size + granularity - 1) / granularity);
+
+ GGML_ASSERT(pool_size + reserve_size <= CUDA_POOL_VMM_MAX_SIZE);
+
+ // allocate more physical memory
+ CUmemAllocationProp prop = {};
+ prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
+ prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
+ prop.location.id = device;
+ CUmemGenericAllocationHandle handle;
+ CU_CHECK(cuMemCreate(&handle, reserve_size, &prop, 0));
+
+ // reserve virtual address space (if not already reserved)
+ if (pool_addr == 0) {
+ CU_CHECK(cuMemAddressReserve(&pool_addr, CUDA_POOL_VMM_MAX_SIZE, 0, 0, 0));
+ }
+
+ // map at the end of the pool
+ CU_CHECK(cuMemMap(pool_addr + pool_size, reserve_size, 0, handle, 0));
+
+ // the memory allocation handle is no longer needed after mapping
+ CU_CHECK(cuMemRelease(handle));
+
+ // set access
+ CUmemAccessDesc access = {};
+ access.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
+ access.location.id = device;
+ access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
+ CU_CHECK(cuMemSetAccess(pool_addr + pool_size, reserve_size, &access, 1));
+
+ // add to the pool
+ pool_size += reserve_size;
+
+ //printf("cuda pool[%d]: size increased to %llu MB (reserved %llu MB)\n",
+ // device, (unsigned long long) (pool_size/1024/1024),
+ // (unsigned long long) (reserve_size/1024/1024));
+ }
+
+ GGML_ASSERT(pool_addr != 0);
+
+ void * ptr = (void *) (pool_addr + pool_used);
+ *actual_size = size;
+ pool_used += size;
+
+#ifdef DEBUG_CUDA_MALLOC
+ printf("cuda pool[%d]: allocated %llu bytes at %llx\n", device, (unsigned long long) size, ptr);
+#endif
+
+ return ptr;
+ }
+
+ void free(void * ptr, size_t size) override {
+#ifdef DEBUG_CUDA_MALLOC
+ printf("cuda pool[%d]: freed %llu bytes at %llx\n", device, (unsigned long long) size, ptr);
+#endif
+
+ pool_used -= size;
+
+ // all deallocations must be in reverse order of the allocations
+ GGML_ASSERT(ptr == (void *) (pool_addr + pool_used));
+ }
+};
+#endif // !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM) && !defined(GGML_USE_MUSA)
+
+std::unique_ptr<ggml_cuda_pool> ggml_backend_cuda_context::new_pool_for_device(int device) {
+#if !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM) && !defined(GGML_USE_MUSA)
+ if (ggml_cuda_info().devices[device].vmm) {
+ return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_vmm(device));
+ }
+#endif // !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM) && !defined(GGML_USE_MUSA)
+ return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_leg(device));
+}
+
+// cuda buffer
+
+struct ggml_backend_cuda_buffer_context {
+ int device;
+ void * dev_ptr = nullptr;
+ std::string name;
+
+ ggml_backend_cuda_buffer_context(int device, void * dev_ptr) :
+ device(device), dev_ptr(dev_ptr),
+ name(GGML_CUDA_NAME + std::to_string(device)) {
+ }
+
+ ~ggml_backend_cuda_buffer_context() {
+ CUDA_CHECK(cudaFree(dev_ptr));
+ }
+};
+
+GGML_CALL static const char * ggml_backend_cuda_buffer_get_name(ggml_backend_buffer_t buffer) {
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+ return ctx->name.c_str();
+}
+
+GGML_CALL static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {
+ return buffer->iface.get_name == ggml_backend_cuda_buffer_get_name;
+}
+
+GGML_CALL static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+ delete ctx;
+
+ // TODO: this needs to be freed in cuda and hipblas backends because
+ // the cuda backend implementation compiled with msvc
+ free(buffer);
+}
+
+GGML_CALL static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) {
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+ return ctx->dev_ptr;
+}
+
+GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+
+ if (tensor->view_src != NULL) {
+ assert(tensor->view_src->buffer->buft == buffer->buft);
+ return;
+ }
+
+ if (ggml_is_quantized(tensor->type) && tensor->view_src == nullptr && ggml_backend_buffer_get_usage(buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
+ // initialize padding to 0 to avoid possible NaN values
+ size_t original_size = ggml_nbytes(tensor);
+ size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor);
+
+ if (padded_size > original_size) {
+ ggml_cuda_set_device(ctx->device);
+ CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size));
+ }
+ }
+}
+
+GGML_CALL static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+
+ ggml_cuda_set_device(ctx->device);
+ CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cudaStreamPerThread));
+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
+}
+
+GGML_CALL static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+
+ ggml_cuda_set_device(ctx->device);
+ CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cudaStreamPerThread));
+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
+}
+
+GGML_CALL static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
+ if (ggml_backend_buffer_is_cuda(src->buffer)) {
+ ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;
+ ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context;
+ if (src_ctx->device == dst_ctx->device) {
+ CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice, cudaStreamPerThread));
+ } else {
+#ifdef GGML_CUDA_NO_PEER_COPY
+ return false;
+#else
+ CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, dst_ctx->device, src->data, src_ctx->device, ggml_nbytes(src), cudaStreamPerThread));
+#endif
+ }
+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
+ return true;
+ }
+ return false;
+
+ GGML_UNUSED(buffer);
+}
+
+GGML_CALL static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+
+ ggml_cuda_set_device(ctx->device);
+ CUDA_CHECK(cudaDeviceSynchronize());
+ CUDA_CHECK(cudaMemset(ctx->dev_ptr, value, buffer->size));
+ CUDA_CHECK(cudaDeviceSynchronize());
+}
+
+static ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
+ /* .get_name = */ ggml_backend_cuda_buffer_get_name,
+ /* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,
+ /* .get_base = */ ggml_backend_cuda_buffer_get_base,
+ /* .init_tensor = */ ggml_backend_cuda_buffer_init_tensor,
+ /* .set_tensor = */ ggml_backend_cuda_buffer_set_tensor,
+ /* .get_tensor = */ ggml_backend_cuda_buffer_get_tensor,
+ /* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,
+ /* .clear = */ ggml_backend_cuda_buffer_clear,
+ /* .reset = */ NULL,
+};
+
+// cuda buffer type
+struct ggml_backend_cuda_buffer_type_context {
+ int device;
+ std::string name;
+};
+
+GGML_CALL static const char * ggml_backend_cuda_buffer_type_name(ggml_backend_buffer_type_t buft) {
+ ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
+
+ return ctx->name.c_str();
+}
+
+static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {
+ return buft->iface.get_name == ggml_backend_cuda_buffer_type_name;
+}
+
+GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+ ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
+
+ ggml_cuda_set_device(buft_ctx->device);
+
+ size = std::max(size, (size_t)1); // cudaMalloc returns null for size 0
+
+ void * dev_ptr;
+ cudaError_t err = ggml_cuda_device_malloc(&dev_ptr, size, buft_ctx->device);
+ if (err != cudaSuccess) {
+ // clear the error
+ cudaGetLastError();
+ GGML_CUDA_LOG_ERROR("%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size / 1024.0 / 1024.0, buft_ctx->device, cudaGetErrorString(err));
+ return nullptr;
+ }
+
+ ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr);
+
+ return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);
+}
+
+GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
+ return 128;
+
+ GGML_UNUSED(buft);
+}
+
+GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
+ size_t size = ggml_nbytes(tensor);
+ int64_t ne0 = tensor->ne[0];
+
+ if (ggml_is_quantized(tensor->type)) {
+ if (ne0 % MATRIX_ROW_PADDING != 0) {
+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);
+ }
+ }
+
+ return size;
+
+ GGML_UNUSED(buft);
+}
+
+static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = {
+ /* .get_name = */ ggml_backend_cuda_buffer_type_name,
+ /* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer,
+ /* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment,
+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX
+ /* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size,
+ /* .is_host = */ NULL,
+};
+
+GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {
+ static std::mutex mutex;
+ std::lock_guard<std::mutex> lock(mutex);
+
+ if (device >= ggml_backend_cuda_get_device_count()) {
+ return nullptr;
+ }
+
+ static ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES];
+
+ static bool ggml_backend_cuda_buffer_type_initialized = false;
+
+ if (!ggml_backend_cuda_buffer_type_initialized) {
+ for (int i = 0; i < GGML_CUDA_MAX_DEVICES; i++) {
+ ggml_backend_cuda_buffer_types[i] = {
+ /* .iface = */ ggml_backend_cuda_buffer_type_interface,
+ /* .context = */ new ggml_backend_cuda_buffer_type_context{i, GGML_CUDA_NAME + std::to_string(i)},
+ };
+ }
+ ggml_backend_cuda_buffer_type_initialized = true;
+ }
+
+ return &ggml_backend_cuda_buffer_types[device];
+}
+
+// cuda split buffer
+
+static int64_t get_row_rounding(const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split) {
+ int64_t row_rounding = 0;
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) {
+ continue;
+ }
+
+ const int cc = ggml_cuda_info().devices[id].cc;
+ row_rounding = std::max(row_rounding, (int64_t)get_mmq_y_host(cc));
+ }
+ return row_rounding;
+}
+
+static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split, int id) {
+ const int64_t nrows = ggml_nrows(tensor);
+ const int64_t rounding = get_row_rounding(tensor_split);
+
+ *row_low = id == 0 ? 0 : nrows*tensor_split[id];
+ *row_low -= *row_low % rounding;
+
+ if (id == ggml_backend_cuda_get_device_count() - 1) {
+ *row_high = nrows;
+ } else {
+ *row_high = nrows*tensor_split[id + 1];
+ *row_high -= *row_high % rounding;
+ }
+}
+
+static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]);
+}
+
+struct ggml_backend_cuda_split_buffer_type_context {
+ std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split;
+};
+
+struct ggml_backend_cuda_split_buffer_context {
+ ~ggml_backend_cuda_split_buffer_context() {
+ for (ggml_tensor_extra_gpu * extra : tensor_extras) {
+ for (int id = 0; id < GGML_CUDA_MAX_DEVICES; ++id) {
+ for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) {
+ if (extra->events[id][is] != nullptr) {
+ CUDA_CHECK(cudaEventDestroy(extra->events[id][is]));
+ }
+ }
+ if (extra->data_device[id] != nullptr) {
+ CUDA_CHECK(cudaFree(extra->data_device[id]));
+ }
+ }
+ delete extra;
+ }
+ }
+
+ std::vector<ggml_tensor_extra_gpu *> tensor_extras;
+};
+
+GGML_CALL static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backend_buffer_t buffer) {
+ return GGML_CUDA_NAME "_Split";
+
+ GGML_UNUSED(buffer);
+}
+
+static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) {
+ return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name;
+ GGML_UNUSED(ggml_backend_buffer_is_cuda_split); // only used in debug builds currently, avoid unused function warning in release builds
+}
+
+GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+ ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;
+ delete ctx;
+}
+
+GGML_CALL static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) {
+ // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced
+ return (void *)0x1000;
+
+ GGML_UNUSED(buffer);
+}
+
+GGML_CALL static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
+ GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported
+
+ ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;
+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;
+
+ const int64_t ne0 = tensor->ne[0];
+
+ ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};
+ ctx->tensor_extras.push_back(extra);
+
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ int64_t row_low, row_high;
+ get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);
+
+ int64_t nrows_split = row_high - row_low;
+ if (nrows_split == 0) {
+ continue;
+ }
+
+ size_t size = ggml_nbytes_split(tensor, nrows_split);
+ const size_t original_size = size;
+
+ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses
+ if (ne0 % MATRIX_ROW_PADDING != 0) {
+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);
+ }
+
+ // FIXME: do not crash if cudaMalloc fails
+ // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first
+ ggml_cuda_set_device(id);
+ char * buf;
+ CUDA_CHECK(ggml_cuda_device_malloc((void**)&buf, size, id));
+
+ // set padding to 0 to avoid possible NaN values
+ if (size > original_size) {
+ CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size));
+ }
+
+ extra->data_device[id] = buf;
+
+ for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) {
+ CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming));
+ }
+ }
+ tensor->extra = extra;
+}
+
+GGML_CALL static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+ // split tensors must always be set in their entirety at once
+ GGML_ASSERT(offset == 0);
+ GGML_ASSERT(size == ggml_nbytes(tensor));
+
+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;
+
+ const int64_t ne0 = tensor->ne[0];
+ const size_t nb1 = tensor->nb[1];
+ ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;
+
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ int64_t row_low, row_high;
+ get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);
+
+ int64_t nrows_split = row_high - row_low;
+ if (nrows_split == 0) {
+ continue;
+ }
+
+ const size_t offset_split = row_low*nb1;
+ size_t size = ggml_nbytes_split(tensor, nrows_split);
+ const size_t original_size = size;
+
+ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses
+ if (ne0 % MATRIX_ROW_PADDING != 0) {
+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);
+ }
+
+ const char * buf_host = (const char *)data + offset_split;
+ CUDA_CHECK(cudaMemcpyAsync(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice, cudaStreamPerThread));
+ }
+
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
+ }
+}
+
+GGML_CALL static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+ // split tensors must always be set in their entirety at once
+ GGML_ASSERT(offset == 0);
+ GGML_ASSERT(size == ggml_nbytes(tensor));
+
+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;
+
+ const int64_t ne0 = tensor->ne[0];
+ const size_t nb1 = tensor->nb[1];
+ ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;
+
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ int64_t row_low, row_high;
+ get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);
+
+ int64_t nrows_split = row_high - row_low;
+ if (nrows_split == 0) {
+ continue;
+ }
+
+ const size_t offset_split = row_low*nb1;
+ size_t size = ggml_nbytes_split(tensor, nrows_split);
+ const size_t original_size = size;
+
+ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses
+ if (ne0 % MATRIX_ROW_PADDING != 0) {
+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);
+ }
+
+ char * buf_host = (char *)data + offset_split;
+ CUDA_CHECK(cudaMemcpyAsync(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost, cudaStreamPerThread));
+ }
+
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
+ }
+}
+
+GGML_CALL static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+ GGML_UNUSED(buffer);
+ GGML_UNUSED(value);
+}
+
+static struct ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {
+ /* .get_name = */ ggml_backend_cuda_split_buffer_get_name,
+ /* .free_buffer = */ ggml_backend_cuda_split_buffer_free_buffer,
+ /* .get_base = */ ggml_backend_cuda_split_buffer_get_base,
+ /* .init_tensor = */ ggml_backend_cuda_split_buffer_init_tensor,
+ /* .set_tensor = */ ggml_backend_cuda_split_buffer_set_tensor,
+ /* .get_tensor = */ ggml_backend_cuda_split_buffer_get_tensor,
+ /* .cpy_tensor = */ NULL,
+ /* .clear = */ ggml_backend_cuda_split_buffer_clear,
+ /* .reset = */ NULL,
+};
+
+// cuda split buffer type
+
+GGML_CALL static const char * ggml_backend_cuda_split_buffer_type_name(ggml_backend_buffer_type_t buft) {
+ return GGML_CUDA_NAME "_Split";
+
+ GGML_UNUSED(buft);
+}
+
+static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {
+ return buft->iface.get_name == ggml_backend_cuda_split_buffer_type_name;
+}
+
+GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+ // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point
+ // instead, we allocate them for each tensor separately in init_tensor
+ // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated,
+ // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct.
+ ggml_backend_cuda_split_buffer_context * ctx = new ggml_backend_cuda_split_buffer_context();
+
+ return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size);
+}
+
+GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
+ return 128;
+
+ GGML_UNUSED(buft);
+}
+
+GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
+ ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;
+
+ size_t total_size = 0;
+
+ const int64_t ne0 = tensor->ne[0];
+
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ int64_t row_low, row_high;
+ get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, id);
+
+ int64_t nrows_split = row_high - row_low;
+ if (nrows_split == 0) {
+ continue;
+ }
+
+ total_size += ggml_nbytes_split(tensor, nrows_split);
+
+ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses
+ if (ne0 % MATRIX_ROW_PADDING != 0) {
+ total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);
+ }
+ }
+
+ return total_size;
+}
+
+GGML_CALL static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
+ return false;
+
+ GGML_UNUSED(buft);
+}
+
+static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface = {
+ /* .get_name = */ ggml_backend_cuda_split_buffer_type_name,
+ /* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer,
+ /* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment,
+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX
+ /* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size,
+ /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host,
+};
+
+GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split) {
+ static std::mutex mutex;
+ std::lock_guard<std::mutex> lock(mutex);
+
+ static std::map<std::array<float, GGML_CUDA_MAX_DEVICES>, struct ggml_backend_buffer_type> buft_map;
+
+ std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split_arr = {};
+
+ bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; });
+ if (all_zero) {
+ tensor_split_arr = ggml_cuda_info().default_tensor_split;
+ } else {
+ float split_sum = 0.0f;
+ for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) {
+ tensor_split_arr[i] = split_sum;
+ split_sum += tensor_split[i];
+ }
+ for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) {
+ tensor_split_arr[i] /= split_sum;
+ }
+ }
+
+ auto it = buft_map.find(tensor_split_arr);
+ if (it != buft_map.end()) {
+ return &it->second;
+ }
+
+ struct ggml_backend_buffer_type buft {
+ /* .iface = */ ggml_backend_cuda_split_buffer_type_interface,
+ /* .context = */ new ggml_backend_cuda_split_buffer_type_context{tensor_split_arr},
+ };
+
+ auto result = buft_map.emplace(tensor_split_arr, buft);
+ return &result.first->second;
+}
+
+// host buffer type
+
+GGML_CALL static const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) {
+ return GGML_CUDA_NAME "_Host";
+
+ GGML_UNUSED(buft);
+}
+
+GGML_CALL static const char * ggml_backend_cuda_host_buffer_name(ggml_backend_buffer_t buffer) {
+ return GGML_CUDA_NAME "_Host";
+
+ GGML_UNUSED(buffer);
+}
+
+GGML_CALL static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+ CUDA_CHECK(cudaFreeHost(buffer->context));
+}
+
+static void * ggml_cuda_host_malloc(size_t size) {
+ if (getenv("GGML_CUDA_NO_PINNED") != nullptr) {
+ return nullptr;
+ }
+
+ void * ptr = nullptr;
+ cudaError_t err = cudaMallocHost((void **) &ptr, size);
+ if (err != cudaSuccess) {
+ // clear the error
+ cudaGetLastError();
+ GGML_CUDA_LOG_WARN("%s: failed to allocate %.2f MiB of pinned memory: %s\n", __func__,
+ size / 1024.0 / 1024.0, cudaGetErrorString(err));
+ return nullptr;
+ }
+
+ return ptr;
+}
+
+GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+ void * ptr = ggml_cuda_host_malloc(size);
+
+ if (ptr == nullptr) {
+ // fallback to cpu buffer
+ return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);
+ }
+
+ ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
+ buffer->buft = buft;
+ buffer->iface.get_name = ggml_backend_cuda_host_buffer_name;
+ buffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer;
+
+ return buffer;
+}
+
+GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {
+ static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = {
+ /* .iface = */ {
+ /* .get_name = */ ggml_backend_cuda_host_buffer_type_name,
+ /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,
+ /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,
+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX
+ /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,
+ /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host,
+ },
+ /* .context = */ nullptr,
+ };
+
+ return &ggml_backend_cuda_buffer_type_host;
+}
+
+//static bool ggml_backend_buffer_is_cuda_host(ggml_backend_buffer_t buffer) {
+// return buffer->buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name;
+//}
+
+/// kernels
+
+typedef void (*ggml_cuda_op_mul_mat_t)(
+ ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
+ const int64_t src1_padded_row_size, cudaStream_t stream);
+
+#ifndef GGML_CUDA_PEER_MAX_BATCH_SIZE
+#define GGML_CUDA_PEER_MAX_BATCH_SIZE 128
+#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE
+
+#define MUL_MAT_SRC1_COL_STRIDE 128
+
+static __global__ void mul_mat_p021_f16_f32(
+ const void * __restrict__ vx, const float * __restrict__ y, float * __restrict__ dst,
+ const int ncols_x, const int nrows_x, const int nchannels_x, const int nchannels_y) {
+
+ const half * x = (const half *) vx;
+
+ const int row_x = blockDim.y*blockIdx.y + threadIdx.y;
+ const int channel = blockDim.z*blockIdx.z + threadIdx.z;
+ const int channel_x = channel / (nchannels_y / nchannels_x);
+
+ const int nrows_y = ncols_x;
+ const int nrows_dst = nrows_x;
+ const int row_dst = row_x;
+
+ float tmp = 0.0f;
+
+ for (int col_x0 = 0; col_x0 < ncols_x; col_x0 += blockDim.x) {
+ const int col_x = col_x0 + threadIdx.x;
+
+ if (col_x >= ncols_x) {
+ break;
+ }
+
+ // x is transposed and permuted
+ const int ix = row_x*nchannels_x*ncols_x + channel_x*ncols_x + col_x;
+ const float xi = __half2float(x[ix]);
+
+ const int row_y = col_x;
+
+ // y is not transposed but permuted
+ const int iy = channel*nrows_y + row_y;
+
+ tmp += xi * y[iy];
+ }
+
+ // dst is not transposed and not permuted
+ const int idst = channel*nrows_dst + row_dst;
+
+ // sum up partial sums and write back result
+ tmp = warp_reduce_sum(tmp);
+
+ if (threadIdx.x == 0) {
+ dst[idst] = tmp;
+ }
+}
+
+static __global__ void mul_mat_vec_nc_f16_f32( // nc == non-contiguous
+ const void * __restrict__ vx, const float * __restrict__ y, float * __restrict__ dst, const int ncols_x, const int nrows_x,
+ const int row_stride_x, const int channel_stride_x, const int channel_x_divisor) {
+
+ const half * x = (const half *) vx;
+
+ const int row_x = blockDim.y*blockIdx.y + threadIdx.y;
+ const int channel = blockDim.z*blockIdx.z + threadIdx.z;
+ const int channel_x = channel / channel_x_divisor;
+
+ const int nrows_y = ncols_x;
+ const int nrows_dst = nrows_x;
+ const int row_dst = row_x;
+
+ const int idst = channel*nrows_dst + row_dst;
+
+ float tmp = 0.0f;
+
+ for (int col_x0 = 0; col_x0 < ncols_x; col_x0 += blockDim.x) {
+ const int col_x = col_x0 + threadIdx.x;
+
+ if (col_x >= ncols_x) {
+ break;
+ }
+
+ const int row_y = col_x;
+
+ const int ix = channel_x*channel_stride_x + row_x*row_stride_x + col_x;
+ const int iy = channel*nrows_y + row_y;
+
+ const float xi = __half2float(x[ix]);
+
+ tmp += xi * y[iy];
+ }
+
+ // sum up partial sums and write back result
+ tmp = warp_reduce_sum(tmp);
+
+ if (threadIdx.x == 0) {
+ dst[idst] = tmp;
+ }
+}
+
+static void ggml_mul_mat_p021_f16_f32_cuda(
+ const void * vx, const float * y, float * dst, const int ncols_x, const int nrows_x,
+ const int nchannels_x, const int nchannels_y, cudaStream_t stream) {
+
+ const dim3 block_nums(1, nrows_x, nchannels_y);
+ const dim3 block_dims(WARP_SIZE, 1, 1);
+ mul_mat_p021_f16_f32<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols_x, nrows_x, nchannels_x, nchannels_y);
+}
+
+static void ggml_mul_mat_vec_nc_f16_f32_cuda(
+ const void * vx, const float * y, float * dst, const int ncols_x, const int nrows_x, const int row_stride_x,
+ const int nchannels_x, const int nchannels_y, const int channel_stride_x, cudaStream_t stream) {
+
+ const dim3 block_nums(1, nrows_x, nchannels_y);
+ const dim3 block_dims(WARP_SIZE, 1, 1);
+ mul_mat_vec_nc_f16_f32<<<block_nums, block_dims, 0, stream>>>
+ (vx, y, dst, ncols_x, nrows_x, row_stride_x, channel_stride_x, nchannels_y/nchannels_x);
+}
+
+static cudaError_t ggml_cuda_cpy_tensor_2d(
+ void * dst, const struct ggml_tensor * src, int64_t i3, int64_t i2, int64_t i1_low, int64_t i1_high, cudaStream_t stream) {
+
+ GGML_ASSERT(ggml_backend_buffer_is_cuda(src->buffer));
+ char * src_ptr = (char *) src->data;
+ char * dst_ptr = (char *) dst;
+
+ const int64_t ne0 = src->ne[0];
+ const int64_t nb0 = src->nb[0];
+ const int64_t nb1 = src->nb[1];
+ const int64_t nb2 = src->nb[2];
+ const int64_t nb3 = src->nb[3];
+ const enum ggml_type type = src->type;
+ const int64_t ts = ggml_type_size(type);
+ const int64_t bs = ggml_blck_size(type);
+ int64_t i1_diff = i1_high - i1_low;
+
+ const char * x = src_ptr + i1_low*nb1 + i2*nb2 + i3*nb3;
+ if (nb0 == ts && nb1 == ts*ne0/bs) {
+ return cudaMemcpyAsync(dst_ptr, x, i1_diff*nb1, cudaMemcpyDeviceToDevice, stream);
+ } else if (nb0 == ts) {
+ return cudaMemcpy2DAsync(dst_ptr, ts*ne0/bs, x, nb1, ts*ne0/bs, i1_diff, cudaMemcpyDeviceToDevice, stream);
+ } else {
+ for (int64_t i1 = 0; i1 < i1_diff; i1++) {
+ const void * rx = (const void *) ((const char *) x + i1*nb1);
+ void * rd = (void *) (dst_ptr + i1*ts*ne0/bs);
+ // pretend the row is a matrix with cols=1
+ cudaError_t r = cudaMemcpy2DAsync(rd, ts/bs, rx, nb0, ts/bs, ne0, cudaMemcpyDeviceToDevice, stream);
+ if (r != cudaSuccess) {
+ return r;
+ }
+ }
+ return cudaSuccess;
+ }
+}
+
+static void ggml_cuda_op_mul_mat_cublas(
+ ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
+ const int64_t src1_padded_row_size, cudaStream_t stream) {
+
+ GGML_ASSERT(src0_dd_i != nullptr);
+ GGML_ASSERT(src1_ddf_i != nullptr);
+ GGML_ASSERT(dst_dd_i != nullptr);
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t ne10 = src1->ne[0];
+
+ const int64_t ne0 = dst->ne[0];
+
+ const int64_t row_diff = row_high - row_low;
+
+ int id = ggml_cuda_get_device();
+
+ // the main device has a larger memory buffer to hold the results from all GPUs
+ // ldc == nrows of the matrix that cuBLAS writes into
+ int64_t ldc = id == ctx.device ? ne0 : row_diff;
+
+ const int compute_capability = ggml_cuda_info().devices[id].cc;
+
+ if (compute_capability >= CC_VOLTA && (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) {
+ // convert src0 and src1 to fp16, multiply as fp16, convert dst to fp32
+ ggml_cuda_pool_alloc<half> src0_as_f16(ctx.pool(id));
+ if (src0->type != GGML_TYPE_F16) {
+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src0->type);
+ GGML_ASSERT(to_fp16_cuda != nullptr);
+ size_t ne = row_diff*ne00;
+ src0_as_f16.alloc(ne);
+ to_fp16_cuda(src0_dd_i, src0_as_f16.get(), ne, stream);
+ }
+ const half * src0_ptr = src0->type == GGML_TYPE_F16 ? (const half *) src0_dd_i : src0_as_f16.get();
+
+ ggml_cuda_pool_alloc<half> src1_as_f16(ctx.pool(id));
+ if (src1->type != GGML_TYPE_F16) {
+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);
+ GGML_ASSERT(to_fp16_cuda != nullptr);
+ size_t ne = src1_ncols*ne10;
+ src1_as_f16.alloc(ne);
+ to_fp16_cuda(src1_ddf_i, src1_as_f16.get(), ne, stream);
+ }
+ const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddf_i : src1_as_f16.get();
+ ggml_cuda_pool_alloc<half> dst_f16(ctx.pool(id), row_diff*src1_ncols);
+
+ const half alpha_f16 = 1.0f;
+ const half beta_f16 = 0.0f;
+
+ CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream));
+ CUBLAS_CHECK(
+ cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N,
+ row_diff, src1_ncols, ne10,
+ &alpha_f16, src0_ptr, CUDA_R_16F, ne00,
+ src1_ptr, CUDA_R_16F, ne10,
+ &beta_f16, dst_f16.get(), CUDA_R_16F, ldc,
+ CUBLAS_COMPUTE_16F,
+ CUBLAS_GEMM_DEFAULT_TENSOR_OP));
+
+ const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);
+ to_fp32_cuda(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream);
+ } else {
+ ggml_cuda_pool_alloc<float> src0_ddq_as_f32(ctx.pool(id));
+ ggml_cuda_pool_alloc<float> src1_ddq_as_f32(ctx.pool(id));
+
+ if (src0->type != GGML_TYPE_F32) {
+ const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src0->type);
+ GGML_ASSERT(to_fp32_cuda != nullptr);
+ src0_ddq_as_f32.alloc(row_diff*ne00);
+ to_fp32_cuda(src0_dd_i, src0_ddq_as_f32.get(), row_diff*ne00, stream);
+ }
+ if (src1->type != GGML_TYPE_F32) {
+ const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src1->type);
+ GGML_ASSERT(to_fp32_cuda != nullptr);
+ src1_ddq_as_f32.alloc(src1_ncols*ne10);
+ to_fp32_cuda(src1_ddf_i, src1_ddq_as_f32.get(), src1_ncols*ne10, stream);
+ }
+
+ const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32.get();
+ const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get();
+
+ const float alpha = 1.0f;
+ const float beta = 0.0f;
+
+ CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream));
+ CUBLAS_CHECK(
+ cublasSgemm(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N,
+ row_diff, src1_ncols, ne10,
+ &alpha, src0_ddf_i, ne00,
+ src1_ddf1_i, ne10,
+ &beta, dst_dd_i, ldc));
+ }
+
+ GGML_UNUSED(dst);
+ GGML_UNUSED(src1_ddq_i);
+ GGML_UNUSED(src1_padded_row_size);
+}
+
+static void ggml_cuda_set_peer_access(const int n_tokens, int main_device) {
+ static bool peer_access_enabled = false;
+
+ const bool enable_peer_access = n_tokens <= GGML_CUDA_PEER_MAX_BATCH_SIZE;
+
+ if (peer_access_enabled == enable_peer_access) {
+ return;
+ }
+
+#ifdef NDEBUG
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ ggml_cuda_set_device(id);
+ CUDA_CHECK(cudaDeviceSynchronize());
+ }
+
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ ggml_cuda_set_device(id);
+
+ for (int id_other = 0; id_other < ggml_backend_cuda_get_device_count(); ++id_other) {
+ if (id == id_other) {
+ continue;
+ }
+ if (id != main_device && id_other != main_device) {
+ continue;
+ }
+
+ int can_access_peer;
+ CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, id, id_other));
+ if (can_access_peer) {
+ if (enable_peer_access) {
+ cudaError_t err = cudaDeviceEnablePeerAccess(id_other, 0);
+ if (err != cudaErrorPeerAccessAlreadyEnabled) {
+ CUDA_CHECK(err);
+ }
+ } else {
+ cudaError_t err = cudaDeviceDisablePeerAccess(id_other);
+ if (err != cudaErrorPeerAccessNotEnabled) {
+ CUDA_CHECK(err);
+ }
+ }
+ }
+ }
+ }
+
+ ggml_cuda_set_device(main_device);
+#endif // NDEBUG
+
+ peer_access_enabled = enable_peer_access;
+
+ GGML_UNUSED(main_device);
+}
+
+static cudaError_t ggml_cuda_Memcpy2DPeerAsync(
+ void * dst, int dstDevice, size_t dpitch, void * src, int srcDevice, size_t spitch, size_t width, size_t height, cudaStream_t stream) {
+
+#if !defined(GGML_USE_HIPBLAS) && !defined(GGML_USE_MUSA)
+ // cudaMemcpy2DAsync may fail with copies between vmm pools of different devices
+ cudaMemcpy3DPeerParms p = {};
+ p.dstDevice = dstDevice;
+ p.dstPtr = make_cudaPitchedPtr(dst, dpitch, dpitch, height);
+ p.srcDevice = srcDevice;
+ p.srcPtr = make_cudaPitchedPtr(src, spitch, spitch, height);
+ p.extent = make_cudaExtent(width, height, 1);
+ return cudaMemcpy3DPeerAsync(&p, stream);
+#else
+ // HIP does not support cudaMemcpy3DPeerAsync or vmm pools
+ GGML_UNUSED(dstDevice);
+ GGML_UNUSED(srcDevice);
+ return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, cudaMemcpyDeviceToDevice, stream);
+#endif // !defined(GGML_USE_HIPBLAS) && !defined(GGML_USE_MUSA)
+}
+
+static void ggml_cuda_op_mul_mat(
+ ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_op_mul_mat_t op,
+ quantize_cuda_t quantize_src1) {
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t ne01 = src0->ne[1];
+ const int64_t ne02 = src0->ne[2];
+ const int64_t ne03 = src0->ne[3];
+
+ const int64_t ne10 = src1->ne[0];
+ const int64_t ne11 = src1->ne[1];
+ const int64_t ne12 = src1->ne[2];
+ const int64_t ne13 = src1->ne[3];
+ const int64_t nrows1 = ggml_nrows(src1);
+
+ GGML_ASSERT(ne03 == ne13);
+
+ const int64_t ne0 = dst->ne[0];
+ const int64_t ne1 = dst->ne[1];
+
+ const int64_t nb2 = dst->nb[2];
+ const int64_t nb3 = dst->nb[3];
+
+ GGML_ASSERT(ggml_backend_buffer_is_cuda(dst->buffer));
+ GGML_ASSERT(ggml_backend_buffer_is_cuda(src1->buffer));
+ ggml_backend_cuda_buffer_context * src1_ctx = (ggml_backend_cuda_buffer_context *) src1->buffer->context;
+ ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *) dst->buffer->context;
+
+ GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1));
+
+ GGML_ASSERT(ne12 >= ne02 && ne12 % ne02 == 0);
+
+ const int64_t i02_divisor = ne12 / ne02;
+
+ const size_t src0_ts = ggml_type_size(src0->type);
+ const size_t src0_bs = ggml_blck_size(src0->type);
+ const size_t q8_1_ts = sizeof(block_q8_1);
+ const size_t q8_1_bs = QK8_1;
+
+ const bool src0_is_contiguous = ggml_is_contiguous(src0);
+ const bool src1_is_contiguous = ggml_is_contiguous(src1);
+
+ const int64_t src1_padded_col_size = GGML_PAD(ne10, MATRIX_ROW_PADDING);
+
+ const bool split = ggml_backend_buffer_is_cuda_split(src0->buffer);
+ GGML_ASSERT(!(split && ne02 > 1));
+ GGML_ASSERT(!(split && ne03 > 1));
+ GGML_ASSERT(!(split && ne02 < ne12));
+
+ ggml_tensor_extra_gpu * src0_extra = split ? (ggml_tensor_extra_gpu *) src0->extra : nullptr;
+
+
+ std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split;
+ if (split) {
+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;
+ tensor_split = buft_ctx->tensor_split;
+ }
+
+ struct dev_data {
+ int cc;
+
+ ggml_cuda_pool_alloc<char> src0_dd_alloc;
+ ggml_cuda_pool_alloc<float> src1_ddf_alloc;
+ ggml_cuda_pool_alloc<char> src1_ddq_alloc;
+ ggml_cuda_pool_alloc<float> dst_dd_alloc;
+
+ char * src0_dd = nullptr;
+ float * src1_ddf = nullptr; // float
+ char * src1_ddq = nullptr; // q8_1
+ float * dst_dd = nullptr;
+
+ int64_t row_low;
+ int64_t row_high;
+ };
+
+ dev_data dev[GGML_CUDA_MAX_DEVICES];
+
+ int used_devices = 0;
+
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ dev[id].cc = ggml_cuda_info().devices[id].cc;
+
+ // by default, use all rows
+ dev[id].row_low = 0;
+ dev[id].row_high = ne01;
+
+ // for multi GPU, get the row boundaries from tensor split
+ // and round to mul_mat_q tile sizes
+ if (split) {
+ const int64_t rounding = get_row_rounding(tensor_split);
+
+ if (id != 0) {
+ dev[id].row_low = ne01*tensor_split[id];
+ if (dev[id].row_low < ne01) {
+ dev[id].row_low -= dev[id].row_low % rounding;
+ }
+ }
+
+ if (id != ggml_backend_cuda_get_device_count() - 1) {
+ dev[id].row_high = ne01*tensor_split[id + 1];
+ if (dev[id].row_high < ne01) {
+ dev[id].row_high -= dev[id].row_high % rounding;
+ }
+ }
+ }
+ }
+
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) {
+ continue;
+ }
+
+ used_devices++;
+
+ const bool src1_on_device = id == src1_ctx->device;
+ const bool dst_on_device = id == dst_ctx->device;
+
+ ggml_cuda_set_device(id);
+ cudaStream_t stream = ctx.stream(id, 0);
+
+ if (src0_is_contiguous) {
+ dev[id].src0_dd = split ? (char *) src0_extra->data_device[id] : (char *) src0->data;
+ } else {
+ dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ctx.pool(id), ggml_nbytes(src0));
+ }
+
+ // If src0 is on a temporary compute buffers (partial offloading) there may be some padding that needs to be cleared:
+ if (ne00 % MATRIX_ROW_PADDING != 0 && ggml_is_quantized(src0->type) && ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && src0->view_src == nullptr) {
+ const int64_t nbytes_data = ggml_row_size(src0->type, (dev[id].row_high - dev[id].row_low)*ne00);
+ const int64_t nbytes_padding = ggml_row_size(src0->type, MATRIX_ROW_PADDING - ne00 % MATRIX_ROW_PADDING);
+ CUDA_CHECK(cudaMemsetAsync(dev[id].src0_dd + nbytes_data , 0, nbytes_padding, stream));
+ }
+
+ if (src1_on_device && src1_is_contiguous) {
+ dev[id].src1_ddf = (float *) src1->data;
+ } else {
+ dev[id].src1_ddf = dev[id].src1_ddf_alloc.alloc(ctx.pool(id), ggml_nelements(src1));
+ }
+
+ if (quantize_src1) {
+ size_t src_1_ddq_size = nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs;
+ if (quantize_src1 == quantize_mmq_q8_1_cuda) {
+ src_1_ddq_size += get_mmq_x_max_host(dev[id].cc)*sizeof(block_q8_1_mmq);
+ }
+ dev[id].src1_ddq = dev[id].src1_ddq_alloc.alloc(ctx.pool(id), src_1_ddq_size);
+
+ if (src1_on_device && src1_is_contiguous) {
+ quantize_src1(dev[id].src1_ddf, dev[id].src1_ddq, ne10, ne11, ne12*ne13, src1_padded_col_size, src0->type, stream);
+ CUDA_CHECK(cudaGetLastError());
+ }
+ }
+
+ if (dst_on_device) {
+ dev[id].dst_dd = (float *) dst->data;
+ } else {
+ const size_t size_dst_ddf = split ? (dev[id].row_high - dev[id].row_low)*ne1 : ggml_nelements(dst);
+ dev[id].dst_dd = dev[id].dst_dd_alloc.alloc(ctx.pool(id), size_dst_ddf);
+ }
+ }
+
+ // if multiple devices are used they need to wait for the main device
+ // here an event is recorded that signals that the main device has finished calculating the input data
+ if (split && used_devices > 1) {
+ ggml_cuda_set_device(ctx.device);
+ CUDA_CHECK(cudaEventRecord(src0_extra->events[ctx.device][0], ctx.stream()));
+ }
+
+ const int64_t src1_col_stride = split && used_devices > 1 ? MUL_MAT_SRC1_COL_STRIDE : ne11;
+ for (int64_t src1_col_0 = 0; src1_col_0 < ne11; src1_col_0 += src1_col_stride) {
+ const int64_t is = split ? (src1_col_0/src1_col_stride) % GGML_CUDA_MAX_STREAMS : 0;
+ const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride;
+
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) {
+ continue;
+ }
+
+ const bool src1_on_device = id == src1_ctx->device;
+ const bool dst_on_device = id == dst_ctx->device;
+ const int64_t row_diff = dev[id].row_high - dev[id].row_low;
+
+ ggml_cuda_set_device(id);
+ cudaStream_t stream = ctx.stream(id, is);
+
+ // wait for main GPU data if necessary
+ if (split && (id != ctx.device || is != 0)) {
+ CUDA_CHECK(cudaStreamWaitEvent(stream, src0_extra->events[ctx.device][0], 0));
+ }
+
+ for (int64_t i0 = 0; i0 < ne13*ne12; ++i0) {
+ const int64_t i03 = i0 / ne12;
+ const int64_t i02 = i0 % ne12;
+
+ size_t src1_ddq_i_offset = i0*ne11 * src1_padded_col_size*q8_1_ts/q8_1_bs;
+ if (quantize_src1 == quantize_mmq_q8_1_cuda) {
+ src1_ddq_i_offset += src1_col_0 * sizeof(block_q8_1_mmq);
+ } else {
+ src1_ddq_i_offset += src1_col_0 * src1_padded_col_size*q8_1_ts/q8_1_bs;
+ }
+
+ // for split tensors the data begins at i0 == i0_offset_low
+ char * src0_dd_i = dev[id].src0_dd + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs;
+ float * src1_ddf_i = dev[id].src1_ddf + (i0*ne11 + src1_col_0) * ne10;
+ char * src1_ddq_i = dev[id].src1_ddq + src1_ddq_i_offset;
+ float * dst_dd_i = dev[id].dst_dd + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff);
+
+ // the main device memory buffer can be on VRAM scratch, with space for all partial results
+ // in that case an offset on dst_ddf_i is needed
+ if (id == ctx.device) {
+ dst_dd_i += dev[id].row_low; // offset is 0 if no tensor split
+ }
+
+ // copy src0, src1 to device if necessary
+ if (src1_is_contiguous) {
+ if (id != ctx.device) {
+ if (quantize_src1) {
+ char * src1_ddq_i_source = dev[ctx.device].src1_ddq + src1_ddq_i_offset;
+ if (quantize_src1 == quantize_mmq_q8_1_cuda) {
+ const size_t pitch = ne11*sizeof(block_q8_1_mmq);
+ const size_t width = src1_ncols*sizeof(block_q8_1_mmq);
+ const size_t height = src1_padded_col_size/(4*QK8_1);
+ CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync(src1_ddq_i, id, pitch, src1_ddq_i_source, ctx.device, pitch, width, height, stream));
+ } else {
+ CUDA_CHECK(cudaMemcpyPeerAsync(
+ src1_ddq_i, id, src1_ddq_i_source, ctx.device, src1_ncols*src1_padded_col_size*q8_1_ts/q8_1_bs, stream));
+ }
+ } else {
+ float * src1_ddf_i_source = (float *) src1->data;
+ src1_ddf_i_source += (i0*ne11 + src1_col_0) * ne10;
+ CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddf_i, id, src1_ddf_i_source, ctx.device,
+ src1_ncols*ne10*sizeof(float), stream));
+ }
+ }
+ } else if (src1_on_device && !src1_is_contiguous) {
+ CUDA_CHECK(ggml_cuda_cpy_tensor_2d(
+ src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream));
+ } else {
+ GGML_ABORT("fatal error");
+ }
+
+ if (quantize_src1 && !src1_is_contiguous) {
+ quantize_src1(src1_ddf_i, src1_ddq_i, ne10, src1_ncols, 1, src1_padded_col_size, src0->type, stream);
+ CUDA_CHECK(cudaGetLastError());
+ }
+
+ if (src1_col_0 == 0 && !src0_is_contiguous && i02 % i02_divisor == 0) {
+ CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src0_dd_i, src0, i03, i02/i02_divisor, dev[id].row_low, dev[id].row_high, stream));
+ }
+
+ // do the computation
+ op(ctx, src0, src1, dst, src0_dd_i, src1_ddf_i, src1_ddq_i, dst_dd_i,
+ dev[id].row_low, dev[id].row_high, src1_ncols, src1_padded_col_size, stream);
+ CUDA_CHECK(cudaGetLastError());
+
+ // copy dst to host or other device if necessary
+ if (!dst_on_device) {
+ void * dst_off_device = dst->data;
+ if (split) {
+ // src0 = weight matrix is saved as a transposed matrix for better memory layout.
+ // dst is NOT transposed.
+ // The outputs of matrix matrix multiplications can therefore NOT simply be concatenated for >1 GPU.
+ // Instead they need to be copied to the correct slice in ne0 = dst row index.
+ // If dst is a vector with ne0 == 1 then you don't have to do this but it still produces correct results.
+ float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);
+ GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));
+ dhf_dst_i += src1_col_0*ne0 + dev[id].row_low;
+ CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync(
+ dhf_dst_i, ctx.device, ne0*sizeof(float), dst_dd_i, id, row_diff*sizeof(float), row_diff*sizeof(float), src1_ncols, stream));
+ } else {
+ float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);
+ GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));
+ dhf_dst_i += src1_col_0*ne0;
+ CUDA_CHECK(cudaMemcpyAsync(dhf_dst_i, dst_dd_i, src1_ncols*ne0*sizeof(float), cudaMemcpyDeviceToDevice, stream));
+ }
+ }
+
+ // add event for the main device to wait on until other device is done
+ if (split && (id != ctx.device || is != 0)) {
+ CUDA_CHECK(cudaEventRecord(src0_extra->events[id][is], stream));
+ }
+ }
+ }
+ }
+
+ // main device waits for all other devices to be finished
+ if (split && ggml_backend_cuda_get_device_count() > 1) {
+ int64_t is_max = (ne11 + MUL_MAT_SRC1_COL_STRIDE - 1) / MUL_MAT_SRC1_COL_STRIDE;
+ is_max = is_max <= GGML_CUDA_MAX_STREAMS ? is_max : GGML_CUDA_MAX_STREAMS;
+
+ ggml_cuda_set_device(ctx.device);
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ if (dev[id].row_low == dev[id].row_high) {
+ continue;
+ }
+ for (int64_t is = 0; is < is_max; ++is) {
+ CUDA_CHECK(cudaStreamWaitEvent(ctx.stream(), src0_extra->events[id][is], 0));
+ }
+ }
+ }
+}
+
+static void ggml_cuda_mul_mat_vec_p021(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1));
+ GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer));
+ GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // 0213 permutation
+ GGML_ASSERT(src1->nb[0] <= src1->nb[1] && src1->nb[2] <= src1->nb[3]); // 0213 permutation
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t ne01 = src0->ne[1];
+ const int64_t ne02 = src0->ne[2];
+
+ const int64_t ne12 = src1->ne[2];
+
+ cudaStream_t main_stream = ctx.stream();
+
+ void * src0_ddq = src0->data;
+ float * src1_ddf = (float *) src1->data;
+ float * dst_ddf = (float *) dst->data;
+
+ ggml_mul_mat_p021_f16_f32_cuda(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, ne02, ne12, main_stream);
+}
+
+static void ggml_cuda_mul_mat_vec_nc(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ GGML_ASSERT(!ggml_is_transposed(src0));
+ GGML_ASSERT(!ggml_is_transposed(src1));
+ GGML_ASSERT(!ggml_is_permuted(src0));
+ GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer));
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t ne01 = src0->ne[1];
+ const int64_t ne02 = src0->ne[2];
+
+ const int64_t nb01 = src0->nb[1];
+ const int64_t nb02 = src0->nb[2];
+
+ const int64_t ne12 = src1->ne[2];
+
+ cudaStream_t main_stream = ctx.stream();
+
+ void * src0_ddq = src0->data;
+ float * src1_ddf = (float *) src1->data;
+ float * dst_ddf = (float *) dst->data;
+
+ const int64_t row_stride_x = nb01 / sizeof(half);
+ const int64_t channel_stride_x = nb02 / sizeof(half);
+
+ ggml_mul_mat_vec_nc_f16_f32_cuda(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, row_stride_x, ne02, ne12, channel_stride_x, main_stream);
+}
+
+static __global__ void k_compute_batched_ptrs(
+ const half * src0_as_f16, const half * src1_as_f16, char * dst,
+ const void ** ptrs_src, void ** ptrs_dst,
+ int64_t ne12, int64_t ne13,
+ int64_t ne23,
+ size_t nb02, size_t nb03,
+ size_t nb12, size_t nb13,
+ size_t nbd2, size_t nbd3,
+ int64_t r2, int64_t r3) {
+ int64_t i13 = blockIdx.x * blockDim.x + threadIdx.x;
+ int64_t i12 = blockIdx.y * blockDim.y + threadIdx.y;
+
+ if (i13 >= ne13 || i12 >= ne12) {
+ return;
+ }
+
+ int64_t i03 = i13 / r3;
+ int64_t i02 = i12 / r2;
+
+ ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_as_f16 + i02*nb02 + i03*nb03;
+ ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_as_f16 + i12*nb12 + i13*nb13;
+ ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3;
+}
+
+static void ggml_cuda_mul_mat_batched_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ GGML_ASSERT(!ggml_is_transposed(src0));
+ GGML_ASSERT(!ggml_is_transposed(src1));
+
+ GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer));
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int64_t ne_dst = ggml_nelements(dst);
+
+ cudaStream_t main_stream = ctx.stream();
+
+ CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(), main_stream));
+
+ void * src0_ddq = src0->data;
+ half * src0_f16 = (half *) src0_ddq;
+ float * src1_ddf = (float *) src1->data;
+ float * dst_ddf = (float *) dst->data;
+
+ // convert src1 to fp16
+ ggml_cuda_pool_alloc<half> src1_f16_alloc(ctx.pool());
+ if (src1->type != GGML_TYPE_F16) {
+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);
+ const int64_t ne_src1 = ggml_nelements(src1);
+ src1_f16_alloc.alloc(ne_src1);
+ GGML_ASSERT(to_fp16_cuda != nullptr);
+ to_fp16_cuda(src1_ddf, src1_f16_alloc.get(), ne_src1, main_stream);
+ }
+ half * src1_f16 = src1->type == GGML_TYPE_F16 ? (half *) src1_ddf : src1_f16_alloc.get();
+
+ ggml_cuda_pool_alloc<half> dst_f16(ctx.pool());
+ char * dst_t;
+
+ cublasComputeType_t cu_compute_type = CUBLAS_COMPUTE_16F;
+ cudaDataType_t cu_data_type = CUDA_R_16F;
+
+ // dst strides
+ size_t nbd2 = dst->nb[2];
+ size_t nbd3 = dst->nb[3];
+
+ const half alpha_f16 = 1.0f;
+ const half beta_f16 = 0.0f;
+
+ const float alpha_f32 = 1.0f;
+ const float beta_f32 = 0.0f;
+
+ const void * alpha = &alpha_f16;
+ const void * beta = &beta_f16;
+
+ if (dst->op_params[0] == GGML_PREC_DEFAULT) {
+ dst_t = (char *) dst_f16.alloc(ne_dst);
+
+ nbd2 /= sizeof(float) / sizeof(half);
+ nbd3 /= sizeof(float) / sizeof(half);
+ } else {
+ dst_t = (char *) dst_ddf;
+
+ cu_compute_type = CUBLAS_COMPUTE_32F;
+ cu_data_type = CUDA_R_32F;
+
+ alpha = &alpha_f32;
+ beta = &beta_f32;
+ }
+
+ GGML_ASSERT(ne12 % ne02 == 0);
+ GGML_ASSERT(ne13 % ne03 == 0);
+
+ // broadcast factors
+ const int64_t r2 = ne12/ne02;
+ const int64_t r3 = ne13/ne03;
+
+#if 0
+ // use cublasGemmEx
+ {
+ for (int i13 = 0; i13 < ne13; ++i13) {
+ for (int i12 = 0; i12 < ne12; ++i12) {
+ int i03 = i13 / r3;
+ int i02 = i12 / r2;
+
+ CUBLAS_CHECK(
+ cublasGemmEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,
+ ne01, ne11, ne10,
+ alpha, (const char *) src0_as_f16 + i02*src0->nb[2] + i03*src0->nb[3] , CUDA_R_16F, nb01/sizeof(half),
+ (const char *) src1_as_f16 + i12*src1->nb[2]/2 + i13*src1->nb[3]/2, CUDA_R_16F, nb11/sizeof(float),
+ beta, ( char *) dst_t + i12*nbd2 + i13*nbd3, cu_data_type, ne01,
+ cu_compute_type,
+ CUBLAS_GEMM_DEFAULT_TENSOR_OP));
+ }
+ }
+ }
+#else
+#ifdef GGML_USE_MUSA
+ GGML_ASSERT(false);
+#else // !GGML_USE_MUSA
+ if (r2 == 1 && r3 == 1 && ggml_is_contiguous_2(src0) && ggml_is_contiguous_2(src1)) {
+ // there is no broadcast and src0, src1 are contiguous across dims 2, 3
+ // use cublasGemmStridedBatchedEx
+ CUBLAS_CHECK(
+ cublasGemmStridedBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,
+ ne01, ne11, ne10,
+ alpha, (const char *) src0_f16, CUDA_R_16F, nb01/nb00, nb02/nb00, // strideA
+ (const char *) src1_f16, CUDA_R_16F, nb11/nb10, nb12/nb10, // strideB
+ beta, ( char *) dst_t, cu_data_type, ne01, nb2/nb0, // strideC
+ ne12*ne13,
+ cu_compute_type,
+ CUBLAS_GEMM_DEFAULT_TENSOR_OP));
+ } else {
+ // use cublasGemmBatchedEx
+ const int ne23 = ne12*ne13;
+
+ ggml_cuda_pool_alloc<const void *> ptrs_src(ctx.pool(), 2*ne23);
+ ggml_cuda_pool_alloc< void *> ptrs_dst(ctx.pool(), 1*ne23);
+
+ dim3 block_dims(ne13, ne12);
+ k_compute_batched_ptrs<<<1, block_dims, 0, main_stream>>>(
+ src0_f16, src1_f16, dst_t,
+ ptrs_src.get(), ptrs_dst.get(),
+ ne12, ne13,
+ ne23,
+ nb02, nb03,
+ src1->type == GGML_TYPE_F16 ? nb12 : nb12/2,
+ src1->type == GGML_TYPE_F16 ? nb13 : nb13/2,
+ nbd2, nbd3,
+ r2, r3);
+ CUDA_CHECK(cudaGetLastError());
+
+ CUBLAS_CHECK(
+ cublasGemmBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,
+ ne01, ne11, ne10,
+ alpha, (const void **) (ptrs_src.get() + 0*ne23), CUDA_R_16F, nb01/nb00,
+ (const void **) (ptrs_src.get() + 1*ne23), CUDA_R_16F, nb11/nb10,
+ beta, ( void **) (ptrs_dst.get() + 0*ne23), cu_data_type, ne01,
+ ne23,
+ cu_compute_type,
+ CUBLAS_GEMM_DEFAULT_TENSOR_OP));
+ }
+#endif // GGML_USE_MUSA
+#endif
+
+ if (dst->op_params[0] == GGML_PREC_DEFAULT) {
+ const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);
+ to_fp32_cuda(dst_f16.get(), dst_ddf, ne_dst, main_stream);
+ }
+}
+
+static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ const bool split = ggml_backend_buffer_is_cuda_split(src0->buffer);
+
+ bool use_dequantize_mul_mat_vec = ggml_cuda_dmmv_type_supported(src0->type)
+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32
+ && src0->ne[0] % (GGML_CUDA_DMMV_X*2) == 0 && src1->ne[1] == 1;
+ bool use_mul_mat_vec_q = ggml_is_quantized(src0->type)
+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32
+ && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;
+ bool use_mul_mat_q = ggml_is_quantized(src0->type)
+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;
+
+ // if mmvq is available it's a better choice than dmmv:
+#ifndef GGML_CUDA_FORCE_DMMV
+ use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && !use_mul_mat_vec_q;
+#endif // GGML_CUDA_FORCE_DMMV
+
+ bool any_gpus_with_slow_fp16 = false;
+
+ if (split) {
+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;
+ auto & tensor_split = buft_ctx->tensor_split;
+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
+ // skip devices that are not going to do any work:
+ if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) {
+ continue;
+ }
+
+ const int cc = ggml_cuda_info().devices[id].cc;
+ use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1]);
+ any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_available(cc);
+ }
+ } else {
+ const int cc = ggml_cuda_info().devices[ctx.device].cc;
+ use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1]);
+ any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_available(cc);
+ }
+
+ // debug helpers
+ //printf("src0: %8d %8d %8d %8d\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]);
+ //printf(" %8d %8d %8d %8d\n", src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]);
+ //printf("src1: %8d %8d %8d %8d\n", src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3]);
+ //printf(" %8d %8d %8d %8d\n", src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]);
+ //printf("src0 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name);
+ //printf("src1 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name);
+
+ if (!split && any_gpus_with_slow_fp16 && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) {
+ // FP32 precision KQ single-batch for batch size 1 without FlashAttention
+ ggml_cuda_mul_mat_vec_p021(ctx, src0, src1, dst);
+ } else if (!split && any_gpus_with_slow_fp16 && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) {
+ // FP32 precision KQV single-batch for batch size 1 without FlashAttention
+ ggml_cuda_mul_mat_vec_nc(ctx, src0, src1, dst);
+ } else if (!split && src0->type == GGML_TYPE_F16 && (src1->type == GGML_TYPE_F16 || !any_gpus_with_slow_fp16)
+ && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) {
+ // KQ + KQV multi-batch without FlashAttention
+ ggml_cuda_mul_mat_batched_cublas(ctx, src0, src1, dst);
+ } else if (use_dequantize_mul_mat_vec) {
+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, nullptr);
+ } else if (use_mul_mat_vec_q) {
+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, quantize_row_q8_1_cuda);
+ } else if (use_mul_mat_q) {
+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_q, quantize_mmq_q8_1_cuda);
+ } else {
+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_cublas, nullptr);
+ }
+}
+
+struct mmid_row_mapping {
+ int32_t i1;
+ int32_t i2;
+};
+
+static __global__ void k_copy_src1_to_contiguous(const char * __restrict__ src1_original, char * __restrict__ src1_contiguous,
+ int * __restrict__ cur_src1_row, mmid_row_mapping * __restrict__ row_mapping,
+ const char * __restrict ids, int64_t i02, size_t ids_nb1, size_t ids_nb0,
+ int64_t ne11, int64_t ne10,
+ size_t nb11, size_t nb12) {
+ int32_t iid1 = blockIdx.x;
+ int32_t id = blockIdx.y;
+
+ const int32_t row_id_i = *(const int32_t *) (ids + iid1*ids_nb1 + id*ids_nb0);
+
+ if (row_id_i != i02) {
+ return;
+ }
+
+ const int64_t i11 = id % ne11;
+ const int64_t i12 = iid1;
+
+ __shared__ int src1_row;
+ if (threadIdx.x == 0) {
+ src1_row = atomicAdd(cur_src1_row, 1);
+ row_mapping[src1_row] = {id, iid1};
+ }
+ __syncthreads();
+
+ const float * src1_row_original = (const float *)(src1_original + i11*nb11 + i12*nb12);
+ float * src1_row_contiguous = (float *)(src1_contiguous + src1_row*nb11);
+
+ for (int i = threadIdx.x; i < ne10; i += blockDim.x) {
+ src1_row_contiguous[i] = src1_row_original[i];
+ }
+}
+
+static __global__ void k_copy_dst_from_contiguous(char * __restrict__ dst_original, const char * __restrict__ dst_contiguous,
+ const mmid_row_mapping * __restrict__ row_mapping,
+ int64_t ne0,
+ size_t nb1, size_t nb2) {
+ int32_t i = blockIdx.x;
+
+ const int32_t i1 = row_mapping[i].i1;
+ const int32_t i2 = row_mapping[i].i2;
+
+ const float * dst_row_contiguous = (const float *)(dst_contiguous + i*nb1);
+ float * dst_row_original = (float *)(dst_original + i1*nb1 + i2*nb2);
+
+ for (int j = threadIdx.x; j < ne0; j += blockDim.x) {
+ dst_row_original[j] = dst_row_contiguous[j];
+ }
+}
+
+static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ const ggml_tensor * ids = dst->src[2];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT(!ggml_backend_buffer_is_cuda_split(src0->buffer) && "mul_mat_id does not support split buffers");
+
+ cudaStream_t stream = ctx.stream();
+
+ const int64_t n_as = ne02;
+ const int64_t n_ids = ids->ne[0];
+
+ std::vector<char> ids_host(ggml_nbytes(ids));
+ const char * ids_dev = (const char *) ids->data;
+ CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids_dev, ggml_nbytes(ids), cudaMemcpyDeviceToHost, stream));
+ CUDA_CHECK(cudaStreamSynchronize(stream));
+
+ ggml_tensor src0_row = *src0;
+ ggml_tensor src1_row = *src1;
+ ggml_tensor dst_row = *dst;
+
+ char * src0_original = (char *) src0->data;
+ char * src1_original = (char *) src1->data;
+ char * dst_original = (char *) dst->data;
+
+ src0_row.ne[2] = 1;
+ src0_row.ne[3] = 1;
+ src0_row.nb[3] = nb02;
+
+ src1_row.ne[1] = 1;
+ src1_row.ne[2] = 1;
+ src1_row.ne[3] = 1;
+ src1_row.nb[2] = nb11;
+ src1_row.nb[3] = nb11;
+
+ dst_row.ne[1] = 1;
+ dst_row.ne[2] = 1;
+ dst_row.ne[3] = 1;
+ dst_row.nb[2] = nb1;
+ dst_row.nb[3] = nb1;
+
+ if (ne12 == 1) {
+ for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
+ for (int64_t id = 0; id < n_ids; id++) {
+ const int32_t i02 = *(const int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]);
+
+ GGML_ASSERT(i02 >= 0 && i02 < n_as);
+
+ const int64_t i11 = id % ne11;
+ const int64_t i12 = iid1;
+
+ const int64_t i1 = id;
+ const int64_t i2 = i12;
+
+ src0_row.data = src0_original + i02*nb02;
+ src1_row.data = src1_original + i11*nb11 + i12*nb12;
+ dst_row.data = dst_original + i1*nb1 + i2*nb2;
+
+ ggml_cuda_mul_mat(ctx, &src0_row, &src1_row, &dst_row);
+ }
+ }
+ } else {
+ ggml_cuda_pool_alloc<char> src1_contiguous(ctx.pool(), sizeof(float)*ggml_nelements(src1));
+ ggml_cuda_pool_alloc<char> dst_contiguous(ctx.pool(), sizeof(float)*ggml_nelements(dst));
+
+ src1_row.data = src1_contiguous.get();
+ dst_row.data = dst_contiguous.get();
+
+ for (int64_t i02 = 0; i02 < n_as; i02++) {
+ int64_t num_src1_rows = 0;
+
+ for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
+ for (int64_t id = 0; id < n_ids; id++) {
+ const int32_t row_id_i = *(const int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]);
+
+ GGML_ASSERT(row_id_i >= 0 && row_id_i < n_as);
+
+ if (row_id_i != i02) {
+ continue;
+ }
+
+ num_src1_rows++;
+ }
+ }
+
+ if (num_src1_rows == 0) {
+ continue;
+ }
+
+ ggml_cuda_pool_alloc<int> dev_cur_src1_row(ctx.pool(), 1);
+ ggml_cuda_pool_alloc<mmid_row_mapping> dev_row_mapping(ctx.pool(), num_src1_rows);
+ CUDA_CHECK(cudaMemsetAsync(dev_cur_src1_row.get(), 0, sizeof(int), stream));
+
+ {
+ dim3 block_dims(std::min((unsigned int)ne10, 768u));
+ dim3 grid_dims(ids->ne[1], n_ids);
+ k_copy_src1_to_contiguous<<<grid_dims, block_dims, 0, stream>>>(
+ src1_original, src1_contiguous.get(),
+ dev_cur_src1_row.get(), dev_row_mapping.get(),
+ ids_dev, i02, ids->nb[1], ids->nb[0],
+ ne11, ne10,
+ nb11, nb12);
+ CUDA_CHECK(cudaGetLastError());
+ }
+
+ src0_row.data = src0_original + i02*nb02;
+
+ GGML_ASSERT(nb11 == sizeof(float)*ne10);
+ GGML_ASSERT(nb1 == sizeof(float)*ne0);
+
+ src1_row.ne[1] = num_src1_rows;
+ src1_row.nb[1] = nb11;
+ src1_row.nb[2] = num_src1_rows*nb11;
+ src1_row.nb[3] = num_src1_rows*nb11;
+
+ dst_row.ne[1] = num_src1_rows;
+ dst_row.nb[1] = nb1;
+ dst_row.nb[2] = num_src1_rows*nb1;
+ dst_row.nb[3] = num_src1_rows*nb1;
+
+ ggml_cuda_mul_mat(ctx, &src0_row, &src1_row, &dst_row);
+
+ {
+ dim3 block_dims(std::min((unsigned int)ne0, 768u));
+ dim3 grid_dims(num_src1_rows);
+ k_copy_dst_from_contiguous<<<grid_dims, block_dims, 0, stream>>>(
+ dst_original, dst_contiguous.get(),
+ dev_row_mapping.get(),
+ ne0,
+ nb1, nb2);
+ CUDA_CHECK(cudaGetLastError());
+ }
+ }
+ }
+}
+
+static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggml_tensor * dst) {
+ // why is this here instead of mul_mat?
+ if (dst->src[0] != nullptr && ggml_backend_buffer_is_cuda_split(dst->src[0]->buffer)) {
+ ggml_cuda_set_peer_access(dst->src[1]->ne[1], ctx.device);
+ }
+
+ switch (dst->op) {
+ case GGML_OP_REPEAT:
+ ggml_cuda_op_repeat(ctx, dst);
+ break;
+ case GGML_OP_GET_ROWS:
+ ggml_cuda_op_get_rows(ctx, dst);
+ break;
+ case GGML_OP_DUP:
+ ggml_cuda_dup(ctx, dst);
+ break;
+ case GGML_OP_CPY:
+ ggml_cuda_cpy(ctx, dst->src[0], dst->src[1]);
+ break;
+ case GGML_OP_CONT:
+ ggml_cuda_dup(ctx, dst);
+ break;
+ case GGML_OP_ADD:
+ ggml_cuda_op_add(ctx, dst);
+ break;
+ case GGML_OP_SUB:
+ ggml_cuda_op_sub(ctx, dst);
+ break;
+ case GGML_OP_ACC:
+ ggml_cuda_op_acc(ctx, dst);
+ break;
+ case GGML_OP_MUL:
+ ggml_cuda_op_mul(ctx, dst);
+ break;
+ case GGML_OP_DIV:
+ ggml_cuda_op_div(ctx, dst);
+ break;
+ case GGML_OP_UNARY:
+ switch (ggml_get_unary_op(dst)) {
+ case GGML_UNARY_OP_GELU:
+ ggml_cuda_op_gelu(ctx, dst);
+ break;
+ case GGML_UNARY_OP_SILU:
+ ggml_cuda_op_silu(ctx, dst);
+ break;
+ case GGML_UNARY_OP_GELU_QUICK:
+ ggml_cuda_op_gelu_quick(ctx, dst);
+ break;
+ case GGML_UNARY_OP_TANH:
+ ggml_cuda_op_tanh(ctx, dst);
+ break;
+ case GGML_UNARY_OP_RELU:
+ ggml_cuda_op_relu(ctx, dst);
+ break;
+ case GGML_UNARY_OP_SIGMOID:
+ ggml_cuda_op_sigmoid(ctx, dst);
+ break;
+ case GGML_UNARY_OP_HARDSIGMOID:
+ ggml_cuda_op_hardsigmoid(ctx, dst);
+ break;
+ case GGML_UNARY_OP_HARDSWISH:
+ ggml_cuda_op_hardswish(ctx, dst);
+ break;
+ default:
+ return false;
+ }
+ break;
+ case GGML_OP_NORM:
+ ggml_cuda_op_norm(ctx, dst);
+ break;
+ case GGML_OP_GROUP_NORM:
+ ggml_cuda_op_group_norm(ctx, dst);
+ break;
+ case GGML_OP_CONCAT:
+ ggml_cuda_op_concat(ctx, dst);
+ break;
+ case GGML_OP_UPSCALE:
+ ggml_cuda_op_upscale(ctx, dst);
+ break;
+ case GGML_OP_PAD:
+ ggml_cuda_op_pad(ctx, dst);
+ break;
+ case GGML_OP_ARANGE:
+ ggml_cuda_op_arange(ctx, dst);
+ break;
+ case GGML_OP_TIMESTEP_EMBEDDING:
+ ggml_cuda_op_timestep_embedding(ctx, dst);
+ break;
+ case GGML_OP_LEAKY_RELU:
+ ggml_cuda_op_leaky_relu(ctx, dst);
+ break;
+ case GGML_OP_RMS_NORM:
+ ggml_cuda_op_rms_norm(ctx, dst);
+ break;
+ case GGML_OP_MUL_MAT:
+ if (dst->src[0]->ne[3] != dst->src[1]->ne[3]) {
+ GGML_CUDA_LOG_ERROR("%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, dst->name, dst->src[0]->ne[3], dst->src[1]->ne[3]);
+ return false;
+ } else {
+ ggml_cuda_mul_mat(ctx, dst->src[0], dst->src[1], dst);
+ }
+ break;
+ case GGML_OP_MUL_MAT_ID:
+ ggml_cuda_mul_mat_id(ctx, dst);
+ break;
+ case GGML_OP_SCALE:
+ ggml_cuda_op_scale(ctx, dst);
+ break;
+ case GGML_OP_SQR:
+ ggml_cuda_op_sqr(ctx, dst);
+ break;
+ case GGML_OP_SQRT:
+ ggml_cuda_op_sqrt(ctx, dst);
+ break;
+ case GGML_OP_SIN:
+ ggml_cuda_op_sin(ctx, dst);
+ break;
+ case GGML_OP_COS:
+ ggml_cuda_op_cos(ctx, dst);
+ break;
+ case GGML_OP_CLAMP:
+ ggml_cuda_op_clamp(ctx, dst);
+ break;
+ case GGML_OP_NONE:
+ case GGML_OP_RESHAPE:
+ case GGML_OP_VIEW:
+ case GGML_OP_PERMUTE:
+ case GGML_OP_TRANSPOSE:
+ break;
+ case GGML_OP_DIAG_MASK_INF:
+ ggml_cuda_op_diag_mask_inf(ctx, dst);
+ break;
+ case GGML_OP_SOFT_MAX:
+ ggml_cuda_op_soft_max(ctx, dst);
+ break;
+ case GGML_OP_ROPE:
+ ggml_cuda_op_rope(ctx, dst);
+ break;
+ case GGML_OP_IM2COL:
+ ggml_cuda_op_im2col(ctx, dst);
+ break;
+ case GGML_OP_CONV_TRANSPOSE_1D:
+ ggml_cuda_op_conv_transpose_1d(ctx,dst);
+ break;
+ case GGML_OP_POOL_2D:
+ ggml_cuda_op_pool2d(ctx, dst);
+ break;
+ case GGML_OP_SUM_ROWS:
+ ggml_cuda_op_sum_rows(ctx, dst);
+ break;
+ case GGML_OP_ARGSORT:
+ ggml_cuda_op_argsort(ctx, dst);
+ break;
+#if !defined(GGML_DISABLE_FLASH_ATTN)
+ case GGML_OP_FLASH_ATTN_EXT:
+ ggml_cuda_flash_attn_ext(ctx, dst);
+ break;
+#endif
+ case GGML_OP_CROSS_ENTROPY_LOSS:
+ ggml_cuda_cross_entropy_loss(ctx, dst);
+ break;
+ default:
+ return false;
+ }
+
+ cudaError_t err = cudaGetLastError();
+ if (err != cudaSuccess) {
+ GGML_CUDA_LOG_ERROR("%s: %s failed\n", __func__, ggml_op_desc(dst));
+ CUDA_CHECK(err);
+ }
+
+ return true;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// backend
+
+GGML_CALL static const char * ggml_backend_cuda_name(ggml_backend_t backend) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+
+ return cuda_ctx->name.c_str();
+}
+
+GGML_CALL static void ggml_backend_cuda_free(ggml_backend_t backend) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+
+ delete cuda_ctx;
+ delete backend;
+}
+
+GGML_CALL static ggml_backend_buffer_type_t ggml_backend_cuda_get_default_buffer_type(ggml_backend_t backend) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+
+ return ggml_backend_cuda_buffer_type(cuda_ctx->device);
+}
+
+GGML_CALL static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
+
+ GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");
+
+ CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));
+}
+
+GGML_CALL static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
+
+ GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");
+
+ CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));
+}
+
+GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {
+ ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;
+ ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;
+
+ if (!ggml_backend_is_cuda(backend_src) || !ggml_backend_is_cuda(backend_dst)) {
+ return false;
+ }
+
+ if (!ggml_backend_buffer_is_cuda(src->buffer) || !ggml_backend_buffer_is_cuda(dst->buffer)) {
+ return false;
+ }
+
+ // device -> device copy
+ ggml_backend_cuda_context * cuda_ctx_src = (ggml_backend_cuda_context *)backend_src->context;
+ ggml_backend_cuda_context * cuda_ctx_dst = (ggml_backend_cuda_context *)backend_dst->context;
+
+ ggml_backend_cuda_buffer_context * buf_ctx_src = (ggml_backend_cuda_buffer_context *)buf_src->context;
+ ggml_backend_cuda_buffer_context * buf_ctx_dst = (ggml_backend_cuda_buffer_context *)buf_dst->context;
+
+ if (cuda_ctx_src->device != buf_ctx_src->device || cuda_ctx_dst->device != buf_ctx_dst->device) {
+#ifndef NDEBUG
+ GGML_CUDA_LOG_WARN("%s: backend and buffer devices do not match\n", __func__);
+#endif
+ return false;
+ }
+
+ if (backend_src != backend_dst) {
+ // copy on src stream
+ if (cuda_ctx_src->device == cuda_ctx_dst->device) {
+ CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_src->stream()));
+ } else {
+#ifdef GGML_CUDA_NO_PEER_COPY
+ return false;
+#else
+ CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, cuda_ctx_dst->device, src->data, cuda_ctx_src->device, ggml_nbytes(dst), cuda_ctx_src->stream()));
+#endif
+ }
+
+ // record event on src stream after the copy
+ if (!cuda_ctx_src->copy_event) {
+ ggml_cuda_set_device(cuda_ctx_src->device);
+ CUDA_CHECK(cudaEventCreateWithFlags(&cuda_ctx_src->copy_event, cudaEventDisableTiming));
+ }
+
+ CUDA_CHECK(cudaEventRecord(cuda_ctx_src->copy_event, cuda_ctx_src->stream()));
+
+ // wait on dst stream for the copy to complete
+ CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx_dst->stream(), cuda_ctx_src->copy_event, 0));
+ } else {
+ // src and dst are on the same backend
+ CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_src->stream()));
+ }
+ return true;
+}
+
+GGML_CALL static void ggml_backend_cuda_synchronize(ggml_backend_t backend) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+
+ CUDA_CHECK(cudaStreamSynchronize(cuda_ctx->stream()));
+
+ GGML_UNUSED(backend);
+}
+
+static void set_ggml_graph_node_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
+ graph_node_properties->node_address = node->data;
+ graph_node_properties->node_op = node->op;
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
+ graph_node_properties->ne[i] = node->ne[i];
+ graph_node_properties->nb[i] = node->nb[i];
+ }
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ graph_node_properties->src_address[i] = node->src[i] ? node->src[i]->data : nullptr;
+ }
+}
+
+static bool ggml_graph_node_has_matching_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
+ if (node->data != graph_node_properties->node_address &&
+ node->op != GGML_OP_CPY &&
+ node->op != GGML_OP_VIEW) {
+ return false;
+ }
+
+ if (node->op != graph_node_properties->node_op) {
+ return false;
+ }
+
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
+ if (node->ne[i] != graph_node_properties->ne[i]) {
+ return false;
+ }
+ if (node->nb[i] != graph_node_properties->nb[i]) {
+ return false;
+ }
+ }
+
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ if (node->src[i] &&
+ node->src[i]->data != graph_node_properties->src_address[i] &&
+ node->op != GGML_OP_CPY &&
+ node->op != GGML_OP_VIEW
+ ) {
+ return false;
+ }
+ }
+ return true;
+}
+
+GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+
+ ggml_cuda_set_device(cuda_ctx->device);
+
+#ifdef USE_CUDA_GRAPH
+ static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr);
+
+ // Objects required for CUDA Graph
+ if (cuda_ctx->cuda_graph == nullptr) {
+ cuda_ctx->cuda_graph.reset(new ggml_cuda_graph());
+ }
+
+ bool use_cuda_graph = true;
+ bool cuda_graph_update_required = false;
+ // vector of pointers to CUDA cpy kernels, which are required to identify
+ // kernel parameters which need updated in the graph for each token
+ std::vector<void *> ggml_cuda_cpy_fn_ptrs;
+
+ if (cuda_ctx->cuda_graph->graph == nullptr) {
+ if (ggml_cuda_info().devices[cuda_ctx->device].cc < CC_AMPERE) {
+ cuda_ctx->cuda_graph->disable_due_to_gpu_arch = true;
+#ifndef NDEBUG
+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to GPU architecture\n", __func__);
+#endif
+ }
+ }
+
+ // Disable CUDA graphs in presence of env var, old GPU, use-case which is changing too rapidly,
+ // or previous graph capture failure.
+ // Also disable for multi-gpu for now. TO DO investigate
+ if (disable_cuda_graphs_due_to_env
+ || cuda_ctx->cuda_graph->disable_due_to_gpu_arch
+ || cuda_ctx->cuda_graph->disable_due_to_too_many_updates
+ || cuda_ctx->cuda_graph->disable_due_to_failed_graph_capture) {
+ use_cuda_graph = false;
+ }
+
+ if (use_cuda_graph) {
+ if (cuda_ctx->cuda_graph->instance == nullptr) {
+ cuda_graph_update_required = true;
+ }
+
+ // Check if the graph size has changed
+ if (cuda_ctx->cuda_graph->ggml_graph_properties.size() != (size_t)cgraph->n_nodes) {
+ cuda_graph_update_required = true;
+ cuda_ctx->cuda_graph->ggml_graph_properties.resize(cgraph->n_nodes);
+ }
+
+ // Loop over nodes in GGML graph to determine if CUDA graph update is required
+ // and store properties to allow this comparison for the next token
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ bool has_matching_properties = true;
+ if (!cuda_graph_update_required) {
+ has_matching_properties = ggml_graph_node_has_matching_properties(cgraph->nodes[i], &cuda_ctx->cuda_graph->ggml_graph_properties[i]);
+ }
+ if (!has_matching_properties) {
+ cuda_graph_update_required = true;
+ }
+ set_ggml_graph_node_properties(cgraph->nodes[i], &cuda_ctx->cuda_graph->ggml_graph_properties[i]);
+ }
+
+ // Loop over nodes in GGML graph to obtain info needed for CUDA graph
+ cuda_ctx->cuda_graph->updated_kernel_arg.clear();
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ ggml_tensor * node = cgraph->nodes[i];
+
+ if (node->src[0] && ggml_backend_buffer_is_cuda_split(node->src[0]->buffer)) {
+ use_cuda_graph = false; // Split buffers are not supported by CUDA graph capture
+#ifndef NDEBUG
+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to split buffer\n", __func__);
+#endif
+ }
+
+ if (node->op == GGML_OP_MUL_MAT_ID) {
+ use_cuda_graph = false; // This node type is not supported by CUDA graph capture
+#ifndef NDEBUG
+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to mul_mat_id\n", __func__);
+#endif
+ }
+
+ if (node->op == GGML_OP_ADD && node->src[1] && node->src[1]->ne[1] > 1) {
+ // disable CUDA graphs for batch size > 1 for now.
+ // Changes in batch size or context size can cause changes to the grid size of some kernels.
+ use_cuda_graph = false;
+#ifndef NDEBUG
+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to batch size > 1 [%s] [%ld %ld %ld %ld]\n", __func__, node->name, node->ne[0], node->ne[1], node->ne[2], node->ne[3]);
+#endif
+ }
+
+ if (node->op == GGML_OP_CPY) {
+ // store the copy op parameter which changes with each token.
+ cuda_ctx->cuda_graph->updated_kernel_arg.push_back((char **) &(node->src[1]->data));
+ // store a pointer to each copy op CUDA kernel to identify it later
+ void * ptr = ggml_cuda_cpy_fn(node->src[0], node->src[1]);
+ if (std::find(ggml_cuda_cpy_fn_ptrs.begin(), ggml_cuda_cpy_fn_ptrs.end(), ptr) == ggml_cuda_cpy_fn_ptrs.end()) {
+ ggml_cuda_cpy_fn_ptrs.push_back(ptr);
+ }
+ }
+
+ if (!use_cuda_graph) {
+ break;
+ }
+ }
+
+ // Disable CUDA graphs (from the next token) if the use-case is demanding too many consecutive graph updates.
+ if (use_cuda_graph && cuda_graph_update_required) {
+ cuda_ctx->cuda_graph->number_consecutive_updates++;
+ } else {
+ cuda_ctx->cuda_graph->number_consecutive_updates = 0;
+ }
+
+ if (cuda_ctx->cuda_graph->number_consecutive_updates >= 4) {
+ cuda_ctx->cuda_graph->disable_due_to_too_many_updates = true;
+#ifndef NDEBUG
+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to too many consecutive updates\n", __func__);
+#endif
+ }
+ }
+
+ if (use_cuda_graph && cuda_graph_update_required) { // Start CUDA graph capture
+ CUDA_CHECK(cudaStreamBeginCapture(cuda_ctx->stream(), cudaStreamCaptureModeRelaxed));
+ }
+
+#else
+ bool use_cuda_graph = false;
+ bool cuda_graph_update_required = false;
+#endif // USE_CUDA_GRAPH
+
+ bool graph_evaluated_or_captured = false;
+
+ while (!graph_evaluated_or_captured) {
+ // Only perform the graph execution if CUDA graphs are not enabled, or we are capturing the graph.
+ // With the use of CUDA graphs, the execution will be performed by the graph launch.
+ if (!use_cuda_graph || cuda_graph_update_required) {
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ ggml_tensor * node = cgraph->nodes[i];
+
+ if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) {
+ continue;
+ }
+
+#ifndef NDEBUG
+ assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ if (node->src[j] != nullptr) {
+ assert(node->src[j]->buffer);
+ assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || ggml_backend_buffer_is_cuda_split(node->src[j]->buffer));
+ }
+ }
+#endif
+
+ bool ok = ggml_cuda_compute_forward(*cuda_ctx, node);
+ if (!ok) {
+ GGML_CUDA_LOG_ERROR("%s: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op));
+ }
+ GGML_ASSERT(ok);
+ }
+ }
+
+#ifdef USE_CUDA_GRAPH
+ if (use_cuda_graph && cuda_graph_update_required) { // End CUDA graph capture
+ if (cuda_ctx->cuda_graph->graph != nullptr) {
+ CUDA_CHECK(cudaGraphDestroy(cuda_ctx->cuda_graph->graph));
+ cuda_ctx->cuda_graph->graph = nullptr;
+ }
+ CUDA_CHECK(cudaStreamEndCapture(cuda_ctx->stream(), &cuda_ctx->cuda_graph->graph));
+
+#if 0
+ if (disable_cuda_graphs_due_to_failed_capture) {
+ use_cuda_graph = false;
+ cuda_ctx->cuda_graph->disable_due_to_failed_graph_capture = true;
+#ifndef NDEBUG
+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to failed graph capture\n", __func__);
+#endif
+ } else {
+ graph_evaluated_or_captured = true; // CUDA graph has been captured
+ }
+#endif
+ graph_evaluated_or_captured = true; // CUDA graph has been captured
+ } else {
+ graph_evaluated_or_captured = true; // ggml graph has been directly evaluated
+ }
+ }
+
+ if (use_cuda_graph) {
+ if (cuda_ctx->cuda_graph->instance == nullptr) { // Create executable graph from captured graph.
+ CUDA_CHECK(cudaGraphInstantiate(&cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, NULL, NULL, 0));
+ }
+
+ // Perform update to graph (if required for this token), and change copy parameter (required for every token)
+
+ if (cuda_graph_update_required) {
+ // Extract nodes from graph
+ // First call with null argument gets number of nodes in graph
+ CUDA_CHECK(cudaGraphGetNodes(cuda_ctx->cuda_graph->graph, nullptr, &cuda_ctx->cuda_graph->num_nodes));
+ // Subsequent call with non-null argument gets nodes
+ cuda_ctx->cuda_graph->nodes.resize(cuda_ctx->cuda_graph->num_nodes);
+ cuda_ctx->cuda_graph->params.resize(cuda_ctx->cuda_graph->num_nodes);
+ if (cuda_ctx->cuda_graph->num_nodes > 0) {
+ CUDA_CHECK(cudaGraphGetNodes(cuda_ctx->cuda_graph->graph, cuda_ctx->cuda_graph->nodes.data(), &cuda_ctx->cuda_graph->num_nodes));
+
+ // Loop over nodes, and extract kernel parameters from each node
+ for (size_t i = 0; i < cuda_ctx->cuda_graph->num_nodes; i++) {
+ cudaGraphNodeType node_type;
+ CUDA_CHECK(cudaGraphNodeGetType(cuda_ctx->cuda_graph->nodes[i], &node_type));
+ if (node_type == cudaGraphNodeTypeKernel) {
+ cudaError_t stat = cudaGraphKernelNodeGetParams(cuda_ctx->cuda_graph->nodes[i], &cuda_ctx->cuda_graph->params[i]); // Get params using runtime
+ if (stat == cudaErrorInvalidDeviceFunction) {
+ // Fails due to incorrect handling by CUDA runtime of CUDA BLAS node.
+ // We don't need to update blas nodes, so clear error and move on.
+ cudaGetLastError();
+ } else {
+ GGML_ASSERT(stat == cudaSuccess);
+ }
+ }
+ }
+ }
+ }
+
+ // One of the arguments to the copy kernel is updated for each token, hence we need to
+ // replace that argument with the updated value in the CUDA graph
+ if (!cuda_graph_update_required) { // on update steps, the live parameters will already be captured
+ int k = 0;
+ for (size_t i = 0; i < cuda_ctx->cuda_graph->num_nodes; i++) {
+ if(count(ggml_cuda_cpy_fn_ptrs.begin(), ggml_cuda_cpy_fn_ptrs.end(), cuda_ctx->cuda_graph->params[i].func) > 0) {
+ char ** updated_kernel_arg_ptr = cuda_ctx->cuda_graph->updated_kernel_arg.at(k++);
+ cuda_ctx->cuda_graph->params[i].kernelParams[1] = updated_kernel_arg_ptr;
+ CUDA_CHECK(cudaGraphKernelNodeSetParams(cuda_ctx->cuda_graph->nodes[i], &cuda_ctx->cuda_graph->params[i]));
+ }
+ }
+ }
+
+ // Update graph executable
+ cudaGraphExecUpdateResultInfo result_info;
+ cudaError_t stat = cudaGraphExecUpdate(cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, &result_info);
+ if (stat == cudaErrorGraphExecUpdateFailure) {
+#ifndef NDEBUG
+ GGML_CUDA_LOG_ERROR("%s: CUDA graph update failed\n", __func__);
+#endif
+ // The pre-existing graph exec cannot be updated due to violated constraints
+ // so instead clear error and re-instantiate
+ cudaGetLastError();
+ CUDA_CHECK(cudaGraphExecDestroy(cuda_ctx->cuda_graph->instance));
+ cuda_ctx->cuda_graph->instance = nullptr;
+ CUDA_CHECK(cudaGraphInstantiate(&cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, NULL, NULL, 0));
+ } else {
+ GGML_ASSERT(stat == cudaSuccess);
+ }
+ // Launch graph
+ CUDA_CHECK(cudaGraphLaunch(cuda_ctx->cuda_graph->instance, cuda_ctx->stream()));
+#else
+ graph_evaluated_or_captured = true;
+#endif // USE_CUDA_GRAPH
+ }
+
+ return GGML_STATUS_SUCCESS;
+}
+
+GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_tensor * op) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
+ switch (op->op) {
+ case GGML_OP_UNARY:
+ switch (ggml_get_unary_op(op)) {
+ case GGML_UNARY_OP_GELU:
+ case GGML_UNARY_OP_SILU:
+ case GGML_UNARY_OP_RELU:
+ case GGML_UNARY_OP_SIGMOID:
+ case GGML_UNARY_OP_HARDSIGMOID:
+ case GGML_UNARY_OP_HARDSWISH:
+ case GGML_UNARY_OP_GELU_QUICK:
+ case GGML_UNARY_OP_TANH:
+ return ggml_is_contiguous(op->src[0]);
+ default:
+ return false;
+ }
+ break;
+ case GGML_OP_MUL_MAT:
+ case GGML_OP_MUL_MAT_ID:
+ {
+ struct ggml_tensor * a = op->src[0];
+ struct ggml_tensor * b = op->src[1];
+ if (b->type == GGML_TYPE_F16 && a->type != GGML_TYPE_F16) {
+ return false;
+ }
+ if (op->op == GGML_OP_MUL_MAT && a->ne[3] != b->ne[3]) {
+ return false;
+ }
+ switch (a->type) {
+ case GGML_TYPE_F32:
+ case GGML_TYPE_F16:
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_Q8_K:
+ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ return true;
+ default:
+ return false;
+ }
+ } break;
+ case GGML_OP_GET_ROWS:
+ {
+ switch (op->src[0]->type) {
+ case GGML_TYPE_F16:
+ case GGML_TYPE_F32:
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ return true;
+ default:
+ return false;
+ }
+ } break;
+ case GGML_OP_CPY:
+ {
+ ggml_type src0_type = op->src[0]->type;
+ ggml_type src1_type = op->src[1]->type;
+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F16) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q8_0) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_0) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_1) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_0) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_1) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_IQ4_NL) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) {
+ return true;
+ }
+ return false;
+ } break;
+ case GGML_OP_DUP:
+ case GGML_OP_REPEAT:
+ case GGML_OP_CONCAT:
+ {
+ ggml_type src0_type = op->src[0]->type;
+ return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16;
+ } break;
+ case GGML_OP_CONV_TRANSPOSE_1D:
+ {
+ ggml_type src0_type = op->src[0]->type;
+ ggml_type src1_type = op->src[1]->type;
+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) {
+ return true;
+ }
+ return false;
+ } break;
+ case GGML_OP_NONE:
+ case GGML_OP_RESHAPE:
+ case GGML_OP_VIEW:
+ case GGML_OP_PERMUTE:
+ case GGML_OP_TRANSPOSE:
+ case GGML_OP_NORM:
+ case GGML_OP_ADD:
+ case GGML_OP_SUB:
+ case GGML_OP_MUL:
+ case GGML_OP_DIV:
+ case GGML_OP_RMS_NORM:
+ case GGML_OP_SCALE:
+ case GGML_OP_SQR:
+ case GGML_OP_SQRT:
+ case GGML_OP_SIN:
+ case GGML_OP_COS:
+ case GGML_OP_CLAMP:
+ case GGML_OP_CONT:
+ case GGML_OP_DIAG_MASK_INF:
+ case GGML_OP_SOFT_MAX:
+ return true;
+ case GGML_OP_ROPE:
+ return ggml_is_contiguous(op->src[0]);
+ case GGML_OP_IM2COL:
+ case GGML_OP_POOL_2D:
+ case GGML_OP_SUM_ROWS:
+ case GGML_OP_ARGSORT:
+ case GGML_OP_ACC:
+ case GGML_OP_GROUP_NORM:
+ case GGML_OP_UPSCALE:
+ case GGML_OP_PAD:
+ case GGML_OP_ARANGE:
+ case GGML_OP_TIMESTEP_EMBEDDING:
+ case GGML_OP_LEAKY_RELU:
+ return true;
+ case GGML_OP_FLASH_ATTN_EXT:
+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+ return (op->src[0]->ne[0] == 64 && op->src[1]->type == GGML_TYPE_F16) || op->src[0]->ne[0] == 128;
+#else
+ if (op->src[0]->ne[0] == 128) {
+ return true;
+ }
+ if (op->src[0]->ne[0] == 64 && op->src[1]->type == GGML_TYPE_F16) {
+ return true;
+ }
+ return ggml_cuda_info().devices[cuda_ctx->device].cc >= CC_VOLTA &&
+ op->src[1]->type == GGML_TYPE_F16 && op->src[2]->type == GGML_TYPE_F16;
+ case GGML_OP_CROSS_ENTROPY_LOSS:
+ return true;
+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+ default:
+ return false;
+ }
+
+ GGML_UNUSED(backend);
+}
+
+GGML_CALL static bool ggml_backend_cuda_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
+ if (ggml_backend_buft_is_cuda_split(buft)) {
+ return true;
+ }
+
+ if (ggml_backend_buft_is_cuda(buft)) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+ ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
+ return buft_ctx->device == cuda_ctx->device;
+ }
+
+ return false;
+}
+
+GGML_CALL static bool ggml_backend_cuda_offload_op(ggml_backend_t backend, const ggml_tensor * op) {
+ const int min_batch_size = 32;
+
+ return (op->ne[1] >= min_batch_size && op->op != GGML_OP_GET_ROWS) ||
+ (op->ne[2] >= min_batch_size && op->op == GGML_OP_MUL_MAT_ID);
+
+ GGML_UNUSED(backend);
+}
+
+static ggml_backend_event_t ggml_backend_cuda_event_new(ggml_backend_t backend) {
+#ifdef GGML_CUDA_NO_PEER_COPY
+ return nullptr;
+#else
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+
+ ggml_cuda_set_device(cuda_ctx->device);
+
+ cudaEvent_t event;
+ CUDA_CHECK(cudaEventCreateWithFlags(&event, cudaEventDisableTiming));
+
+ return new ggml_backend_event {
+ /* .backend = */ backend,
+ /* .context = */ event,
+ };
+#endif
+}
+
+static void ggml_backend_cuda_event_free(ggml_backend_event_t event) {
+ CUDA_CHECK(cudaEventDestroy((cudaEvent_t)event->context));
+
+ delete event;
+}
+
+static void ggml_backend_cuda_event_record(ggml_backend_event_t event) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)event->backend->context;
+
+ CUDA_CHECK(cudaEventRecord((cudaEvent_t)event->context, cuda_ctx->stream()));
+}
+
+static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+
+ if (ggml_backend_is_cuda(event->backend)) {
+ CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx->stream(), (cudaEvent_t)event->context, 0));
+ } else {
+#if 0
+ // untested
+ auto wait_fn = [](void * user_data) {
+ ggml_backend_event_t event = (ggml_backend_event_t)user_data;
+ ggml_backend_event_synchronize(event);
+ };
+
+ CUDA_CHECK(cudaLaunchHostFunc(cuda_ctx->stream(), wait_fn, event));
+#endif
+ GGML_ABORT("fatal error");
+ }
+}
+
+static void ggml_backend_cuda_event_synchronize(ggml_backend_event_t event) {
+ CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));
+}
+
+static ggml_backend_i ggml_backend_cuda_interface = {
+ /* .get_name = */ ggml_backend_cuda_name,
+ /* .free = */ ggml_backend_cuda_free,
+ /* .get_default_buffer_type = */ ggml_backend_cuda_get_default_buffer_type,
+ /* .set_tensor_async = */ ggml_backend_cuda_set_tensor_async,
+ /* .get_tensor_async = */ ggml_backend_cuda_get_tensor_async,
+ /* .cpy_tensor_async = */ ggml_backend_cuda_cpy_tensor_async,
+ /* .synchronize = */ ggml_backend_cuda_synchronize,
+ /* .graph_plan_create = */ NULL,
+ /* .graph_plan_free = */ NULL,
+ /* .graph_plan_update = */ NULL,
+ /* .graph_plan_compute = */ NULL,
+ /* .graph_compute = */ ggml_backend_cuda_graph_compute,
+ /* .supports_op = */ ggml_backend_cuda_supports_op,
+ /* .supports_buft = */ ggml_backend_cuda_supports_buft,
+ /* .offload_op = */ ggml_backend_cuda_offload_op,
+ /* .event_new = */ ggml_backend_cuda_event_new,
+ /* .event_free = */ ggml_backend_cuda_event_free,
+ /* .event_record = */ ggml_backend_cuda_event_record,
+ /* .event_wait = */ ggml_backend_cuda_event_wait,
+ /* .event_synchronize = */ ggml_backend_cuda_event_synchronize,
+};
+
+static ggml_guid_t ggml_backend_cuda_guid() {
+ static ggml_guid guid = { 0x2c, 0xdd, 0xe8, 0x1c, 0x65, 0xb3, 0x65, 0x73, 0x6a, 0x12, 0x88, 0x61, 0x1c, 0xc9, 0xdc, 0x25 };
+ return &guid;
+}
+
+GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {
+ if (device < 0 || device >= ggml_backend_cuda_get_device_count()) {
+ GGML_CUDA_LOG_ERROR("%s: invalid device %d\n", __func__, device);
+ return nullptr;
+ }
+
+ ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context(device);
+ if (ctx == nullptr) {
+ GGML_CUDA_LOG_ERROR("%s: failed to allocate context\n", __func__);
+ return nullptr;
+ }
+
+ ggml_backend_t cuda_backend = new ggml_backend {
+ /* .guid = */ ggml_backend_cuda_guid(),
+ /* .interface = */ ggml_backend_cuda_interface,
+ /* .context = */ ctx
+ };
+
+ return cuda_backend;
+}
+
+GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend) {
+ return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cuda_guid());
+}
+
+GGML_CALL int ggml_backend_cuda_get_device_count() {
+ return ggml_cuda_info().device_count;
+}
+
+GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) {
+ cudaDeviceProp prop;
+ CUDA_CHECK(cudaGetDeviceProperties(&prop, device));
+ snprintf(description, description_size, "%s", prop.name);
+}
+
+GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) {
+ ggml_cuda_set_device(device);
+
+ CUDA_CHECK(cudaMemGetInfo(free, total));
+}
+
+GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
+ if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
+ return false;
+ }
+
+#if CUDART_VERSION >= 11100 || defined(GGML_USE_MUSA)
+ cudaError_t err = cudaHostRegister(buffer, size, cudaHostRegisterPortable | cudaHostRegisterReadOnly);
+ if (err != cudaSuccess) {
+ // clear the error
+ cudaGetLastError();
+
+ GGML_CUDA_LOG_WARN("%s: failed to register %.2f MiB of pinned memory: %s\n", __func__,
+ size / 1024.0 / 1024.0, cudaGetErrorString(err));
+ return false;
+ }
+ return true;
+#else
+ return false;
+#endif
+}
+
+GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer) {
+ if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
+ return;
+ }
+
+ cudaError_t err = cudaHostUnregister(buffer);
+ if (err != cudaSuccess) {
+ // clear the error
+ cudaGetLastError();
+ }
+}
+
+// backend registry
+GGML_CALL static ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) {
+ ggml_backend_t cuda_backend = ggml_backend_cuda_init((int) (intptr_t) user_data);
+ return cuda_backend;
+
+ GGML_UNUSED(params);
+}
+
+GGML_CALL int ggml_backend_cuda_reg_devices() {
+ int device_count = ggml_backend_cuda_get_device_count();
+ //int device_count = 1; // DEBUG: some tools require delaying CUDA initialization
+ for (int i = 0; i < device_count; i++) {
+ char name[128];
+ snprintf(name, sizeof(name), "%s%d", GGML_CUDA_NAME, i);
+ ggml_backend_register(name, ggml_backend_reg_cuda_init, ggml_backend_cuda_buffer_type(i), (void *) (intptr_t) i);
+ }
+ return device_count;
+}
diff --git a/llama/ggml-cuda.h b/llama/ggml-cuda.h
new file mode 100644
index 00000000..a08cd816
--- /dev/null
+++ b/llama/ggml-cuda.h
@@ -0,0 +1,75 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#ifdef GGML_USE_HIPBLAS
+#define GGML_CUDA_NAME "ROCm"
+#define GGML_CUBLAS_NAME "hipBLAS"
+#elif defined(GGML_USE_MUSA)
+#define GGML_CUDA_NAME "MUSA"
+#define GGML_CUBLAS_NAME "muBLAS"
+#else
+#define GGML_CUDA_NAME "CUDA"
+#define GGML_CUBLAS_NAME "cuBLAS"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define GGML_CUDA_MAX_DEVICES 16
+
+// backend API
+GGML_API GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device);
+
+GGML_API GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend);
+
+// device buffer
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);
+
+// split tensor buffer that splits matrices by rows across multiple devices
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split);
+
+// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);
+
+GGML_API GGML_CALL int ggml_backend_cuda_reg_devices();
+
+GGML_API GGML_CALL int ggml_backend_cuda_get_device_count(void);
+GGML_API GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);
+GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);
+
+GGML_API GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);
+GGML_API GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer);
+
+GGML_API void ggml_backend_cuda_log_set_callback(ggml_log_callback log_callback, void * user_data);
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama/ggml-cuda/acc.cu b/llama/ggml-cuda/acc.cu
new file mode 100644
index 00000000..7c6d6ea0
--- /dev/null
+++ b/llama/ggml-cuda/acc.cu
@@ -0,0 +1,73 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "acc.cuh"
+
+static __global__ void acc_f32(const float * x, const float * y, float * dst, const int ne,
+ const int ne10, const int ne11, const int ne12,
+ const int nb1, const int nb2, int offset) {
+ const int i = blockDim.x * blockIdx.x + threadIdx.x;
+ if (i >= ne) {
+ return;
+ }
+ int src1_idx = i - offset;
+ int oz = src1_idx / nb2;
+ int oy = (src1_idx - (oz * nb2)) / nb1;
+ int ox = src1_idx % nb1;
+ if (src1_idx >= 0 && ox < ne10 && oy < ne11 && oz < ne12) {
+ dst[i] = x[i] + y[ox + oy * ne10 + oz * ne10 * ne11];
+ } else {
+ dst[i] = x[i];
+ }
+}
+
+static void acc_f32_cuda(const float * x, const float * y, float * dst, const int n_elements,
+ const int ne10, const int ne11, const int ne12,
+ const int nb1, const int nb2, const int offset, cudaStream_t stream) {
+ int num_blocks = (n_elements + CUDA_ACC_BLOCK_SIZE - 1) / CUDA_ACC_BLOCK_SIZE;
+ acc_f32<<<num_blocks, CUDA_ACC_BLOCK_SIZE, 0, stream>>>(x, y, dst, n_elements, ne10, ne11, ne12, nb1, nb2, offset);
+}
+
+void ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ const float * src0_d = (const float *)src0->data;
+ const float * src1_d = (const float *)src1->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+ GGML_ASSERT(dst->ne[3] == 1); // just 3D tensors supported
+
+ int nb1 = dst->op_params[0] / 4; // 4 bytes of float32
+ int nb2 = dst->op_params[1] / 4; // 4 bytes of float32
+ // int nb3 = dst->op_params[2] / 4; // 4 bytes of float32 - unused
+ int offset = dst->op_params[3] / 4; // offset in bytes
+
+ acc_f32_cuda(src0_d, src1_d, dst_d, ggml_nelements(dst), src1->ne[0], src1->ne[1], src1->ne[2], nb1, nb2, offset, stream);
+}
diff --git a/llama/ggml-cuda/acc.cuh b/llama/ggml-cuda/acc.cuh
new file mode 100644
index 00000000..23f09703
--- /dev/null
+++ b/llama/ggml-cuda/acc.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_ACC_BLOCK_SIZE 256
+
+void ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/arange.cu b/llama/ggml-cuda/arange.cu
new file mode 100644
index 00000000..c36ac823
--- /dev/null
+++ b/llama/ggml-cuda/arange.cu
@@ -0,0 +1,60 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "arange.cuh"
+
+static __global__ void arange_f32(float * dst, const int ne0, const float start, const float step) {
+ // blockIDx.x: idx of ne0 / BLOCK_SIZE
+ int nidx = threadIdx.x + blockIdx.x * blockDim.x;
+ if (nidx >= ne0) {
+ return;
+ }
+ dst[nidx] = start + step * nidx;
+}
+
+static void arange_f32_cuda(float * dst, const int ne0, const float start, const float step, cudaStream_t stream) {
+ int num_blocks = (ne0 + CUDA_ARANGE_BLOCK_SIZE - 1) / CUDA_ARANGE_BLOCK_SIZE;
+ arange_f32<<<num_blocks, CUDA_ARANGE_BLOCK_SIZE, 0, stream>>>(dst, ne0, start, step);
+}
+
+void ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
+
+ float start;
+ float stop;
+ float step;
+ memcpy(&start, (float *)dst->op_params + 0, sizeof(float));
+ memcpy(&stop, (float *)dst->op_params + 1, sizeof(float));
+ memcpy(&step, (float *)dst->op_params + 2, sizeof(float));
+
+ int64_t steps = (int64_t)ceil((stop - start) / step);
+ GGML_ASSERT(ggml_nelements(dst) == steps);
+
+ arange_f32_cuda(dst_d, dst->ne[0], start, step, stream);
+}
diff --git a/llama/ggml-cuda/arange.cuh b/llama/ggml-cuda/arange.cuh
new file mode 100644
index 00000000..186f90c0
--- /dev/null
+++ b/llama/ggml-cuda/arange.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_ARANGE_BLOCK_SIZE 256
+
+void ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/argsort.cu b/llama/ggml-cuda/argsort.cu
new file mode 100644
index 00000000..85f62458
--- /dev/null
+++ b/llama/ggml-cuda/argsort.cu
@@ -0,0 +1,130 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "argsort.cuh"
+
+template<typename T>
+static inline __device__ void ggml_cuda_swap(T & a, T & b) {
+ T tmp = a;
+ a = b;
+ b = tmp;
+}
+
+template<ggml_sort_order order>
+static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols, int ncols_pad) {
+ // bitonic sort
+ int col = threadIdx.x;
+ int row = blockIdx.y;
+
+ if (col >= ncols_pad) {
+ return;
+ }
+
+ const float * x_row = x + row * ncols;
+ extern __shared__ int dst_row[];
+
+ // initialize indices
+ dst_row[col] = col;
+
+ __syncthreads();
+
+ for (int k = 2; k <= ncols_pad; k *= 2) {
+ for (int j = k / 2; j > 0; j /= 2) {
+ int ixj = col ^ j;
+ if (ixj > col) {
+ if ((col & k) == 0) {
+ if (dst_row[col] >= ncols ||
+ (dst_row[ixj] < ncols && (order == GGML_SORT_ORDER_ASC ?
+ x_row[dst_row[col]] > x_row[dst_row[ixj]] :
+ x_row[dst_row[col]] < x_row[dst_row[ixj]]))
+ ) {
+ ggml_cuda_swap(dst_row[col], dst_row[ixj]);
+ }
+ } else {
+ if (dst_row[ixj] >= ncols ||
+ (dst_row[col] < ncols && (order == GGML_SORT_ORDER_ASC ?
+ x_row[dst_row[col]] < x_row[dst_row[ixj]] :
+ x_row[dst_row[col]] > x_row[dst_row[ixj]]))
+ ) {
+ ggml_cuda_swap(dst_row[col], dst_row[ixj]);
+ }
+ }
+ }
+ __syncthreads();
+ }
+ }
+
+ // copy the result to dst without the padding
+ if (col < ncols) {
+ dst[row * ncols + col] = dst_row[col];
+ }
+}
+
+static int next_power_of_2(int x) {
+ int n = 1;
+ while (n < x) {
+ n *= 2;
+ }
+ return n;
+}
+
+static void argsort_f32_i32_cuda(const float * x, int * dst, const int ncols, const int nrows, ggml_sort_order order, cudaStream_t stream) {
+ // bitonic sort requires ncols to be power of 2
+ const int ncols_pad = next_power_of_2(ncols);
+
+ const dim3 block_dims(ncols_pad, 1, 1);
+ const dim3 block_nums(1, nrows, 1);
+ const size_t shared_mem = ncols_pad * sizeof(int);
+
+ // FIXME: this limit could be raised by ~2-4x on Ampere or newer
+ GGML_ASSERT(shared_mem <= ggml_cuda_info().devices[ggml_cuda_get_device()].smpb);
+
+ if (order == GGML_SORT_ORDER_ASC) {
+ k_argsort_f32_i32<GGML_SORT_ORDER_ASC><<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);
+ } else if (order == GGML_SORT_ORDER_DESC) {
+ k_argsort_f32_i32<GGML_SORT_ORDER_DESC><<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);
+ } else {
+ GGML_ABORT("fatal error");
+ }
+}
+
+void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_I32);
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ const int64_t ncols = src0->ne[0];
+ const int64_t nrows = ggml_nrows(src0);
+
+ enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];
+
+ argsort_f32_i32_cuda(src0_d, (int *)dst_d, ncols, nrows, order, stream);
+}
diff --git a/llama/ggml-cuda/argsort.cuh b/llama/ggml-cuda/argsort.cuh
new file mode 100644
index 00000000..7fc2c9d7
--- /dev/null
+++ b/llama/ggml-cuda/argsort.cuh
@@ -0,0 +1,29 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/binbcast.cu b/llama/ggml-cuda/binbcast.cu
new file mode 100644
index 00000000..4dc774d0
--- /dev/null
+++ b/llama/ggml-cuda/binbcast.cu
@@ -0,0 +1,314 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "binbcast.cuh"
+
+static __device__ __forceinline__ float op_repeat(const float a, const float b) {
+ return b;
+ GGML_UNUSED(a);
+}
+
+static __device__ __forceinline__ float op_add(const float a, const float b) {
+ return a + b;
+}
+
+static __device__ __forceinline__ float op_sub(const float a, const float b) {
+ return a - b;
+}
+
+static __device__ __forceinline__ float op_mul(const float a, const float b) {
+ return a * b;
+}
+
+static __device__ __forceinline__ float op_div(const float a, const float b) {
+ return a / b;
+}
+
+template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>
+static __global__ void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst,
+ int ne0, int ne1, int ne2, int ne3,
+ int ne10, int ne11, int ne12, int ne13,
+ /*int s0, */ int s1, int s2, int s3,
+ /*int s00,*/ int s01, int s02, int s03,
+ /*int s10,*/ int s11, int s12, int s13) {
+ const int i0s = blockDim.x*blockIdx.x + threadIdx.x;
+ const int i1 = (blockDim.y*blockIdx.y + threadIdx.y);
+ const int i2 = (blockDim.z*blockIdx.z + threadIdx.z) / ne3;
+ const int i3 = (blockDim.z*blockIdx.z + threadIdx.z) % ne3;
+
+ if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {
+ return;
+ }
+
+ const int i11 = i1 % ne11;
+ const int i12 = i2 % ne12;
+ const int i13 = i3 % ne13;
+
+ const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;
+ const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;
+ const size_t i_dst = i3*s3 + i2*s2 + i1*s1;
+
+ const src0_t * src0_row = src0 + i_src0;
+ const src1_t * src1_row = src1 + i_src1;
+ dst_t * dst_row = dst + i_dst;
+
+ for (int i0 = i0s; i0 < ne0; i0 += blockDim.x*gridDim.x) {
+ const int i10 = i0 % ne10;
+ dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);
+ }
+}
+
+template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>
+static __global__ void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t * dst,
+ int ne0, int ne1, int ne2, int ne3,
+ int ne10, int ne11, int ne12, int ne13,
+ /*int s0, */ int s1, int s2, int s3,
+ /*int s00,*/ int s01, int s02, int s03,
+ /*int s10,*/ int s11, int s12, int s13) {
+
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ const int i3 = i/(ne2*ne1*ne0);
+ const int i2 = (i/(ne1*ne0)) % ne2;
+ const int i1 = (i/ne0) % ne1;
+ const int i0 = i % ne0;
+
+ if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {
+ return;
+ }
+
+ const int i11 = i1 % ne11;
+ const int i12 = i2 % ne12;
+ const int i13 = i3 % ne13;
+
+ const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;
+ const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;
+ const size_t i_dst = i3*s3 + i2*s2 + i1*s1;
+
+ const src0_t * src0_row = src0 + i_src0;
+ const src1_t * src1_row = src1 + i_src1;
+ dst_t * dst_row = dst + i_dst;
+
+ const int i10 = i0 % ne10;
+ dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);
+}
+
+template<float (*bin_op)(const float, const float)>
+struct bin_bcast_cuda {
+ template<typename src0_t, typename src1_t, typename dst_t>
+ void operator()(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst,
+ const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,
+ cudaStream_t stream) {
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ int nr0 = ne10/ne0;
+ int nr1 = ne11/ne1;
+ int nr2 = ne12/ne2;
+ int nr3 = ne13/ne3;
+
+ int nr[4] = { nr0, nr1, nr2, nr3 };
+
+ // collapse dimensions until first broadcast dimension
+ int64_t cne[] = {ne0, ne1, ne2, ne3};
+ int64_t cne0[] = {ne00, ne01, ne02, ne03};
+ int64_t cne1[] = {ne10, ne11, ne12, ne13};
+
+ size_t cnb[] = {nb0, nb1, nb2, nb3};
+ size_t cnb0[] = {nb00, nb01, nb02, nb03};
+ size_t cnb1[] = {nb10, nb11, nb12, nb13};
+
+ auto collapse = [](int64_t cne[]) {
+ cne[0] *= cne[1];
+ cne[1] = cne[2];
+ cne[2] = cne[3];
+ cne[3] = 1;
+ };
+
+ auto collapse_nb = [](size_t cnb[], const int64_t cne[]) {
+ cnb[1] *= cne[1];
+ cnb[2] *= cne[2];
+ cnb[3] *= cne[3];
+ };
+
+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)) {
+ for (int i = 0; i < 4; i++) {
+ if (nr[i] != 1) {
+ break;
+ }
+ if (i > 0) {
+ collapse_nb(cnb, cne);
+ collapse_nb(cnb0, cne0);
+ collapse_nb(cnb1, cne1);
+ collapse(cne);
+ collapse(cne0);
+ collapse(cne1);
+ }
+ }
+ }
+
+ {
+ int64_t ne0 = cne[0];
+ int64_t ne1 = cne[1];
+ int64_t ne2 = cne[2];
+ int64_t ne3 = cne[3];
+
+ //int64_t ne00 = cne0[0]; GGML_UNUSED(ne00);
+ //int64_t ne01 = cne0[1]; GGML_UNUSED(ne01);
+ //int64_t ne02 = cne0[2]; GGML_UNUSED(ne02);
+ //int64_t ne03 = cne0[3]; GGML_UNUSED(ne03);
+
+ int64_t ne10 = cne1[0];
+ int64_t ne11 = cne1[1];
+ int64_t ne12 = cne1[2];
+ int64_t ne13 = cne1[3];
+
+ size_t nb0 = cnb[0];
+ size_t nb1 = cnb[1];
+ size_t nb2 = cnb[2];
+ size_t nb3 = cnb[3];
+
+ size_t nb00 = cnb0[0];
+ size_t nb01 = cnb0[1];
+ size_t nb02 = cnb0[2];
+ size_t nb03 = cnb0[3];
+
+ size_t nb10 = cnb1[0];
+ size_t nb11 = cnb1[1];
+ size_t nb12 = cnb1[2];
+ size_t nb13 = cnb1[3];
+
+ size_t s0 = nb0 / sizeof(dst_t);
+ size_t s1 = nb1 / sizeof(dst_t);
+ size_t s2 = nb2 / sizeof(dst_t);
+ size_t s3 = nb3 / sizeof(dst_t);
+
+ size_t s10 = nb10 / sizeof(src1_t);
+ size_t s11 = nb11 / sizeof(src1_t);
+ size_t s12 = nb12 / sizeof(src1_t);
+ size_t s13 = nb13 / sizeof(src1_t);
+
+ size_t s00 = nb00 / sizeof(src0_t);
+ size_t s01 = nb01 / sizeof(src0_t);
+ size_t s02 = nb02 / sizeof(src0_t);
+ size_t s03 = nb03 / sizeof(src0_t);
+
+ GGML_ASSERT(nb0 % sizeof(dst_t) == 0);
+ GGML_ASSERT(nb1 % sizeof(dst_t) == 0);
+ GGML_ASSERT(nb2 % sizeof(dst_t) == 0);
+ GGML_ASSERT(nb3 % sizeof(dst_t) == 0);
+
+ GGML_ASSERT(nb00 % sizeof(src0_t) == 0);
+ GGML_ASSERT(nb01 % sizeof(src0_t) == 0);
+ GGML_ASSERT(nb02 % sizeof(src0_t) == 0);
+ GGML_ASSERT(nb03 % sizeof(src0_t) == 0);
+
+ GGML_ASSERT(nb10 % sizeof(src1_t) == 0);
+ GGML_ASSERT(nb11 % sizeof(src1_t) == 0);
+ GGML_ASSERT(nb12 % sizeof(src1_t) == 0);
+ GGML_ASSERT(nb13 % sizeof(src1_t) == 0);
+
+ GGML_ASSERT(s0 == 1);
+ GGML_ASSERT(s00 == 1);
+ GGML_ASSERT(s10 == 1);
+
+ const int block_size = 128;
+
+ int64_t hne0 = std::max(ne0/2LL, 1LL);
+
+ dim3 block_dims;
+ block_dims.x = std::min<unsigned int>(hne0, block_size);
+ block_dims.y = std::min<unsigned int>(ne1, block_size / block_dims.x);
+ block_dims.z = std::min(std::min<unsigned int>(ne2*ne3, block_size / block_dims.x / block_dims.y), 64U);
+
+ dim3 block_nums(
+ (hne0 + block_dims.x - 1) / block_dims.x,
+ (ne1 + block_dims.y - 1) / block_dims.y,
+ (ne2*ne3 + block_dims.z - 1) / block_dims.z
+ );
+
+ if (block_nums.z > 65535) {
+ // this is the maximum number of blocks in z dimension, fallback to 1D grid kernel
+ int block_num = (ne0*ne1*ne2*ne3 + block_size - 1) / block_size;
+ k_bin_bcast_unravel<bin_op><<<block_num, block_size, 0, stream>>>(
+ src0_dd, src1_dd, dst_dd,
+ ne0, ne1, ne2, ne3,
+ ne10, ne11, ne12, ne13,
+ /* s0, */ s1, s2, s3,
+ /* s00, */ s01, s02, s03,
+ /* s10, */ s11, s12, s13);
+ } else {
+ k_bin_bcast<bin_op><<<block_nums, block_dims, 0, stream>>>(
+ src0_dd, src1_dd, dst_dd,
+ ne0, ne1, ne2, ne3,
+ ne10, ne11, ne12, ne13,
+ /* s0, */ s1, s2, s3,
+ /* s00, */ s01, s02, s03,
+ /* s10, */ s11, s12, s13);
+ }
+ }
+ }
+};
+
+template<class op>
+static void ggml_cuda_op_bin_bcast(
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
+ const void * src0_dd, const void * src1_dd, void * dst_dd, cudaStream_t stream) {
+
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
+ op()(src0, src1, dst, (const float *)src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);
+ } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {
+ op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (half *) dst_dd, stream);
+ } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {
+ op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);
+ } else {
+ fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__,
+ ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));
+ GGML_ABORT("fatal error");
+ }
+}
+
+void ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_repeat>>(dst, dst->src[0], dst, nullptr, dst->src[0]->data, dst->data, ctx.stream());
+}
+
+void ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_add>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
+}
+
+void ggml_cuda_op_sub(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_sub>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
+}
+
+void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_mul>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
+}
+
+void ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_div>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
+}
diff --git a/llama/ggml-cuda/binbcast.cuh b/llama/ggml-cuda/binbcast.cuh
new file mode 100644
index 00000000..e433e538
--- /dev/null
+++ b/llama/ggml-cuda/binbcast.cuh
@@ -0,0 +1,33 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+void ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+void ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+void ggml_cuda_op_sub(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+void ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/clamp.cu b/llama/ggml-cuda/clamp.cu
new file mode 100644
index 00000000..8f5f2eda
--- /dev/null
+++ b/llama/ggml-cuda/clamp.cu
@@ -0,0 +1,60 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "clamp.cuh"
+
+static __global__ void clamp_f32(const float * x, float * dst, const float min, const float max, const int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+
+ dst[i] = x[i] < min ? min : (x[i] > max ? max : x[i]);
+}
+
+static void clamp_f32_cuda(const float * x, float * dst, const float min, const float max, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_CLAMP_BLOCK_SIZE - 1) / CUDA_CLAMP_BLOCK_SIZE;
+ clamp_f32<<<num_blocks, CUDA_CLAMP_BLOCK_SIZE, 0, stream>>>(x, dst, min, max, k);
+}
+
+
+void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ float min;
+ float max;
+ memcpy(&min, dst->op_params, sizeof(float));
+ memcpy(&max, (float *) dst->op_params + 1, sizeof(float));
+
+ clamp_f32_cuda(src0_d, dst_d, min, max, ggml_nelements(src0), stream);
+}
diff --git a/llama/ggml-cuda/clamp.cuh b/llama/ggml-cuda/clamp.cuh
new file mode 100644
index 00000000..e0a3c3e4
--- /dev/null
+++ b/llama/ggml-cuda/clamp.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_CLAMP_BLOCK_SIZE 256
+
+void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/common.cuh b/llama/ggml-cuda/common.cuh
new file mode 100644
index 00000000..bd3838b1
--- /dev/null
+++ b/llama/ggml-cuda/common.cuh
@@ -0,0 +1,695 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "ggml.h"
+#include "ggml-cuda.h"
+
+#include <cstdint>
+#include <memory>
+
+#if defined(GGML_USE_HIPBLAS)
+#define GGML_COMMON_DECL_HIP
+#define GGML_COMMON_IMPL_HIP
+#else
+#define GGML_COMMON_DECL_CUDA
+#define GGML_COMMON_IMPL_CUDA
+#if defined(GGML_USE_MUSA)
+#define GGML_COMMON_DECL_MUSA
+#define GGML_COMMON_IMPL_MUSA
+#endif
+#endif
+#include "ggml-common.h"
+
+#include <cstdio>
+#include <array>
+#include <cassert>
+#include <cfloat>
+#include <string>
+#include <vector>
+
+#if defined(GGML_USE_HIPBLAS)
+#include "vendors/hip.h"
+#elif defined(GGML_USE_MUSA)
+#include "vendors/musa.h"
+#else
+#include "vendors/cuda.h"
+#endif // defined(GGML_USE_HIPBLAS)
+
+#define STRINGIZE_IMPL(...) #__VA_ARGS__
+#define STRINGIZE(...) STRINGIZE_IMPL(__VA_ARGS__)
+
+#define WARP_SIZE 32
+#define CUDART_HMAX 11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed)
+#define CUDART_HMASK 12000 // CUDA 12.0, min. ver. for half2 -> uint mask comparisons
+
+#define CC_PASCAL 600
+#define MIN_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products
+#define CC_VOLTA 700
+#define CC_TURING 750
+#define CC_AMPERE 800
+#define CC_OFFSET_AMD 1000000
+#define CC_RDNA1 (CC_OFFSET_AMD + 1010)
+#define CC_RDNA2 (CC_OFFSET_AMD + 1030)
+#define CC_RDNA3 (CC_OFFSET_AMD + 1100)
+
+#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses
+
+#if defined(_MSC_VER)
+#pragma warning(disable: 4244 4267) // possible loss of data
+#endif
+
+#define GGML_CUDA_MAX_STREAMS 8
+
+[[noreturn]]
+void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg);
+
+#define CUDA_CHECK_GEN(err, success, error_fn) \
+ do { \
+ auto err_ = (err); \
+ if (err_ != (success)) { \
+ ggml_cuda_error(#err, __func__, __FILE__, __LINE__, error_fn(err_)); \
+ } \
+ } while (0)
+
+#define CUDA_CHECK(err) CUDA_CHECK_GEN(err, cudaSuccess, cudaGetErrorString)
+
+#if CUDART_VERSION >= 12000 || defined(GGML_USE_MUSA)
+ static const char * cublas_get_error_str(const cublasStatus_t err) {
+ return cublasGetStatusString(err);
+ }
+#else
+ static const char * cublas_get_error_str(const cublasStatus_t err) {
+ switch (err) {
+ case CUBLAS_STATUS_SUCCESS: return "CUBLAS_STATUS_SUCCESS";
+ case CUBLAS_STATUS_NOT_INITIALIZED: return "CUBLAS_STATUS_NOT_INITIALIZED";
+ case CUBLAS_STATUS_ALLOC_FAILED: return "CUBLAS_STATUS_ALLOC_FAILED";
+ case CUBLAS_STATUS_INVALID_VALUE: return "CUBLAS_STATUS_INVALID_VALUE";
+ case CUBLAS_STATUS_ARCH_MISMATCH: return "CUBLAS_STATUS_ARCH_MISMATCH";
+ case CUBLAS_STATUS_MAPPING_ERROR: return "CUBLAS_STATUS_MAPPING_ERROR";
+ case CUBLAS_STATUS_EXECUTION_FAILED: return "CUBLAS_STATUS_EXECUTION_FAILED";
+ case CUBLAS_STATUS_INTERNAL_ERROR: return "CUBLAS_STATUS_INTERNAL_ERROR";
+ case CUBLAS_STATUS_NOT_SUPPORTED: return "CUBLAS_STATUS_NOT_SUPPORTED";
+ default: return "unknown error";
+ }
+ }
+#endif // CUDART_VERSION >= 12000
+
+#define CUBLAS_CHECK(err) CUDA_CHECK_GEN(err, CUBLAS_STATUS_SUCCESS, cublas_get_error_str)
+
+#if !defined(GGML_USE_HIPBLAS)
+static const char * cu_get_error_str(CUresult err) {
+ const char * err_str;
+ cuGetErrorString(err, &err_str);
+ return err_str;
+}
+#define CU_CHECK(err) CUDA_CHECK_GEN(err, CUDA_SUCCESS, cu_get_error_str)
+#endif
+
+#if CUDART_VERSION >= 11100 || defined(GGML_USE_MUSA)
+#define GGML_CUDA_ASSUME(x) __builtin_assume(x)
+#else
+#define GGML_CUDA_ASSUME(x)
+#endif // CUDART_VERSION >= 11100
+
+#ifdef GGML_CUDA_F16
+typedef half dfloat; // dequantize float
+typedef half2 dfloat2;
+#else
+typedef float dfloat; // dequantize float
+typedef float2 dfloat2;
+#endif // GGML_CUDA_F16
+
+#if (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) || __CUDA_ARCH__ >= CC_PASCAL
+#define FP16_AVAILABLE
+#endif // (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) || __CUDA_ARCH__ >= CC_PASCAL
+
+#if defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610
+#define FAST_FP16_AVAILABLE
+#endif // defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610
+
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_VOLTA
+#define FP16_MMA_AVAILABLE
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_VOLTA
+
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_TURING
+#define INT8_MMA_AVAILABLE
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_TURING
+
+static constexpr bool fast_fp16_available(const int cc) {
+ return cc >= CC_PASCAL && cc != 610;
+}
+
+static constexpr bool fp16_mma_available(const int cc) {
+ return cc < CC_OFFSET_AMD && cc >= CC_VOLTA;
+}
+
+static constexpr bool int8_mma_available(const int cc) {
+ return cc < CC_OFFSET_AMD && cc >= CC_TURING;
+}
+
+[[noreturn]]
+static __device__ void no_device_code(
+ const char * file_name, const int line, const char * function_name, const int arch, const char * arch_list) {
+
+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+ printf("%s:%d: ERROR: HIP kernel %s has no device code compatible with HIP arch %d.\n",
+ file_name, line, function_name, arch);
+ GGML_UNUSED(arch_list);
+#else
+ printf("%s:%d: ERROR: CUDA kernel %s has no device code compatible with CUDA arch %d. ggml-cuda.cu was compiled for: %s\n",
+ file_name, line, function_name, arch, arch_list);
+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+ __trap();
+
+ GGML_UNUSED(no_device_code); // suppress unused function warning
+}
+
+#ifdef __CUDA_ARCH__
+#define NO_DEVICE_CODE no_device_code(__FILE__, __LINE__, __FUNCTION__, __CUDA_ARCH__, STRINGIZE(__CUDA_ARCH_LIST__))
+#else
+#define NO_DEVICE_CODE //GGML_ABORT("NO_DEVICE_CODE not valid in host code.")
+#endif // __CUDA_ARCH__
+
+static __device__ __forceinline__ float warp_reduce_sum(float x) {
+#pragma unroll
+ for (int mask = 16; mask > 0; mask >>= 1) {
+ x += __shfl_xor_sync(0xffffffff, x, mask, 32);
+ }
+ return x;
+}
+
+static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {
+#pragma unroll
+ for (int mask = 16; mask > 0; mask >>= 1) {
+ a.x += __shfl_xor_sync(0xffffffff, a.x, mask, 32);
+ a.y += __shfl_xor_sync(0xffffffff, a.y, mask, 32);
+ }
+ return a;
+}
+
+static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {
+#ifdef FP16_AVAILABLE
+
+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+#pragma unroll
+ for (int mask = 16; mask > 0; mask >>= 1) {
+ const half2 a_other = __shfl_xor_sync(0xffffffff, a, mask, 32);
+ reinterpret_cast<half&>(a.x) += __low2half(a_other);
+ reinterpret_cast<half&>(a.y) += __high2half(a_other);
+ }
+ return a;
+#else
+#pragma unroll
+ for (int mask = 16; mask > 0; mask >>= 1) {
+ a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32));
+ }
+ return a;
+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+
+#else
+ NO_DEVICE_CODE;
+ return a;
+#endif // FP16_AVAILABLE
+}
+
+static __device__ __forceinline__ float warp_reduce_max(float x) {
+#pragma unroll
+ for (int mask = 16; mask > 0; mask >>= 1) {
+ x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, mask, 32));
+ }
+ return x;
+}
+
+static __device__ __forceinline__ half ggml_cuda_hmax(const half a, const half b) {
+#ifdef FP16_AVAILABLE
+
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && CUDART_VERSION < CUDART_HMAX
+ return __float2half(fmaxf(__half2float(a), __half2float(b)));
+#else
+ return __hmax(a, b);
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && CUDART_VERSION < CUDART_HMAX
+
+#else
+ NO_DEVICE_CODE;
+ GGML_UNUSED(b);
+ return a;
+#endif // FP16_AVAILABLE
+}
+
+static __device__ __forceinline__ half2 ggml_cuda_hmax2(const half2 a, const half2 b) {
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+
+#if CUDART_VERSION >= CUDART_HMAX
+ return __hmax2(a, b);
+#else
+ half2 ret;
+ reinterpret_cast<half&>(ret.x) = __float2half(fmaxf( __low2float(a), __low2float(b)));
+ reinterpret_cast<half&>(ret.y) = __float2half(fmaxf(__high2float(a), __high2float(b)));
+ return ret;
+#endif // CUDART_VERSION >= CUDART_HMAX
+
+#else
+ GGML_UNUSED(a);
+ GGML_UNUSED(b);
+ NO_DEVICE_CODE;
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+}
+
+static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL
+#pragma unroll
+ for (int mask = 16; mask > 0; mask >>= 1) {
+ x = ggml_cuda_hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32));
+ }
+ return x;
+#else
+ GGML_UNUSED(x);
+ NO_DEVICE_CODE;
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL
+}
+
+#if CUDART_VERSION < CUDART_HMASK
+static __device__ __forceinline__ uint32_t __hgt2_mask(const half2 a, const half2 b) {
+ const uint32_t mask_low = 0x0000FFFF * (float( __low2half(a)) > float( __low2half(b)));
+ const uint32_t mask_high = 0xFFFF0000 * (float(__high2half(a)) > float(__high2half(b)));
+ return mask_low | mask_high;
+}
+#endif // CUDART_VERSION < CUDART_HMASK
+
+static __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, int c) {
+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+#if defined(__gfx906__) || defined(__gfx908__) || defined(__gfx90a__) || defined(RDNA2)
+ c = __builtin_amdgcn_sdot4(a, b, c, false);
+#elif defined(RDNA3)
+ c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);
+#elif defined(__gfx1010__) || defined(__gfx900__)
+ int tmp1;
+ int tmp2;
+ asm("\n \
+ v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 \n \
+ v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 \n \
+ v_add3_u32 %0, %1, %2, %0 \n \
+ v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 \n \
+ v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 \n \
+ v_add3_u32 %0, %1, %2, %0 \n \
+ "
+ : "+v"(c), "=&v"(tmp1), "=&v"(tmp2)
+ : "v"(a), "v"(b)
+ );
+#else
+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
+ c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];
+#endif
+ return c;
+
+#else // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+
+#if __CUDA_ARCH__ >= MIN_CC_DP4A
+ return __dp4a(a, b, c);
+#else // __CUDA_ARCH__ >= MIN_CC_DP4A
+ const int8_t * a8 = (const int8_t *) &a;
+ const int8_t * b8 = (const int8_t *) &b;
+ return c + a8[0]*b8[0] + a8[1]*b8[1] + a8[2]*b8[2] + a8[3]*b8[3];
+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A
+
+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+}
+
+// TODO: move to ggml-common.h
+static constexpr __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
+
+typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, dfloat2 & v);
+
+static __device__ __forceinline__ float get_alibi_slope(
+ const float max_bias, const uint32_t h, const uint32_t n_head_log2, const float m0, const float m1
+) {
+ if (max_bias <= 0.0f) {
+ return 1.0f;
+ }
+ const float base = h < n_head_log2 ? m0 : m1;
+ const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;
+
+ return powf(base, exph);
+}
+
+template <ggml_type type>
+struct ggml_cuda_type_traits;
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_F16> {
+ static constexpr int qk = 1;
+ static constexpr int qr = 1;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q4_0> {
+ static constexpr int qk = QK4_0;
+ static constexpr int qr = QR4_0;
+ static constexpr int qi = QI4_0;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q4_1> {
+ static constexpr int qk = QK4_1;
+ static constexpr int qr = QR4_1;
+ static constexpr int qi = QI4_1;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q5_0> {
+ static constexpr int qk = QK5_0;
+ static constexpr int qr = QR5_0;
+ static constexpr int qi = QI5_0;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q5_1> {
+ static constexpr int qk = QK5_1;
+ static constexpr int qr = QR5_1;
+ static constexpr int qi = QI5_1;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q8_0> {
+ static constexpr int qk = QK8_0;
+ static constexpr int qr = QR8_0;
+ static constexpr int qi = QI8_0;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q2_K> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR2_K;
+ static constexpr int qi = QI2_K;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q3_K> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR3_K;
+ static constexpr int qi = QI3_K;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q4_K> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR4_K;
+ static constexpr int qi = QI4_K;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q5_K> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR5_K;
+ static constexpr int qi = QI5_K;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q6_K> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR6_K;
+ static constexpr int qi = QI6_K;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ2_XXS> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR2_XXS;
+ static constexpr int qi = QI2_XXS;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ2_XS> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR2_XS;
+ static constexpr int qi = QI2_XS;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ2_S> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR2_S;
+ static constexpr int qi = QI2_S;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ3_XXS> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR3_XXS;
+ static constexpr int qi = QI3_XXS;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ1_S> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR1_S;
+ static constexpr int qi = QI1_S;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ1_M> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR1_M;
+ static constexpr int qi = QI1_M;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ4_NL> {
+ static constexpr int qk = QK4_NL;
+ static constexpr int qr = QR4_NL;
+ static constexpr int qi = QI4_NL;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ4_XS> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR4_XS;
+ static constexpr int qi = QI4_XS;
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ3_S> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR3_S;
+ static constexpr int qi = QI3_S;
+};
+
+//////////////////////
+
+struct ggml_cuda_device_info {
+ int device_count;
+
+ struct cuda_device_info {
+ int cc; // compute capability
+ int nsm; // number of streaming multiprocessors
+ size_t smpb; // max. shared memory per block
+ size_t smpbo; // max. shared memory per block (with opt-in)
+ bool vmm; // virtual memory support
+ size_t vmm_granularity; // granularity of virtual memory
+ size_t total_vram;
+ };
+
+ cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {};
+
+ std::array<float, GGML_CUDA_MAX_DEVICES> default_tensor_split = {};
+};
+
+const ggml_cuda_device_info & ggml_cuda_info();
+
+void ggml_cuda_set_device(int device);
+int ggml_cuda_get_device();
+
+struct ggml_cuda_pool {
+ virtual ~ggml_cuda_pool() = default;
+
+ virtual void * alloc(size_t size, size_t * actual_size) = 0;
+ virtual void free(void * ptr, size_t size) = 0;
+};
+
+template<typename T>
+struct ggml_cuda_pool_alloc {
+ ggml_cuda_pool * pool = nullptr;
+ T * ptr = nullptr;
+ size_t actual_size = 0;
+
+ ggml_cuda_pool_alloc() = default;
+
+ explicit ggml_cuda_pool_alloc(ggml_cuda_pool & pool) : pool(&pool) {
+ }
+
+ ggml_cuda_pool_alloc(ggml_cuda_pool & pool, size_t size) : pool(&pool) {
+ alloc(size);
+ }
+
+ ~ggml_cuda_pool_alloc() {
+ if (ptr != nullptr) {
+ pool->free(ptr, actual_size);
+ }
+ }
+
+ // size is in number of elements
+ T * alloc(size_t size) {
+ GGML_ASSERT(pool != nullptr);
+ GGML_ASSERT(ptr == nullptr);
+ ptr = (T *) pool->alloc(size * sizeof(T), &this->actual_size);
+ return ptr;
+ }
+
+ T * alloc(ggml_cuda_pool & pool, size_t size) {
+ this->pool = &pool;
+ return alloc(size);
+ }
+
+ T * get() {
+ return ptr;
+ }
+
+ ggml_cuda_pool_alloc(const ggml_cuda_pool_alloc &) = delete;
+ ggml_cuda_pool_alloc(ggml_cuda_pool_alloc &&) = delete;
+ ggml_cuda_pool_alloc& operator=(const ggml_cuda_pool_alloc &) = delete;
+ ggml_cuda_pool_alloc& operator=(ggml_cuda_pool_alloc &&) = delete;
+};
+
+
+// backend interface
+
+struct ggml_tensor_extra_gpu {
+ void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors
+ cudaEvent_t events[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; // events for synchronizing multiple GPUs
+};
+
+
+#if (CUDART_VERSION >= 12000) && defined(GGML_CUDA_USE_GRAPHS)
+#define USE_CUDA_GRAPH
+#endif
+
+struct ggml_graph_node_properties {
+ void * node_address;
+ ggml_op node_op;
+ int64_t ne[GGML_MAX_DIMS];
+ size_t nb[GGML_MAX_DIMS];
+ void * src_address[GGML_MAX_SRC];
+};
+
+struct ggml_cuda_graph {
+#ifdef USE_CUDA_GRAPH
+ ~ggml_cuda_graph() {
+ if (instance != nullptr) {
+ CUDA_CHECK(cudaGraphExecDestroy(instance));
+ }
+ if (graph != nullptr) {
+ CUDA_CHECK(cudaGraphDestroy(graph));
+ }
+ }
+ cudaGraph_t graph = nullptr;
+ cudaGraphExec_t instance = nullptr;
+ size_t num_nodes = 0;
+ std::vector<cudaGraphNode_t> nodes;
+ std::vector<cudaKernelNodeParams> params;
+ bool disable_due_to_gpu_arch = false;
+ bool disable_due_to_too_many_updates = false;
+ bool disable_due_to_failed_graph_capture = false;
+ int number_consecutive_updates = 0;
+ std::vector<ggml_graph_node_properties> ggml_graph_properties;
+ std::vector<char **> updated_kernel_arg;
+#endif
+};
+
+struct ggml_backend_cuda_context {
+ int device;
+ std::string name;
+ cudaEvent_t copy_event = nullptr;
+
+ cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } };
+ cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr};
+
+ std::unique_ptr<ggml_cuda_graph> cuda_graph;
+
+ explicit ggml_backend_cuda_context(int device) :
+ device(device),
+ name(GGML_CUDA_NAME + std::to_string(device)) {
+ }
+
+ ~ggml_backend_cuda_context() {
+ if (copy_event != nullptr) {
+ CUDA_CHECK(cudaEventDestroy(copy_event));
+ }
+ for (int i = 0; i < GGML_CUDA_MAX_DEVICES; ++i) {
+ for (int j = 0; j < GGML_CUDA_MAX_STREAMS; ++j) {
+ if (streams[i][j] != nullptr) {
+ CUDA_CHECK(cudaStreamDestroy(streams[i][j]));
+ }
+ }
+ if (cublas_handles[i] != nullptr) {
+ CUBLAS_CHECK(cublasDestroy(cublas_handles[i]));
+ }
+ }
+ }
+
+ cudaStream_t stream(int device, int stream) {
+ if (streams[device][stream] == nullptr) {
+ ggml_cuda_set_device(device);
+ CUDA_CHECK(cudaStreamCreateWithFlags(&streams[device][stream], cudaStreamNonBlocking));
+ }
+ return streams[device][stream];
+ }
+
+ cudaStream_t stream() {
+ return stream(device, 0);
+ }
+
+ cublasHandle_t cublas_handle(int device) {
+ if (cublas_handles[device] == nullptr) {
+ ggml_cuda_set_device(device);
+ CUBLAS_CHECK(cublasCreate(&cublas_handles[device]));
+ CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device], CUBLAS_TF32_TENSOR_OP_MATH));
+ }
+ return cublas_handles[device];
+ }
+
+ cublasHandle_t cublas_handle() {
+ return cublas_handle(device);
+ }
+
+ // pool
+ std::unique_ptr<ggml_cuda_pool> pools[GGML_CUDA_MAX_DEVICES];
+
+ static std::unique_ptr<ggml_cuda_pool> new_pool_for_device(int device);
+
+ ggml_cuda_pool & pool(int device) {
+ if (pools[device] == nullptr) {
+ pools[device] = new_pool_for_device(device);
+ }
+ return *pools[device];
+ }
+
+ ggml_cuda_pool & pool() {
+ return pool(device);
+ }
+};
diff --git a/llama/ggml-cuda/concat.cu b/llama/ggml-cuda/concat.cu
new file mode 100644
index 00000000..72c36e40
--- /dev/null
+++ b/llama/ggml-cuda/concat.cu
@@ -0,0 +1,222 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "concat.cuh"
+
+// contiguous kernels
+static __global__ void concat_f32_dim0(const float * x, const float * y, float * dst, const int ne0, const int ne00) {
+ int nidx = threadIdx.x + blockIdx.x * blockDim.x;
+ if (nidx >= ne0) {
+ return;
+ }
+
+ int offset_dst =
+ nidx +
+ blockIdx.y * ne0 +
+ blockIdx.z * ne0 * gridDim.y;
+
+ if (nidx < ne00) { // src0
+ int offset_src =
+ nidx +
+ blockIdx.y * ne00 +
+ blockIdx.z * ne00 * gridDim.y;
+ dst[offset_dst] = x[offset_src];
+ } else {
+ int offset_src =
+ (nidx - ne00) +
+ blockIdx.y * (ne0 - ne00) +
+ blockIdx.z * (ne0 - ne00) * gridDim.y;
+ dst[offset_dst] = y[offset_src];
+ }
+}
+
+static __global__ void concat_f32_dim1(const float * x, const float * y, float * dst, const int ne0, const int ne01) {
+ int nidx = threadIdx.x + blockIdx.x * blockDim.x;
+ if (nidx >= ne0) {
+ return;
+ }
+
+ int offset_dst =
+ nidx +
+ blockIdx.y * ne0 +
+ blockIdx.z * ne0 * gridDim.y;
+
+ if (blockIdx.y < ne01) { // src0
+ int offset_src =
+ nidx +
+ blockIdx.y * ne0 +
+ blockIdx.z * ne0 * ne01;
+ dst[offset_dst] = x[offset_src];
+ } else {
+ int offset_src =
+ nidx +
+ (blockIdx.y - ne01) * ne0 +
+ blockIdx.z * ne0 * (gridDim.y - ne01);
+ dst[offset_dst] = y[offset_src];
+ }
+}
+
+static __global__ void concat_f32_dim2(const float * x, const float * y, float * dst, const int ne0, const int ne02) {
+ int nidx = threadIdx.x + blockIdx.x * blockDim.x;
+ if (nidx >= ne0) {
+ return;
+ }
+
+ int offset_dst =
+ nidx +
+ blockIdx.y * ne0 +
+ blockIdx.z * ne0 * gridDim.y;
+
+ if (blockIdx.z < ne02) { // src0
+ int offset_src =
+ nidx +
+ blockIdx.y * ne0 +
+ blockIdx.z * ne0 * gridDim.y;
+ dst[offset_dst] = x[offset_src];
+ } else {
+ int offset_src =
+ nidx +
+ blockIdx.y * ne0 +
+ (blockIdx.z - ne02) * ne0 * gridDim.y;
+ dst[offset_dst] = y[offset_src];
+ }
+}
+
+static void concat_f32_cuda(const float * x, const float * y, float * dst, int ne00, int ne01, int ne02, int ne0, int ne1, int ne2, int dim, cudaStream_t stream) {
+ int num_blocks = (ne0 + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE;
+ dim3 gridDim(num_blocks, ne1, ne2);
+ if (dim == 0) {
+ concat_f32_dim0<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne00);
+ return;
+ }
+ if (dim == 1) {
+ concat_f32_dim1<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne01);
+ return;
+ }
+ concat_f32_dim2<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne02);
+}
+
+// non-contiguous kernel (slow)
+static __global__ void concat_f32_non_cont(
+ const char * src0,
+ const char * src1,
+ char * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne03,
+ uint64_t nb00,
+ uint64_t nb01,
+ uint64_t nb02,
+ uint64_t nb03,
+ int64_t /*ne10*/,
+ int64_t /*ne11*/,
+ int64_t /*ne12*/,
+ int64_t /*ne13*/,
+ uint64_t nb10,
+ uint64_t nb11,
+ uint64_t nb12,
+ uint64_t nb13,
+ int64_t ne0,
+ int64_t /*ne1*/,
+ int64_t /*ne2*/,
+ int64_t /*ne3*/,
+ uint64_t nb0,
+ uint64_t nb1,
+ uint64_t nb2,
+ uint64_t nb3,
+ int32_t dim) {
+ const int64_t i3 = blockIdx.z;
+ const int64_t i2 = blockIdx.y;
+ const int64_t i1 = blockIdx.x;
+
+ int64_t o[4] = {0, 0, 0, 0};
+ o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));
+
+ const float * x;
+
+ for (int i0 = threadIdx.x; i0 < ne0; i0 += blockDim.x) {
+ if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
+ x = (const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);
+ } else {
+ x = (const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
+ }
+
+ float * y = (float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ *y = *x;
+ }
+}
+
+
+void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+
+ cudaStream_t stream = ctx.stream();
+
+ const int32_t dim = ((int32_t *) dst->op_params)[0];
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
+
+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
+ const float * src0_d = (const float *)src0->data;
+ const float * src1_d = (const float *)src1->data;
+
+ float * dst_d = (float *)dst->data;
+
+ if (dim != 3) {
+ for (int i3 = 0; i3 < dst->ne[3]; i3++) {
+ concat_f32_cuda(
+ src0_d + i3 * (src0->nb[3] / 4),
+ src1_d + i3 * (src1->nb[3] / 4),
+ dst_d + i3 * ( dst->nb[3] / 4),
+ src0->ne[0], src0->ne[1], src0->ne[2],
+ dst->ne[0], dst->ne[1], dst->ne[2], dim, stream);
+ }
+ } else {
+ const size_t size0 = ggml_nbytes(src0);
+ const size_t size1 = ggml_nbytes(src1);
+
+ CUDA_CHECK(cudaMemcpyAsync(dst_d, src0_d, size0, cudaMemcpyDeviceToDevice, stream));
+ CUDA_CHECK(cudaMemcpyAsync(dst_d + size0/4, src1_d, size1, cudaMemcpyDeviceToDevice, stream));
+ }
+ } else {
+ dim3 grid_dim(dst->ne[1], dst->ne[2], dst->ne[3]);
+ concat_f32_non_cont<<<grid_dim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(
+ (const char *)src0->data,
+ (const char *)src1->data,
+ ( char *)dst->data,
+ src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
+ src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
+ src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],
+ src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
+ dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
+ dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
+ }
+}
diff --git a/llama/ggml-cuda/concat.cuh b/llama/ggml-cuda/concat.cuh
new file mode 100644
index 00000000..c03cd437
--- /dev/null
+++ b/llama/ggml-cuda/concat.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_CONCAT_BLOCK_SIZE 256
+
+void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/conv-transpose-1d.cu b/llama/ggml-cuda/conv-transpose-1d.cu
new file mode 100644
index 00000000..ab5e6bdf
--- /dev/null
+++ b/llama/ggml-cuda/conv-transpose-1d.cu
@@ -0,0 +1,113 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "conv-transpose-1d.cuh"
+
+static __global__ void conv_transpose_1d_kernel(
+ const int s0, const int p0, const int d0, const int output_size,
+ const int src0_ne0, const int src0_ne1, const int src0_ne2, const int src0_ne3,
+ const int src1_ne0, const int src1_ne1, const int src1_ne2, const int src1_ne3,
+ const int dst_ne0, const int dst_ne1, const int dst_ne2, const int dst_ne3,
+ const float * src0, const float * src1, float * dst) {
+ int global_index = threadIdx.x + blockIdx.x * blockDim.x;
+ if (global_index >= output_size) {
+ return;
+ }
+
+ int out_index = global_index / dst_ne0;
+
+ float accumulator = 0;
+
+ for (int c = 0; c < src0_ne2; c++) {
+ int idx = global_index % dst_ne0;
+
+ int kernel_offset = (src0_ne0 * src0_ne1 * c) + (out_index * src0_ne0);
+ int input_offset = src1_ne0 * c;
+
+ for (int i = 0; i < src1_ne0; i++) {
+ if (!(idx >= i*s0 && idx < i*s0 + src0_ne0)) {
+ continue;
+ }
+ int weight_idx = idx - i*s0;
+
+ float kernel_weight = src0[kernel_offset + weight_idx];
+ float input_value = src1[input_offset+i];
+
+ accumulator += kernel_weight * input_value;
+ }
+ }
+ dst[global_index] = accumulator;
+}
+
+static void conv_transpose_1d_f32_f32_cuda(
+ const int s0, const int p0, const int d0, const int output_size,
+ const int src0_ne0, const int src0_ne1, const int src0_ne2, const int src0_ne3,
+ const int src1_ne0, const int src1_ne1, const int src1_ne2, const int src1_ne3,
+ const int dst_ne0, const int dst_ne1, const int dst_ne2, const int dst_ne3,
+ const float * src0, const float * src1, float * dst,
+ cudaStream_t stream) {
+
+ const int num_blocks = (output_size + CUDA_CONV_TRANPOSE_1D_BLOCK_SIZE - 1) / CUDA_CONV_TRANPOSE_1D_BLOCK_SIZE;
+ conv_transpose_1d_kernel<<<num_blocks,CUDA_CONV_TRANPOSE_1D_BLOCK_SIZE, 0, stream>>>(
+ s0,p0,d0,output_size,
+ src0_ne0, src0_ne1, src0_ne2, src0_ne3,
+ src1_ne0, src1_ne1, src1_ne2, src1_ne3,
+ dst_ne0, dst_ne1, dst_ne2, dst_ne3,
+ src0,src1, dst);
+}
+
+void ggml_cuda_op_conv_transpose_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+
+ const ggml_tensor * src1 = dst->src[1];
+ const float * src1_d = (const float *)src1->data;
+
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+ GGML_ASSERT(ggml_is_contiguous(src1));
+
+ const int32_t * opts = (const int32_t *)dst->op_params;
+
+ const int s0 = opts[0];
+ const int p0 = 0;//opts[3];
+ const int d0 = 1;//opts[4];
+
+ const int64_t kernel_size = ggml_nelements(src0);
+ const int64_t input_size = ggml_nelements(src1);
+ const int64_t output_size = ggml_nelements(dst);
+
+ conv_transpose_1d_f32_f32_cuda(s0, p0, d0, output_size,
+ src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
+ src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],
+ dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
+ src0_d, src1_d, dst_d, stream);
+}
diff --git a/llama/ggml-cuda/conv-transpose-1d.cuh b/llama/ggml-cuda/conv-transpose-1d.cuh
new file mode 100644
index 00000000..891917b2
--- /dev/null
+++ b/llama/ggml-cuda/conv-transpose-1d.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_CONV_TRANPOSE_1D_BLOCK_SIZE 256
+
+void ggml_cuda_op_conv_transpose_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/convert.cu b/llama/ggml-cuda/convert.cu
new file mode 100644
index 00000000..046d5b74
--- /dev/null
+++ b/llama/ggml-cuda/convert.cu
@@ -0,0 +1,712 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "convert.cuh"
+#include "dequantize.cuh"
+
+#define CUDA_Q8_0_NE_ALIGN 2048
+
+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
+static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k) {
+ const int64_t i = (int64_t)2*(blockDim.x*blockIdx.x + threadIdx.x);
+
+ if (i >= k) {
+ return;
+ }
+
+ const int64_t ib = i/qk; // block index
+ const int64_t iqs = (i%qk)/qr; // quant index
+ const int64_t iybs = i - i%qk; // y block start index
+ const int64_t y_offset = qr == 1 ? 1 : qk/2;
+
+ // dequantize
+ dfloat2 v;
+ dequantize_kernel(vx, ib, iqs, v);
+
+ y[iybs + iqs + 0] = v.x;
+ y[iybs + iqs + y_offset] = v.y;
+}
+
+template <bool need_check>
+static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, half * __restrict__ y, const int64_t k) {
+#if __CUDA_ARCH__ >= CC_PASCAL
+ constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;
+
+ const int64_t i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;
+ const int * x0 = ((int *) vx) + blockIdx.x * nint;
+ half2 * y2 = (half2 *) (y + i0);
+
+ __shared__ int vals[nint];
+
+#pragma unroll
+ for (int ix0 = 0; ix0 < nint; ix0 += WARP_SIZE) {
+ if (need_check && i0*sizeof(block_q8_0)/QK8_0 + sizeof(int)*(ix0 + threadIdx.x) >= k*sizeof(block_q8_0)/QK8_0) {
+ break;
+ }
+
+ const int ix = ix0 + threadIdx.x;
+ vals[ix] = x0[ix];
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int iy = 0; iy < CUDA_Q8_0_NE_ALIGN; iy += 2*WARP_SIZE) {
+ if (need_check && i0 + iy + 2*threadIdx.x >= k) {
+ return;
+ }
+
+ const half * b0 = ((const half *) vals) + (sizeof(block_q8_0)/sizeof(half)) * ((iy + 2*threadIdx.x)/QK8_0);
+ const half d = *b0;
+ const char2 qs = ((const char2 *) (b0 + 1))[threadIdx.x % (QK8_0/2)];
+
+ y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d));
+ }
+#else
+ GGML_UNUSED(vx);
+ GGML_UNUSED(y);
+ GGML_UNUSED(k);
+ NO_DEVICE_CODE;
+#endif // __CUDA_ARCH__ >= CC_PASCAL
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {
+
+ const int64_t i = blockIdx.x;
+
+ // assume 32 threads
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8;
+ const int64_t ir = tid%8;
+ const int64_t ib = 8*i + ir;
+ if (ib >= nb32) {
+ return;
+ }
+
+ dst_t * y = yy + 256*i + 32*ir + 4*il;
+
+ const block_q4_0 * x = (const block_q4_0 *)vx + ib;
+ const float d = __half2float(x->d);
+ const float dm = -8*d;
+
+ const uint8_t * q = x->qs + 4*il;
+
+ for (int l = 0; l < 4; ++l) {
+ y[l+ 0] = d * (q[l] & 0xF) + dm;
+ y[l+16] = d * (q[l] >> 4) + dm;
+ }
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {
+
+ const int64_t i = blockIdx.x;
+
+ // assume 32 threads
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8;
+ const int64_t ir = tid%8;
+ const int64_t ib = 8*i + ir;
+ if (ib >= nb32) {
+ return;
+ }
+
+ dst_t * y = yy + 256*i + 32*ir + 4*il;
+
+ const block_q4_1 * x = (const block_q4_1 *)vx + ib;
+ const float2 d = __half22float2(x->dm);
+
+ const uint8_t * q = x->qs + 4*il;
+
+ for (int l = 0; l < 4; ++l) {
+ y[l+ 0] = d.x * (q[l] & 0xF) + d.y;
+ y[l+16] = d.x * (q[l] >> 4) + d.y;
+ }
+}
+
+//================================== k-quants
+
+template<typename dst_t>
+static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+
+ const int64_t i = blockIdx.x;
+ const block_q2_K * x = (const block_q2_K *) vx;
+
+ const int64_t tid = threadIdx.x;
+ const int64_t n = tid/32;
+ const int64_t l = tid - 32*n;
+ const int64_t is = 8*n + l/16;
+
+ const uint8_t q = x[i].qs[32*n + l];
+ dst_t * y = yy + i*QK_K + 128*n;
+
+ float dall = __low2half(x[i].dm);
+ float dmin = __high2half(x[i].dm);
+ y[l+ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);
+ y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4);
+ y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4);
+ y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+
+ const int64_t i = blockIdx.x;
+ const block_q3_K * x = (const block_q3_K *) vx;
+
+ const int64_t r = threadIdx.x/4;
+ const int64_t tid = r/2;
+ const int64_t is0 = r%2;
+ const int64_t l0 = 16*is0 + 4*(threadIdx.x%4);
+ const int64_t n = tid / 4;
+ const int64_t j = tid - 4*n;
+
+ uint8_t m = 1 << (4*n + j);
+ int64_t is = 8*n + 2*j + is0;
+ int shift = 2*j;
+
+ int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :
+ is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :
+ is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :
+ (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4);
+ float d_all = x[i].d;
+ float dl = d_all * (us - 32);
+
+ dst_t * y = yy + i*QK_K + 128*n + 32*j;
+ const uint8_t * q = x[i].qs + 32*n;
+ const uint8_t * hm = x[i].hmask;
+
+ for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));
+}
+
+static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {
+ if (j < 4) {
+ d = q[j] & 63; m = q[j + 4] & 63;
+ } else {
+ d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);
+ m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);
+ }
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const block_q4_K * x = (const block_q4_K *) vx;
+
+ const int64_t i = blockIdx.x;
+
+ // assume 32 threads
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8;
+ const int64_t ir = tid%8;
+ const int64_t is = 2*il;
+ const int64_t n = 4;
+
+ dst_t * y = yy + i*QK_K + 64*il + n*ir;
+
+ const float dall = __low2half(x[i].dm);
+ const float dmin = __high2half(x[i].dm);
+
+ const uint8_t * q = x[i].qs + 32*il + n*ir;
+
+ uint8_t sc, m;
+ get_scale_min_k4(is + 0, x[i].scales, sc, m);
+ const float d1 = dall * sc; const float m1 = dmin * m;
+ get_scale_min_k4(is + 1, x[i].scales, sc, m);
+ const float d2 = dall * sc; const float m2 = dmin * m;
+ for (int l = 0; l < n; ++l) {
+ y[l + 0] = d1 * (q[l] & 0xF) - m1;
+ y[l +32] = d2 * (q[l] >> 4) - m2;
+ }
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const block_q5_K * x = (const block_q5_K *) vx;
+
+ const int64_t i = blockIdx.x;
+
+ // assume 64 threads - this is very slightly better than the one below
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/16; // il is in 0...3
+ const int64_t ir = tid%16; // ir is in 0...15
+ const int64_t is = 2*il; // is is in 0...6
+
+ dst_t * y = yy + i*QK_K + 64*il + 2*ir;
+
+ const float dall = __low2half(x[i].dm);
+ const float dmin = __high2half(x[i].dm);
+
+ const uint8_t * ql = x[i].qs + 32*il + 2*ir;
+ const uint8_t * qh = x[i].qh + 2*ir;
+
+ uint8_t sc, m;
+ get_scale_min_k4(is + 0, x[i].scales, sc, m);
+ const float d1 = dall * sc; const float m1 = dmin * m;
+ get_scale_min_k4(is + 1, x[i].scales, sc, m);
+ const float d2 = dall * sc; const float m2 = dmin * m;
+
+ uint8_t hm = 1 << (2*il);
+ y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1;
+ y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1;
+ hm <<= 1;
+ y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2;
+ y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2;
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const block_q6_K * x = (const block_q6_K *) vx;
+
+ const int64_t i = blockIdx.x;
+
+ // assume 64 threads - this is very slightly better than the one below
+ const int64_t tid = threadIdx.x;
+ const int64_t ip = tid/32; // ip is 0 or 1
+ const int64_t il = tid - 32*ip; // 0...32
+ const int64_t is = 8*ip + il/16;
+
+ dst_t * y = yy + i*QK_K + 128*ip + il;
+
+ const float d = x[i].d;
+
+ const uint8_t * ql = x[i].ql + 64*ip + il;
+ const uint8_t qh = x[i].qh[32*ip + il];
+ const int8_t * sc = x[i].scales + is;
+
+ y[ 0] = d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32);
+ y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32);
+ y[64] = d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32);
+ y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+
+ const int64_t i = blockIdx.x;
+ const block_iq2_xxs * x = (const block_iq2_xxs *) vx;
+
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
+ const uint16_t * q2 = x[i].qs + 4*ib;
+ const uint8_t * aux8 = (const uint8_t *)q2;
+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);
+ const uint32_t aux32 = q2[2] | (q2[3] << 16);
+ const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f;
+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
+ for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+
+ const int64_t i = blockIdx.x;
+ const block_iq2_xs * x = (const block_iq2_xs *) vx;
+
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
+ const uint16_t * q2 = x[i].qs + 4*ib;
+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));
+ const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
+ const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];
+ for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+
+ const int64_t i = blockIdx.x;
+ const block_iq2_s * x = (const block_iq2_s *) vx;
+
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));
+ const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
+ const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];
+ for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+
+ const int64_t i = blockIdx.x;
+ const block_iq3_xxs * x = (const block_iq3_xxs *) vx;
+
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
+ const uint8_t * q3 = x[i].qs + 8*ib;
+ const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;
+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);
+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);
+ const uint32_t aux32 = gas[0] | (gas[1] << 16);
+ const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f;
+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
+ for (int j = 0; j < 4; ++j) {
+ y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
+ y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
+ }
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+
+ const int64_t i = blockIdx.x;
+ const block_iq3_s * x = (const block_iq3_s *) vx;
+
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
+ const uint8_t * qs = x[i].qs + 8*ib;
+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));
+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));
+ const float d = (float)x[i].d * (1 + 2*((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf));
+ const uint8_t signs = x[i].signs[4*ib + il];
+ for (int j = 0; j < 4; ++j) {
+ y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
+ y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
+ }
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+
+ const int64_t i = blockIdx.x;
+ const block_iq1_s * x = (const block_iq1_s *) vx;
+
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
+ const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;
+ const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 7) + 1);
+ uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;
+ grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)];
+ grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;
+ grid32[0] &= 0x0f0f0f0f;
+ for (int j = 0; j < 8; ++j) {
+ y[j] = d * (q[j] + delta);
+ }
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+
+ const int64_t i = blockIdx.x;
+ const block_iq1_m * x = (const block_iq1_m *) vx;
+
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
+ const uint16_t * sc = (const uint16_t *)x[i].scales;
+ iq1m_scale_t scale;
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+ const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);
+ const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);
+ const float delta = x[i].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;
+ uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;
+ grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)];
+ grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;
+ grid32[0] &= 0x0f0f0f0f;
+ for (int j = 0; j < 8; ++j) {
+ y[j] = d * (q[j] + delta);
+ }
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+
+ const int64_t i = blockIdx.x;
+ const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);
+
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + i*QK_K + 32*ib + 4*il;
+ const uint8_t * q4 = x[ib].qs + 4*il;
+ const float d = (float)x[ib].d;
+ for (int j = 0; j < 4; ++j) {
+ y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];
+ y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];
+ }
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+ const block_iq4_xs * x = (const block_iq4_xs *)vx;
+
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + i*QK_K + 32*ib + 4*il;
+ const uint8_t * q4 = x[i].qs + 16*ib + 4*il;
+ const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);
+ for (int j = 0; j < 4; ++j) {
+ y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];
+ y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];
+ }
+}
+
+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
+static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {
+ const int num_blocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE);
+ dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);
+}
+
+static void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * __restrict__ y, const int64_t k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_Q8_0_NE_ALIGN - 1) / CUDA_Q8_0_NE_ALIGN;
+ if (k % CUDA_Q8_0_NE_ALIGN == 0) {
+ const bool need_check = false;
+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);
+ } else {
+ const bool need_check = true;
+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);
+ }
+}
+
+template<typename dst_t>
+static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb32 = k / 32;
+ const int nb = (k + 255) / 256;
+ dequantize_block_q4_0<<<nb, 32, 0, stream>>>(vx, y, nb32);
+}
+
+template<typename dst_t>
+static void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb32 = k / 32;
+ const int nb = (k + 255) / 256;
+ dequantize_block_q4_1<<<nb, 32, 0, stream>>>(vx, y, nb32);
+}
+
+template<typename dst_t>
+static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = (k + QK_K - 1) / QK_K;
+ dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq1_m_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq1_m<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = (k + QK_K - 1) / QK_K;
+ dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template <typename src_t, typename dst_t>
+static __global__ void convert_unary(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k) {
+ const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+
+ const src_t * x = (src_t *) vx;
+
+ y[i] = x[i];
+}
+
+template <typename src_t, typename dst_t>
+static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;
+ convert_unary<src_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);
+}
+
+to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_Q4_0:
+ return dequantize_row_q4_0_cuda;
+ case GGML_TYPE_Q4_1:
+ return dequantize_row_q4_1_cuda;
+ case GGML_TYPE_Q5_0:
+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;
+ case GGML_TYPE_Q5_1:
+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
+ case GGML_TYPE_Q8_0:
+ if (ggml_cuda_info().devices[ggml_cuda_get_device()].cc >= CC_PASCAL) {
+ return dequantize_block_q8_0_f16_cuda;
+ }
+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
+ case GGML_TYPE_Q2_K:
+ return dequantize_row_q2_K_cuda;
+ case GGML_TYPE_Q3_K:
+ return dequantize_row_q3_K_cuda;
+ case GGML_TYPE_Q4_K:
+ return dequantize_row_q4_K_cuda;
+ case GGML_TYPE_Q5_K:
+ return dequantize_row_q5_K_cuda;
+ case GGML_TYPE_Q6_K:
+ return dequantize_row_q6_K_cuda;
+ case GGML_TYPE_IQ2_XXS:
+ return dequantize_row_iq2_xxs_cuda;
+ case GGML_TYPE_IQ2_XS:
+ return dequantize_row_iq2_xs_cuda;
+ case GGML_TYPE_IQ2_S:
+ return dequantize_row_iq2_s_cuda;
+ case GGML_TYPE_IQ3_XXS:
+ return dequantize_row_iq3_xxs_cuda;
+ case GGML_TYPE_IQ1_S:
+ return dequantize_row_iq1_s_cuda;
+ case GGML_TYPE_IQ1_M:
+ return dequantize_row_iq1_m_cuda;
+ case GGML_TYPE_IQ4_NL:
+ return dequantize_row_iq4_nl_cuda;
+ case GGML_TYPE_IQ4_XS:
+ return dequantize_row_iq4_xs_cuda;
+ case GGML_TYPE_IQ3_S:
+ return dequantize_row_iq3_s_cuda;
+ case GGML_TYPE_F32:
+ return convert_unary_cuda<float>;
+ default:
+ return nullptr;
+ }
+}
+
+to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_Q4_0:
+ return dequantize_row_q4_0_cuda;
+ case GGML_TYPE_Q4_1:
+ return dequantize_row_q4_1_cuda;
+ case GGML_TYPE_Q5_0:
+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;
+ case GGML_TYPE_Q5_1:
+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
+ case GGML_TYPE_Q8_0:
+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
+ case GGML_TYPE_Q2_K:
+ return dequantize_row_q2_K_cuda;
+ case GGML_TYPE_Q3_K:
+ return dequantize_row_q3_K_cuda;
+ case GGML_TYPE_Q4_K:
+ return dequantize_row_q4_K_cuda;
+ case GGML_TYPE_Q5_K:
+ return dequantize_row_q5_K_cuda;
+ case GGML_TYPE_Q6_K:
+ return dequantize_row_q6_K_cuda;
+ case GGML_TYPE_IQ2_XXS:
+ return dequantize_row_iq2_xxs_cuda;
+ case GGML_TYPE_IQ2_XS:
+ return dequantize_row_iq2_xs_cuda;
+ case GGML_TYPE_IQ2_S:
+ return dequantize_row_iq2_s_cuda;
+ case GGML_TYPE_IQ3_XXS:
+ return dequantize_row_iq3_xxs_cuda;
+ case GGML_TYPE_IQ1_S:
+ return dequantize_row_iq1_s_cuda;
+ case GGML_TYPE_IQ1_M:
+ return dequantize_row_iq1_m_cuda;
+ case GGML_TYPE_IQ4_NL:
+ return dequantize_row_iq4_nl_cuda;
+ case GGML_TYPE_IQ4_XS:
+ return dequantize_row_iq4_xs_cuda;
+ case GGML_TYPE_IQ3_S:
+ return dequantize_row_iq3_s_cuda;
+ case GGML_TYPE_F16:
+ return convert_unary_cuda<half>;
+ default:
+ return nullptr;
+ }
+}
diff --git a/llama/ggml-cuda/convert.cuh b/llama/ggml-cuda/convert.cuh
new file mode 100644
index 00000000..e3f31c3c
--- /dev/null
+++ b/llama/ggml-cuda/convert.cuh
@@ -0,0 +1,39 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_DEQUANTIZE_BLOCK_SIZE 256
+
+template<typename T>
+using to_t_cuda_t = void (*)(const void * __restrict__ x, T * __restrict__ y, int64_t k, cudaStream_t stream);
+
+typedef to_t_cuda_t<float> to_fp32_cuda_t;
+typedef to_t_cuda_t<half> to_fp16_cuda_t;
+
+to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type);
+
+to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type);
diff --git a/llama/ggml-cuda/cpy.cu b/llama/ggml-cuda/cpy.cu
new file mode 100644
index 00000000..c4334735
--- /dev/null
+++ b/llama/ggml-cuda/cpy.cu
@@ -0,0 +1,515 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "cpy.cuh"
+
+typedef void (*cpy_kernel_t)(const char * cx, char * cdst);
+
+static __device__ void cpy_1_f32_f32(const char * cxi, char * cdsti) {
+ const float * xi = (const float *) cxi;
+ float * dsti = (float *) cdsti;
+
+ *dsti = *xi;
+}
+
+static __device__ void cpy_1_f32_f16(const char * cxi, char * cdsti) {
+ const float * xi = (const float *) cxi;
+ half * dsti = (half *) cdsti;
+
+ *dsti = __float2half(*xi);
+}
+
+static __device__ void cpy_1_f16_f16(const char * cxi, char * cdsti) {
+ const half * xi = (const half *) cxi;
+ half * dsti = (half *) cdsti;
+
+ *dsti = *xi;
+}
+
+static __device__ void cpy_1_f16_f32(const char * cxi, char * cdsti) {
+ const half * xi = (const half *) cxi;
+ float * dsti = (float *) cdsti;
+
+ *dsti = *xi;
+}
+
+template <cpy_kernel_t cpy_1>
+static __global__ void cpy_f32_f16(const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,
+ const int nb12, const int nb13) {
+ const int64_t i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= ne) {
+ return;
+ }
+
+ // determine indices i03/i13, i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor
+ // then combine those indices with the corresponding byte offsets to get the total offsets
+ const int64_t i03 = i/(ne00 * ne01 * ne02);
+ const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);
+ const int64_t i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;
+ const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;
+ const int64_t x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;
+
+ const int64_t i13 = i/(ne10 * ne11 * ne12);
+ const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);
+ const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;
+ const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;
+ const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13;
+
+ cpy_1(cx + x_offset, cdst + dst_offset);
+}
+
+static __device__ void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) {
+ const float * xi = (const float *) cxi;
+ block_q8_0 * dsti = (block_q8_0 *) cdsti;
+
+ float amax = 0.0f; // absolute max
+
+ for (int j = 0; j < QK8_0; j++) {
+ const float v = xi[j];
+ amax = fmaxf(amax, fabsf(v));
+ }
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ dsti->d = d;
+
+ for (int j = 0; j < QK8_0; ++j) {
+ const float x0 = xi[j]*id;
+
+ dsti->qs[j] = roundf(x0);
+ }
+}
+
+static __device__ void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {
+ const float * xi = (const float *) cxi;
+ block_q4_0 * dsti = (block_q4_0 *) cdsti;
+
+ float amax = 0.0f;
+ float vmax = 0.0f;
+
+ for (int j = 0; j < QK4_0; ++j) {
+ const float v = xi[j];
+ if (amax < fabsf(v)) {
+ amax = fabsf(v);
+ vmax = v;
+ }
+ }
+
+ const float d = vmax / -8;
+ const float id = d ? 1.0f/d : 0.0f;
+
+ dsti->d = d;
+
+ for (int j = 0; j < QK4_0/2; ++j) {
+ const float x0 = xi[0 + j]*id;
+ const float x1 = xi[QK4_0/2 + j]*id;
+
+ const uint8_t xi0 = min(15, (int8_t)(x0 + 8.5f));
+ const uint8_t xi1 = min(15, (int8_t)(x1 + 8.5f));
+
+ dsti->qs[j] = xi0;
+ dsti->qs[j] |= xi1 << 4;
+ }
+}
+
+static __device__ void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) {
+ const float * xi = (const float *) cxi;
+ block_q4_1 * dsti = (block_q4_1 *) cdsti;
+
+ float vmin = FLT_MAX;
+ float vmax = -FLT_MAX;
+
+ for (int j = 0; j < QK4_1; ++j) {
+ const float v = xi[j];
+
+ if (v < vmin) vmin = v;
+ if (v > vmax) vmax = v;
+ }
+
+ const float d = (vmax - vmin) / ((1 << 4) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ dsti->dm.x = d;
+ dsti->dm.y = vmin;
+
+ for (int j = 0; j < QK4_1/2; ++j) {
+ const float x0 = (xi[0 + j] - vmin)*id;
+ const float x1 = (xi[QK4_1/2 + j] - vmin)*id;
+
+ const uint8_t xi0 = min(15, (int8_t)(x0 + 0.5f));
+ const uint8_t xi1 = min(15, (int8_t)(x1 + 0.5f));
+
+ dsti->qs[j] = xi0;
+ dsti->qs[j] |= xi1 << 4;
+ }
+}
+
+static __device__ void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) {
+ const float * xi = (const float *) cxi;
+ block_q5_0 * dsti = (block_q5_0 *) cdsti;
+
+ float amax = 0.0f;
+ float vmax = 0.0f;
+
+ for (int j = 0; j < QK5_0; ++j) {
+ const float v = xi[j];
+ if (amax < fabsf(v)) {
+ amax = fabsf(v);
+ vmax = v;
+ }
+ }
+
+ const float d = vmax / -16;
+ const float id = d ? 1.0f/d : 0.0f;
+
+ dsti->d = d;
+
+ uint32_t qh = 0;
+ for (int j = 0; j < QK5_0/2; ++j) {
+ const float x0 = xi[0 + j]*id;
+ const float x1 = xi[QK5_0/2 + j]*id;
+
+ const uint8_t xi0 = min(31, (int8_t)(x0 + 16.5f));
+ const uint8_t xi1 = min(31, (int8_t)(x1 + 16.5f));
+
+ dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);
+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);
+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);
+ }
+ memcpy(dsti->qh, &qh, sizeof(qh));
+}
+
+static __device__ void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) {
+ const float * xi = (const float *) cxi;
+ block_q5_1 * dsti = (block_q5_1 *) cdsti;
+
+ float min = xi[0];
+ float max = xi[0];
+
+ for (int j = 1; j < QK5_1; ++j) {
+ const float v = xi[j];
+ min = v < min ? v : min;
+ max = v > max ? v : max;
+ }
+
+ const float d = (max - min) / 31;
+ const float id = d ? 1.0f/d : 0.0f;
+
+ dsti->dm.x = d;
+ dsti->dm.y = min;
+
+ uint32_t qh = 0;
+ for (int j = 0; j < QK5_1/2; ++j) {
+ const float x0 = (xi[0 + j] - min)*id;
+ const float x1 = (xi[QK5_1/2 + j] - min)*id;
+
+ const uint8_t xi0 = (uint8_t)(x0 + 0.5f);
+ const uint8_t xi1 = (uint8_t)(x1 + 0.5f);
+
+ dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);
+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);
+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2);
+ }
+ memcpy(dsti->qh, &qh, sizeof(qh));
+}
+
+
+static __device__ __forceinline__ int best_index_int8(int n, const int8_t * val, float x) {
+ if (x <= val[0]) return 0;
+ if (x >= val[n-1]) return n-1;
+ int ml = 0, mu = n-1;
+ while (mu-ml > 1) {
+ int mav = (ml+mu)/2;
+ if (x < val[mav]) mu = mav; else ml = mav;
+ }
+ return x - val[mu-1] < val[mu] - x ? mu-1 : mu;
+}
+
+static __device__ void cpy_blck_f32_iq4_nl(const char * cxi, char * cdsti) {
+ const float * xi = (const float *) cxi;
+ block_iq4_nl * dsti = (block_iq4_nl *) cdsti;
+
+ float amax = 0.0f;
+ float vmax = 0.0f;
+
+ for (int j = 0; j < QK4_NL; ++j) {
+ const float v = xi[j];
+ if (amax < fabsf(v)) {
+ amax = fabsf(v);
+ vmax = v;
+ }
+ }
+
+ float d = vmax / kvalues_iq4nl[0];
+ const float id = d ? 1.0f/d : 0.0f;
+
+ float sumqx = 0, sumq2 = 0;
+ for (int j = 0; j < QK4_NL/2; ++j) {
+ const float x0 = xi[0 + j]*id;
+ const float x1 = xi[QK4_NL/2 + j]*id;
+ const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl, x0);
+ const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl, x1);
+ dsti->qs[j] = xi0 | (xi1 << 4);
+ const float v0 = kvalues_iq4nl[xi0];
+ const float v1 = kvalues_iq4nl[xi1];
+ const float w0 = xi[0 + j]*xi[0 + j];
+ const float w1 = xi[QK4_NL/2 + j]*xi[QK4_NL/2 + j];
+ sumqx += w0*v0*xi[j] + w1*v1*xi[QK4_NL/2 + j];
+ sumq2 += w0*v0*v0 + w1*v1*v1;
+ }
+
+ dsti->d = sumq2 > 0 ? sumqx/sumq2 : d;
+}
+
+template <cpy_kernel_t cpy_blck, int qk>
+static __global__ void cpy_f32_q(const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,
+ const int nb12, const int nb13) {
+ const int i = (blockDim.x*blockIdx.x + threadIdx.x)*qk;
+
+ if (i >= ne) {
+ return;
+ }
+
+ const int i03 = i/(ne00 * ne01 * ne02);
+ const int i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);
+ const int i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;
+ const int i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;
+ const int x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;
+
+ const int i13 = i/(ne10 * ne11 * ne12);
+ const int i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);
+ const int i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;
+ const int i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;
+ const int dst_offset = (i10/qk)*nb10 + i11*nb11 + i12*nb12 + i13*nb13;
+
+ cpy_blck(cx + x_offset, cdst + dst_offset);
+}
+
+static void ggml_cpy_f16_f32_cuda(
+ const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {
+
+ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
+ cpy_f32_f16<cpy_1_f16_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>
+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+}
+
+static void ggml_cpy_f32_f32_cuda(
+ const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {
+
+ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
+ cpy_f32_f16<cpy_1_f32_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>
+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+}
+
+static void ggml_cpy_f32_f16_cuda(
+ const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {
+
+ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
+ cpy_f32_f16<cpy_1_f32_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>
+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+}
+
+static void ggml_cpy_f32_q8_0_cuda(
+ const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {
+
+ GGML_ASSERT(ne % QK8_0 == 0);
+ const int num_blocks = ne / QK8_0;
+ cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, 1, 0, stream>>>
+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+}
+
+static void ggml_cpy_f32_q4_0_cuda(
+ const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {
+
+ GGML_ASSERT(ne % QK4_0 == 0);
+ const int num_blocks = ne / QK4_0;
+ cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, 1, 0, stream>>>
+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+}
+
+static void ggml_cpy_f32_q4_1_cuda(
+ const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {
+
+ GGML_ASSERT(ne % QK4_1 == 0);
+ const int num_blocks = ne / QK4_1;
+ cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, 1, 0, stream>>>
+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+}
+
+static void ggml_cpy_f32_q5_0_cuda(
+ const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {
+
+ GGML_ASSERT(ne % QK5_0 == 0);
+ const int num_blocks = ne / QK5_0;
+ cpy_f32_q<cpy_blck_f32_q5_0, QK5_0><<<num_blocks, 1, 0, stream>>>
+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+}
+
+static void ggml_cpy_f32_q5_1_cuda(
+ const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {
+
+ GGML_ASSERT(ne % QK5_1 == 0);
+ const int num_blocks = ne / QK5_1;
+ cpy_f32_q<cpy_blck_f32_q5_1, QK5_1><<<num_blocks, 1, 0, stream>>>
+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+}
+
+static void ggml_cpy_f32_iq4_nl_cuda(
+ const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {
+
+ GGML_ASSERT(ne % QK4_NL == 0);
+ const int num_blocks = ne / QK4_NL;
+ cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, 1, 0, stream>>>
+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+}
+
+static void ggml_cpy_f16_f16_cuda(
+ const char * cx, char * cdst, const int ne,
+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,
+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {
+
+ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
+ cpy_f32_f16<cpy_1_f16_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>
+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+}
+
+void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1) {
+ const int64_t ne = ggml_nelements(src0);
+ GGML_ASSERT(ne == ggml_nelements(src1));
+
+ GGML_ASSERT(ggml_nbytes(src0) <= INT_MAX);
+ GGML_ASSERT(ggml_nbytes(src1) <= INT_MAX);
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t ne01 = src0->ne[1];
+ const int64_t ne02 = src0->ne[2];
+
+ //GGML_ASSERT(src0->ne[3] == 1);
+
+ const int64_t nb00 = src0->nb[0];
+ const int64_t nb01 = src0->nb[1];
+ const int64_t nb02 = src0->nb[2];
+ const int64_t nb03 = src0->nb[3];
+
+ const int64_t ne10 = src1->ne[0];
+ const int64_t ne11 = src1->ne[1];
+ const int64_t ne12 = src1->ne[2];
+
+ //GGML_ASSERT(src1->ne[3] == 1);
+
+ const int64_t nb10 = src1->nb[0];
+ const int64_t nb11 = src1->nb[1];
+ const int64_t nb12 = src1->nb[2];
+ const int64_t nb13 = src1->nb[3];
+
+ cudaStream_t main_stream = ctx.stream();
+
+ char * src0_ddc = (char *) src0->data;
+ char * src1_ddc = (char *) src1->data;
+
+ if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {
+ ggml_cpy_f32_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {
+ ggml_cpy_f32_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {
+ ggml_cpy_f32_q8_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {
+ ggml_cpy_f32_q4_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {
+ ggml_cpy_f32_q4_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) {
+ ggml_cpy_f32_q5_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) {
+ ggml_cpy_f32_iq4_nl_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) {
+ ggml_cpy_f32_q5_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {
+ ggml_cpy_f16_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {
+ ggml_cpy_f16_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ } else {
+ fprintf(stderr, "%s: unsupported type combination (%s to %s)\n", __func__,
+ ggml_type_name(src0->type), ggml_type_name(src1->type));
+ GGML_ABORT("fatal error");
+ }
+}
+
+void ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ ggml_cuda_cpy(ctx, src0, dst);
+}
+
+void* ggml_cuda_cpy_fn(const ggml_tensor * src0, ggml_tensor * src1) {
+ if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {
+ return (void*) cpy_f32_f16<cpy_1_f32_f32>;
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {
+ return (void*) cpy_f32_f16<cpy_1_f32_f16>;
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {
+ return (void*) cpy_f32_q<cpy_blck_f32_q8_0, QK8_0>;
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {
+ return (void*) cpy_f32_q<cpy_blck_f32_q4_0, QK4_0>;
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {
+ return (void*) cpy_f32_q<cpy_blck_f32_q4_1, QK4_1>;
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) {
+ return (void*) cpy_f32_q<cpy_blck_f32_q5_0, QK5_0>;
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) {
+ return (void*) cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL>;
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) {
+ return (void*) cpy_f32_q<cpy_blck_f32_q5_1, QK5_1>;
+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {
+ return (void*) cpy_f32_f16<cpy_1_f32_f16>;
+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {
+ return (void*) cpy_f32_f16<cpy_1_f16_f32>;
+ } else {
+ fprintf(stderr, "%s: unsupported type combination (%s to %s)\n", __func__,
+ ggml_type_name(src0->type), ggml_type_name(src1->type));
+ GGML_ABORT("fatal error");
+ }
+}
diff --git a/llama/ggml-cuda/cpy.cuh b/llama/ggml-cuda/cpy.cuh
new file mode 100644
index 00000000..7b140e30
--- /dev/null
+++ b/llama/ggml-cuda/cpy.cuh
@@ -0,0 +1,35 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_CPY_BLOCK_SIZE 32
+
+void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1);
+
+void ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void* ggml_cuda_cpy_fn(const ggml_tensor * src0, ggml_tensor * src1);
diff --git a/llama/ggml-cuda/cross-entropy-loss.cu b/llama/ggml-cuda/cross-entropy-loss.cu
new file mode 100644
index 00000000..55b9416f
--- /dev/null
+++ b/llama/ggml-cuda/cross-entropy-loss.cu
@@ -0,0 +1,132 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+#include "cross-entropy-loss.cuh"
+#include "sumrows.cuh"
+
+#include <cmath>
+#include <cstdint>
+
+static __global__ void cross_entropy_loss_f32(const float * logits, const float * labels, float * dst, const int nclasses, const int k) {
+ const int warp_id = threadIdx.x / WARP_SIZE;
+ const int lane_id = threadIdx.x % WARP_SIZE;
+ const int i0 = blockDim.x*blockIdx.x + warp_id*WARP_SIZE;
+
+ const int ne_tmp = WARP_SIZE*nclasses;
+
+ extern __shared__ float tmp_all[];
+ float * tmp_logits = tmp_all + (2*warp_id + 0)*ne_tmp;
+ float * tmp_labels = tmp_all + (2*warp_id + 1)*ne_tmp;
+
+ // Each warp first loads ne_tmp logits/labels into shared memory:
+ for (int i = lane_id; i < ne_tmp; i += WARP_SIZE) {
+ const int ig = i0*nclasses + i; // ig == i global
+
+ tmp_logits[i] = ig < k*nclasses ? logits[ig] : 0.0f;
+ tmp_labels[i] = ig < k*nclasses ? labels[ig] : 0.0f;
+ }
+
+ // Each thread in the warp then calculates the cross entropy loss for a single row.
+ // TODO: pad in order to avoid shared memory bank conflicts.
+
+ // Find maximum for softmax:
+ float max = -INFINITY;
+ for (int i = 0; i < nclasses; ++i) {
+ max = fmaxf(max, tmp_logits[lane_id*nclasses + i]);
+ }
+
+ // Calculate log(softmax(logits)) which is just logits - max:
+ float sum = 0.0f;
+ for (int i = 0; i < nclasses; ++i) {
+ float val = tmp_logits[lane_id*nclasses + i] - max;
+ sum += expf(val);
+ tmp_logits[lane_id*nclasses + i] = val;
+ }
+ sum = logf(sum);
+
+ // log(exp(logits - max) / sum) = (logits - max) - log(sum)
+ float loss = 0.0f;
+ for (int i = 0; i < nclasses; ++i) {
+ loss += (tmp_logits[lane_id*nclasses + i] - sum) * tmp_labels[lane_id*nclasses + i];
+ }
+ loss = -warp_reduce_sum(loss) / (float)k;
+
+ __syncthreads();
+
+ if (lane_id == 0) {
+ tmp_all[warp_id] = loss;
+ }
+
+ __syncthreads();
+
+ if (warp_id != 0) {
+ return;
+ }
+
+ loss = lane_id < CUDA_CROSS_ENTROPY_LOSS_BLOCK_SIZE/WARP_SIZE ? tmp_all[lane_id] : 0.0f;
+ loss = warp_reduce_sum(loss);
+
+ if (lane_id != 0) {
+ return;
+ }
+
+ dst[blockIdx.x] = loss;
+}
+
+void ggml_cuda_cross_entropy_loss(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+ GGML_ASSERT(ggml_is_contiguous(src1));
+ GGML_ASSERT(ggml_is_contiguous(dst));
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t nrows = ggml_nrows(src0);
+
+ const float * src0_d = (const float *) src0->data;
+ const float * src1_d = (const float *) src1->data;
+ float * dst_d = (float *) dst->data;
+
+ ggml_cuda_pool & pool = ctx.pool();
+ cudaStream_t stream = ctx.stream();
+
+ const dim3 blocks_dim(CUDA_CROSS_ENTROPY_LOSS_BLOCK_SIZE, 1, 1);
+ const dim3 blocks_num((nrows + CUDA_CROSS_ENTROPY_LOSS_BLOCK_SIZE - 1) / CUDA_CROSS_ENTROPY_LOSS_BLOCK_SIZE, 1, 1);
+ const int shmem = 2*CUDA_CROSS_ENTROPY_LOSS_BLOCK_SIZE*ne00*sizeof(float);
+
+ ggml_cuda_pool_alloc<float> dst_tmp(pool, blocks_num.x);
+
+ cross_entropy_loss_f32<<<blocks_num, blocks_dim, shmem, stream>>>(src0_d, src1_d, dst_tmp.ptr, ne00, nrows);
+
+ // Combine results from individual blocks:
+ sum_rows_f32_cuda(dst_tmp.ptr, dst_d, blocks_num.x, 1, stream);
+}
diff --git a/llama/ggml-cuda/cross-entropy-loss.cuh b/llama/ggml-cuda/cross-entropy-loss.cuh
new file mode 100644
index 00000000..7f76ae6c
--- /dev/null
+++ b/llama/ggml-cuda/cross-entropy-loss.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_CROSS_ENTROPY_LOSS_BLOCK_SIZE 256
+
+void ggml_cuda_cross_entropy_loss(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/dequantize.cuh b/llama/ggml-cuda/dequantize.cuh
new file mode 100644
index 00000000..c464bc6b
--- /dev/null
+++ b/llama/ggml-cuda/dequantize.cuh
@@ -0,0 +1,129 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
+ const block_q4_0 * x = (const block_q4_0 *) vx;
+
+ const dfloat d = x[ib].d;
+
+ const int vui = x[ib].qs[iqs];
+
+ v.x = vui & 0xF;
+ v.y = vui >> 4;
+
+#ifdef GGML_CUDA_F16
+ v = __hsub2(v, {8.0f, 8.0f});
+ v = __hmul2(v, {d, d});
+#else
+ v.x = (v.x - 8.0f) * d;
+ v.y = (v.y - 8.0f) * d;
+#endif // GGML_CUDA_F16
+}
+
+static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
+ const block_q4_1 * x = (const block_q4_1 *) vx;
+
+ const dfloat d = __low2half(x[ib].dm);
+ const dfloat m = __high2half(x[ib].dm);
+
+ const int vui = x[ib].qs[iqs];
+
+ v.x = vui & 0xF;
+ v.y = vui >> 4;
+
+#ifdef GGML_CUDA_F16
+ v = __hmul2(v, {d, d});
+ v = __hadd2(v, {m, m});
+#else
+ v.x = (v.x * d) + m;
+ v.y = (v.y * d) + m;
+#endif // GGML_CUDA_F16
+}
+
+static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
+ const block_q5_0 * x = (const block_q5_0 *) vx;
+
+ const dfloat d = x[ib].d;
+
+ uint32_t qh;
+ memcpy(&qh, x[ib].qh, sizeof(qh));
+
+ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
+
+ v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);
+ v.y = ((x[ib].qs[iqs] >> 4) | xh_1);
+
+#ifdef GGML_CUDA_F16
+ v = __hsub2(v, {16.0f, 16.0f});
+ v = __hmul2(v, {d, d});
+#else
+ v.x = (v.x - 16.0f) * d;
+ v.y = (v.y - 16.0f) * d;
+#endif // GGML_CUDA_F16
+}
+
+static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
+ const block_q5_1 * x = (const block_q5_1 *) vx;
+
+ const dfloat d = __low2half(x[ib].dm);
+ const dfloat m = __high2half(x[ib].dm);
+
+ uint32_t qh;
+ memcpy(&qh, x[ib].qh, sizeof(qh));
+
+ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
+
+ v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);
+ v.y = ((x[ib].qs[iqs] >> 4) | xh_1);
+
+#ifdef GGML_CUDA_F16
+ v = __hmul2(v, {d, d});
+ v = __hadd2(v, {m, m});
+#else
+ v.x = (v.x * d) + m;
+ v.y = (v.y * d) + m;
+#endif // GGML_CUDA_F16
+}
+
+static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
+ const block_q8_0 * x = (const block_q8_0 *) vx;
+
+ const dfloat d = x[ib].d;
+
+ v.x = x[ib].qs[iqs + 0];
+ v.y = x[ib].qs[iqs + 1];
+
+#ifdef GGML_CUDA_F16
+ v = __hmul2(v, {d, d});
+#else
+ v.x *= d;
+ v.y *= d;
+#endif // GGML_CUDA_F16
+}
diff --git a/llama/ggml-cuda/diagmask.cu b/llama/ggml-cuda/diagmask.cu
new file mode 100644
index 00000000..43433451
--- /dev/null
+++ b/llama/ggml-cuda/diagmask.cu
@@ -0,0 +1,66 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "diagmask.cuh"
+
+static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past) {
+ const int col = blockDim.y*blockIdx.y + threadIdx.y;
+ const int row = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (col >= ncols) {
+ return;
+ }
+
+ const int i = row*ncols + col;
+ //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i];
+ //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU
+ dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX;
+}
+
+static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, const int rows_per_channel, const int n_past, cudaStream_t stream) {
+ const dim3 block_dims(1, CUDA_DIAG_MASK_INF_BLOCK_SIZE, 1);
+ const int block_num_x = (ncols_x + CUDA_DIAG_MASK_INF_BLOCK_SIZE - 1) / CUDA_DIAG_MASK_INF_BLOCK_SIZE;
+ const dim3 block_nums(nrows_x, block_num_x, 1);
+ diag_mask_inf_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols_x, rows_per_channel, n_past);
+}
+
+void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t ne01 = src0->ne[1];
+ const int nrows0 = ggml_nrows(src0);
+
+ const int n_past = ((int32_t *) dst->op_params)[0];
+
+ diag_mask_inf_f32_cuda(src0_d, dst_d, ne00, nrows0, ne01, n_past, stream);
+}
diff --git a/llama/ggml-cuda/diagmask.cuh b/llama/ggml-cuda/diagmask.cuh
new file mode 100644
index 00000000..ee332322
--- /dev/null
+++ b/llama/ggml-cuda/diagmask.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32
+
+void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/dmmv.cu b/llama/ggml-cuda/dmmv.cu
new file mode 100644
index 00000000..0430f1fb
--- /dev/null
+++ b/llama/ggml-cuda/dmmv.cu
@@ -0,0 +1,709 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "dmmv.cuh"
+#include "dequantize.cuh"
+#include "convert.cuh"
+
+#ifndef K_QUANTS_PER_ITERATION
+#define K_QUANTS_PER_ITERATION 2
+#else
+static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUANTS_PER_ITERATION must be 1 or 2");
+#endif
+
+static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {
+
+ static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");
+
+ const int row = blockIdx.x*blockDim.y + threadIdx.y;
+ if (row > nrows) return;
+
+ const int num_blocks_per_row = ncols / QK_K;
+ const int ib0 = row*num_blocks_per_row;
+
+ const block_q2_K * x = (const block_q2_K *)vx + ib0;
+
+ float tmp = 0; // partial sum for thread in warp
+
+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...15
+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1
+
+ const int step = 16/K_QUANTS_PER_ITERATION;
+
+ const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...
+ const int in = tid - step*im; // 0...15 or 0...7
+
+ const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 or 0...14 in steps of 2
+ const int q_offset = 32*im + l0;
+ const int s_offset = 8*im;
+ const int y_offset = 128*im + l0;
+
+ uint32_t aux[4];
+ const uint8_t * d = (const uint8_t *)aux;
+ const uint8_t * m = (const uint8_t *)(aux + 2);
+
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
+
+ const float * y = yy + i * QK_K + y_offset;
+ const uint8_t * q = x[i].qs + q_offset;
+
+ const float dall = __low2half(x[i].dm);
+ const float dmin = __high2half(x[i].dm);
+
+ const uint32_t * a = (const uint32_t *)(x[i].scales + s_offset);
+ aux[0] = a[0] & 0x0f0f0f0f;
+ aux[1] = a[1] & 0x0f0f0f0f;
+ aux[2] = (a[0] >> 4) & 0x0f0f0f0f;
+ aux[3] = (a[1] >> 4) & 0x0f0f0f0f;
+
+ float sum1 = 0, sum2 = 0;
+ for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {
+ sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3)
+ + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3)
+ + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3)
+ + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3)
+ + y[l+16] * d[1] * ((q[l+16] >> 0) & 3)
+ + y[l+48] * d[3] * ((q[l+16] >> 2) & 3)
+ + y[l+80] * d[5] * ((q[l+16] >> 4) & 3)
+ +y[l+112] * d[7] * ((q[l+16] >> 6) & 3);
+ sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6]
+ + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7];
+
+ }
+ tmp += dall * sum1 - dmin * sum2;
+
+ }
+
+ // sum up partial sums and write back result
+ tmp = warp_reduce_sum(tmp);
+
+ if (threadIdx.x == 0) {
+ dst[row] = tmp;
+ }
+}
+
+static __global__ void dequantize_mul_mat_vec_q3_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {
+
+ const int row = blockIdx.x*blockDim.y + threadIdx.y;
+ if (row > nrows) return;
+
+ const int num_blocks_per_row = ncols / QK_K;
+ const int ib0 = row*num_blocks_per_row;
+
+ const block_q3_K * x = (const block_q3_K *)vx + ib0;
+
+ float tmp = 0; // partial sum for thread in warp
+
+ const uint16_t kmask1 = 0x0303;
+ const uint16_t kmask2 = 0x0f0f;
+
+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16
+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1
+
+ const int n = K_QUANTS_PER_ITERATION; // iterations in the inner loop
+ const int step = 16/K_QUANTS_PER_ITERATION;
+ const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...
+ const int in = tid - step*im; // 0....15 or 0...7
+
+ const uint8_t m = 1 << (4*im);
+
+ const int l0 = n*in; // 0...15 or 0...14 in steps of 2
+ const int q_offset = 32*im + l0;
+ const int y_offset = 128*im + l0;
+
+ uint16_t utmp[4];
+ const int8_t * s = (const int8_t *)utmp;
+
+ const uint16_t s_shift = 4*im;
+
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
+
+ const float * y = yy + i * QK_K + y_offset;
+ const uint8_t * q = x[i].qs + q_offset;
+ const uint8_t * h = x[i].hmask + l0;
+
+ const uint16_t * a = (const uint16_t *)x[i].scales;
+ utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4);
+ utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4);
+ utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4);
+ utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4);
+
+ const float d = x[i].d;
+
+ float sum = 0;
+ for (int l = 0; l < n; ++l) {
+ sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4))
+ + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4))
+ + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4))
+ + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4));
+ sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4))
+ + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4))
+ + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4))
+ + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4));
+ }
+ tmp += d * sum;
+
+ }
+
+ // sum up partial sums and write back result
+ tmp = warp_reduce_sum(tmp);
+
+ if (threadIdx.x == 0) {
+ dst[row] = tmp;
+ }
+}
+
+static __global__ void dequantize_mul_mat_vec_q4_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {
+
+ const int row = blockIdx.x*blockDim.y + threadIdx.y;
+ if (row > nrows) return;
+ const int num_blocks_per_row = ncols / QK_K;
+ const int ib0 = row*num_blocks_per_row;
+
+ const block_q4_K * x = (const block_q4_K *)vx + ib0;
+
+ const uint16_t kmask1 = 0x3f3f;
+ const uint16_t kmask2 = 0x0f0f;
+ const uint16_t kmask3 = 0xc0c0;
+
+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16
+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1
+
+ const int step = 8/K_QUANTS_PER_ITERATION; // 8 or 4
+
+ const int il = tid/step; // 0...3
+ const int ir = tid - step*il; // 0...7 or 0...3
+ const int n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4
+
+ const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224
+ const int in = il%2;
+
+ const int l0 = n*(2*ir + in);
+ const int q_offset = 32*im + l0;
+ const int y_offset = 64*im + l0;
+
+ uint16_t aux[4];
+ const uint8_t * sc = (const uint8_t *)aux;
+
+#if K_QUANTS_PER_ITERATION == 2
+ uint32_t q32[4];
+ const uint8_t * q4 = (const uint8_t *)q32;
+#else
+ uint16_t q16[4];
+ const uint8_t * q4 = (const uint8_t *)q16;
+#endif
+
+ float tmp = 0; // partial sum for thread in warp
+
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
+
+ const float * y1 = yy + i*QK_K + y_offset;
+ const float * y2 = y1 + 128;
+
+ const float dall = __low2half(x[i].dm);
+ const float dmin = __high2half(x[i].dm);
+
+ const uint16_t * a = (const uint16_t *)x[i].scales;
+ aux[0] = a[im+0] & kmask1;
+ aux[1] = a[im+2] & kmask1;
+ aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);
+ aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);
+
+#if K_QUANTS_PER_ITERATION == 2
+ const uint32_t * q1 = (const uint32_t *)(x[i].qs + q_offset);
+ const uint32_t * q2 = q1 + 16;
+
+ q32[0] = q1[0] & 0x0f0f0f0f;
+ q32[1] = q1[0] & 0xf0f0f0f0;
+ q32[2] = q2[0] & 0x0f0f0f0f;
+ q32[3] = q2[0] & 0xf0f0f0f0;
+
+ float4 s = {0.f, 0.f, 0.f, 0.f};
+ float smin = 0;
+ for (int l = 0; l < 4; ++l) {
+ s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+ 4];
+ s.z += y2[l] * q4[l+8]; s.w += y2[l+32] * q4[l+12];
+ smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];
+ }
+ tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin;
+#else
+ const uint16_t * q1 = (const uint16_t *)(x[i].qs + q_offset);
+ const uint16_t * q2 = q1 + 32;
+
+ q16[0] = q1[0] & 0x0f0f;
+ q16[1] = q1[0] & 0xf0f0;
+ q16[2] = q2[0] & 0x0f0f;
+ q16[3] = q2[0] & 0xf0f0;
+
+ float4 s = {0.f, 0.f, 0.f, 0.f};
+ float smin = 0;
+ for (int l = 0; l < 2; ++l) {
+ s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+2];
+ s.z += y2[l] * q4[l+4]; s.w += y2[l+32] * q4[l+6];
+ smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];
+ }
+ tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin;
+#endif
+
+ }
+
+ // sum up partial sums and write back result
+ tmp = warp_reduce_sum(tmp);
+
+ if (tid == 0) {
+ dst[row] = tmp;
+ }
+}
+
+static __global__ void dequantize_mul_mat_vec_q5_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols) {
+
+ const int row = blockIdx.x;
+ const int num_blocks_per_row = ncols / QK_K;
+ const int ib0 = row*num_blocks_per_row;
+
+ const block_q5_K * x = (const block_q5_K *)vx + ib0;
+
+ float tmp = 0; // partial sum for thread in warp
+
+ const uint16_t kmask1 = 0x3f3f;
+ const uint16_t kmask2 = 0x0f0f;
+ const uint16_t kmask3 = 0xc0c0;
+
+ const int tid = threadIdx.x/2; // 0...15
+ const int ix = threadIdx.x%2;
+
+ const int il = tid/4; // 0...3
+ const int ir = tid - 4*il;// 0...3
+ const int n = 2;
+
+ const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224
+ const int in = il%2;
+
+ const int l0 = n*(2*ir + in);
+ const int q_offset = 32*im + l0;
+ const int y_offset = 64*im + l0;
+
+ const uint8_t hm1 = 1 << (2*im);
+ const uint8_t hm2 = hm1 << 4;
+
+ uint16_t aux[4];
+ const uint8_t * sc = (const uint8_t *)aux;
+
+ uint16_t q16[8];
+ const uint8_t * q4 = (const uint8_t *)q16;
+
+ for (int i = ix; i < num_blocks_per_row; i += 2) {
+
+ const uint8_t * ql1 = x[i].qs + q_offset;
+ const uint8_t * qh = x[i].qh + l0;
+ const float * y1 = yy + i*QK_K + y_offset;
+ const float * y2 = y1 + 128;
+
+ const float dall = __low2half(x[i].dm);
+ const float dmin = __high2half(x[i].dm);
+
+ const uint16_t * a = (const uint16_t *)x[i].scales;
+ aux[0] = a[im+0] & kmask1;
+ aux[1] = a[im+2] & kmask1;
+ aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);
+ aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);
+
+ float4 sum = {0.f, 0.f, 0.f, 0.f};
+ float smin = 0;
+ const uint16_t * q1 = (const uint16_t *)ql1;
+ const uint16_t * q2 = q1 + 32;
+ q16[0] = q1[0] & 0x0f0f;
+ q16[1] = q1[8] & 0x0f0f;
+ q16[2] = (q1[0] >> 4) & 0x0f0f;
+ q16[3] = (q1[8] >> 4) & 0x0f0f;
+ q16[4] = q2[0] & 0x0f0f;
+ q16[5] = q2[8] & 0x0f0f;
+ q16[6] = (q2[0] >> 4) & 0x0f0f;
+ q16[7] = (q2[8] >> 4) & 0x0f0f;
+ for (int l = 0; l < n; ++l) {
+ sum.x += y1[l+ 0] * (q4[l +0] + (qh[l+ 0] & (hm1 << 0) ? 16 : 0))
+ + y1[l+16] * (q4[l +2] + (qh[l+16] & (hm1 << 0) ? 16 : 0));
+ sum.y += y1[l+32] * (q4[l +4] + (qh[l+ 0] & (hm1 << 1) ? 16 : 0))
+ + y1[l+48] * (q4[l +6] + (qh[l+16] & (hm1 << 1) ? 16 : 0));
+ sum.z += y2[l+ 0] * (q4[l +8] + (qh[l+ 0] & (hm2 << 0) ? 16 : 0))
+ + y2[l+16] * (q4[l+10] + (qh[l+16] & (hm2 << 0) ? 16 : 0));
+ sum.w += y2[l+32] * (q4[l+12] + (qh[l+ 0] & (hm2 << 1) ? 16 : 0))
+ + y2[l+48] * (q4[l+14] + (qh[l+16] & (hm2 << 1) ? 16 : 0));
+ smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3]
+ + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7];
+ }
+ tmp += dall * (sum.x * sc[0] + sum.y * sc[1] + sum.z * sc[4] + sum.w * sc[5]) - dmin * smin;
+ }
+
+ // sum up partial sums and write back result
+ tmp = warp_reduce_sum(tmp);
+
+ if (threadIdx.x == 0) {
+ dst[row] = tmp;
+ }
+}
+
+static __global__ void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {
+
+ static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");
+
+ const int row = blockIdx.x*blockDim.y + threadIdx.y;
+ if (row > nrows) return;
+
+ const int num_blocks_per_row = ncols / QK_K;
+ const int ib0 = row*num_blocks_per_row;
+
+ const block_q6_K * x = (const block_q6_K *)vx + ib0;
+
+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16
+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0, 1
+
+ const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8
+
+ const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...
+ const int in = tid - step*im; // 0...15 or 0...7
+
+#if K_QUANTS_PER_ITERATION == 1
+ const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15
+ const int is = 0;
+#else
+ const int l0 = 4 * in; // 0, 4, 8, ..., 28
+ const int is = in / 4;
+#endif
+ const int ql_offset = 64*im + l0;
+ const int qh_offset = 32*im + l0;
+ const int s_offset = 8*im + is;
+ const int y_offset = 128*im + l0;
+
+ float tmp = 0; // partial sum for thread in warp
+
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
+
+ const float * y = yy + i * QK_K + y_offset;
+ const uint8_t * ql = x[i].ql + ql_offset;
+ const uint8_t * qh = x[i].qh + qh_offset;
+ const int8_t * s = x[i].scales + s_offset;
+
+ const float d = x[i].d;
+
+#if K_QUANTS_PER_ITERATION == 1
+ float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32)
+ + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32)
+ + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32)
+ + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32)
+ + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32)
+ + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32)
+ + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32)
+ +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32);
+ tmp += sum;
+#else
+ float sum = 0;
+ for (int l = 0; l < 4; ++l) {
+ sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32)
+ + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32)
+ + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32)
+ + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32);
+ }
+ tmp += sum;
+#endif
+
+ }
+
+ // sum up partial sums and write back result
+ tmp = warp_reduce_sum(tmp);
+
+ if (tid == 0) {
+ dst[row] = tmp;
+ }
+}
+
+static __device__ void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
+ const half * x = (const half *) vx;
+
+ // automatic half -> float type cast if dfloat == float
+ v.x = x[ib + iqs + 0];
+ v.y = x[ib + iqs + 1];
+}
+
+static constexpr __device__ dequantize_kernel_t get_dequantize_kernel(ggml_type type) {
+ return type == GGML_TYPE_Q4_0 ? dequantize_q4_0 :
+ type == GGML_TYPE_Q4_1 ? dequantize_q4_1 :
+ type == GGML_TYPE_Q5_0 ? dequantize_q5_0 :
+ type == GGML_TYPE_Q5_1 ? dequantize_q5_1 :
+ type == GGML_TYPE_Q8_0 ? dequantize_q8_0 :
+ type == GGML_TYPE_F16 ? convert_f16 :
+ nullptr;
+}
+
+template <ggml_type type>
+static __global__ void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows) {
+ constexpr int qk = ggml_cuda_type_traits<type>::qk; // quantized weights per x block
+ constexpr int qr = ggml_cuda_type_traits<type>::qr; // number of quantized weights per data value in x block
+ constexpr dequantize_kernel_t dequantize_kernel = get_dequantize_kernel(type);
+
+ const int64_t row = (int64_t)blockIdx.x*blockDim.y + threadIdx.y;
+
+ if (row >= nrows) {
+ return;
+ }
+
+ const int tid = threadIdx.x;
+
+ const int iter_stride = 2*GGML_CUDA_DMMV_X;
+ const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter
+ const int y_offset = qr == 1 ? 1 : qk/2;
+
+// partial sum for each thread
+#ifdef GGML_CUDA_F16
+ half2 tmp = {0.0f, 0.0f}; // two sums for f16 to take advantage of half2 intrinsics
+#else
+ float tmp = 0.0f;
+#endif // GGML_CUDA_F16
+
+ for (int i = 0; i < ncols; i += iter_stride) {
+ const int col = i + vals_per_iter*tid;
+ const int64_t ib = ((int64_t)row*ncols + col)/qk; // x block index
+ const int iqs = (col%qk)/qr; // x quant index
+ const int iybs = col - col%qk; // y block start index
+
+// processing >2 values per i iter is faster for fast GPUs
+#pragma unroll
+ for (int j = 0; j < vals_per_iter; j += 2) {
+ // process 2 vals per j iter
+
+ // dequantize
+ // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val
+ dfloat2 v;
+ dequantize_kernel(vx, ib, iqs + j/qr, v);
+
+ // matrix multiplication
+ // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2
+#ifdef GGML_CUDA_F16
+ tmp += __hmul2(v, {
+ y[iybs + iqs + j/qr + 0],
+ y[iybs + iqs + j/qr + y_offset]
+ });
+#else
+ tmp += v.x * y[iybs + iqs + j/qr + 0];
+ tmp += v.y * y[iybs + iqs + j/qr + y_offset];
+#endif // GGML_CUDA_F16
+ }
+ }
+
+ // sum up partial sums and write back result
+ tmp = warp_reduce_sum(tmp);
+
+ if (tid == 0) {
+#ifdef GGML_CUDA_F16
+ dst[row] = tmp.x + tmp.y;
+#else
+ dst[row] = tmp;
+#endif // GGML_CUDA_F16
+ }
+}
+
+static void dequantize_mul_mat_vec_q4_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ GGML_ASSERT(ncols % (GGML_CUDA_DMMV_X*2) == 0);
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
+ // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
+ const dim3 block_nums(block_num_y, 1, 1);
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
+ dequantize_mul_mat_vec<GGML_TYPE_Q4_0>
+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);
+}
+
+static void dequantize_mul_mat_vec_q4_1_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ GGML_ASSERT(ncols % (GGML_CUDA_DMMV_X*2) == 0);
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
+ const dim3 block_nums(block_num_y, 1, 1);
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
+ dequantize_mul_mat_vec<GGML_TYPE_Q4_1>
+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);
+}
+
+static void dequantize_mul_mat_vec_q5_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ GGML_ASSERT(ncols % (GGML_CUDA_DMMV_X*2) == 0);
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
+ const dim3 block_nums(block_num_y, 1, 1);
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
+ dequantize_mul_mat_vec<GGML_TYPE_Q5_0>
+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);
+}
+
+static void dequantize_mul_mat_vec_q5_1_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ GGML_ASSERT(ncols % (GGML_CUDA_DMMV_X*2) == 0);
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
+ const dim3 block_nums(block_num_y, 1, 1);
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
+ dequantize_mul_mat_vec<GGML_TYPE_Q5_1>
+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);
+}
+
+static void dequantize_mul_mat_vec_q8_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ GGML_ASSERT(ncols % (GGML_CUDA_DMMV_X*2) == 0);
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
+ const dim3 block_nums(block_num_y, 1, 1);
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
+ dequantize_mul_mat_vec<GGML_TYPE_Q8_0>
+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);
+}
+
+static void dequantize_mul_mat_vec_q2_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ GGML_ASSERT(ncols % QK_K == 0);
+ const int ny = 2; // very slightly faster than 1 even when K_QUANTS_PER_ITERATION = 2
+ const int block_num_y = (nrows + ny - 1) / ny;
+ const dim3 block_nums(block_num_y, 1, 1);
+ const dim3 block_dims(32, ny, 1);
+ dequantize_mul_mat_vec_q2_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);
+}
+
+static void dequantize_mul_mat_vec_q3_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ GGML_ASSERT(ncols % QK_K == 0);
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
+ const int block_num_y = (nrows + ny - 1) / ny;
+ const dim3 block_nums(block_num_y, 1, 1);
+ const dim3 block_dims(32, ny, 1);
+ dequantize_mul_mat_vec_q3_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);
+}
+
+static void dequantize_mul_mat_vec_q4_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ GGML_ASSERT(ncols % QK_K == 0);
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
+ const int block_num_y = (nrows + ny - 1) / ny;
+ const dim3 block_nums(block_num_y, 1, 1);
+ const dim3 block_dims(32, ny, 1);
+ dequantize_mul_mat_vec_q4_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);
+}
+
+static void dequantize_mul_mat_vec_q5_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ GGML_ASSERT(ncols % QK_K == 0);
+ const dim3 block_dims(32, 1, 1);
+ dequantize_mul_mat_vec_q5_k<<<nrows, block_dims, 0, stream>>>(vx, y, dst, ncols);
+}
+
+static void dequantize_mul_mat_vec_q6_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ GGML_ASSERT(ncols % QK_K == 0);
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
+ const int block_num_y = (nrows + ny - 1) / ny;
+ const dim3 block_nums(block_num_y, 1, 1);
+ const dim3 block_dims(32, ny, 1);
+ dequantize_mul_mat_vec_q6_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);
+}
+
+static void convert_mul_mat_vec_f16_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ GGML_ASSERT(ncols % (GGML_CUDA_DMMV_X*2) == 0);
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
+ const dim3 block_nums(block_num_y, 1, 1);
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
+ dequantize_mul_mat_vec<GGML_TYPE_F16>
+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);
+}
+
+void ggml_cuda_op_dequantize_mul_mat_vec(
+ ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
+ const int64_t src1_padded_row_size, cudaStream_t stream) {
+ GGML_UNUSED(ctx);
+ const int64_t ne00 = src0->ne[0];
+ const int64_t row_diff = row_high - row_low;
+
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ // on some GPUs it is faster to convert src1 to half and to use half precision intrinsics
+#ifdef GGML_CUDA_F16
+ ggml_cuda_pool_alloc<half> src1_dfloat_a(ctx.pool());
+ half * src1_dfloat = nullptr; // dfloat == half
+
+ bool src1_convert_f16 =
+ src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 ||
+ src0->type == GGML_TYPE_Q5_0 || src0->type == GGML_TYPE_Q5_1 ||
+ src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16;
+
+ if (src1_convert_f16) {
+ src1_dfloat = src1_dfloat_a.alloc(ne00);
+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);
+ GGML_ASSERT(to_fp16_cuda != nullptr);
+ to_fp16_cuda(src1_ddf_i, src1_dfloat, ne00, stream);
+ }
+#else
+ const dfloat * src1_dfloat = (const dfloat *) src1_ddf_i; // dfloat == float, no conversion
+#endif // GGML_CUDA_F16
+
+ switch (src0->type) {
+ case GGML_TYPE_Q4_0:
+ dequantize_mul_mat_vec_q4_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+ break;
+ case GGML_TYPE_Q4_1:
+ dequantize_mul_mat_vec_q4_1_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+ break;
+ case GGML_TYPE_Q5_0:
+ dequantize_mul_mat_vec_q5_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+ break;
+ case GGML_TYPE_Q5_1:
+ dequantize_mul_mat_vec_q5_1_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+ break;
+ case GGML_TYPE_Q8_0:
+ dequantize_mul_mat_vec_q8_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+ break;
+ case GGML_TYPE_Q2_K:
+ dequantize_mul_mat_vec_q2_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+ break;
+ case GGML_TYPE_Q3_K:
+ dequantize_mul_mat_vec_q3_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+ break;
+ case GGML_TYPE_Q4_K:
+ dequantize_mul_mat_vec_q4_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+ break;
+ case GGML_TYPE_Q5_K:
+ dequantize_mul_mat_vec_q5_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+ break;
+ case GGML_TYPE_Q6_K:
+ dequantize_mul_mat_vec_q6_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+ break;
+ case GGML_TYPE_F16:
+ convert_mul_mat_vec_f16_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+
+ GGML_UNUSED(src1);
+ GGML_UNUSED(dst);
+ GGML_UNUSED(src1_ddq_i);
+ GGML_UNUSED(src1_ncols);
+ GGML_UNUSED(src1_padded_row_size);
+}
+
+bool ggml_cuda_dmmv_type_supported(ggml_type src0_type) {
+ return src0_type == GGML_TYPE_Q4_0 || src0_type == GGML_TYPE_Q4_1 ||
+ src0_type == GGML_TYPE_Q5_0 || src0_type == GGML_TYPE_Q5_1 ||
+ src0_type == GGML_TYPE_Q8_0 || src0_type == GGML_TYPE_Q2_K ||
+ src0_type == GGML_TYPE_Q3_K || src0_type == GGML_TYPE_Q4_K ||
+ src0_type == GGML_TYPE_Q5_K || src0_type == GGML_TYPE_Q6_K ||
+ src0_type == GGML_TYPE_F16;
+}
diff --git a/llama/ggml-cuda/dmmv.cuh b/llama/ggml-cuda/dmmv.cuh
new file mode 100644
index 00000000..4e8c0f37
--- /dev/null
+++ b/llama/ggml-cuda/dmmv.cuh
@@ -0,0 +1,46 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+// dmmv = dequantize_mul_mat_vec
+
+// TODO: remove this?
+#ifndef GGML_CUDA_DMMV_X
+#define GGML_CUDA_DMMV_X 32
+#endif
+
+#ifndef GGML_CUDA_MMV_Y
+#define GGML_CUDA_MMV_Y 1
+#endif
+
+void ggml_cuda_op_dequantize_mul_mat_vec(
+ ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
+ const int64_t src1_padded_row_size, cudaStream_t stream);
+
+bool ggml_cuda_dmmv_type_supported(ggml_type src0_type);
diff --git a/llama/ggml-cuda/fattn-common.cuh b/llama/ggml-cuda/fattn-common.cuh
new file mode 100644
index 00000000..21fa5903
--- /dev/null
+++ b/llama/ggml-cuda/fattn-common.cuh
@@ -0,0 +1,734 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "common.cuh"
+#include "convert.cuh"
+#include "vecdotq.cuh"
+
+#include <cstdint>
+
+#define FATTN_KQ_STRIDE 256
+#define HALF_MAX_HALF __float2half(65504.0f/2) // Use neg. of this instead of -INFINITY to initialize KQ max vals to avoid NaN upon subtraction.
+#define SOFTMAX_FTZ_THRESHOLD -20.0f // Softmax exp. of values smaller than this are flushed to zero to avoid NaNs.
+
+typedef void (* fattn_kernel_t)(
+ const char * __restrict__ Q,
+ const char * __restrict__ K,
+ const char * __restrict__ V,
+ const char * __restrict__ mask,
+ float * __restrict__ dst,
+ float2 * __restrict__ dst_meta,
+ const float scale,
+ const float max_bias,
+ const float m0,
+ const float m1,
+ const uint32_t n_head_log2,
+ const float logit_softcap,
+ const int ne00,
+ const int ne01,
+ const int ne02,
+ const int ne03,
+ const int ne10,
+ const int ne11,
+ const int ne12,
+ const int ne13,
+ const int ne31,
+ const int nb31,
+ const int nb01,
+ const int nb02,
+ const int nb03,
+ const int nb11,
+ const int nb12,
+ const int nb13,
+ const int nb21,
+ const int nb22,
+ const int nb23,
+ const int ne0,
+ const int ne1,
+ const int ne2,
+ const int ne3);
+
+typedef half (*vec_dot_KQ_f16_t)(
+ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds);
+typedef float (*vec_dot_KQ_f32_t)(
+ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds);
+
+template<typename T, int D>
+static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q4_0(
+ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {
+
+ const block_q4_0 * K_q4_0 = (const block_q4_0 *) K_c;
+ GGML_UNUSED(Q_v);
+
+ T sum = 0.0f;
+
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D/sizeof(int); k_KQ_0 += WARP_SIZE) {
+ const int k_KQ = k_KQ_0 + threadIdx.x;
+
+ const int ib = k_KQ / QI8_1;
+ const int iqs4 = k_KQ % QI4_0;
+ const int shift = k_KQ & (QI8_1/2);
+
+ const int v = (get_int_b2(K_q4_0[ib].qs, iqs4) >> shift) & 0x0F0F0F0F;
+ const int u = Q_q8[k_KQ_0/WARP_SIZE];
+
+ const int sumi = ggml_cuda_dp4a(v, u, 0);
+
+#ifdef FP16_AVAILABLE
+ if (std::is_same<T, half>::value) {
+ const half2 * Q_ds = (const half2 *) Q_ds_v;
+
+ const half2 sum2 = __half2half2(K_q4_0[ib].d) * Q_ds[k_KQ_0/WARP_SIZE];
+ sum += (T) (((half) sumi)*__low2half(sum2) - __high2half(sum2) /* *8/QI8_1 == 1 */);
+ } else
+#endif // FP16_AVAILABLE
+ {
+ const float2 * Q_ds = (const float2 *) Q_ds_v;
+
+ sum += (T) (__half2float(K_q4_0[ib].d) * (sumi*Q_ds[k_KQ_0/WARP_SIZE].x - (8/QI8_1)*Q_ds[k_KQ_0/WARP_SIZE].y));
+ }
+ }
+
+ return sum;
+}
+
+template<typename T, int D>
+static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q4_1(
+ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {
+
+ const block_q4_1 * K_q4_1 = (const block_q4_1 *) K_c;
+ GGML_UNUSED(Q_v);
+
+ T sum = 0.0f;
+
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D/sizeof(int); k_KQ_0 += WARP_SIZE) {
+ const int k_KQ = k_KQ_0 + threadIdx.x;
+
+ const int ib = k_KQ / QI8_1;
+ const int iqs4 = k_KQ % QI4_1;
+ const int shift = k_KQ & (QI8_1/2);
+
+ const int v = (get_int_b4(K_q4_1[ib].qs, iqs4) >> shift) & 0x0F0F0F0F;
+ const int u = Q_q8[k_KQ_0/WARP_SIZE];
+
+ const int sumi = ggml_cuda_dp4a(v, u, 0);
+
+#ifdef FP16_AVAILABLE
+ if (std::is_same<T, half>::value) {
+ const half2 * Q_ds = (const half2 *) Q_ds_v;
+
+ const half2 d4d8_m4s8 = K_q4_1[ib].dm * Q_ds[k_KQ_0/WARP_SIZE];
+ const half2 sumid4d8_m4s8scaled = d4d8_m4s8 * make_half2(sumi, 1.0f/QI8_1);
+ sum += (T) (__low2half(sumid4d8_m4s8scaled) + __high2half(sumid4d8_m4s8scaled));
+ } else
+#endif // FP16_AVAILABLE
+ {
+ const float2 * Q_ds = (const float2 *) Q_ds_v;
+
+ const float sumid4d8 = __low2float(K_q4_1[ib].dm)*Q_ds[k_KQ_0/WARP_SIZE].x * sumi;
+ const float m4s8scaled = __high2float(K_q4_1[ib].dm)*Q_ds[k_KQ_0/WARP_SIZE].y / QI8_1;
+
+ sum += (T) (sumid4d8 + m4s8scaled);
+ }
+ }
+
+ return sum;
+}
+
+template<typename T, int D>
+static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q5_0(
+ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {
+
+ const block_q5_0 * K_q5_0 = (const block_q5_0 *) K_c;
+ GGML_UNUSED(Q_v);
+
+ T sum = 0.0f;
+
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D/sizeof(int); k_KQ_0 += WARP_SIZE) {
+ const int k_KQ = k_KQ_0 + threadIdx.x;
+
+ const int ib = k_KQ / QI8_1;
+ const int iqs4 = k_KQ % QI5_0;
+ const int iqs8 = k_KQ % QI8_1;
+ const int shift = k_KQ & (QI8_1/2);
+
+ int v = (get_int_b2(K_q5_0[ib].qs, iqs4) >> shift) & 0x0F0F0F0F;
+ const int vh = get_int_b2(K_q5_0[ib].qh, 0) >> (iqs8 * QI5_0);
+ v |= (vh << 4) & 0x00000010; // 0 -> 4
+ v |= (vh << 11) & 0x00001000; // 1 -> 12
+ v |= (vh << 18) & 0x00100000; // 2 -> 20
+ v |= (vh << 25) & 0x10000000; // 3 -> 28
+
+ const int u = Q_q8[k_KQ_0/WARP_SIZE];
+
+ const int sumi = ggml_cuda_dp4a(v, u, 0);
+
+#ifdef FP16_AVAILABLE
+ if (std::is_same<T, half>::value) {
+ const half2 * Q_ds = (const half2 *) Q_ds_v;
+
+ const half2 sum2 = __half2half2(K_q5_0[ib].d) * Q_ds[k_KQ_0/WARP_SIZE];
+ sum += (T) (((half) sumi)*__low2half(sum2) - __high2half(sum2)*__float2half(2.0f)) /* *16/QI8_1 == 2 */;
+ } else
+#endif // FP16_AVAILABLE
+ {
+ const float2 * Q_ds = (const float2 *) Q_ds_v;
+
+ sum += (T) (__half2float(K_q5_0[ib].d) * (sumi*Q_ds[k_KQ_0/WARP_SIZE].x - (16/QI8_1)*Q_ds[k_KQ_0/WARP_SIZE].y));
+ }
+ }
+
+ return sum;
+}
+
+template<typename T, int D>
+static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q5_1(
+ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {
+
+ const block_q5_1 * K_q5_1 = (const block_q5_1 *) K_c;
+ GGML_UNUSED(Q_v);
+
+ T sum = 0.0f;
+
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D/sizeof(int); k_KQ_0 += WARP_SIZE) {
+ const int k_KQ = k_KQ_0 + threadIdx.x;
+
+ const int ib = k_KQ / QI8_1;
+ const int iqs4 = k_KQ % QI5_1;
+ const int iqs8 = k_KQ % QI8_1;
+ const int shift = k_KQ & (QI8_1/2);
+
+ int v = (get_int_b2(K_q5_1[ib].qs, iqs4) >> shift) & 0x0F0F0F0F;
+ const int vh = get_int_b2(K_q5_1[ib].qh, 0) >> (iqs8 * QI5_1);
+ v |= (vh << 4) & 0x00000010; // 0 -> 4
+ v |= (vh << 11) & 0x00001000; // 1 -> 12
+ v |= (vh << 18) & 0x00100000; // 2 -> 20
+ v |= (vh << 25) & 0x10000000; // 3 -> 28
+
+ const int u = Q_q8[k_KQ_0/WARP_SIZE];
+
+ const int sumi = ggml_cuda_dp4a(v, u, 0);
+
+#ifdef FP16_AVAILABLE
+ if (std::is_same<T, half>::value) {
+ const half2 * Q_ds = (const half2 *) Q_ds_v;
+
+ const half2 d5d8_m5s8 = K_q5_1[ib].dm * Q_ds[k_KQ_0/WARP_SIZE];
+ const half2 sumid5d8_m5s8scaled = d5d8_m5s8 * make_half2(sumi, 1.0f/QI8_1);
+ sum += (T) (__low2half(sumid5d8_m5s8scaled) + __high2half(sumid5d8_m5s8scaled));
+ } else
+#endif // FP16_AVAILABLE
+ {
+ const float2 * Q_ds = (const float2 *) Q_ds_v;
+
+ const float sumid5d8 = __low2float(K_q5_1[ib].dm)*Q_ds[k_KQ_0/WARP_SIZE].x * sumi;
+ const float m5s8scaled = __high2float(K_q5_1[ib].dm)*Q_ds[k_KQ_0/WARP_SIZE].y / QI8_1;
+
+ sum += (T) (sumid5d8 + m5s8scaled);
+ }
+ }
+
+ return sum;
+}
+
+template <typename T, int D>
+static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q8_0(
+ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {
+
+ const block_q8_0 * K_q8_0 = (const block_q8_0 *) K_c;
+ GGML_UNUSED(Q_v);
+
+ T sum = 0.0f;
+
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D/sizeof(int); k_KQ_0 += WARP_SIZE) {
+ const int k_KQ = k_KQ_0 + threadIdx.x;
+
+ const int ib = k_KQ / QI8_0;
+ const int iqs = k_KQ % QI8_0;
+
+ const int v = get_int_b2(K_q8_0[ib].qs, iqs);
+
+ T Q_d;
+ if (std::is_same<T, half>::value) {
+ const half2 * Q_ds = (const half2 *) Q_ds_v;
+ Q_d = __low2half(Q_ds[k_KQ_0/WARP_SIZE]);
+ } else {
+ const float2 * Q_ds = (const float2 *) Q_ds_v;
+ Q_d = Q_ds[k_KQ_0/WARP_SIZE].x;
+ }
+
+ sum += vec_dot_q8_0_q8_1_impl<T, 1>(&v, &Q_q8[k_KQ_0/WARP_SIZE], K_q8_0[ib].d, Q_d);
+ }
+
+ return sum;
+}
+
+template <typename T, int D>
+static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_f16(
+ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds_v) {
+
+ const half2 * K_h2 = (const half2 *) K_c;
+ GGML_UNUSED(Q_q8);
+ GGML_UNUSED(Q_ds_v);
+
+#ifdef FP16_AVAILABLE
+ if (std::is_same<T, half>::value) {
+ const half2 * Q_h2 = (const half2 *) Q_v;
+
+ half2 sum2 = make_half2(0.0f, 0.0f);
+
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) {
+ const int k_KQ = k_KQ_0 + threadIdx.x;
+
+ const half2 K_ik = K_h2[k_KQ];
+ sum2 += K_ik * Q_h2[k_KQ_0/WARP_SIZE];
+ }
+
+ return __low2half(sum2) + __high2half(sum2);
+ }
+#endif // FP16_AVAILABLE
+
+ const float2 * Q_f2 = (const float2 *) Q_v;
+
+ float sum = 0.0f;
+
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) {
+ const int k_KQ = k_KQ_0 + threadIdx.x;
+
+ const half2 K_ik = K_h2[k_KQ];
+ sum += __low2float(K_ik) * Q_f2[k_KQ_0/WARP_SIZE].x;
+ sum += __high2float(K_ik) * Q_f2[k_KQ_0/WARP_SIZE].y;
+ }
+
+ return sum;
+}
+
+template <typename Tds>
+static __device__ __forceinline__ void quantize_q8_1_to_shared(
+ const float * __restrict__ x, const float scale, int * __restrict__ yq32, void * __restrict__ yds) {
+
+ float vals[sizeof(int)] = {0.0f};
+#pragma unroll
+ for (int l = 0; l < sizeof(int); ++l) {
+ vals[l] = scale * x[4*threadIdx.x + l];
+ }
+
+ float amax = fabsf(vals[0]);
+ float sum = vals[0];
+#pragma unroll
+ for (int l = 1; l < sizeof(int); ++l) {
+ amax = fmaxf(amax, fabsf(vals[l]));
+ sum += vals[l];
+ }
+#pragma unroll
+ for (int mask = QI8_1/2; mask > 0; mask >>= 1) {
+ amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, mask, 32));
+ sum += __shfl_xor_sync(0xFFFFFFFF, sum, mask, 32);
+ }
+
+ const float d = amax / 127;
+ int q32 = 0;
+ int8_t * q8 = (int8_t *) &q32;
+
+ if (d != 0.0f) {
+#pragma unroll
+ for (int l = 0; l < sizeof(int); ++l) {
+ q8[l] = roundf(vals[l] / d);
+ }
+ }
+
+ yq32[threadIdx.x] = q32;
+ if (threadIdx.x % QI8_1 == 0) {
+ if (std::is_same<Tds, half2>::value) {
+ ((half2 *) yds)[threadIdx.x/QI8_1] = make_half2(d, sum);
+ } else {
+ ((float2 *) yds)[threadIdx.x/QI8_1] = make_float2(d, sum);
+ }
+ }
+}
+
+typedef half (*dequantize_1_f16_t)(const void *, const int64_t);
+typedef float (*dequantize_1_f32_t)(const void *, const int64_t);
+
+template <typename T>
+static __device__ __forceinline__ T dequantize_1_q4_0(const void * __restrict__ vx, const int64_t i) {
+ const block_q4_0 * x = (const block_q4_0 *) vx;
+
+ const int64_t ib = i / QK4_0;
+ const int iqs = i % (QK4_0/2);
+ const int shift = (i % QK4_0) / (QK4_0/2);
+
+ const T d = x[ib].d;
+ const int q0 = x[ib].qs[iqs];
+ const int q = ((q0 >> (4*shift)) & 0x0F) - 8;
+
+#ifdef FP16_AVAILABLE
+ if (std::is_same<T, half>::value) {
+ return ((half) d)*((half) q);
+ }
+#endif // FP16_AVAILABLE
+
+ return ((float) d)*((float) q);
+}
+
+template <typename T>
+static __device__ __forceinline__ T dequantize_1_q4_1(const void * __restrict__ vx, const int64_t i) {
+ const block_q4_1 * x = (const block_q4_1 *) vx;
+
+ const int64_t ib = i / QK4_1;
+ const int iqs = i % (QK4_1/2);
+ const int shift = (i % QK4_1) / (QK4_1/2);
+
+ const half2 dm = x[ib].dm;
+ const int q0 = x[ib].qs[iqs];
+ const int q = ((q0 >> (4*shift)) & 0x0F);
+
+#ifdef FP16_AVAILABLE
+ if (std::is_same<T, half>::value) {
+ return __low2half(dm)*((half) q) + __high2half(dm);
+ }
+#endif // FP16_AVAILABLE
+
+ return __low2float(dm)*((float) q) + __high2float(dm);
+}
+
+template <typename T>
+static __device__ __forceinline__ T dequantize_1_q5_0(const void * __restrict__ vx, const int64_t i) {
+ const block_q5_0 * x = (const block_q5_0 *) vx;
+
+ const int64_t ib = i / QK5_0;
+ const int idq = i % QK5_0;
+ const int iqs = i % (QK5_0/2);
+ const int shift = (i % QK5_0) / (QK5_0/2);
+
+ const T d = x[ib].d;
+ const int ql0 = x[ib].qs[iqs];
+ const int qh0 = get_int_b2(x[ib].qh, 0);
+ const int ql = ((ql0 >> (4*shift)) & 0x0F);
+ const int qh = ((qh0 >> idq) << 4) & 0x10;
+ const int q = (ql | qh) - 16;
+
+#ifdef FP16_AVAILABLE
+ if (std::is_same<T, half>::value) {
+ return ((half) d)*((half) q);
+ }
+#endif // FP16_AVAILABLE
+
+ return ((float) d)*((float) q);
+}
+
+template <typename T>
+static __device__ __forceinline__ T dequantize_1_q5_1(const void * __restrict__ vx, const int64_t i) {
+ const block_q5_1 * x = (const block_q5_1 *) vx;
+
+ const int64_t ib = i / QK5_1;
+ const int idq = i % QK5_1;
+ const int iqs = i % (QK5_1/2);
+ const int shift = (i % QK5_1) / (QK5_1/2);
+
+ const half2 dm = x[ib].dm;
+ const int ql0 = x[ib].qs[iqs];
+ const int qh0 = get_int_b4(x[ib].qh, 0);
+ const int ql = ((ql0 >> (4*shift)) & 0x0F);
+ const int qh = ((qh0 >> idq) << 4) & 0x10;
+ const int q = (ql | qh);
+
+#ifdef FP16_AVAILABLE
+ if (std::is_same<T, half>::value) {
+ return __low2half(dm)*((half) q) + __high2half(dm);
+ }
+#endif // FP16_AVAILABLE
+
+ return __low2float(dm)*((float) q) + __high2float(dm);
+}
+
+template <typename T>
+static __device__ __forceinline__ T dequantize_1_q8_0(const void * __restrict__ vx, const int64_t i) {
+ const block_q8_0 * x = (const block_q8_0 *) vx;
+
+ const int64_t ib = i / QK8_0;
+ const int iqs = i % QK8_0;
+
+ const T d = x[ib].d;
+ const int q = x[ib].qs[iqs];
+
+#ifdef FP16_AVAILABLE
+ if (std::is_same<T, half>::value) {
+ return ((half) d)*((half) q);
+ }
+#endif // FP16_AVAILABLE
+
+ return ((float) d)*((float) q);
+}
+
+template <typename T>
+static __device__ __forceinline__ T dequantize_1_f16(const void * __restrict__ vx, const int64_t i) {
+ const half * x = (const half *) vx;
+
+ return x[i];
+}
+
+template <int D>
+constexpr __device__ vec_dot_KQ_f16_t get_vec_dot_KQ_f16(ggml_type type_K) {
+ return type_K == GGML_TYPE_Q4_0 ? vec_dot_fattn_vec_KQ_q4_0<half, D> :
+ type_K == GGML_TYPE_Q4_1 ? vec_dot_fattn_vec_KQ_q4_1<half, D> :
+ type_K == GGML_TYPE_Q5_0 ? vec_dot_fattn_vec_KQ_q5_0<half, D> :
+ type_K == GGML_TYPE_Q5_1 ? vec_dot_fattn_vec_KQ_q5_1<half, D> :
+ type_K == GGML_TYPE_Q8_0 ? vec_dot_fattn_vec_KQ_q8_0<half, D> :
+ type_K == GGML_TYPE_F16 ? vec_dot_fattn_vec_KQ_f16<half, D> :
+ nullptr;
+}
+
+template <int D>
+constexpr __device__ vec_dot_KQ_f32_t get_vec_dot_KQ_f32(ggml_type type_K) {
+ return type_K == GGML_TYPE_Q4_0 ? vec_dot_fattn_vec_KQ_q4_0<float, D> :
+ type_K == GGML_TYPE_Q4_1 ? vec_dot_fattn_vec_KQ_q4_1<float, D> :
+ type_K == GGML_TYPE_Q5_0 ? vec_dot_fattn_vec_KQ_q5_0<float, D> :
+ type_K == GGML_TYPE_Q5_1 ? vec_dot_fattn_vec_KQ_q5_1<float, D> :
+ type_K == GGML_TYPE_Q8_0 ? vec_dot_fattn_vec_KQ_q8_0<float, D> :
+ type_K == GGML_TYPE_F16 ? vec_dot_fattn_vec_KQ_f16<float, D> :
+ nullptr;
+}
+
+constexpr __device__ dequantize_1_f16_t get_dequantize_1_f16(ggml_type type_V) {
+ return type_V == GGML_TYPE_Q4_0 ? dequantize_1_q4_0<half> :
+ type_V == GGML_TYPE_Q4_1 ? dequantize_1_q4_1<half> :
+ type_V == GGML_TYPE_Q5_0 ? dequantize_1_q5_0<half> :
+ type_V == GGML_TYPE_Q5_1 ? dequantize_1_q5_1<half> :
+ type_V == GGML_TYPE_Q8_0 ? dequantize_1_q8_0<half> :
+ type_V == GGML_TYPE_F16 ? dequantize_1_f16<half> :
+ nullptr;
+}
+
+constexpr __device__ dequantize_1_f32_t get_dequantize_1_f32(ggml_type type_V) {
+ return type_V == GGML_TYPE_Q4_0 ? dequantize_1_q4_0<float> :
+ type_V == GGML_TYPE_Q4_1 ? dequantize_1_q4_1<float> :
+ type_V == GGML_TYPE_Q5_0 ? dequantize_1_q5_0<float> :
+ type_V == GGML_TYPE_Q5_1 ? dequantize_1_q5_1<float> :
+ type_V == GGML_TYPE_Q8_0 ? dequantize_1_q8_0<float> :
+ type_V == GGML_TYPE_F16 ? dequantize_1_f16<float> :
+ nullptr;
+}
+
+template<int D, int parallel_blocks> // D == head size
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+__launch_bounds__(D, 1)
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+static __global__ void flash_attn_combine_results(
+ const float * __restrict__ VKQ_parts,
+ const float2 * __restrict__ VKQ_meta,
+ float * __restrict__ dst) {
+ VKQ_parts += parallel_blocks*D * gridDim.y*blockIdx.x;
+ VKQ_meta += parallel_blocks * gridDim.y*blockIdx.x;
+ dst += D * gridDim.y*blockIdx.x;
+
+ const int tid = threadIdx.x;
+ __builtin_assume(tid < D);
+
+ __shared__ float2 meta[parallel_blocks];
+ if (tid < 2*parallel_blocks) {
+ ((float *) meta)[threadIdx.x] = ((const float *)VKQ_meta) [blockIdx.y*(2*parallel_blocks) + tid];
+ }
+
+ __syncthreads();
+
+ float kqmax = meta[0].x;
+#pragma unroll
+ for (int l = 1; l < parallel_blocks; ++l) {
+ kqmax = max(kqmax, meta[l].x);
+ }
+
+ float VKQ_numerator = 0.0f;
+ float VKQ_denominator = 0.0f;
+#pragma unroll
+ for (int l = 0; l < parallel_blocks; ++l) {
+ const float diff = meta[l].x - kqmax;
+ const float KQ_max_scale = expf(diff);
+ const uint32_t ftz_mask = 0xFFFFFFFF * (diff > SOFTMAX_FTZ_THRESHOLD);
+ *((uint32_t *) &KQ_max_scale) &= ftz_mask;
+
+ VKQ_numerator += KQ_max_scale * VKQ_parts[l*gridDim.y*D + blockIdx.y*D + tid];
+ VKQ_denominator += KQ_max_scale * meta[l].y;
+ }
+
+ dst[blockIdx.y*D + tid] = VKQ_numerator / VKQ_denominator;
+}
+
+static void on_no_fattn_vec_case(const int D) {
+ if (D == 64) {
+ fprintf(stderr, "Unsupported KV type combination for head_size 64.\n");
+ fprintf(stderr, "By default only f16 KV cache is supported.\n");
+ fprintf(stderr, "Compile with GGML_CUDA_FA_ALL_QUANTS for V cache quantization support.\n");
+ GGML_ABORT("fatal error");
+ } else if (D == 128) {
+ fprintf(stderr, "Unsupported KV type combination for head_size 128.\n");
+ fprintf(stderr, "Supported combinations:\n");
+ fprintf(stderr, " - K == q4_0, V == q4_0, 4.50 BPV\n");
+ fprintf(stderr, " - K == q8_0, V == q8_0, 8.50 BPV\n");
+ fprintf(stderr, " - K == f16, V == f16, 16.00 BPV\n");
+ fprintf(stderr, "Compile with GGML_CUDA_FA_ALL_QUANTS for all combinations of q4_0, q4_1, q5_0, q5_1, q8_0, and f16.\n");
+ GGML_ABORT("fatal error");
+ } else {
+ fprintf(stderr, "Unsupported KV type combination for head_size 256.\n");
+ fprintf(stderr, "Only f16 is supported.\n");
+ GGML_ABORT("fatal error");
+ }
+}
+
+template <int D, int parallel_blocks>
+void launch_fattn(
+ ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel,
+ const int nwarps, const int cols_per_block, const bool need_f16_K, const bool need_f16_V
+) {
+ const ggml_tensor * Q = dst->src[0];
+ const ggml_tensor * K = dst->src[1];
+ const ggml_tensor * V = dst->src[2];
+
+ const ggml_tensor * mask = dst->src[3];
+
+ ggml_tensor * KQV = dst;
+
+ GGML_ASSERT(Q->type == GGML_TYPE_F32);
+ GGML_ASSERT(KQV->type == GGML_TYPE_F32);
+
+ GGML_ASSERT(!mask || mask->type == GGML_TYPE_F16);
+ GGML_ASSERT(!mask || mask->ne[1] >= GGML_PAD(Q->ne[1], 16) &&
+ "the Flash-Attention CUDA kernel requires the mask to be padded to 16 and at least n_queries big");
+
+ GGML_ASSERT(K->ne[1] % FATTN_KQ_STRIDE == 0 && "Incorrect KV cache padding.");
+
+ ggml_cuda_pool & pool = ctx.pool();
+ cudaStream_t main_stream = ctx.stream();
+
+ ggml_cuda_pool_alloc<half> K_f16(pool);
+ ggml_cuda_pool_alloc<half> V_f16(pool);
+ ggml_cuda_pool_alloc<float> dst_tmp(pool);
+ ggml_cuda_pool_alloc<float2> dst_tmp_meta(pool);
+
+ char * K_data = (char *) K->data;
+ size_t nb11 = K->nb[1];
+ size_t nb12 = K->nb[2];
+ size_t nb13 = K->nb[3];
+
+ char * V_data = (char *) V->data;
+ size_t nb21 = V->nb[1];
+ size_t nb22 = V->nb[2];
+ size_t nb23 = V->nb[3];
+
+ if (need_f16_K && K->type != GGML_TYPE_F16) {
+ K_f16.alloc(ggml_nelements(K));
+ to_fp16_cuda_t to_fp16 = ggml_get_to_fp16_cuda(K->type);
+ to_fp16(K_data, K_f16.ptr, ggml_nelements(K), main_stream);
+ K_data = (char *) K_f16.ptr;
+
+ const size_t bs = ggml_blck_size(K->type);
+ const size_t ts = ggml_type_size(K->type);
+
+ nb11 = nb11*bs*sizeof(half)/ts;
+ nb12 = nb12*bs*sizeof(half)/ts;
+ nb13 = nb13*bs*sizeof(half)/ts;
+ }
+
+ if (need_f16_V && V->type != GGML_TYPE_F16) {
+ V_f16.alloc(ggml_nelements(V));
+ to_fp16_cuda_t to_fp16 = ggml_get_to_fp16_cuda(V->type);
+ to_fp16(V_data, V_f16.ptr, ggml_nelements(V), main_stream);
+ V_data = (char *) V_f16.ptr;
+
+ const size_t bs = ggml_blck_size(V->type);
+ const size_t ts = ggml_type_size(V->type);
+
+ nb21 = nb21*bs*sizeof(half)/ts;
+ nb22 = nb22*bs*sizeof(half)/ts;
+ nb23 = nb23*bs*sizeof(half)/ts;
+ }
+
+ if (parallel_blocks > 1) {
+ dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV));
+ dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV));
+ }
+
+ const dim3 block_dim(WARP_SIZE, nwarps, 1);
+ const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]);
+ const int shmem = 0;
+
+ float scale = 1.0f;
+ float max_bias = 0.0f;
+ float logit_softcap = 0.0f;
+
+ memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float));
+ memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float));
+ memcpy(&logit_softcap, (float *) KQV->op_params + 2, sizeof(float));
+
+ if (logit_softcap != 0.0f) {
+ scale /= logit_softcap;
+ }
+
+ const uint32_t n_head = Q->ne[2];
+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));
+
+ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
+
+ fattn_kernel<<<blocks_num, block_dim, shmem, main_stream>>>(
+ (const char *) Q->data,
+ K_data,
+ V_data,
+ mask ? ((const char *) mask->data) : nullptr,
+ (parallel_blocks) == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr,
+ scale, max_bias, m0, m1, n_head_log2, logit_softcap,
+ Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3],
+ K->ne[0], K->ne[1], K->ne[2], K->ne[3],
+ mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0,
+ Q->nb[1], Q->nb[2], Q->nb[3],
+ nb11, nb12, nb13,
+ nb21, nb22, nb23,
+ KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3]
+ );
+ CUDA_CHECK(cudaGetLastError());
+
+ if ((parallel_blocks) == 1) {
+ return;
+ }
+
+ const dim3 block_dim_combine(D, 1, 1);
+ const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z);
+ const int shmem_combine = 0;
+
+ flash_attn_combine_results<D, parallel_blocks>
+ <<<blocks_num_combine, block_dim_combine, shmem_combine, main_stream>>>
+ (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data);
+ CUDA_CHECK(cudaGetLastError());
+}
diff --git a/llama/ggml-cuda/fattn-tile-f16.cu b/llama/ggml-cuda/fattn-tile-f16.cu
new file mode 100644
index 00000000..c77de58e
--- /dev/null
+++ b/llama/ggml-cuda/fattn-tile-f16.cu
@@ -0,0 +1,379 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+#include "fattn-common.cuh"
+#include "fattn-tile-f16.cuh"
+
+#define FATTN_KQ_STRIDE_TILE_F16 64
+
+template<int D, int ncols, int nwarps, int parallel_blocks, bool use_logit_softcap> // D == head size
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+__launch_bounds__(nwarps*WARP_SIZE, 1)
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+static __global__ void flash_attn_tile_ext_f16(
+ const char * __restrict__ Q,
+ const char * __restrict__ K,
+ const char * __restrict__ V,
+ const char * __restrict__ mask,
+ float * __restrict__ dst,
+ float2 * __restrict__ dst_meta,
+ const float scale,
+ const float max_bias,
+ const float m0,
+ const float m1,
+ const uint32_t n_head_log2,
+ const float logit_softcap,
+ const int ne00,
+ const int ne01,
+ const int ne02,
+ const int ne03,
+ const int ne10,
+ const int ne11,
+ const int ne12,
+ const int ne13,
+ const int ne31,
+ const int nb31,
+ const int nb01,
+ const int nb02,
+ const int nb03,
+ const int nb11,
+ const int nb12,
+ const int nb13,
+ const int nb21,
+ const int nb22,
+ const int nb23,
+ const int ne0,
+ const int ne1,
+ const int ne2,
+ const int ne3) {
+#ifdef FP16_AVAILABLE
+ // Skip unused kernel variants for faster compilation:
+ if (use_logit_softcap && !(D == 128 || D == 256)) {
+ NO_DEVICE_CODE;
+ return;
+ }
+
+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.
+
+ const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on.
+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.
+
+ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.
+ const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0);
+ const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio));
+ const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape
+ const half * maskh = (const half *) mask + ne11*ic0;
+
+ const int stride_KV2 = nb11 / sizeof(half2);
+
+ const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1);
+ const half slopeh = __float2half(slopef);
+
+ static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64.");
+
+ __shared__ half KQ[ncols*FATTN_KQ_STRIDE_TILE_F16];
+ half2 * KQ2 = (half2 *) KQ;
+
+ __shared__ half2 KV_tmp[FATTN_KQ_STRIDE_TILE_F16][D/2 + 1]; // Pad D to avoid memory bank conflicts.
+
+ half kqmax[ncols/nwarps];
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ kqmax[j0/nwarps] = -HALF_MAX_HALF;
+ }
+ half2 kqsum[ncols/nwarps] = {{0.0f, 0.0f}};
+
+ half2 VKQ[ncols/nwarps][(D/2)/WARP_SIZE] = {{{0.0f, 0.0f}}};
+
+ // Convert Q to half2 and store in registers:
+ __shared__ half2 Q_h2[ncols][D/2];
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ const float2 tmp = ic0 + j < ne01 ? Q_f2[j*(nb01/sizeof(float2)) + i] : make_float2(0.0f, 0.0f);
+ Q_h2[j][i] = make_half2(scale, scale) * make_half2(tmp.x, tmp.y);
+ }
+ }
+
+ __syncthreads();
+
+ const int k_start = parallel_blocks == 1 ? 0 : ip*FATTN_KQ_STRIDE_TILE_F16;
+ for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE_TILE_F16) {
+ // Calculate KQ tile and keep track of new maximum KQ values:
+
+ half kqmax_new[ncols/nwarps];
+#pragma unroll
+ for (int j = 0; j < ncols/nwarps; ++j) {
+ kqmax_new[j] = kqmax[j];
+ }
+
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += nwarps) {
+ const int i_KQ = i_KQ_0 + threadIdx.y;
+
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) {
+ const int k_KQ = k_KQ_0 + threadIdx.x;
+
+ KV_tmp[i_KQ][k_KQ] = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ];
+ }
+ }
+
+ __syncthreads();
+
+ half2 sum2[FATTN_KQ_STRIDE_TILE_F16/WARP_SIZE][ncols/nwarps] = {{{0.0f, 0.0f}}};
+
+#pragma unroll
+ for (int k_KQ = 0; k_KQ < D/2; ++k_KQ) {
+ half2 K_k[FATTN_KQ_STRIDE_TILE_F16/WARP_SIZE];
+ half2 Q_k[ncols/nwarps];
+
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += WARP_SIZE) {
+ const int i_KQ = i_KQ_0 + threadIdx.x;
+
+ K_k[i_KQ_0/WARP_SIZE] = KV_tmp[i_KQ][k_KQ];
+ }
+#pragma unroll
+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {
+ const int j_KQ = j_KQ_0 + threadIdx.y;
+
+ Q_k[j_KQ_0/nwarps] = Q_h2[j_KQ][k_KQ];
+ }
+
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += WARP_SIZE) {
+#pragma unroll
+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {
+ sum2[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps] += K_k[i_KQ_0/WARP_SIZE]*Q_k[j_KQ_0/nwarps];
+ }
+ }
+ }
+
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += WARP_SIZE) {
+ const int i_KQ = i_KQ_0 + threadIdx.x;
+
+#pragma unroll
+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {
+ const int j_KQ = j_KQ_0 + threadIdx.y;
+
+ half sum;
+ if (use_logit_softcap) {
+ const float2 tmp = __half22float2(sum2[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]);
+ sum = logit_softcap * tanhf(tmp.x + tmp.y);
+ } else {
+ sum = __low2half(sum2[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]) + __high2half(sum2[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]);
+ }
+ sum += mask ? slopeh*maskh[j_KQ*ne11 + k_VKQ_0 + i_KQ] : __float2half(0.0f);
+
+ kqmax_new[j_KQ_0/nwarps] = ggml_cuda_hmax(kqmax_new[j_KQ_0/nwarps], sum);
+
+ KQ[j_KQ*FATTN_KQ_STRIDE_TILE_F16 + i_KQ] = sum;
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ kqmax_new[j0/nwarps] = warp_reduce_max(kqmax_new[j0/nwarps]);
+ const half2 KQ_max_scale = __half2half2(hexp(kqmax[j0/nwarps] - kqmax_new[j0/nwarps]));
+ kqmax[j0/nwarps] = kqmax_new[j0/nwarps];
+
+#pragma unroll
+ for (int i0 = 0; i0 < FATTN_KQ_STRIDE_TILE_F16/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ const half2 diff = KQ2[j*(FATTN_KQ_STRIDE_TILE_F16/2) + i] - __half2half2(kqmax[j0/nwarps]);
+ const half2 val = h2exp(diff);
+ kqsum[j0/nwarps] = kqsum[j0/nwarps]*KQ_max_scale + val;
+ KQ2[j*(FATTN_KQ_STRIDE_TILE_F16/2) + i] = val;
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ VKQ[j0/nwarps][i0/WARP_SIZE] *= KQ_max_scale;
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE_TILE_F16; k0 += nwarps) {
+ const int k = k0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ KV_tmp[k][i] = V_h2[(k_VKQ_0 + k)*stride_KV2 + i];
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE_TILE_F16; k0 += 2) {
+ half2 V_k[(D/2)/WARP_SIZE][2];
+ half2 KQ_k[ncols/nwarps];
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ V_k[i0/WARP_SIZE][0] = KV_tmp[k0 + 0][i];
+ V_k[i0/WARP_SIZE][1] = KV_tmp[k0 + 1][i];
+ }
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ KQ_k[j0/nwarps] = KQ2[j*(FATTN_KQ_STRIDE_TILE_F16/2) + k0/2];
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ VKQ[j0/nwarps][i0/WARP_SIZE] += V_k[i0/WARP_SIZE][0]* __low2half2(KQ_k[j0/nwarps]);
+ VKQ[j0/nwarps][i0/WARP_SIZE] += V_k[i0/WARP_SIZE][1]*__high2half2(KQ_k[j0/nwarps]);
+ }
+ }
+ }
+
+ __syncthreads();
+ }
+
+#pragma unroll
+ for (int j_VKQ_0 = 0; j_VKQ_0 < ncols; j_VKQ_0 += nwarps) {
+ const int j_VKQ = j_VKQ_0 + threadIdx.y;
+
+ if (ic0 + j_VKQ >= ne01) {
+ return;
+ }
+
+ half kqsum_j = __low2half(kqsum[j_VKQ_0/nwarps]) + __high2half(kqsum[j_VKQ_0/nwarps]);
+ kqsum_j = warp_reduce_sum(kqsum_j);
+
+#pragma unroll
+ for (int i00 = 0; i00 < D; i00 += 2*WARP_SIZE) {
+ const int i0 = i00 + 2*threadIdx.x;
+
+ half2 dst_val = VKQ[j_VKQ_0/nwarps][i0/(2*WARP_SIZE)];
+ if (parallel_blocks == 1) {
+ dst_val /= __half2half2(kqsum_j);
+ }
+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;
+ dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 0] = __low2float(dst_val);
+ dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 1] = __high2float(dst_val);
+ }
+
+ if (parallel_blocks != 1 && threadIdx.x == 0) {
+ dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j);
+ }
+ }
+#else
+ NO_DEVICE_CODE;
+#endif // FP16_AVAILABLE
+}
+
+template <int cols_per_block, int parallel_blocks, bool use_logit_softcap>
+void launch_fattn_tile_f16_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * Q = dst->src[0];
+ switch (Q->ne[0]) {
+ case 64: {
+ constexpr int D = 64;
+ constexpr int nwarps = 8;
+ fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16<D, cols_per_block, nwarps, parallel_blocks, use_logit_softcap>;
+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block, true, true);
+ } break;
+ case 128: {
+ constexpr int D = 128;
+ constexpr int nwarps = 8;
+ fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16<D, cols_per_block, nwarps, parallel_blocks, use_logit_softcap>;
+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block, true, true);
+ } break;
+ default: {
+ GGML_ABORT("FlashAttention without tensor cores only supports head sizes 64 and 128.");
+ } break;
+ }
+}
+
+void ggml_cuda_flash_attn_ext_tile_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * KQV = dst;
+ const ggml_tensor * Q = dst->src[0];
+
+ const int32_t precision = KQV->op_params[3];
+ GGML_ASSERT(precision == GGML_PREC_DEFAULT);
+
+ float logit_softcap;
+ memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));
+
+ if (Q->ne[1] <= 16) {
+ constexpr int cols_per_block = 16;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ launch_fattn_tile_f16_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ launch_fattn_tile_f16_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ if (Q->ne[1] <= 32) {
+ constexpr int cols_per_block = 32;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ launch_fattn_tile_f16_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ launch_fattn_tile_f16_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ constexpr int cols_per_block = 32;
+ constexpr int parallel_blocks = 1;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ launch_fattn_tile_f16_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ launch_fattn_tile_f16_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ }
+}
diff --git a/llama/ggml-cuda/fattn-tile-f16.cuh b/llama/ggml-cuda/fattn-tile-f16.cuh
new file mode 100644
index 00000000..0ac19a67
--- /dev/null
+++ b/llama/ggml-cuda/fattn-tile-f16.cuh
@@ -0,0 +1,29 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+void ggml_cuda_flash_attn_ext_tile_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/fattn-tile-f32.cu b/llama/ggml-cuda/fattn-tile-f32.cu
new file mode 100644
index 00000000..ad3b1243
--- /dev/null
+++ b/llama/ggml-cuda/fattn-tile-f32.cu
@@ -0,0 +1,371 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+#include "fattn-common.cuh"
+#include "fattn-tile-f32.cuh"
+
+#define FATTN_KQ_STRIDE_TILE_F32 32
+
+template<int D, int ncols, int nwarps, int parallel_blocks, bool use_logit_softcap> // D == head size
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+__launch_bounds__(nwarps*WARP_SIZE, 1)
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+static __global__ void flash_attn_tile_ext_f32(
+ const char * __restrict__ Q,
+ const char * __restrict__ K,
+ const char * __restrict__ V,
+ const char * __restrict__ mask,
+ float * __restrict__ dst,
+ float2 * __restrict__ dst_meta,
+ const float scale,
+ const float max_bias,
+ const float m0,
+ const float m1,
+ const uint32_t n_head_log2,
+ const float logit_softcap,
+ const int ne00,
+ const int ne01,
+ const int ne02,
+ const int ne03,
+ const int ne10,
+ const int ne11,
+ const int ne12,
+ const int ne13,
+ const int ne31,
+ const int nb31,
+ const int nb01,
+ const int nb02,
+ const int nb03,
+ const int nb11,
+ const int nb12,
+ const int nb13,
+ const int nb21,
+ const int nb22,
+ const int nb23,
+ const int ne0,
+ const int ne1,
+ const int ne2,
+ const int ne3) {
+ // Skip unused kernel variants for faster compilation:
+ if (use_logit_softcap && !(D == 128 || D == 256)) {
+ NO_DEVICE_CODE;
+ return;
+ }
+
+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.
+
+ const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on.
+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.
+
+ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.
+ const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0);
+ const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio));
+ const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape
+ const half * maskh = (const half *) mask + ne11*ic0;
+
+ const int stride_KV2 = nb11 / sizeof(half2);
+
+ const float slope = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1);
+
+ static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64.");
+
+ __shared__ float KQ[ncols*FATTN_KQ_STRIDE_TILE_F32];
+
+ __shared__ float KV_tmp[FATTN_KQ_STRIDE_TILE_F32][D + 1]; // Pad D to avoid memory bank conflicts.
+ float2 * KV_tmp2 = (float2 *) KV_tmp;
+
+ float kqmax[ncols/nwarps];
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ kqmax[j0/nwarps] = -FLT_MAX/2.0f;
+ }
+ float kqsum[ncols/nwarps] = {0.0f};
+
+ float2 VKQ[ncols/nwarps][(D/2)/WARP_SIZE] = {{{0.0f, 0.0f}}};
+
+ // Convert Q to half2 and store in registers:
+ __shared__ float Q_f[ncols][D];
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < D; i0 += 2*WARP_SIZE) {
+ float2 tmp = ic0 + j < ne01 ? Q_f2[j*(nb01/sizeof(float2)) + i0/2 + threadIdx.x] : make_float2(0.0f, 0.0f);
+ Q_f[j][i0 + 0*WARP_SIZE + threadIdx.x] = tmp.x * scale;
+ Q_f[j][i0 + 1*WARP_SIZE + threadIdx.x] = tmp.y * scale;
+ }
+ }
+
+ __syncthreads();
+
+ const int k_start = parallel_blocks == 1 ? 0 : ip*FATTN_KQ_STRIDE_TILE_F32;
+ for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE_TILE_F32) {
+ // Calculate KQ tile and keep track of new maximum KQ values:
+
+ float kqmax_new[ncols/nwarps];
+#pragma unroll
+ for (int j = 0; j < ncols/nwarps; ++j) {
+ kqmax_new[j] = kqmax[j];
+ }
+
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += nwarps) {
+ const int i_KQ = i_KQ_0 + threadIdx.y;
+
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += 2*WARP_SIZE) {
+ const half2 tmp = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ_0/2 + threadIdx.x];
+ KV_tmp[i_KQ][k_KQ_0 + 0*WARP_SIZE + threadIdx.x] = __low2float(tmp);
+ KV_tmp[i_KQ][k_KQ_0 + 1*WARP_SIZE + threadIdx.x] = __high2float(tmp);
+ }
+ }
+
+ __syncthreads();
+
+ float sum[FATTN_KQ_STRIDE_TILE_F32/WARP_SIZE][ncols/nwarps] = {{0.0f}};
+
+#pragma unroll
+ for (int k_KQ = 0; k_KQ < D; ++k_KQ) {
+ float K_k[FATTN_KQ_STRIDE_TILE_F32/WARP_SIZE];
+ float Q_k[ncols/nwarps];
+
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += WARP_SIZE) {
+ const int i_KQ = i_KQ_0 + threadIdx.x;
+
+ K_k[i_KQ_0/WARP_SIZE] = KV_tmp[i_KQ][k_KQ];
+ }
+#pragma unroll
+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {
+ const int j_KQ = j_KQ_0 + threadIdx.y;
+
+ Q_k[j_KQ_0/nwarps] = Q_f[j_KQ][k_KQ];
+ }
+
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += WARP_SIZE) {
+#pragma unroll
+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {
+ sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps] += K_k[i_KQ_0/WARP_SIZE] * Q_k[j_KQ_0/nwarps];
+ }
+ }
+ }
+
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += WARP_SIZE) {
+ const int i_KQ = i_KQ_0 + threadIdx.x;
+
+#pragma unroll
+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {
+ const int j_KQ = j_KQ_0 + threadIdx.y;
+
+ if (use_logit_softcap) {
+ sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps] = logit_softcap * tanhf(sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]);
+ }
+
+ sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps] += mask ? slope*__half2float(maskh[j_KQ*ne11 + k_VKQ_0 + i_KQ]) : 0.0f;
+
+ kqmax_new[j_KQ_0/nwarps] = fmaxf(kqmax_new[j_KQ_0/nwarps], sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]);
+
+ KQ[j_KQ*FATTN_KQ_STRIDE_TILE_F32 + i_KQ] = sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps];
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ kqmax_new[j0/nwarps] = warp_reduce_max(kqmax_new[j0/nwarps]);
+ const float KQ_max_scale = expf(kqmax[j0/nwarps] - kqmax_new[j0/nwarps]);
+ kqmax[j0/nwarps] = kqmax_new[j0/nwarps];
+
+ float kqsum_add = 0.0f;
+#pragma unroll
+ for (int i0 = 0; i0 < FATTN_KQ_STRIDE_TILE_F32; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ const float diff = KQ[j*FATTN_KQ_STRIDE_TILE_F32 + i] - kqmax[j0/nwarps];
+ const float val = expf(diff);
+ kqsum_add += val;
+ KQ[j*FATTN_KQ_STRIDE_TILE_F32 + i] = val;
+ }
+ kqsum[j0/nwarps] = kqsum[j0/nwarps]*KQ_max_scale + kqsum_add;
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ VKQ[j0/nwarps][i0/WARP_SIZE].x *= KQ_max_scale;
+ VKQ[j0/nwarps][i0/WARP_SIZE].y *= KQ_max_scale;
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE_TILE_F32; k0 += nwarps) {
+ const int k = k0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ KV_tmp2[k*(D/2) + i].x = __low2float(V_h2[(k_VKQ_0 + k)*stride_KV2 + i]);
+ KV_tmp2[k*(D/2) + i].y = __high2float(V_h2[(k_VKQ_0 + k)*stride_KV2 + i]);
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int k = 0; k < FATTN_KQ_STRIDE_TILE_F32; ++k) {
+ float2 V_k[(D/2)/WARP_SIZE];
+ float KQ_k[ncols/nwarps];
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ V_k[i0/WARP_SIZE] = KV_tmp2[k*(D/2) + i];
+ }
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ KQ_k[j0/nwarps] = KQ[j*FATTN_KQ_STRIDE_TILE_F32 + k];
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ VKQ[j0/nwarps][i0/WARP_SIZE].x += V_k[i0/WARP_SIZE].x*KQ_k[j0/nwarps];
+ VKQ[j0/nwarps][i0/WARP_SIZE].y += V_k[i0/WARP_SIZE].y*KQ_k[j0/nwarps];
+ }
+ }
+ }
+
+ __syncthreads();
+ }
+
+#pragma unroll
+ for (int j_VKQ_0 = 0; j_VKQ_0 < ncols; j_VKQ_0 += nwarps) {
+ const int j_VKQ = j_VKQ_0 + threadIdx.y;
+
+ if (ic0 + j_VKQ >= ne01) {
+ return;
+ }
+
+ float kqsum_j = kqsum[j_VKQ_0/nwarps];
+ kqsum_j = warp_reduce_sum(kqsum_j);
+
+#pragma unroll
+ for (int i00 = 0; i00 < D; i00 += 2*WARP_SIZE) {
+ const int i0 = i00 + 2*threadIdx.x;
+
+ float2 dst_val = VKQ[j_VKQ_0/nwarps][i0/(2*WARP_SIZE)];
+ if (parallel_blocks == 1) {
+ dst_val.x /= kqsum_j;
+ dst_val.y /= kqsum_j;
+ }
+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;
+ dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 0] = dst_val.x;
+ dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 1] = dst_val.y;
+ }
+
+ if (parallel_blocks != 1 && threadIdx.x == 0) {
+ dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j);
+ }
+ }
+}
+
+template <int cols_per_block, int parallel_blocks, bool use_logit_softcap>
+void launch_fattn_tile_f32_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * Q = dst->src[0];
+ switch (Q->ne[0]) {
+ case 64: {
+ constexpr int D = 64;
+ constexpr int nwarps = 8;
+ fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32<D, cols_per_block, nwarps, parallel_blocks, use_logit_softcap>;
+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block, true, true);
+ } break;
+ case 128: {
+ constexpr int D = 128;
+ constexpr int nwarps = 8;
+ fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32<D, cols_per_block, nwarps, parallel_blocks, use_logit_softcap>;
+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block, true, true);
+ } break;
+ default: {
+ GGML_ABORT("FlashAttention without tensor cores only supports head sizes 64 and 128.");
+ } break;
+ }
+}
+
+void ggml_cuda_flash_attn_ext_tile_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * KQV = dst;
+ const ggml_tensor * Q = dst->src[0];
+
+ float logit_softcap;
+ memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));
+
+ if (Q->ne[1] <= 16) {
+ constexpr int cols_per_block = 16;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ launch_fattn_tile_f32_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ launch_fattn_tile_f32_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ if (Q->ne[1] <= 32) {
+ constexpr int cols_per_block = 32;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ launch_fattn_tile_f32_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ launch_fattn_tile_f32_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ constexpr int cols_per_block = 32;
+ constexpr int parallel_blocks = 1;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ launch_fattn_tile_f32_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ launch_fattn_tile_f32_64_128<cols_per_block, parallel_blocks, use_logit_softcap>(ctx, dst);
+ }
+}
diff --git a/llama/ggml-cuda/fattn-tile-f32.cuh b/llama/ggml-cuda/fattn-tile-f32.cuh
new file mode 100644
index 00000000..ff8fa4f4
--- /dev/null
+++ b/llama/ggml-cuda/fattn-tile-f32.cuh
@@ -0,0 +1,29 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+void ggml_cuda_flash_attn_ext_tile_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/fattn-vec-f16.cuh b/llama/ggml-cuda/fattn-vec-f16.cuh
new file mode 100644
index 00000000..d10f5df8
--- /dev/null
+++ b/llama/ggml-cuda/fattn-vec-f16.cuh
@@ -0,0 +1,468 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+#include "fattn-common.cuh"
+
+template<int D, int ncols, int parallel_blocks, ggml_type type_K, ggml_type type_V, bool use_logit_softcap> // D == head size
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+__launch_bounds__(D, 1)
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+static __global__ void flash_attn_vec_ext_f16(
+ const char * __restrict__ Q,
+ const char * __restrict__ K,
+ const char * __restrict__ V,
+ const char * __restrict__ mask,
+ float * __restrict__ dst,
+ float2 * __restrict__ dst_meta,
+ const float scale,
+ const float max_bias,
+ const float m0,
+ const float m1,
+ const uint32_t n_head_log2,
+ const float logit_softcap,
+ const int ne00,
+ const int ne01,
+ const int ne02,
+ const int ne03,
+ const int ne10,
+ const int ne11,
+ const int ne12,
+ const int ne13,
+ const int ne31,
+ const int nb31,
+ const int nb01,
+ const int nb02,
+ const int nb03,
+ const int nb11,
+ const int nb12,
+ const int nb13,
+ const int nb21,
+ const int nb22,
+ const int nb23,
+ const int ne0,
+ const int ne1,
+ const int ne2,
+ const int ne3) {
+#ifdef FP16_AVAILABLE
+ // Skip unused kernel variants for faster compilation:
+ if (use_logit_softcap && !(D == 128 || D == 256)) {
+ NO_DEVICE_CODE;
+ return;
+ }
+
+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.
+
+ constexpr vec_dot_KQ_f16_t vec_dot_KQ = get_vec_dot_KQ_f16<D>(type_K);
+ constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16;
+ constexpr dequantize_1_f16_t dequantize_1_v = get_dequantize_1_f16(type_V);
+
+ const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on.
+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.
+
+ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.
+ Q += nb02* blockIdx.y + nb01*ic0;
+ K += nb12*(blockIdx.y / gqa_ratio);
+ V += nb22*(blockIdx.y / gqa_ratio);
+
+ const half * maskh = (const half *) mask + ne11*ic0;
+
+ const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1);
+ const half slopeh = __float2half(slopef);
+
+ static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64.");
+ constexpr int nwarps = D / WARP_SIZE;
+ const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;
+ __builtin_assume(tid < D);
+
+ __shared__ half KQ[ncols*D];
+ half2 * KQ2 = (half2 *) KQ;
+
+ half kqmax[ncols];
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ kqmax[j] = -HALF_MAX_HALF;
+ }
+ half kqsum[ncols] = {0.0f};
+
+ __shared__ half kqmax_shared[ncols][WARP_SIZE];
+ __shared__ half kqsum_shared[ncols][WARP_SIZE];
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ if (threadIdx.y == 0) {
+ kqmax_shared[j][threadIdx.x] = -HALF_MAX_HALF;
+ kqsum_shared[j][threadIdx.x] = 0.0f;
+ }
+ }
+ __syncthreads();
+
+ // Convert Q to half2 (f16 K) or q8_1 (quantized K) and store in registers:
+ half2 Q_h2[ncols][D/(2*WARP_SIZE)];
+ int Q_i32[ncols][D/(sizeof(int)*QK8_1) == 0 ? 1 : D/(sizeof(int)*QK8_1)];
+ half2 Q_ds[ncols][D/QK8_1 == 0 ? 1 : D/QK8_1];
+ if (Q_q8_1) {
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ if (j0 + nwarps > ncols && j >= ncols) {
+ break;
+ }
+
+ // Reuse KQ as temporary storage for converting Q to q8_1:
+ int * tmp_q_i32 = (int *) &KQ[j*D];
+ half2 * tmp_q_ds = (half2 *) (tmp_q_i32 + D/sizeof(int));
+
+ // Set memory to zero if out of bounds:
+ if (ncols > 2 && ic0 + j >= ne01) {
+#pragma unroll
+ for (int i0 = 0; i0 < D/sizeof(int); i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ tmp_q_i32[i] = 0;
+ }
+ if (threadIdx.x < D/QK8_1) {
+ tmp_q_ds[threadIdx.x] = make_half2(0.0f, 0.0f);
+ }
+ continue;
+ }
+
+ const float * Q_f = (const float *) (Q + j*nb01);
+#pragma unroll
+ for (int i0 = 0; i0 < D/sizeof(int); i0 += WARP_SIZE) {
+ quantize_q8_1_to_shared<half2>(Q_f + 4*i0, scale, tmp_q_i32, tmp_q_ds);
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ int * tmp_q_i32 = (int *) &KQ[j*D];
+ half2 * tmp_q_ds = (half2 *) (tmp_q_i32 + D/sizeof(int));
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/sizeof(int); i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ Q_i32[j][i0/WARP_SIZE] = tmp_q_i32[i];
+ Q_ds[j][i0/WARP_SIZE] = tmp_q_ds[i/QI8_1];
+ }
+ }
+
+ __syncthreads();
+ } else {
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ const float2 * Q_f2_j = (const float2 *) (Q + j*nb01);
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ const float2 tmp = ncols <= 2 || ic0 + j < ne01 ? Q_f2_j[i] : make_float2(0.0f, 0.0f);
+ Q_h2[j][i0/WARP_SIZE] = make_half2(scale, scale) * make_half2(tmp.x, tmp.y);
+ }
+ }
+ }
+
+
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ KQ[j*D + tid] = -HALF_MAX_HALF;
+ }
+
+ half2 VKQ[ncols] = {{0.0f, 0.0f}};
+
+ const int k_start = parallel_blocks == 1 ? 0 : ip*D;
+ for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) {
+ // Calculate KQ tile and keep track of new maximum KQ values:
+
+ // For unknown reasons using a half array of size 1 for kqmax_new causes a performance regression,
+ // see https://github.com/ggerganov/llama.cpp/pull/7061 .
+ // Therefore this variable is defined twice but only used once (so that the compiler can optimize out the unused variable).
+ half kqmax_new = kqmax[0];
+ half kqmax_new_arr[ncols];
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ kqmax_new_arr[j] = kqmax[j];
+ }
+
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += nwarps) {
+ const int i_KQ = i_KQ_0 + threadIdx.y;
+
+ if ((i_KQ_0 + nwarps > D && i_KQ >= D) || (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + i_KQ >= ne11)) {
+ break;
+ }
+
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ half sum = vec_dot_KQ(K + (k_VKQ_0 + i_KQ)*nb11, Q_h2[j], Q_i32[j], Q_ds[j]);
+ sum = warp_reduce_sum(sum);
+
+ if (use_logit_softcap) {
+ sum = logit_softcap*tanhf(sum);
+ }
+
+ sum += mask ? slopeh*maskh[j*ne11 + k_VKQ_0 + i_KQ] : __float2half(0.0f);
+
+ if (ncols == 1) {
+ kqmax_new = ggml_cuda_hmax(kqmax_new, sum);
+ } else {
+ kqmax_new_arr[j] = ggml_cuda_hmax(kqmax_new_arr[j], sum);
+ }
+
+ if (threadIdx.x == 0) {
+ KQ[j*D + i_KQ] = sum;
+ }
+ }
+ }
+
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ half kqmax_new_j = ncols == 1 ? kqmax_new : kqmax_new_arr[j];
+
+ kqmax_new_j = warp_reduce_max(kqmax_new_j);
+ if (threadIdx.x == 0) {
+ kqmax_shared[j][threadIdx.y] = kqmax_new_j;
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ half kqmax_new_j = kqmax_shared[j][threadIdx.x];
+ kqmax_new_j = warp_reduce_max(kqmax_new_j);
+
+ const half KQ_max_scale = hexp(kqmax[j] - kqmax_new_j);
+ kqmax[j] = kqmax_new_j;
+
+ const half val = hexp(KQ[j*D + tid] - kqmax[j]);
+ kqsum[j] = kqsum[j]*KQ_max_scale + val;
+ KQ[j*D + tid] = val;
+
+ VKQ[j] *= __half2half2(KQ_max_scale);
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int k0 = 0; k0 < D; k0 += 2) {
+ if (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + k0 >= ne11) {
+ break;
+ }
+
+ half2 V_k;
+ reinterpret_cast<half&>(V_k.x) = dequantize_1_v(V + (k_VKQ_0 + k0 + 0)*nb21, tid);
+ reinterpret_cast<half&>(V_k.y) = dequantize_1_v(V + (k_VKQ_0 + k0 + 1)*nb21, tid);
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ VKQ[j] += V_k*KQ2[j*(D/2) + k0/2];
+ }
+ }
+
+ __syncthreads();
+ }
+
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ kqsum[j] = warp_reduce_sum(kqsum[j]);
+ if (threadIdx.x == 0) {
+ kqsum_shared[j][threadIdx.y] = kqsum[j];
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) {
+ if (ncols > 2 && ic0 + j_VKQ >= ne01) {
+ break;
+ }
+
+ kqsum[j_VKQ] = kqsum_shared[j_VKQ][threadIdx.x];
+ kqsum[j_VKQ] = warp_reduce_sum(kqsum[j_VKQ]);
+
+ half dst_val = (__low2half(VKQ[j_VKQ]) + __high2half(VKQ[j_VKQ]));
+ if (parallel_blocks == 1) {
+ dst_val /= kqsum[j_VKQ];
+ }
+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;
+ dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val;
+ }
+
+ if (parallel_blocks != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) {
+ dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]);
+ }
+#else
+ NO_DEVICE_CODE;
+#endif // FP16_AVAILABLE
+}
+
+template <int D, int cols_per_block, int parallel_blocks, ggml_type type_K, ggml_type type_V, bool use_logit_softcap>
+void ggml_cuda_flash_attn_ext_vec_f16_case_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ constexpr int nwarps = D/WARP_SIZE;
+ fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>;
+ constexpr bool need_f16_K = D != 128;
+ constexpr bool need_f16_V = D != 128 && D != 64;
+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block, need_f16_K, need_f16_V);
+}
+
+template <int D, ggml_type type_K, ggml_type type_V>
+void ggml_cuda_flash_attn_ext_vec_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * KQV = dst;
+ const ggml_tensor * Q = dst->src[0];
+ const ggml_tensor * K = dst->src[1];
+ const ggml_tensor * V = dst->src[2];
+
+ const int32_t precision = KQV->op_params[3];
+ GGML_ASSERT(precision == GGML_PREC_DEFAULT);
+
+ GGML_ASSERT(K->type == type_K);
+ GGML_ASSERT(V->type == type_V);
+
+ float logit_softcap;
+ memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));
+
+ if (Q->ne[1] == 1) {
+ constexpr int cols_per_block = 1;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ ggml_cuda_flash_attn_ext_vec_f16_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ ggml_cuda_flash_attn_ext_vec_f16_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ if (Q->ne[1] == 2) {
+ constexpr int cols_per_block = 2;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ ggml_cuda_flash_attn_ext_vec_f16_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ ggml_cuda_flash_attn_ext_vec_f16_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ if (Q->ne[1] <= 4) {
+ constexpr int cols_per_block = 4;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ ggml_cuda_flash_attn_ext_vec_f16_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ ggml_cuda_flash_attn_ext_vec_f16_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ if (Q->ne[1] <= 8) {
+ constexpr int cols_per_block = 8;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ ggml_cuda_flash_attn_ext_vec_f16_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ ggml_cuda_flash_attn_ext_vec_f16_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ constexpr int cols_per_block = 8;
+ constexpr int parallel_blocks = 1;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ ggml_cuda_flash_attn_ext_vec_f16_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ ggml_cuda_flash_attn_ext_vec_f16_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ }
+}
+
+#define DECL_FATTN_VEC_F16_CASE(D, type_K, type_V) \
+ template void ggml_cuda_flash_attn_ext_vec_f16_case \
+ <D, type_K, type_V>(ggml_backend_cuda_context & ctx, ggml_tensor * dst) \
+
+extern DECL_FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F16);
+
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_0);
+
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_1);
+
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_0);
+
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_1);
+
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q8_0);
+
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F16);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F16);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F16);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_F16);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_F16);
+extern DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16);
+
+extern DECL_FATTN_VEC_F16_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/fattn-vec-f32.cuh b/llama/ggml-cuda/fattn-vec-f32.cuh
new file mode 100644
index 00000000..5f42c314
--- /dev/null
+++ b/llama/ggml-cuda/fattn-vec-f32.cuh
@@ -0,0 +1,446 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+#include "fattn-common.cuh"
+
+template<int D, int ncols, int parallel_blocks, ggml_type type_K, ggml_type type_V, bool use_logit_softcap> // D == head size
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+__launch_bounds__(D, 1)
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+static __global__ void flash_attn_vec_ext_f32(
+ const char * __restrict__ Q,
+ const char * __restrict__ K,
+ const char * __restrict__ V,
+ const char * __restrict__ mask,
+ float * __restrict__ dst,
+ float2 * __restrict__ dst_meta,
+ const float scale,
+ const float max_bias,
+ const float m0,
+ const float m1,
+ const uint32_t n_head_log2,
+ const float logit_softcap,
+ const int ne00,
+ const int ne01,
+ const int ne02,
+ const int ne03,
+ const int ne10,
+ const int ne11,
+ const int ne12,
+ const int ne13,
+ const int ne31,
+ const int nb31,
+ const int nb01,
+ const int nb02,
+ const int nb03,
+ const int nb11,
+ const int nb12,
+ const int nb13,
+ const int nb21,
+ const int nb22,
+ const int nb23,
+ const int ne0,
+ const int ne1,
+ const int ne2,
+ const int ne3) {
+ // Skip unused kernel variants for faster compilation:
+ if (use_logit_softcap && !(D == 128 || D == 256)) {
+ NO_DEVICE_CODE;
+ return;
+ }
+
+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.
+
+ constexpr vec_dot_KQ_f32_t vec_dot_KQ = get_vec_dot_KQ_f32<D>(type_K);
+ constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16;
+ constexpr dequantize_1_f32_t dequantize_1_v = get_dequantize_1_f32(type_V);
+
+ const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on.
+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.
+
+ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.
+ Q += nb02* blockIdx.y + nb01*ic0;
+ K += nb12*(blockIdx.y / gqa_ratio);
+ V += nb22*(blockIdx.y / gqa_ratio); // K and V have same shape
+ const half * maskh = (const half *) mask + ne11*ic0;
+
+ const float slope = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1);
+
+ static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64.");
+ constexpr int nwarps = D / WARP_SIZE;
+ const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;
+ __builtin_assume(tid < D);
+
+ __shared__ float KQ[ncols*D];
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ KQ[j*D + tid] = -FLT_MAX/2.0f;
+ }
+
+ float kqmax[ncols];
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ kqmax[j] = -FLT_MAX/2.0f;
+ }
+ float kqsum[ncols] = {0.0f};
+
+ __shared__ float kqmax_shared[ncols][WARP_SIZE];
+ __shared__ float kqsum_shared[ncols][WARP_SIZE];
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ if (threadIdx.y == 0) {
+ kqmax_shared[j][threadIdx.x] = -FLT_MAX/2.0f;
+ kqsum_shared[j][threadIdx.x] = 0.0f;
+ }
+ }
+ __syncthreads();
+
+ // Convert Q to float2 (f16 K) or q8_1 (quantized K) and store in registers:
+ float2 Q_f2[ncols][D/(2*WARP_SIZE)];
+ int Q_i32[ncols][D/(sizeof(int)*QK8_1) == 0 ? 1 : D >= D/(sizeof(int)*QK8_1)];
+ float2 Q_ds[ncols][D/QK8_1 == 0 ? 1 : D/QK8_1];
+ if (Q_q8_1) {
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ if (j0 + nwarps > ncols && j >= ncols) {
+ break;
+ }
+
+ // Reuse KQ as temporary storage for converting Q to q8_1:
+ int * tmp_q_i32 = (int *) &KQ[j*D];
+ float2 * tmp_q_ds = (float2 *) (tmp_q_i32 + D/sizeof(int));
+
+ // Set memory to zero if out of bounds:
+ if (ncols > 2 && ic0 + j >= ne01) {
+#pragma unroll
+ for (int i0 = 0; i0 < D/sizeof(int); i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ tmp_q_i32[i] = 0;
+ }
+ if (threadIdx.x < D/QK8_1) {
+ tmp_q_ds[threadIdx.x] = make_float2(0.0f, 0.0f);
+ }
+ continue;
+ }
+
+ const float * Q_f = (const float *) (Q + j*nb01);
+#pragma unroll
+ for (int i0 = 0; i0 < D/sizeof(int); i0 += WARP_SIZE) {
+ quantize_q8_1_to_shared<float2>(Q_f + 4*i0, scale, tmp_q_i32, tmp_q_ds);
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ int * tmp_q_i32 = (int *) &KQ[j*D];
+ float2 * tmp_q_ds = (float2 *) (tmp_q_i32 + D/sizeof(int));
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/sizeof(int); i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ Q_i32[j][i0/WARP_SIZE] = tmp_q_i32[i];
+ Q_ds[j][i0/WARP_SIZE] = tmp_q_ds[i/QI8_1];
+ }
+ }
+
+ __syncthreads();
+ } else {
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ const float2 * Q_f2_j = (const float2 *) (Q + j*nb01);
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ Q_f2[j][i0/WARP_SIZE] = ncols <= 2 || ic0 + j < ne01 ? Q_f2_j[i] : make_float2(0.0f, 0.0f);
+ Q_f2[j][i0/WARP_SIZE].x *= scale;
+ Q_f2[j][i0/WARP_SIZE].y *= scale;
+ }
+ }
+ }
+
+ float VKQ[ncols] = {0.0f};
+
+ const int k_start = parallel_blocks == 1 ? 0 : ip*D;
+ for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) {
+ // Calculate KQ tile and keep track of new maximum KQ values:
+
+ float kqmax_new_arr[ncols];
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ kqmax_new_arr[j] = kqmax[j];
+ }
+
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += nwarps) {
+ const int i_KQ = i_KQ_0 + threadIdx.y;
+
+ if ((i_KQ_0 + nwarps > D && i_KQ >= D) || (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + i_KQ >= ne11)) {
+ break;
+ }
+
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ float sum = vec_dot_KQ(K + (k_VKQ_0 + i_KQ)*nb11, Q_f2[j], Q_i32[j], Q_ds[j]);
+ sum = warp_reduce_sum(sum);
+
+ if (use_logit_softcap) {
+ sum = logit_softcap*tanhf(sum);
+ }
+
+ sum += mask ? slope*__half2float(maskh[j*ne11 + k_VKQ_0 + i_KQ]) : 0.0f;
+
+ kqmax_new_arr[j] = fmaxf(kqmax_new_arr[j], sum);
+
+ if (threadIdx.x == 0) {
+ KQ[j*D + i_KQ] = sum;
+ }
+ }
+ }
+
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ float kqmax_new_j = kqmax_new_arr[j];
+
+ kqmax_new_j = warp_reduce_max(kqmax_new_j);
+ if (threadIdx.x == 0) {
+ kqmax_shared[j][threadIdx.y] = kqmax_new_j;
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ float kqmax_new_j = kqmax_shared[j][threadIdx.x];
+ kqmax_new_j = warp_reduce_max(kqmax_new_j);
+
+ const float KQ_max_scale = expf(kqmax[j] - kqmax_new_j);
+ kqmax[j] = kqmax_new_j;
+
+ const float val = expf(KQ[j*D + tid] - kqmax[j]);
+ kqsum[j] = kqsum[j]*KQ_max_scale + val;
+ KQ[j*D + tid] = val;
+
+ VKQ[j] *= KQ_max_scale;
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int k = 0; k < D; ++k) {
+ if (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + k >= ne11) {
+ break;
+ }
+
+ const float V_ki = dequantize_1_v(V + (k_VKQ_0 + k)*nb21, tid);
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ VKQ[j] += V_ki*KQ[j*D + k];
+ }
+ }
+
+ __syncthreads();
+ }
+
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ kqsum[j] = warp_reduce_sum(kqsum[j]);
+ if (threadIdx.x == 0) {
+ kqsum_shared[j][threadIdx.y] = kqsum[j];
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) {
+ if (ncols > 2 && ic0 + j_VKQ >= ne01) {
+ break;
+ }
+
+ kqsum[j_VKQ] = kqsum_shared[j_VKQ][threadIdx.x];
+ kqsum[j_VKQ] = warp_reduce_sum(kqsum[j_VKQ]);
+
+ float dst_val = VKQ[j_VKQ];
+ if (parallel_blocks == 1) {
+ dst_val /= kqsum[j_VKQ];
+ }
+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;
+ dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val;
+ }
+
+ if (parallel_blocks != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) {
+ dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]);
+ }
+}
+
+template <int D, int cols_per_block, int parallel_blocks, ggml_type type_K, ggml_type type_V, bool use_logit_softcap>
+void ggml_cuda_flash_attn_ext_vec_f32_case_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ constexpr int nwarps = D/WARP_SIZE;
+ fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f32<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>;
+ constexpr bool need_f16_K = D != 128;
+ constexpr bool need_f16_V = D != 128 && D != 64;
+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block, need_f16_K, need_f16_V);
+}
+
+template <int D, ggml_type type_K, ggml_type type_V>
+void ggml_cuda_flash_attn_ext_vec_f32_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * KQV = dst;
+ const ggml_tensor * Q = dst->src[0];
+ const ggml_tensor * K = dst->src[1];
+ const ggml_tensor * V = dst->src[2];
+
+ GGML_ASSERT(K->type == type_K);
+ GGML_ASSERT(V->type == type_V);
+
+ float logit_softcap;
+ memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));
+
+ if (Q->ne[1] == 1) {
+ constexpr int cols_per_block = 1;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ ggml_cuda_flash_attn_ext_vec_f32_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ ggml_cuda_flash_attn_ext_vec_f32_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ if (Q->ne[1] == 2) {
+ constexpr int cols_per_block = 2;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ ggml_cuda_flash_attn_ext_vec_f32_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ ggml_cuda_flash_attn_ext_vec_f32_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ if (Q->ne[1] <= 4) {
+ constexpr int cols_per_block = 4;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ ggml_cuda_flash_attn_ext_vec_f32_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ ggml_cuda_flash_attn_ext_vec_f32_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ if (Q->ne[1] <= 8) {
+ constexpr int cols_per_block = 8;
+ constexpr int parallel_blocks = 4;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ ggml_cuda_flash_attn_ext_vec_f32_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ ggml_cuda_flash_attn_ext_vec_f32_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ }
+ return;
+ }
+
+ constexpr int cols_per_block = 8;
+ constexpr int parallel_blocks = 1;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ ggml_cuda_flash_attn_ext_vec_f32_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ } else {
+ constexpr bool use_logit_softcap = true;
+ ggml_cuda_flash_attn_ext_vec_f32_case_impl<D, cols_per_block, parallel_blocks, type_K, type_V, use_logit_softcap>(ctx, dst);
+ }
+}
+
+#define DECL_FATTN_VEC_F32_CASE(D, type_K, type_V) \
+ template void ggml_cuda_flash_attn_ext_vec_f32_case \
+ <D, type_K, type_V>(ggml_backend_cuda_context & ctx, ggml_tensor * dst) \
+
+extern DECL_FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F16);
+
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_0);
+
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_1);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_1);
+
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_0);
+
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_1);
+
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q8_0);
+
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F16);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F16);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F16);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_F16);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_F16);
+extern DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16);
+
+extern DECL_FATTN_VEC_F32_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/fattn-wmma-f16.cuh b/llama/ggml-cuda/fattn-wmma-f16.cuh
new file mode 100644
index 00000000..0c34fb93
--- /dev/null
+++ b/llama/ggml-cuda/fattn-wmma-f16.cuh
@@ -0,0 +1,569 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+#include "fattn-common.cuh"
+
+#ifdef FP16_MMA_AVAILABLE
+#include <mma.h>
+#endif // FP16_MMA_AVAILABLE
+
+// D == head size, VKQ_stride == num VKQ rows calculated in parallel:
+template<int D, int ncols, int nwarps, int VKQ_stride, int parallel_blocks, typename KQ_acc_t, bool use_logit_softcap>
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+__launch_bounds__(nwarps*WARP_SIZE, 1)
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+static __global__ void flash_attn_ext_f16(
+ const char * __restrict__ Q,
+ const char * __restrict__ K,
+ const char * __restrict__ V,
+ const char * __restrict__ mask,
+ float * __restrict__ dst,
+ float2 * __restrict__ dst_meta,
+ const float scale,
+ const float max_bias,
+ const float m0,
+ const float m1,
+ const uint32_t n_head_log2,
+ const float logit_softcap,
+ const int ne00,
+ const int ne01,
+ const int ne02,
+ const int ne03,
+ const int ne10,
+ const int ne11,
+ const int ne12,
+ const int ne13,
+ const int ne31,
+ const int nb31,
+ const int nb01,
+ const int nb02,
+ const int nb03,
+ const int nb11,
+ const int nb12,
+ const int nb13,
+ const int nb21,
+ const int nb22,
+ const int nb23,
+ const int ne0,
+ const int ne1,
+ const int ne2,
+ const int ne3) {
+#ifdef FP16_MMA_AVAILABLE
+ // Skip unused kernel variants for faster compilation:
+ if (use_logit_softcap && !(D == 128 || D == 256)) {
+ NO_DEVICE_CODE;
+ return;
+ }
+
+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.
+
+ const int ic0 = ncols*(blockIdx.x / parallel_blocks); // Index of the first Q/QKV column to work on.
+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.
+
+ static_assert(D <= FATTN_KQ_STRIDE, "D must be <= FATTN_KQ_STRIDE.");
+ static_assert(ncols == 8 || ncols % 16 == 0, "ncols must be 8 or a multiple of 16.");
+ constexpr int frag_m = ncols == 8 ? 32 : 16;
+ constexpr int frag_n = ncols == 8 ? 8 : 16;
+ static_assert(D % frag_m == 0, "If ncols == 8 then D % frag_m must be 0.");
+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, frag_m, frag_n, 16, half, nvcuda::wmma::row_major> frag_a_K;
+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, frag_m, frag_n, 16, half, nvcuda::wmma::col_major> frag_a_V;
+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, frag_m, frag_n, 16, half, nvcuda::wmma::col_major> frag_b;
+ typedef nvcuda::wmma::fragment<nvcuda::wmma::accumulator, frag_m, frag_n, 16, KQ_acc_t> frag_c_KQ;
+ typedef nvcuda::wmma::fragment<nvcuda::wmma::accumulator, frag_m, frag_n, 16, half> frag_c_VKQ;
+
+ constexpr int KQ_stride_tc = nwarps*frag_m; // Number of KQ rows calculated in parallel.
+ constexpr int VKQ_ratio = KQ_stride_tc/VKQ_stride; // Number of parallel VKQ accumulators needed to keep all warps busy.
+ static_assert(VKQ_ratio <= nwarps, "VKQ_ratio must be <= nwarps.");
+
+ // Pad internal representation of KQ, KQV to reduce shared memory bank conflicts:
+ constexpr int D_padded = D + 8;
+ constexpr int kqs_padded = FATTN_KQ_STRIDE + 8;
+ constexpr int kqar = sizeof(KQ_acc_t)/sizeof(half);
+
+ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.
+ const float * Q_f = (const float *) (Q + nb02* blockIdx.y + nb01*ic0);
+ const half * K_h = (const half *) (K + nb12*(blockIdx.y / gqa_ratio));
+ const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape
+ const half * maskh = (const half *) mask + (nb31/sizeof(half))* ic0;
+ const half2 * mask2 = (const half2 *) mask + (nb31/sizeof(half))*(ic0/2);
+
+ const int stride_Q = nb01 / sizeof(float);
+ const int stride_KV = nb11 / sizeof(half);
+
+ const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1);
+ const half slopeh = __float2half(slopef);
+ const half2 slope2 = make_half2(slopef, slopef);
+
+ const half2 logit_softcap_2 = make_half2(logit_softcap, logit_softcap);
+
+ frag_b Q_b[D/16][ncols/frag_n];
+
+ // A single buffer for temporarily holding tiles of KQ and VKQ parts:
+ constexpr int mem_KQ = ncols*kqs_padded*kqar;
+ constexpr int mem_VKQ_parts = VKQ_ratio*ncols*D_padded;
+ __shared__ half KQ[mem_KQ >= mem_VKQ_parts ? mem_KQ : mem_VKQ_parts];
+ float * KQ_f = (float *) KQ;
+ half2 * KQ2 = (half2 *) KQ;
+
+ float KQ_rowsum_f[ncols/nwarps] = {0.0f};
+ float KQ_max_f[ncols/nwarps];
+ float KQ_max_scale_f[ncols/nwarps] = {0.0f};
+
+#pragma unroll
+ for (int j = 0; j < ncols/nwarps; ++j) {
+ KQ_max_f[j] = -FLT_MAX/2.0f;
+ }
+
+ half2 KQ_rowsum_h2[ncols/nwarps] = {{0.0f, 0.0f}};
+ half2 KQ_max_h2[ncols/nwarps];
+ half2 KQ_max_scale_h2[ncols/nwarps] = {{0.0f, 0.0f}};
+
+#pragma unroll
+ for (int j = 0; j < ncols/nwarps; ++j) {
+ KQ_max_h2[j] = make_half2(-HALF_MAX_HALF, -HALF_MAX_HALF);
+ }
+
+ __shared__ half VKQ[ncols*D_padded]; // Accumulator for final VKQ slice.
+ half2 * VKQ2 = (half2 *) VKQ;
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+ if (i0 + WARP_SIZE > D/2 && i >= D/2) {
+ break;
+ }
+ VKQ2[j*(D_padded/2) + i] = make_half2(0.0f, 0.0f);
+ }
+ }
+
+ // Convert Q to half and apply scale, temporarily store in KQ:
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+#pragma unroll
+ for (int i0 = 0; i0 < D; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+ if (i0 + WARP_SIZE > D && i >= D) {
+ break;
+ }
+ KQ[j*D_padded + i] = ic0 + j < ne01 ? Q_f[j*stride_Q + i] * scale : 0.0f;
+ }
+ }
+
+ __syncthreads();
+
+ // Load Q into tensor core fragments/registers since it will be used frequently:
+#pragma unroll
+ for (int i0 = 0; i0 < D; i0 += 16) {
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {
+ nvcuda::wmma::load_matrix_sync(Q_b[i0/16][j0/frag_n], KQ + j0*D_padded + i0, D_padded);
+ }
+ }
+
+ __syncthreads();
+
+ // Iterate over ne11 == previous tokens:
+ for (int k_VKQ_0 = ip*FATTN_KQ_STRIDE; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE) {
+ // Calculate tile of KQ:
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE; i_KQ_0 += KQ_stride_tc) {
+ frag_c_KQ KQ_c[ncols/frag_n];
+#pragma unroll
+ for (int j = 0; j < ncols/frag_n; ++j) {
+ nvcuda::wmma::fill_fragment(KQ_c[j], 0.0f);
+ }
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += 16) {
+ frag_a_K K_a;
+ nvcuda::wmma::load_matrix_sync(K_a, K_h + (k_VKQ_0 + i_KQ_0 + frag_m*threadIdx.y)*stride_KV + k_KQ_0, stride_KV);
+#pragma unroll
+ for (int j = 0; j < ncols/frag_n; ++j) {
+ nvcuda::wmma::mma_sync(KQ_c[j], K_a, Q_b[k_KQ_0/16][j], KQ_c[j]);
+ }
+ }
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {
+ nvcuda::wmma::store_matrix_sync((KQ_acc_t *) KQ + j0*kqs_padded + i_KQ_0 + frag_m*threadIdx.y, KQ_c[j0/frag_n], kqs_padded, nvcuda::wmma::mem_col_major);
+ }
+ }
+
+ __syncthreads();
+
+ // Calculate softmax for each KQ column using the current max. value.
+ // The divisor is stored in KQ_rowsum and will be applied at the end.
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ if (std::is_same<KQ_acc_t, float>::value) {
+ float KQ_f_tmp[FATTN_KQ_STRIDE / WARP_SIZE];
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ KQ_f_tmp[k0/WARP_SIZE] = KQ_f[j*kqs_padded + k];
+
+ if (use_logit_softcap) {
+ KQ_f_tmp[k0/WARP_SIZE] = logit_softcap*tanhf(KQ_f_tmp[k0/WARP_SIZE]);
+ }
+ }
+
+ float KQ_max_new = KQ_max_f[j0/nwarps];
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ KQ_f_tmp[k0/WARP_SIZE] += mask ? __half2float(slopeh*maskh[j*(nb31/sizeof(half)) + k_VKQ_0 + k]) : 0.0f;
+ KQ_max_new = max(KQ_max_new, KQ_f_tmp[k0/WARP_SIZE]);
+ }
+ KQ_max_new = warp_reduce_max(KQ_max_new);
+
+ const float diff = KQ_max_f[j0/nwarps] - KQ_max_new;
+ KQ_max_scale_f[j0/nwarps] = expf(diff);
+ if (diff <= SOFTMAX_FTZ_THRESHOLD) {
+ KQ_max_scale_f[j0/nwarps] = 0.0f;
+ }
+ KQ_max_f[j0/nwarps] = KQ_max_new;
+
+ float KQ_rowsum_add = 0.0f;
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ const float diff = KQ_f_tmp[k0/WARP_SIZE] - KQ_max_f[j0/nwarps];
+ KQ_f_tmp[k0/WARP_SIZE] = expf(diff);
+ if (diff <= SOFTMAX_FTZ_THRESHOLD) {
+ KQ_f_tmp[k0/WARP_SIZE] = 0.0f;
+ }
+ KQ_rowsum_add += KQ_f_tmp[k0/WARP_SIZE];
+ KQ[j*(kqar*kqs_padded) + k] = KQ_f_tmp[k0/WARP_SIZE];
+ }
+ KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);
+
+ // Scale previous KQ_rowsum to account for a potential increase in KQ_max:
+ KQ_rowsum_f[j0/nwarps] = KQ_max_scale_f[j0/nwarps]*KQ_rowsum_f[j0/nwarps] + KQ_rowsum_add;
+ } else {
+ half2 KQ2_tmp[FATTN_KQ_STRIDE/(2*WARP_SIZE)];
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ KQ2_tmp[k0/WARP_SIZE] = KQ2[j*(kqs_padded/2) + k];
+
+ if (use_logit_softcap) {
+ // There is no dedicated tangens hyperbolicus function for half2.
+ KQ2_tmp[k0/WARP_SIZE] = h2exp(KQ2_tmp[k0/WARP_SIZE]*make_half2(2.0f, 2.0f));
+ KQ2_tmp[k0/WARP_SIZE] = (KQ2_tmp[k0/WARP_SIZE] - make_half2(1.0f, 1.0f))
+ /(KQ2_tmp[k0/WARP_SIZE] + make_half2(1.0f, 1.0f));
+
+ KQ2_tmp[k0/WARP_SIZE] *= logit_softcap_2;
+ }
+ }
+
+ half2 KQ_max_new = KQ_max_h2[j0/nwarps];
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ KQ2_tmp[k0/WARP_SIZE] += mask ? slope2*mask2[(j*ne11 + k_VKQ_0)/2 + k] : make_half2(0.0f, 0.0f);
+ KQ_max_new = ggml_cuda_hmax2(KQ_max_new, KQ2_tmp[k0/WARP_SIZE]);
+ }
+ KQ_max_new = __half2half2(warp_reduce_max(ggml_cuda_hmax(__low2half(KQ_max_new), __high2half(KQ_max_new))));
+ const half2 diff = KQ_max_h2[j0/nwarps] - KQ_max_new;
+ KQ_max_scale_h2[j0/nwarps] = h2exp(diff);
+ const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));
+ *((uint32_t *) &KQ_max_scale_h2[j0/nwarps]) &= ftz_mask;
+ KQ_max_h2[j0/nwarps] = KQ_max_new;
+
+ half2 KQ_rowsum_add = make_half2(0.0f, 0.0f);
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ const half2 diff = KQ2_tmp[k0/WARP_SIZE] - KQ_max_h2[j0/nwarps];
+ KQ2_tmp[k0/WARP_SIZE] = h2exp(diff);
+ const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));
+ *((uint32_t *) &KQ2_tmp[k0/WARP_SIZE]) &= ftz_mask;
+ KQ_rowsum_add += KQ2_tmp[k0/WARP_SIZE];
+ KQ2[j*(kqs_padded/2) + k] = KQ2_tmp[k0/WARP_SIZE];
+ }
+ KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);
+
+ // Scale previous KQ_rowsum to account for a potential increase in KQ_max:
+ KQ_rowsum_h2[j0/nwarps] = KQ_max_scale_h2[j0/nwarps]*KQ_rowsum_h2[j0/nwarps] + KQ_rowsum_add;
+ }
+ }
+
+ __syncthreads();
+
+ frag_b KQ_b[FATTN_KQ_STRIDE/(VKQ_ratio*16)][ncols/frag_n];
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {
+ const int k = k0 + (threadIdx.y % VKQ_ratio)*16;
+ nvcuda::wmma::load_matrix_sync(
+ KQ_b[k0/(VKQ_ratio*16)][j0/frag_n],
+ KQ + j0*(kqar*kqs_padded) + k,
+ kqar*kqs_padded);
+ }
+ }
+
+ frag_c_VKQ VKQ_c[D/VKQ_stride][ncols/frag_n];
+#pragma unroll
+ for (int i_VKQ_0 = 0; i_VKQ_0 < D; i_VKQ_0 += VKQ_stride) {
+#pragma unroll
+ for (int j = 0; j < ncols/frag_n; ++j) {
+ nvcuda::wmma::fill_fragment(VKQ_c[i_VKQ_0/VKQ_stride][j], 0.0f);
+ }
+
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {
+ const int k = k0 + (threadIdx.y % VKQ_ratio)*16;
+
+ frag_a_V v_a;
+ nvcuda::wmma::load_matrix_sync(v_a, V_h + (k_VKQ_0 + k)*stride_KV + i_VKQ_0 + frag_m*(threadIdx.y/VKQ_ratio), stride_KV);
+#pragma unroll
+ for (int j = 0; j < ncols/frag_n; ++j) {
+ nvcuda::wmma::mma_sync(VKQ_c[i_VKQ_0/VKQ_stride][j], v_a, KQ_b[k0/(VKQ_ratio*16)][j], VKQ_c[i_VKQ_0/VKQ_stride][j]);
+ }
+ }
+ }
+
+ __syncthreads();
+
+ const int offset_k = (threadIdx.y % VKQ_ratio) * (ncols*D_padded);
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += VKQ_stride) {
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {
+ nvcuda::wmma::store_matrix_sync(
+ KQ + offset_k + j0*D_padded + i_KQ_0 + frag_m*(threadIdx.y/VKQ_ratio),
+ VKQ_c[i_KQ_0/VKQ_stride][j0/frag_n],
+ D_padded, nvcuda::wmma::mem_col_major);
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ half2 VKQ_scale;
+ if (std::is_same<KQ_acc_t, float>::value) {
+ VKQ_scale = make_half2(KQ_max_scale_f[j0/nwarps], KQ_max_scale_f[j0/nwarps]);
+ } else {
+ VKQ_scale = KQ_max_scale_h2[j0/nwarps];
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+ if (i0 + WARP_SIZE > D/2 && i >= D/2) {
+ break;
+ }
+
+ half2 VKQ_add = make_half2(0.0f, 0.0f);
+#pragma unroll
+ for (int l = 0; l < VKQ_ratio; ++l) {
+ VKQ_add += KQ2[l*(ncols*D_padded/2) + j*(D_padded/2) + i];
+ }
+ VKQ2[j*(D_padded/2) + i] = VKQ_scale*VKQ2[j*(D_padded/2) + i] + VKQ_add;
+ }
+ }
+
+ __syncthreads();
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j_VKQ = j0 + threadIdx.y;
+ if (ic0 + j_VKQ >= ne01) {
+ return;
+ }
+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;
+
+ float KQ_rowsum_j;
+ if (std::is_same<KQ_acc_t, float>::value) {
+ KQ_rowsum_j = KQ_rowsum_f[j0/nwarps];
+ } else {
+ KQ_rowsum_j = __low2float(KQ_rowsum_h2[j0/nwarps]) + __high2float(KQ_rowsum_h2[j0/nwarps]);
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < D; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+ if (i0 + WARP_SIZE > D && i >= D) {
+ break;
+ }
+ float dst_val = VKQ[j_VKQ*D_padded + i];
+ if (parallel_blocks == 1) {
+ dst_val /= KQ_rowsum_j;
+ }
+ dst[j_dst*gridDim.y*D + blockIdx.y*D + i] = dst_val;
+ }
+
+ if (parallel_blocks == 1 || threadIdx.x != 0) {
+ continue;
+ }
+
+ float2 dst_meta_val;
+ if (std::is_same<KQ_acc_t, float>::value) {
+ dst_meta_val.x = KQ_max_f[j0/nwarps];
+ } else {
+ dst_meta_val.x = __low2float(KQ_max_h2[j0/nwarps]);
+ }
+ dst_meta_val.y = KQ_rowsum_j;
+ dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = dst_meta_val;
+ }
+#else
+ NO_DEVICE_CODE;
+#endif // FP16_MMA_AVAILABLE
+}
+
+constexpr int get_max_power_of_2(int x) {
+ return x % 2 == 0 ? 2*get_max_power_of_2(x/2) : 1;
+}
+
+static_assert(get_max_power_of_2(1) == 1, "Test failed.");
+static_assert(get_max_power_of_2(2) == 2, "Test failed.");
+static_assert(get_max_power_of_2(4) == 4, "Test failed.");
+static_assert(get_max_power_of_2(6) == 2, "Test failed.");
+
+// Number of VKQ rows calculated in parallel:
+constexpr int get_VKQ_stride(int D, int nwarps, int frag_m) {
+ return (get_max_power_of_2(D/frag_m) < nwarps ? get_max_power_of_2(D/frag_m) : nwarps)*frag_m;
+}
+
+static_assert(get_VKQ_stride(128, 1, 32) == 32, "Test failed.");
+static_assert(get_VKQ_stride(128, 2, 32) == 64, "Test failed.");
+static_assert(get_VKQ_stride(128, 4, 32) == 128, "Test failed.");
+static_assert(get_VKQ_stride( 64, 1, 32) == 32, "Test failed.");
+static_assert(get_VKQ_stride( 64, 2, 32) == 64, "Test failed.");
+static_assert(get_VKQ_stride( 64, 4, 32) == 64, "Test failed.");
+static_assert(get_VKQ_stride( 80, 1, 16) == 16, "Test failed.");
+static_assert(get_VKQ_stride( 80, 2, 16) == 16, "Test failed.");
+static_assert(get_VKQ_stride( 80, 4, 16) == 16, "Test failed.");
+
+template <int D, int cols_per_block, typename KQ_acc_t>
+void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * KQV = dst;
+ const ggml_tensor * Q = dst->src[0];
+
+ constexpr int nwarps = 4;
+
+ constexpr int frag_m = cols_per_block == 8 && D % 32 == 0 ? 32 : 16;
+ const int blocks_num_pb1 = ((Q->ne[1] + cols_per_block - 1) / cols_per_block)*Q->ne[2]*Q->ne[3];
+ const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;
+
+ float logit_softcap;
+ memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));
+
+ if (4*blocks_num_pb1 < 2*nsm) {
+ constexpr int parallel_blocks = 4;
+ fattn_kernel_t fattn_kernel;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ fattn_kernel = flash_attn_ext_f16<
+ D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>;
+ } else {
+ constexpr bool use_logit_softcap = true;
+ fattn_kernel = flash_attn_ext_f16<
+ D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>;
+ }
+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block, true, true);
+ return;
+ }
+ if (2*blocks_num_pb1 < 2*nsm) {
+ constexpr int parallel_blocks = 2;
+ fattn_kernel_t fattn_kernel;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ fattn_kernel = flash_attn_ext_f16<
+ D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>;
+ } else {
+ constexpr bool use_logit_softcap = true;
+ fattn_kernel = flash_attn_ext_f16<
+ D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>;
+ }
+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block, true, true);
+ return;
+ }
+ constexpr int parallel_blocks = 1;
+ fattn_kernel_t fattn_kernel;
+ if (logit_softcap == 0.0f) {
+ constexpr bool use_logit_softcap = false;
+ fattn_kernel = flash_attn_ext_f16<
+ D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>;
+ } else {
+ constexpr bool use_logit_softcap = true;
+ fattn_kernel = flash_attn_ext_f16<
+ D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>;
+ }
+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block, true, true);
+}
+
+#define DECL_FATTN_WMMA_F16_CASE(D, cols_per_block, KQ_acc_t) \
+ template void ggml_cuda_flash_attn_ext_wmma_f16_case \
+ <D, cols_per_block, KQ_acc_t>(ggml_backend_cuda_context & ctx, ggml_tensor * dst) \
+
+extern DECL_FATTN_WMMA_F16_CASE( 64, 16, float);
+extern DECL_FATTN_WMMA_F16_CASE( 80, 16, float);
+extern DECL_FATTN_WMMA_F16_CASE( 96, 16, float);
+extern DECL_FATTN_WMMA_F16_CASE(112, 16, float);
+extern DECL_FATTN_WMMA_F16_CASE(128, 16, float);
+extern DECL_FATTN_WMMA_F16_CASE(256, 16, float);
+
+extern DECL_FATTN_WMMA_F16_CASE( 64, 32, float);
+extern DECL_FATTN_WMMA_F16_CASE( 80, 32, float);
+extern DECL_FATTN_WMMA_F16_CASE( 96, 32, float);
+extern DECL_FATTN_WMMA_F16_CASE(112, 32, float);
+extern DECL_FATTN_WMMA_F16_CASE(128, 32, float);
+// extern DECL_FATTN_WMMA_F16_CASE(256, 16, float);
+
+extern DECL_FATTN_WMMA_F16_CASE( 64, 8, half);
+extern DECL_FATTN_WMMA_F16_CASE( 96, 8, half);
+extern DECL_FATTN_WMMA_F16_CASE(128, 8, half);
+extern DECL_FATTN_WMMA_F16_CASE(256, 8, half);
+
+extern DECL_FATTN_WMMA_F16_CASE( 64, 16, half);
+extern DECL_FATTN_WMMA_F16_CASE( 80, 16, half);
+extern DECL_FATTN_WMMA_F16_CASE( 96, 16, half);
+extern DECL_FATTN_WMMA_F16_CASE(112, 16, half);
+extern DECL_FATTN_WMMA_F16_CASE(128, 16, half);
+extern DECL_FATTN_WMMA_F16_CASE(256, 16, half);
+
+extern DECL_FATTN_WMMA_F16_CASE( 64, 32, half);
+extern DECL_FATTN_WMMA_F16_CASE( 80, 32, half);
+extern DECL_FATTN_WMMA_F16_CASE( 96, 32, half);
+extern DECL_FATTN_WMMA_F16_CASE(112, 32, half);
+extern DECL_FATTN_WMMA_F16_CASE(128, 32, half);
+extern DECL_FATTN_WMMA_F16_CASE(256, 16, half);
diff --git a/llama/ggml-cuda/fattn.cu b/llama/ggml-cuda/fattn.cu
new file mode 100644
index 00000000..5bd553e9
--- /dev/null
+++ b/llama/ggml-cuda/fattn.cu
@@ -0,0 +1,371 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+#include "fattn-common.cuh"
+#include "fattn-tile-f16.cuh"
+#include "fattn-tile-f32.cuh"
+#include "fattn-vec-f16.cuh"
+#include "fattn-vec-f32.cuh"
+#include "fattn-wmma-f16.cuh"
+#include "fattn.cuh"
+
+#include <cstdint>
+
+static void ggml_cuda_flash_attn_ext_wmma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * KQV = dst;
+ const ggml_tensor * Q = dst->src[0];
+
+ const int32_t precision = KQV->op_params[3];
+
+ if (precision != GGML_PREC_DEFAULT) {
+ if (Q->ne[1] <= 32 || Q->ne[0] > 128) {
+ constexpr int cols_per_block = 16;
+ switch (Q->ne[0]) {
+ case 64:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, float>(ctx, dst);
+ break;
+ case 80:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, float>(ctx, dst);
+ break;
+ case 96:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, float>(ctx, dst);
+ break;
+ case 112:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, float>(ctx, dst);
+ break;
+ case 128:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, float>(ctx, dst);
+ break;
+ case 256:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, float>(ctx, dst);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+ } else {
+ constexpr int cols_per_block = 32;
+ switch (Q->ne[0]) {
+ case 64:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, float>(ctx, dst);
+ break;
+ case 80:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, float>(ctx, dst);
+ break;
+ case 96:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, float>(ctx, dst);
+ break;
+ case 112:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, float>(ctx, dst);
+ break;
+ case 128:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, float>(ctx, dst);
+ break;
+ // case 256:
+ // ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, float>(ctx, dst);
+ // break;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+ }
+ return;
+ }
+
+ if (Q->ne[1] <= 8 && Q->ne[0] % WARP_SIZE == 0) {
+ constexpr int cols_per_block = 8;
+ switch (Q->ne[0]) {
+ case 64:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst);
+ break;
+ case 96:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst);
+ break;
+ case 128:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst);
+ break;
+ case 256:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+ return;
+ }
+
+ if (Q->ne[1] <= 32) {
+ constexpr int cols_per_block = 16;
+ switch (Q->ne[0]) {
+ case 64:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst);
+ break;
+ case 80:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, half>(ctx, dst);
+ break;
+ case 96:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst);
+ break;
+ case 112:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, half>(ctx, dst);
+ break;
+ case 128:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst);
+ break;
+ case 256:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+ return;
+ }
+
+ constexpr int cols_per_block = 32;
+ switch (Q->ne[0]) {
+ case 64:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst);
+ break;
+ case 80:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, half>(ctx, dst);
+ break;
+ case 96:
+ ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst);
+ break;
+ case 112:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, half>(ctx, dst);
+ break;
+ case 128:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst);
+ break;
+ case 256:
+ ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+}
+#define FATTN_VEC_F16_CASE(D, type_K, type_V) \
+ if (Q->ne[0] == (D) && K->type == (type_K) && V->type == (type_V)) { \
+ ggml_cuda_flash_attn_ext_vec_f16_case<D, type_K, type_V>(ctx, dst); \
+ return; \
+ } \
+
+static void ggml_cuda_flash_attn_ext_vec_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ ggml_tensor * Q = dst->src[1];
+ ggml_tensor * K = dst->src[1];
+ ggml_tensor * V = dst->src[2];
+
+#ifdef GGML_CUDA_FA_ALL_QUANTS
+ FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_0)
+ FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_1)
+ FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_0)
+ FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_1)
+ FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q8_0)
+ FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F16 )
+
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_0)
+
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_1)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_1)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_1)
+
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_0)
+
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_1)
+
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q8_0)
+
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F16)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F16)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F16)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_F16)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_F16)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16)
+
+ FATTN_VEC_F16_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16)
+#else
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)
+
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)
+
+ FATTN_VEC_F16_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F16)
+ FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16)
+ FATTN_VEC_F16_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16)
+#endif // GGML_CUDA_FA_ALL_QUANTS
+
+ on_no_fattn_vec_case(Q->ne[0]);
+}
+
+#define FATTN_VEC_F32_CASE(D, type_K, type_V) \
+ if (Q->ne[0] == (D) && K->type == (type_K) && V->type == (type_V)) { \
+ ggml_cuda_flash_attn_ext_vec_f32_case<D, type_K, type_V>(ctx, dst); \
+ return; \
+ } \
+
+static void ggml_cuda_flash_attn_ext_vec_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ ggml_tensor * Q = dst->src[1];
+ ggml_tensor * K = dst->src[1];
+ ggml_tensor * V = dst->src[2];
+
+#ifdef GGML_CUDA_FA_ALL_QUANTS
+ FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_0)
+ FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_1)
+ FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_0)
+ FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_1)
+ FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q8_0)
+ FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F16)
+
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_0)
+
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_1)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_1)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_1)
+
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_0)
+
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_1)
+
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q8_0)
+
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F16)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F16)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F16)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_F16)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_F16)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16)
+
+ FATTN_VEC_F32_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16)
+#else
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)
+
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)
+
+ FATTN_VEC_F32_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F16)
+ FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16)
+ FATTN_VEC_F32_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16)
+#endif // GGML_CUDA_FA_ALL_QUANTS
+
+ on_no_fattn_vec_case(Q->ne[0]);
+}
+
+void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * KQV = dst;
+ const ggml_tensor * Q = dst->src[0];
+
+ ggml_cuda_set_device(ctx.device);
+ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+ const int32_t precision = KQV->op_params[3];
+
+ // On AMD the tile kernels perform poorly, use the vec kernel instead:
+ if (cc >= CC_OFFSET_AMD) {
+ if (precision == GGML_PREC_DEFAULT && fast_fp16_available(cc)) {
+ ggml_cuda_flash_attn_ext_vec_f16(ctx, dst);
+ } else {
+ ggml_cuda_flash_attn_ext_vec_f32(ctx, dst);
+ }
+ return;
+ }
+
+ if (!fast_fp16_available(cc)) {
+ if (Q->ne[1] <= 8) {
+ ggml_cuda_flash_attn_ext_vec_f32(ctx, dst);
+ } else {
+ ggml_cuda_flash_attn_ext_tile_f32(ctx, dst);
+ }
+ return;
+ }
+
+ if (!fp16_mma_available(cc)) {
+ if (Q->ne[1] <= 8) {
+ ggml_cuda_flash_attn_ext_vec_f16(ctx, dst);
+ } else {
+ ggml_cuda_flash_attn_ext_tile_f16(ctx, dst);
+ }
+ return;
+ }
+
+ if (Q->ne[1] == 1 && Q->ne[0] % (2*WARP_SIZE) == 0) {
+ if (precision == GGML_PREC_DEFAULT) {
+ ggml_cuda_flash_attn_ext_vec_f16(ctx, dst);
+ return;
+ } else if(Q->ne[0] <= 128) {
+ ggml_cuda_flash_attn_ext_vec_f32(ctx, dst);
+ return;
+ }
+ }
+
+ ggml_cuda_flash_attn_ext_wmma_f16(ctx, dst);
+}
diff --git a/llama/ggml-cuda/fattn.cuh b/llama/ggml-cuda/fattn.cuh
new file mode 100644
index 00000000..f502eaaa
--- /dev/null
+++ b/llama/ggml-cuda/fattn.cuh
@@ -0,0 +1,29 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/getrows.cu b/llama/ggml-cuda/getrows.cu
new file mode 100644
index 00000000..4c627947
--- /dev/null
+++ b/llama/ggml-cuda/getrows.cu
@@ -0,0 +1,203 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "getrows.cuh"
+#include "dequantize.cuh"
+
+template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
+static __global__ void k_get_rows(
+ const void * src0, const int32_t * src1, dst_t * dst,
+ int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/
+ /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/
+ /*size_t s0,*/ size_t s1, size_t s2, size_t s3,
+ /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,
+ size_t s10, size_t s11, size_t s12/*, size_t s13*/) {
+
+ const int i00 = (blockIdx.x*blockDim.x + threadIdx.x)*2;
+ const int i10 = blockDim.y*blockIdx.y + threadIdx.y;
+ const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;
+ const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;
+
+ if (i00 >= ne00) {
+ return;
+ }
+
+ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];
+
+ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;
+ const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03;
+
+ const int ib = i00/qk; // block index
+ const int iqs = (i00%qk)/qr; // quant index
+ const int iybs = i00 - i00%qk; // dst block start index
+ const int y_offset = qr == 1 ? 1 : qk/2;
+
+ // dequantize
+ dfloat2 v;
+ dequantize_kernel(src0_row, ib, iqs, v);
+
+ dst_row[iybs + iqs + 0] = v.x;
+ dst_row[iybs + iqs + y_offset] = v.y;
+}
+
+template<typename src0_t, typename dst_t>
+static __global__ void k_get_rows_float(
+ const src0_t * src0, const int32_t * src1, dst_t * dst,
+ int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/
+ /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/
+ /*size_t s0,*/ size_t s1, size_t s2, size_t s3,
+ /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,
+ size_t s10, size_t s11, size_t s12/*, size_t s13*/) {
+
+ const int i00 = blockIdx.x*blockDim.x + threadIdx.x;
+ const int i10 = blockDim.y*blockIdx.y + threadIdx.y;
+ const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;
+ const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;
+
+ if (i00 >= ne00) {
+ return;
+ }
+
+ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];
+
+ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;
+ const src0_t * src0_row = (const src0_t *)((const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03);
+
+ dst_row[i00] = src0_row[i00];
+}
+
+template<int qk, int qr, dequantize_kernel_t dq>
+static void get_rows_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
+ const void * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);
+ const int block_num_x = (ne00 + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);
+ const dim3 block_nums(block_num_x, ne10, ne11*ne12);
+
+ // strides in elements
+ //const size_t s0 = nb0 / ggml_element_size(dst);
+ const size_t s1 = nb1 / ggml_element_size(dst);
+ const size_t s2 = nb2 / ggml_element_size(dst);
+ const size_t s3 = nb3 / ggml_element_size(dst);
+
+ const size_t s10 = nb10 / ggml_element_size(src1);
+ const size_t s11 = nb11 / ggml_element_size(src1);
+ const size_t s12 = nb12 / ggml_element_size(src1);
+ //const size_t s13 = nb13 / ggml_element_size(src1);
+
+ GGML_ASSERT(ne00 % 2 == 0);
+
+ k_get_rows<qk, qr, dq><<<block_nums, block_dims, 0, stream>>>(
+ src0_dd, src1_dd, dst_dd,
+ ne00, /*ne01, ne02, ne03,*/
+ /*ne10, ne11,*/ ne12, /*ne13,*/
+ /* s0,*/ s1, s2, s3,
+ /* nb00,*/ nb01, nb02, nb03,
+ s10, s11, s12/*, s13*/);
+
+ GGML_UNUSED(dst);
+}
+
+template<typename src0_t>
+static void get_rows_cuda_float(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
+ const src0_t * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);
+ const int block_num_x = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;
+ const dim3 block_nums(block_num_x, ne10, ne11*ne12);
+
+ // strides in elements
+ //const size_t s0 = nb0 / ggml_element_size(dst);
+ const size_t s1 = nb1 / ggml_element_size(dst);
+ const size_t s2 = nb2 / ggml_element_size(dst);
+ const size_t s3 = nb3 / ggml_element_size(dst);
+
+ const size_t s10 = nb10 / ggml_element_size(src1);
+ const size_t s11 = nb11 / ggml_element_size(src1);
+ const size_t s12 = nb12 / ggml_element_size(src1);
+ //const size_t s13 = nb13 / ggml_element_size(src1);
+
+ k_get_rows_float<<<block_nums, block_dims, 0, stream>>>(
+ src0_dd, src1_dd, dst_dd,
+ ne00, /*ne01, ne02, ne03,*/
+ /*ne10, ne11,*/ ne12, /*ne13,*/
+ /* s0,*/ s1, s2, s3,
+ /* nb00,*/ nb01, nb02, nb03,
+ s10, s11, s12/*, s13*/);
+
+ GGML_UNUSED(dst);
+}
+
+void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ const float * src0_d = (const float *)src0->data;
+ const float * src1_d = (const float *)src1->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+
+ GGML_ASSERT(src1->type == GGML_TYPE_I32);
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
+
+ GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));
+ GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));
+ GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));
+
+ const int32_t * src1_i32 = (const int32_t *) src1_d;
+
+ switch (src0->type) {
+ case GGML_TYPE_F16:
+ get_rows_cuda_float(src0, src1, dst, (const half *)src0_d, src1_i32, dst_d, stream);
+ break;
+ case GGML_TYPE_F32:
+ get_rows_cuda_float(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
+ break;
+ case GGML_TYPE_Q4_0:
+ get_rows_cuda<QK4_0, QR4_0, dequantize_q4_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
+ break;
+ case GGML_TYPE_Q4_1:
+ get_rows_cuda<QK4_1, QR4_1, dequantize_q4_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
+ break;
+ case GGML_TYPE_Q5_0:
+ get_rows_cuda<QK5_0, QR5_0, dequantize_q5_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
+ break;
+ case GGML_TYPE_Q5_1:
+ get_rows_cuda<QK5_1, QR5_1, dequantize_q5_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
+ break;
+ case GGML_TYPE_Q8_0:
+ get_rows_cuda<QK8_0, QR8_0, dequantize_q8_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
+ break;
+ default:
+ // TODO: k-quants
+ GGML_ABORT("%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type));
+ break;
+ }
+}
diff --git a/llama/ggml-cuda/getrows.cuh b/llama/ggml-cuda/getrows.cuh
new file mode 100644
index 00000000..e2bc477e
--- /dev/null
+++ b/llama/ggml-cuda/getrows.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_GET_ROWS_BLOCK_SIZE 256
+
+void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/im2col.cu b/llama/ggml-cuda/im2col.cu
new file mode 100644
index 00000000..247cf147
--- /dev/null
+++ b/llama/ggml-cuda/im2col.cu
@@ -0,0 +1,130 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "im2col.cuh"
+
+template <typename T>
+static __global__ void im2col_kernel(
+ const float * x, T * dst, int64_t batch_offset,
+ int64_t offset_delta, int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, int64_t pelements, int64_t CHW,
+ int s0, int s1, int p0, int p1, int d0, int d1) {
+ const int64_t i = threadIdx.x + blockIdx.x * blockDim.x;
+ if (i >= pelements) {
+ return;
+ }
+
+ const int64_t ksize = OW * (KH > 1 ? KW : 1);
+ const int64_t kx = i / ksize;
+ const int64_t kd = kx * ksize;
+ const int64_t ky = (i - kd) / OW;
+ const int64_t ix = i % OW;
+
+ const int64_t oh = blockIdx.y;
+ const int64_t batch = blockIdx.z / IC;
+ const int64_t ic = blockIdx.z % IC;
+
+ const int64_t iiw = ix * s0 + kx * d0 - p0;
+ const int64_t iih = oh * s1 + ky * d1 - p1;
+
+ const int64_t offset_dst =
+ ((batch * OH + oh) * OW + ix) * CHW +
+ (ic * (KW * KH) + ky * KW + kx);
+
+ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
+ dst[offset_dst] = 0.0f;
+ } else {
+ const int64_t offset_src = ic * offset_delta + batch * batch_offset;
+ dst[offset_dst] = x[offset_src + iih * IW + iiw];
+ }
+}
+
+template <typename T>
+static void im2col_cuda(const float * x, T* dst,
+ int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,
+ int64_t batch, int64_t batch_offset, int64_t offset_delta,
+ int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {
+ const int parallel_elements = OW * KW * KH;
+ const int num_blocks = (parallel_elements + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;
+ dim3 block_nums(num_blocks, OH, batch * IC);
+ im2col_kernel<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, stream>>>(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1);
+}
+
+static void im2col_cuda_f16(const float * x, half * dst,
+ int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,
+ int64_t batch, int64_t batch_offset, int64_t offset_delta,
+ int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {
+
+ im2col_cuda<half>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1, d0, d1, stream);
+}
+
+static void im2col_cuda_f32(const float * x, float * dst,
+ int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,
+ int64_t batch, int64_t batch_offset, int64_t offset_delta,
+ int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {
+
+ im2col_cuda<float>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1, d0, d1, stream);
+}
+
+void ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ const float * src1_d = (const float *)src1->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);
+
+ const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
+ const int32_t s1 = ((const int32_t*)(dst->op_params))[1];
+ const int32_t p0 = ((const int32_t*)(dst->op_params))[2];
+ const int32_t p1 = ((const int32_t*)(dst->op_params))[3];
+ const int32_t d0 = ((const int32_t*)(dst->op_params))[4];
+ const int32_t d1 = ((const int32_t*)(dst->op_params))[5];
+
+ const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;
+
+ const int64_t IC = src1->ne[is_2D ? 2 : 1];
+ const int64_t IH = is_2D ? src1->ne[1] : 1;
+ const int64_t IW = src1->ne[0];
+
+ const int64_t KH = is_2D ? src0->ne[1] : 1;
+ const int64_t KW = src0->ne[0];
+
+ const int64_t OH = is_2D ? dst->ne[2] : 1;
+ const int64_t OW = dst->ne[1];
+
+ const size_t delta_offset = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32
+ const int64_t batch = src1->ne[3];
+ const size_t batch_offset = src1->nb[3] / 4; // nb is byte offset, src is type float32
+
+ if(dst->type == GGML_TYPE_F16) {
+ im2col_cuda_f16(src1_d, (half *) dst_d, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);
+ } else {
+ im2col_cuda_f32(src1_d, (float *) dst_d, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);
+ }
+}
diff --git a/llama/ggml-cuda/im2col.cuh b/llama/ggml-cuda/im2col.cuh
new file mode 100644
index 00000000..df1a8c64
--- /dev/null
+++ b/llama/ggml-cuda/im2col.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_IM2COL_BLOCK_SIZE 256
+
+void ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/mma.cuh b/llama/ggml-cuda/mma.cuh
new file mode 100644
index 00000000..7f6ea994
--- /dev/null
+++ b/llama/ggml-cuda/mma.cuh
@@ -0,0 +1,247 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+struct mma_int_A_I16K4 {
+ static constexpr int I = 16;
+ static constexpr int K = 4;
+ static constexpr int ne = 2;
+
+ int x[ne] = {0};
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ const int ret = (l%2) * (I/2) + threadIdx.x / K;
+ GGML_CUDA_ASSUME(ret >= 0);
+ GGML_CUDA_ASSUME(ret < I);
+ return ret;
+ }
+
+ static __device__ __forceinline__ int get_k(const int /* l */) {
+ const int ret = threadIdx.x % K;
+ GGML_CUDA_ASSUME(ret >= 0);
+ GGML_CUDA_ASSUME(ret < K);
+ return ret;
+ }
+
+ __device__ __forceinline__ void load(const int * __restrict__ xs0, const int & stride) {
+#if defined(INT8_MMA_AVAILABLE)
+ const int * xs = xs0 + (threadIdx.x%I)*stride;
+ asm("ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];"
+ : "+r"(x[0]), "+r"(x[1])
+ : "l"(xs));
+#else
+#pragma unroll
+ for (int l = 0; l < ne; ++l) {
+ x[l] = xs0[get_i(l)*stride + get_k(l)];
+ }
+#endif // defined(INT8_MMA_AVAILABLE)
+ }
+};
+
+struct mma_int_A_I16K8 {
+ static constexpr int I = 16;
+ static constexpr int K = 8;
+ static constexpr int ne = 4;
+
+ int x[ne] = {0};
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ const int ret = (l%2) * (I/2) + threadIdx.x / (K/2);
+ GGML_CUDA_ASSUME(ret >= 0);
+ GGML_CUDA_ASSUME(ret < I);
+ return ret;
+ }
+
+ static __device__ __forceinline__ int get_k(const int l) {
+ const int ret = (l/2) * (K/2) + threadIdx.x % (K/2);
+ GGML_CUDA_ASSUME(ret >= 0);
+ GGML_CUDA_ASSUME(ret < K);
+ return ret;
+ }
+
+ __device__ __forceinline__ void load(const int * __restrict__ xs0, const int & stride) {
+#if defined(INT8_MMA_AVAILABLE)
+ const int * xs = xs0 + (threadIdx.x%I)*stride + (threadIdx.x/I)*(K/2);
+ asm("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];"
+ : "+r"(x[0]), "+r"(x[1]), "+r"(x[2]), "+r"(x[3])
+ : "l"(xs));
+#else
+#pragma unroll
+ for (int l = 0; l < ne; ++l) {
+ x[l] = xs0[get_i(l)*stride + get_k(l)];
+ }
+#endif // defined(INT8_MMA_AVAILABLE)
+ }
+
+ __device__ __forceinline__ void load_low(const int * __restrict__ xs0, const int & stride) {
+ ((mma_int_A_I16K4 *) x)[0].load(xs0, stride);
+ }
+};
+
+struct mma_int_B_J8K4 {
+ static constexpr int J = 8;
+ static constexpr int K = 4;
+ static constexpr int ne = 1;
+
+ int x[ne] = {0};
+
+ static __device__ __forceinline__ int get_j(const int /* l */) {
+ const int ret = threadIdx.x / K;
+ GGML_CUDA_ASSUME(ret >= 0);
+ GGML_CUDA_ASSUME(ret < J);
+ return ret;
+ }
+
+ static __device__ __forceinline__ int get_k(const int /* l */) {
+ const int ret = threadIdx.x % K;
+ GGML_CUDA_ASSUME(ret >= 0);
+ GGML_CUDA_ASSUME(ret < K);
+ return ret;
+ }
+
+ __device__ __forceinline__ void load(const int * __restrict__ xs0, const int & stride) {
+#if defined(INT8_MMA_AVAILABLE) && false // Loading as 4 byte values is faster
+ const int * xs = xs0 + (threadIdx.x%J)*stride;
+ asm("ldmatrix.sync.aligned.m8n8.x1.b16 {%0}, [%1];"
+ : "+r"(x[0])
+ : "l"(xs));
+#else
+#pragma unroll
+ for (int l = 0; l < ne; ++l) {
+ x[l] = xs0[get_j(l)*stride + get_k(l)];
+ }
+#endif // defined(INT8_MMA_AVAILABLE)
+ }
+};
+
+struct mma_int_B_J8K8 {
+ static constexpr int J = 8;
+ static constexpr int K = 8;
+ static constexpr int ne = 2;
+
+ int x[ne] = {0};
+
+ static __device__ __forceinline__ int get_j(const int /* l */) {
+ const int ret = threadIdx.x / (K/2);
+ GGML_CUDA_ASSUME(ret >= 0);
+ GGML_CUDA_ASSUME(ret < J);
+ return ret;
+ }
+
+ static __device__ __forceinline__ int get_k(const int l) {
+ const int ret = l * (K/2) + threadIdx.x % (K/2);
+ GGML_CUDA_ASSUME(ret >= 0);
+ GGML_CUDA_ASSUME(ret < K);
+ return ret;
+ }
+
+ __device__ __forceinline__ void load(const int * __restrict__ xs0, const int & stride) {
+#if defined(INT8_MMA_AVAILABLE) && false // Loading as 4 byte values is faster
+ const int * xs = xs0 + (threadIdx.x%J)*stride + ((threadIdx.x/J)*(K/2)) % K;
+ asm("ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];"
+ : "+r"(x[0]), "+r"(x[1])
+ : "l"(xs));
+#else
+#pragma unroll
+ for (int l = 0; l < ne; ++l) {
+ x[l] = xs0[get_j(l)*stride + get_k(l)];
+ }
+#endif // defined(INT8_MMA_AVAILABLE)
+ }
+};
+
+struct mma_int_C_I16J8 {
+ static constexpr int I = 16;
+ static constexpr int J = 8;
+ static constexpr int ne = 4;
+
+ int x[ne] = {0};
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ const int ret = (l/2) * (I/2) + threadIdx.x / (J/2);
+ GGML_CUDA_ASSUME(ret >= 0);
+ GGML_CUDA_ASSUME(ret < I);
+ return ret;
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ const int ret = 2 * (threadIdx.x % (J/2)) + l%2;
+ GGML_CUDA_ASSUME(ret >= 0);
+ GGML_CUDA_ASSUME(ret < J);
+ return ret;
+ }
+
+ __device__ __forceinline__ void mma_K4(const mma_int_A_I16K4 & mma_A, const mma_int_B_J8K4 & mma_B) {
+#ifdef INT8_MMA_AVAILABLE
+#if __CUDA_ARCH__ >= CC_AMPERE
+ asm("mma.sync.aligned.m16n8k16.row.col.s32.s8.s8.s32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"
+ : "+r"(x[0]), "+r"(x[1]), "+r"(x[2]), "+r"(x[3])
+ : "r"(mma_A.x[0]), "r"(mma_A.x[1]), "r"(mma_B.x[0]));
+#else
+ // On Turing m16n8k16 mma is not available, use 2x m8n8k16 mma instead:
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(x[0]), "+r"(x[1])
+ : "r"(mma_A.x[0]), "r"(mma_B.x[0]));
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(x[2]), "+r"(x[3])
+ : "r"(mma_A.x[1]), "r"(mma_B.x[0]));
+#endif // __CUDA_ARCH__ >= CC_AMPERE
+#else
+ GGML_UNUSED(mma_A);
+ GGML_UNUSED(mma_B);
+ NO_DEVICE_CODE;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ __device__ __forceinline__ void mma_K8(const mma_int_A_I16K8 & mma_A, const mma_int_B_J8K8 & mma_B) {
+#ifdef INT8_MMA_AVAILABLE
+#if __CUDA_ARCH__ >= CC_AMPERE
+ asm("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"
+ : "+r"(x[0]), "+r"(x[1]), "+r"(x[2]), "+r"(x[3])
+ : "r"(mma_A.x[0]), "r"(mma_A.x[1]), "r"(mma_A.x[2]), "r"(mma_A.x[3]), "r"(mma_B.x[0]), "r"(mma_B.x[1]));
+#else
+ // On Turing m16n8k32 mma is not available, use 4x m8n8k16 mma instead:
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(x[0]), "+r"(x[1])
+ : "r"(mma_A.x[0]), "r"(mma_B.x[0]));
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(x[2]), "+r"(x[3])
+ : "r"(mma_A.x[1]), "r"(mma_B.x[0]));
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(x[0]), "+r"(x[1])
+ : "r"(mma_A.x[2]), "r"(mma_B.x[1]));
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(x[2]), "+r"(x[3])
+ : "r"(mma_A.x[3]), "r"(mma_B.x[1]));
+#endif // __CUDA_ARCH__ >= CC_AMPERE
+#else
+ GGML_UNUSED(mma_A);
+ GGML_UNUSED(mma_B);
+ NO_DEVICE_CODE;
+#endif // INT8_MMA_AVAILABLE
+ }
+};
diff --git a/llama/ggml-cuda/mmq.cu b/llama/ggml-cuda/mmq.cu
new file mode 100644
index 00000000..bdc6a49f
--- /dev/null
+++ b/llama/ggml-cuda/mmq.cu
@@ -0,0 +1,176 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "mmq.cuh"
+
+void ggml_cuda_op_mul_mat_q(
+ ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
+ const int64_t src1_padded_row_size, cudaStream_t stream) {
+
+ const int64_t ne00 = src0->ne[0];
+
+ const int64_t nb01 = src0->nb[1];
+
+ const int64_t ne10 = src1->ne[0];
+ const int64_t ne11 = src1->ne[1];
+ GGML_ASSERT(ne10 % QK8_1 == 0);
+
+ const int64_t ne0 = dst->ne[0];
+
+ const int64_t row_diff = row_high - row_low;
+ const int64_t stride00 = nb01 / ggml_type_size(src0->type);
+
+ int id = ggml_cuda_get_device();
+ const int compute_capability = ggml_cuda_info().devices[id].cc;
+
+ // the main device has a larger memory buffer to hold the results from all GPUs
+ // nrows_dst == nrows of the matrix that the kernel writes into
+ const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff;
+
+ const mmq_args args = {src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stride00, src1_padded_row_size, src1_ncols, ne11, nrows_dst};
+
+ switch (src0->type) {
+ case GGML_TYPE_Q4_0:
+ mul_mat_q_case<GGML_TYPE_Q4_0>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q4_1:
+ mul_mat_q_case<GGML_TYPE_Q4_1>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q5_0:
+ mul_mat_q_case<GGML_TYPE_Q5_0>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q5_1:
+ mul_mat_q_case<GGML_TYPE_Q5_1>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q8_0:
+ mul_mat_q_case<GGML_TYPE_Q8_0>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q2_K:
+ mul_mat_q_case<GGML_TYPE_Q2_K>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q3_K:
+ mul_mat_q_case<GGML_TYPE_Q3_K>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q4_K:
+ mul_mat_q_case<GGML_TYPE_Q4_K>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q5_K:
+ mul_mat_q_case<GGML_TYPE_Q5_K>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q6_K:
+ mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ2_XXS:
+ mul_mat_q_case<GGML_TYPE_IQ2_XXS>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ2_XS:
+ mul_mat_q_case<GGML_TYPE_IQ2_XS>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ2_S:
+ mul_mat_q_case<GGML_TYPE_IQ2_S>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ3_XXS:
+ mul_mat_q_case<GGML_TYPE_IQ3_XXS>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ3_S:
+ mul_mat_q_case<GGML_TYPE_IQ3_S>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ1_S:
+ mul_mat_q_case<GGML_TYPE_IQ1_S>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ4_XS:
+ mul_mat_q_case<GGML_TYPE_IQ4_XS>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ4_NL:
+ mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, args, stream);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+
+ GGML_UNUSED(src1);
+ GGML_UNUSED(dst);
+ GGML_UNUSED(src1_ddf_i);
+}
+
+bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11) {
+#ifdef GGML_CUDA_FORCE_CUBLAS
+ return false;
+#endif // GGML_CUDA_FORCE_CUBLAS
+
+ bool mmq_supported;
+
+ switch (type) {
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ4_NL:
+ mmq_supported = true;
+ break;
+ default:
+ mmq_supported = false;
+ break;
+ }
+
+ if (!mmq_supported) {
+ return false;
+ }
+
+ if (int8_mma_available(cc)) {
+ return true;
+ }
+
+ if (cc < MIN_CC_DP4A) {
+ return false;
+ }
+
+#ifdef GGML_CUDA_FORCE_MMQ
+ return true;
+#endif //GGML_CUDA_FORCE_MMQ
+
+ if (cc < CC_OFFSET_AMD) {
+ return cc < CC_VOLTA || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;
+ }
+
+ return cc < CC_RDNA3 || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;
+}
diff --git a/llama/ggml-cuda/mmq.cuh b/llama/ggml-cuda/mmq.cuh
new file mode 100644
index 00000000..86fa62a1
--- /dev/null
+++ b/llama/ggml-cuda/mmq.cuh
@@ -0,0 +1,2962 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "common.cuh"
+#include "vecdotq.cuh"
+#include "mma.cuh"
+
+#include <climits>
+#include <cstdint>
+
+#define MMQ_DP4A_MAX_BATCH_SIZE 64 // Max. batch size to use for dp4a MMQ kernels when FP16 tensor cores are available.
+#define MMQ_ITER_K 256
+#define MMQ_NWARPS 8
+
+typedef void (*load_tiles_mmq_t)(const char * __restrict__ x, int * x_tile, const int & kbx0, const int & i_max, const int & stride);
+typedef void (*vec_dot_mmq_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00);
+typedef void (*mmq_write_back_t)(const float * __restrict__ sum, float * __restrict__ dst, const int & stride, const int & i_max, const int & j_max);
+
+enum mmq_q8_1_ds_layout {
+ MMQ_Q8_1_DS_LAYOUT_D4,
+ MMQ_Q8_1_DS_LAYOUT_DS4,
+ MMQ_Q8_1_DS_LAYOUT_D2S6,
+};
+
+struct block_q8_1_mmq {
+ // The y float data is converted to a data layout that can simply be copied to shared memory as a contiguous block.
+ // The y float data is first grouped as blocks of 128 values.
+ // These blocks are then treated as individual data values and transposed.
+ //
+ // To avoid shared memory bank conflicts each block is padded with 16 bytes.
+ // This padding is also used to store block scales/partial sums.
+ // The scales multiplied with the quantized data are equal to the unquantized values.
+ // The partial sums are obtained by summing up a subgroup of the contained values (prior to quantization)
+ // and are only needed for performance reasons.
+ //
+ // The exact data stored depends on the x data type.
+ union {
+ float d4[4]; // 1 32 bit scale per 32 values, stored as d0,d1,d2,d3
+ half2 ds4[4]; // 1 16 bit scale + 1 16 bit partial sum per 32 values, stored as d0,s0,d1,s1,d2,s2,d3,s3
+ half d2s6[8]; // 1 16 bit scale per 64 values + 1 16 bit partial sum per 16 values for the first 96 values,
+ // stored as d0,d1,s1,s2,s3,s4,s5
+ };
+ int8_t qs[4*QK8_1]; // 128 values quantized to 8 bit each
+};
+static_assert(sizeof(block_q8_1_mmq) == 4*QK8_1 + 4*sizeof(half2), "Unexpected block_q8_1_mmq size");
+static_assert(sizeof(block_q8_1_mmq) == 4*sizeof(block_q8_1), "Unexpected block_q8_1_mmq size");
+
+static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) {
+ switch (type_x) {
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ return MMQ_Q8_1_DS_LAYOUT_DS4;
+ case GGML_TYPE_Q5_0:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_Q5_1:
+ return MMQ_Q8_1_DS_LAYOUT_DS4;
+ case GGML_TYPE_Q8_0:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_Q2_K:
+ return MMQ_Q8_1_DS_LAYOUT_D2S6;
+ case GGML_TYPE_Q3_K:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ return MMQ_Q8_1_DS_LAYOUT_DS4;
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ3_S:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_IQ1_S:
+ return MMQ_Q8_1_DS_LAYOUT_DS4;
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ4_NL:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+}
+
+struct tile_x_sizes {
+ int qs;
+ int dm;
+ int sc;
+};
+
+static constexpr int get_mmq_x_max_host(const int cc) {
+ return int8_mma_available(cc) ? 128 :
+#ifdef GGML_CUDA_FORCE_MMQ
+ cc >= CC_VOLTA && cc < CC_OFFSET_AMD ? 128 : 64;
+#else
+ cc >= CC_VOLTA && cc < CC_OFFSET_AMD ? MMQ_DP4A_MAX_BATCH_SIZE : 64;
+#endif // GGML_CUDA_FORCE_MMQ
+}
+
+static constexpr __device__ int get_mmq_x_max_device() {
+#ifdef INT8_MMA_AVAILABLE
+ return 128;
+#else // INT8_MMA_AVAILABLE
+
+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+ return 128;
+#else // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+
+#if __CUDA_ARCH__ >= CC_VOLTA
+#ifdef GGML_CUDA_FORCE_MMQ
+ return MMQ_DP4A_MAX_BATCH_SIZE;
+#else // GGML_CUDA_FORCE_MMQ
+ return 128;
+#endif // GGML_CUDA_FORCE_MMQ
+#else // __CUDA_ARCH__ >= CC_VOLTA
+
+ return 64;
+#endif // __CUDA_ARCH__ >= CC_VOLTA
+
+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+#endif // INT8_MMA_AVAILABLE
+}
+
+static constexpr int get_mmq_y_host(const int cc) {
+ return cc >= CC_OFFSET_AMD ? (cc == CC_RDNA1 ? 64 : 128) : (cc >= CC_VOLTA ? 128 : 64);
+}
+
+static constexpr __device__ int get_mmq_y_device() {
+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+#if defined(RDNA1)
+ return 64;
+#else
+ return 128;
+#endif // defined RDNA1
+#else
+#if __CUDA_ARCH__ >= CC_VOLTA
+ return 128;
+#else
+ return 64;
+#endif // __CUDA_ARCH__ >= CC_VOLTA
+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+}
+
+#define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{mmq_y*WARP_SIZE + mmq_y, mmq_y*WARP_SIZE/QI4_0 + mmq_y/QI4_0, 0}
+#define MMQ_DP4A_TXS_Q4_1 tile_x_sizes{mmq_y*WARP_SIZE + mmq_y, mmq_y*WARP_SIZE/QI4_1 + mmq_y/QI4_1, 0}
+#define MMQ_DP4A_TXS_Q8_0 tile_x_sizes{mmq_y*WARP_SIZE*2 + mmq_y, mmq_y*WARP_SIZE*2/QI8_0 + mmq_y/(QI8_0/2), 0}
+#define MMQ_DP4A_TXS_Q8_0_16 tile_x_sizes{mmq_y*WARP_SIZE*2 + mmq_y, mmq_y*WARP_SIZE*4/QI8_0 + mmq_y/(QI8_0/4), 0}
+#define MMQ_DP4A_TXS_Q8_1 tile_x_sizes{mmq_y*WARP_SIZE*2 + mmq_y, mmq_y*WARP_SIZE*2/QI8_1 + mmq_y/(QI8_1/2), 0}
+#define MMQ_DP4A_TXS_Q2_K tile_x_sizes{mmq_y*WARP_SIZE*2 + mmq_y, mmq_y*WARP_SIZE + mmq_y, 0}
+#define MMQ_DP4A_TXS_Q3_K tile_x_sizes{mmq_y*WARP_SIZE*2 + mmq_y, mmq_y, mmq_y*WARP_SIZE/8 + mmq_y/8}
+#define MMQ_DP4A_TXS_Q4_K tile_x_sizes{mmq_y*WARP_SIZE + mmq_y, mmq_y*WARP_SIZE/QI4_K, mmq_y*WARP_SIZE/8 + mmq_y/8}
+#define MMQ_DP4A_TXS_Q5_K tile_x_sizes{mmq_y*WARP_SIZE*2 + mmq_y, mmq_y*WARP_SIZE/QI5_K + mmq_y/QI5_K, mmq_y*WARP_SIZE/8 + mmq_y/8}
+#define MMQ_DP4A_TXS_Q6_K tile_x_sizes{mmq_y*WARP_SIZE*2 + mmq_y, mmq_y*WARP_SIZE/QI6_K + mmq_y/QI6_K, mmq_y*WARP_SIZE/8 + mmq_y/8}
+
+static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int mmq_y) {
+ return type == GGML_TYPE_Q4_0 ? MMQ_DP4A_TXS_Q4_0 :
+ type == GGML_TYPE_Q4_1 ? MMQ_DP4A_TXS_Q4_1 :
+ type == GGML_TYPE_Q5_0 ? MMQ_DP4A_TXS_Q8_0 :
+ type == GGML_TYPE_Q5_1 ? MMQ_DP4A_TXS_Q8_1 :
+ type == GGML_TYPE_Q8_0 ? MMQ_DP4A_TXS_Q8_0 :
+ type == GGML_TYPE_Q2_K ? MMQ_DP4A_TXS_Q2_K :
+ type == GGML_TYPE_Q3_K ? MMQ_DP4A_TXS_Q3_K :
+ type == GGML_TYPE_Q4_K ? MMQ_DP4A_TXS_Q4_K :
+ type == GGML_TYPE_Q5_K ? MMQ_DP4A_TXS_Q5_K :
+ type == GGML_TYPE_Q6_K ? MMQ_DP4A_TXS_Q6_K :
+ type == GGML_TYPE_IQ2_XXS ? MMQ_DP4A_TXS_Q8_0 :
+ type == GGML_TYPE_IQ2_XS ? MMQ_DP4A_TXS_Q8_0_16 :
+ type == GGML_TYPE_IQ2_S ? MMQ_DP4A_TXS_Q8_0_16 :
+ type == GGML_TYPE_IQ3_XXS ? MMQ_DP4A_TXS_Q8_0 :
+ type == GGML_TYPE_IQ3_S ? MMQ_DP4A_TXS_Q8_0 :
+ type == GGML_TYPE_IQ1_S ? MMQ_DP4A_TXS_Q8_0 :
+ type == GGML_TYPE_IQ4_XS ? MMQ_DP4A_TXS_Q8_0 :
+ type == GGML_TYPE_IQ4_NL ? MMQ_DP4A_TXS_Q8_0 :
+ tile_x_sizes{0, 0, 0};
+}
+
+#define MMQ_MMA_TILE_X_K_Q8_0 (2*WARP_SIZE + 2*WARP_SIZE/QI8_0 + 4)
+#define MMQ_MMA_TILE_X_K_Q8_1 (2*WARP_SIZE + 2*WARP_SIZE/QI8_0 + 4)
+#define MMQ_MMA_TILE_X_K_Q2_K (2*WARP_SIZE + WARP_SIZE + 4)
+#define MMQ_MMA_TILE_X_K_Q3_K (2*WARP_SIZE + WARP_SIZE/2 + 4)
+#define MMQ_MMA_TILE_X_K_Q6_K (2*WARP_SIZE + WARP_SIZE/QI6_K + WARP_SIZE/8 + 7)
+
+static_assert(MMQ_MMA_TILE_X_K_Q8_0 % 8 == 4, "Wrong padding.");
+static_assert(MMQ_MMA_TILE_X_K_Q8_1 % 8 == 4, "Wrong padding.");
+static_assert(MMQ_MMA_TILE_X_K_Q2_K % 8 == 4, "Wrong padding.");
+static_assert(MMQ_MMA_TILE_X_K_Q3_K % 8 == 4, "Wrong padding.");
+static_assert(MMQ_MMA_TILE_X_K_Q6_K % 8 == 4, "Wrong padding.");
+
+static constexpr __host__ __device__ int mmq_get_mma_tile_x_k(ggml_type type) {
+ return type == GGML_TYPE_Q4_0 ? MMQ_MMA_TILE_X_K_Q8_0 :
+ type == GGML_TYPE_Q4_1 ? MMQ_MMA_TILE_X_K_Q8_1 :
+ type == GGML_TYPE_Q5_0 ? MMQ_MMA_TILE_X_K_Q8_0 :
+ type == GGML_TYPE_Q5_1 ? MMQ_MMA_TILE_X_K_Q8_1 :
+ type == GGML_TYPE_Q8_0 ? MMQ_MMA_TILE_X_K_Q8_0 :
+ type == GGML_TYPE_Q2_K ? MMQ_MMA_TILE_X_K_Q2_K :
+ type == GGML_TYPE_Q3_K ? MMQ_MMA_TILE_X_K_Q3_K :
+ type == GGML_TYPE_Q4_K ? MMQ_MMA_TILE_X_K_Q8_1 :
+ type == GGML_TYPE_Q5_K ? MMQ_MMA_TILE_X_K_Q8_1 :
+ type == GGML_TYPE_Q6_K ? MMQ_MMA_TILE_X_K_Q6_K :
+ type == GGML_TYPE_IQ2_XXS ? MMQ_MMA_TILE_X_K_Q8_0 :
+ type == GGML_TYPE_IQ2_XS ? MMQ_MMA_TILE_X_K_Q3_K :
+ type == GGML_TYPE_IQ2_S ? MMQ_MMA_TILE_X_K_Q3_K :
+ type == GGML_TYPE_IQ3_XXS ? MMQ_MMA_TILE_X_K_Q8_0 :
+ type == GGML_TYPE_IQ3_S ? MMQ_MMA_TILE_X_K_Q8_0 :
+ type == GGML_TYPE_IQ1_S ? MMQ_MMA_TILE_X_K_Q8_0 :
+ type == GGML_TYPE_IQ4_XS ? MMQ_MMA_TILE_X_K_Q8_0 :
+ type == GGML_TYPE_IQ4_NL ? MMQ_MMA_TILE_X_K_Q8_0 :
+ 0;
+}
+
+#define MMQ_TILE_Y_K (WARP_SIZE + WARP_SIZE/QI8_1)
+
+static int mmq_get_granularity_host(const int mmq_x, const int cc) {
+ return int8_mma_available(cc) && mmq_x >= 48 ? 16 : 8;
+}
+
+#ifdef INT8_MMA_AVAILABLE
+static constexpr __device__ int mmq_get_granularity_device(const int mmq_x) {
+ return mmq_x >= 48 ? 16 : 8;
+}
+#else
+static constexpr __device__ int mmq_get_granularity_device(const int /* mmq_x */) {
+ return 8;
+}
+#endif // INT8_MMA_AVAILABLE
+
+// ------------------------------------------------------------
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_0(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + 2*WARP_SIZE);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, mmq_y);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kbx = threadIdx.x / QI4_0;
+ const int kqsx = threadIdx.x % QI4_0;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbx;
+ const int qs0 = get_int_b2(bxi->qs, kqsx);
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI4_0) + kqsx + 0] = __vsubss4((qs0 >> 0) & 0x0F0F0F0F, 0x08080808);
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI4_0) + kqsx + QI4_0] = __vsubss4((qs0 >> 4) & 0x0F0F0F0F, 0x08080808);
+#else
+ x_qs[i*(WARP_SIZE + 1) + threadIdx.x] = qs0;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int blocks_per_tile_x_row = WARP_SIZE / QI4_0;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_0) {
+ int i = i0 + threadIdx.y * QI4_0 + threadIdx.x / blocks_per_tile_x_row;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbxd;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d;
+#else
+ x_df[i*(WARP_SIZE/QI4_0) + i/QI4_0 + kbxd] = bxi->d;
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q4_0_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, mmq_y);
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QR4_0*VDR_Q4_0_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);
+
+ int u[2*VDR_Q4_0_Q8_1_MMQ];
+
+#pragma unroll
+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) {
+ u[2*l+0] = y_qs[j*MMQ_TILE_Y_K + kyqs + l];
+ u[2*l+1] = y_qs[j*MMQ_TILE_Y_K + kyqs + (l + QI4_0)];
+ }
+
+ sum[j0/nwarps*mmq_y/WARP_SIZE + i0/WARP_SIZE] += vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>
+ (&x_qs[i*(WARP_SIZE + 1) + k0/QR4_0], u,
+ x_df[i*(WARP_SIZE/QI4_0) + i/QI4_0 + k0/(QR4_0*QI4_0)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_1(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + 2*WARP_SIZE);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, mmq_y);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kbx = threadIdx.x / QI4_1;
+ const int kqsx = threadIdx.x % QI4_1;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbx;
+ const int qs0 = get_int_b4(bxi->qs, kqsx);
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI4_1) + kqsx + 0] = (qs0 >> 0) & 0x0F0F0F0F;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI4_1) + kqsx + QI4_1] = (qs0 >> 4) & 0x0F0F0F0F;
+#else
+ x_qs[i*(WARP_SIZE + 1) + threadIdx.x] = qs0;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int blocks_per_tile_x_row = WARP_SIZE / QI4_1;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_1) {
+ int i = i0 + threadIdx.y * QI4_1 + threadIdx.x / blocks_per_tile_x_row;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbxd;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + kbxd] = bxi->dm;
+#else
+ x_dm[i*(WARP_SIZE/QI4_1) + i/QI4_1 + kbxd] = bxi->dm;
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q4_1_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, mmq_y);
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QR4_1*VDR_Q4_1_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);
+
+ int u[2*VDR_Q4_1_Q8_1_MMQ];
+
+#pragma unroll
+ for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) {
+ u[2*l+0] = y_qs[j*MMQ_TILE_Y_K + kyqs + l];
+ u[2*l+1] = y_qs[j*MMQ_TILE_Y_K + kyqs + (l + QI4_1)];
+ }
+
+ sum[j0/nwarps*mmq_y/WARP_SIZE + i0/WARP_SIZE] += vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>
+ (&x_qs[i*(WARP_SIZE + 1) + k0/QR4_1], u,
+ x_dm[i*(WARP_SIZE/QI4_1) + i/QI4_1 + k0/(QR4_1*QI4_1)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_0(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + WARP_SIZE*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_0, mmq_y);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kbx = threadIdx.x / QI5_0;
+ const int kqsx = threadIdx.x % QI5_0;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbx;
+
+ const int ql = get_int_b2(bxi->qs, kqsx);
+ const int qh = get_int_b2(bxi->qh, 0) >> (4 * (threadIdx.x % QI5_0));
+
+ int qs0 = (ql >> 0) & 0x0F0F0F0F;
+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
+ qs0 = __vsubss4(qs0, 0x10101010); // subtract 16
+
+ int qs1 = (ql >> 4) & 0x0F0F0F0F;
+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
+ qs1 = __vsubss4(qs1, 0x10101010); // subtract 16
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI5_0) + kqsx + 0] = qs0;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + kbx*(2*QI5_0) + kqsx + 0] = qs0;
+ x_qs[i*(2*WARP_SIZE + 1) + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int blocks_per_tile_x_row = WARP_SIZE / QI5_0;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_0) {
+ int i = i0 + threadIdx.y * QI5_0 + threadIdx.x / blocks_per_tile_x_row;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbxd;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d;
+#else
+ x_df[i*(WARP_SIZE/QI5_0) + i/QI5_0 + kbxd] = bxi->d;
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_1(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + 2*WARP_SIZE);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, mmq_y);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kbx = threadIdx.x / QI5_1;
+ const int kqsx = threadIdx.x % QI5_1;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbx;
+
+ const int ql = get_int_b4(bxi->qs, kqsx);
+ const int qh = get_int_b4(bxi->qh, 0) >> (4 * (threadIdx.x % QI5_1));
+
+ int qs0 = (ql >> 0) & 0x0F0F0F0F;
+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
+
+ int qs1 = (ql >> 4) & 0x0F0F0F0F;
+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI5_1) + kqsx + 0] = qs0;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + kbx*(2*QI5_1) + kqsx + 0] = qs0;
+ x_qs[i*(2*WARP_SIZE + 1) + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int blocks_per_tile_x_row = WARP_SIZE / QI5_1;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_1) {
+ int i = i0 + threadIdx.y * QI5_1 + threadIdx.x / blocks_per_tile_x_row;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbxd;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + kbxd] = bxi->dm;
+#else
+ x_dm[i*(WARP_SIZE/QI5_1) + i/QI5_1 + kbxd] = bxi->dm;
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q8_0(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_tile + 2*WARP_SIZE);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, mmq_y);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kbx = threadIdx.x / QI8_0;
+ const int kqsx = threadIdx.x % QI8_0;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 0 + threadIdx.x] = get_int_b2(bxi[0].qs, kqsx);
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + WARP_SIZE + threadIdx.x] = get_int_b2(bxi[WARP_SIZE/QI8_0].qs, kqsx);
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + 0 + threadIdx.x] = get_int_b2(bxi[0].qs, kqsx);
+ x_qs[i*(2*WARP_SIZE + 1) + WARP_SIZE + threadIdx.x] = get_int_b2(bxi[WARP_SIZE/QI8_0].qs, kqsx);
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int blocks_per_tile_x_row = 2*WARP_SIZE / QI8_0;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI8_0/2) {
+ int i = i0 + threadIdx.y * (QI8_0/2) + threadIdx.x / blocks_per_tile_x_row;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d;
+#else
+ x_df[i*(2*WARP_SIZE/QI8_0) + i/(QI8_0/2) + kbxd] = bxi->d;
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q8_0_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, mmq_y);
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += VDR_Q8_0_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ sum[j0/nwarps*mmq_y/WARP_SIZE + i0/WARP_SIZE] += vec_dot_q8_0_q8_1_impl<float, VDR_Q8_0_Q8_1_MMQ>
+ (&x_qs[i*(2*WARP_SIZE + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k0 % WARP_SIZE],
+ x_df[i*(2*WARP_SIZE/QI8_0) + i/(QI8_0/2) + k0/QI8_0], y_df[j*MMQ_TILE_Y_K + (k0/QI8_1) % (WARP_SIZE/QI8_1)]);
+ }
+ }
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps, mmq_q8_1_ds_layout ds_layout>
+static __device__ __forceinline__ void vec_dot_q8_0_q8_1_mma(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ typedef mma_int_A_I16K8 mma_A;
+ typedef mma_int_B_J8K8 mma_B;
+ typedef mma_int_C_I16J8 mma_C;
+
+ constexpr int granularity = mmq_get_granularity_device(mmq_x);
+ constexpr int rows_per_warp = 2 * granularity;
+ constexpr int ntx = rows_per_warp/mma_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (mma_B::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + 2*WARP_SIZE;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+ const half2 * y_ds = (const half2 *) y;
+
+ mma_A A[ntx][WARP_SIZE/QI8_0];
+ float dA[ntx][mma_C::ne/2][WARP_SIZE/QI8_0];
+
+ const int i0 = (threadIdx.y/ntx)*rows_per_warp;
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QI8_0) {
+ const int k0 = k00 + k01;
+
+ A[n][k01/QI8_0].load(x_qs + (i0 + n*mma_A::I)*MMQ_MMA_TILE_X_K_Q8_0 + k0, MMQ_MMA_TILE_X_K_Q8_0);
+ }
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int i = i0 + n*mma_A::I + mma_C::get_i(2*l);
+
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QI8_0) {
+ const int k0 = k00 + k01;
+
+ dA[n][l][k01/QI8_0] = x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + k0/QI8_0];
+ }
+ }
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += ntx*mma_C::J) {
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QI8_0) {
+ mma_B B;
+ float dB[mma_C::ne/2];
+
+ B.load(y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int j = j0 + mma_C::get_j(l);
+
+ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {
+ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];
+ } else {
+ dB[l] = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ mma_C C;
+ C.mma_K8(A[n][k01/QI8_0], B);
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne; ++l) {
+ sum[(j0/mma_C::J + n)*mma_C::ne + l] += C.x[l]*dA[n][l/2][k01/QI8_0]*dB[l%2];
+ }
+ }
+ }
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q8_1_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, mmq_y);
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += VDR_Q8_0_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ sum[j0/nwarps*mmq_y/WARP_SIZE + i0/WARP_SIZE] += vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>
+ (&x_qs[i*(2*WARP_SIZE + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],
+ x_dm[i*(WARP_SIZE/QI5_1) + i/QI5_1 + k0/QI8_1], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q8_1_q8_1_mma(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ typedef mma_int_A_I16K8 mma_A;
+ typedef mma_int_B_J8K8 mma_B;
+ typedef mma_int_C_I16J8 mma_C;
+
+ constexpr int granularity = mmq_get_granularity_device(mmq_x);
+ constexpr int rows_per_warp = 2 * granularity;
+ constexpr int ntx = rows_per_warp/mma_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (mma_B::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + 2*WARP_SIZE;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_dm = (const half2 *) y;
+
+ mma_A A[ntx][WARP_SIZE/QI8_1];
+ float2 dmA[ntx][mma_C::ne/2][WARP_SIZE/QI8_1];
+
+ const int i0 = (threadIdx.y/ntx)*rows_per_warp;
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QI8_1) {
+ const int k0 = k00 + k01;
+
+ A[n][k01/QI8_1].load(x_qs + (i0 + n*mma_A::I)*MMQ_MMA_TILE_X_K_Q8_1 + k0, MMQ_MMA_TILE_X_K_Q8_1);
+ }
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int i = i0 + n*mma_A::I + mma_C::get_i(2*l);
+
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QI8_1) {
+ const int k0 = k00 + k01;
+
+ dmA[n][l][k01/QI8_1] = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + k0/QI8_1]);
+ }
+ }
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += ntx*mma_C::J) {
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QI8_1) {
+ mma_B B;
+ float2 dsB[mma_C::ne/2];
+
+ B.load(y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int j = j0 + mma_C::get_j(l);
+
+ dsB[l] = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ mma_C C;
+ C.mma_K8(A[n][k01/QI8_1], B);
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne; ++l) {
+ sum[(j0/mma_C::J + n)*mma_C::ne + l] += dmA[n][l/2][k01/QI8_1].x*dsB[l%2].x*C.x[l];
+ sum[(j0/mma_C::J + n)*mma_C::ne + l] += dmA[n][l/2][k01/QI8_1].y*dsB[l%2].y;
+ }
+ }
+ }
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q8_0_16_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ constexpr tile_x_sizes txs = MMQ_DP4A_TXS_Q8_0_16;
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QI8_0) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ sum[j0/nwarps*mmq_y/WARP_SIZE + i0/WARP_SIZE] += vec_dot_q8_0_16_q8_1_impl<QI8_0>(
+ &x_qs[i*(2*WARP_SIZE + 1) + k0],
+ &y_qs[j*MMQ_TILE_Y_K + k01],
+ &x_df[i*(2*WARP_SIZE*2/QI8_0) + i/(QI8_0/4) + k0/(QI8_0/2)],
+ y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q8_0_16_q8_1_mma(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+#ifdef INT8_MMA_AVAILABLE
+
+ typedef mma_int_A_I16K4 mma_A;
+ typedef mma_int_A_I16K8 mma_A_K8;
+ typedef mma_int_B_J8K4 mma_B;
+ typedef mma_int_C_I16J8 mma_C;
+
+ constexpr int granularity = mmq_get_granularity_device(mmq_x);
+ constexpr int rows_per_warp = 2 * granularity;
+ constexpr int ntx = rows_per_warp/mma_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (mma_B::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + WARP_SIZE*2;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+ const int i0 = (threadIdx.y / ntx) * (ntx*mma_A::I);
+
+ mma_A A[ntx][8];
+ float dA[ntx][mma_C::ne/2][8];
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += 8) {
+ const int k0 = k00 + k01;
+
+ ((mma_A_K8 *) A[n])[k01/8].load(x_qs + (i0 + n*mma_A::I)*MMQ_MMA_TILE_X_K_Q3_K + k0, MMQ_MMA_TILE_X_K_Q3_K);
+ }
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int i = i0 + n*mma_C::I + mma_C::get_i(2*l);
+
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += 4) {
+ const int k0 = k00 + k01;
+
+ dA[n][l][k01/4] = x_df[i*MMQ_MMA_TILE_X_K_Q3_K + k0/4];
+ }
+ }
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += ntx*mma_C::J) {
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {
+ mma_B B[2];
+ float dB[mma_C::ne/2];
+
+ B[0].load(y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);
+ B[1].load(y_qs + j0*MMQ_TILE_Y_K + (k01 + mma_B::K), MMQ_TILE_Y_K);
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int j = j0 + mma_C::get_j(l);
+
+ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ mma_C C[2];
+ C[0].mma_K4(A[n][k01/4 + 0], B[0]);
+ C[1].mma_K4(A[n][k01/4 + 1], B[1]);
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne; ++l) {
+ sum[(j0/mma_C::J + n)*mma_C::ne + l] += dB[l%2]*(C[0].x[l]*dA[n][l/2][k01/4 + 0] + C[1].x[l]*dA[n][l/2][k01/4 + 1]);
+ }
+ }
+ }
+ }
+#else
+ GGML_UNUSED(x); GGML_UNUSED(y); GGML_UNUSED(sum);
+ NO_DEVICE_CODE;
+#endif // INT8_MMA_AVAILABLE
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q2_K(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + 2*WARP_SIZE);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, mmq_y);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kqsx = threadIdx.x % QI2_K;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * WARP_SIZE/QI2_K) {
+ int i = i0 + threadIdx.y*(WARP_SIZE/QI2_K) + threadIdx.x/QI2_K;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q2_K * bxi = (const block_q2_K *) x + kbx0 + i*stride;
+
+ const int x_ql_0 = get_int_b2(bxi->qs, kqsx);
+
+#pragma unroll
+ for (int l = 0; l < QR2_K; ++l) {
+ const int k = (kqsx/8)*32 + l*8 + kqsx % 8;
+
+ const int x_qs_k = (x_ql_0 >> (2*l)) & 0x03030303;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q2_K + k] = x_qs_k;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + k] = x_qs_k;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int sc_m = bxi->scales[kqsx];
+#ifdef FAST_FP16_AVAILABLE
+ const half2 x_dm_ik = __hmul2(bxi->dm, make_half2(sc_m & 0x0F, sc_m >> 4));
+#else
+ const float2 bxi_dmf = __half22float2(bxi->dm);
+ const half2 x_dm_ik = make_half2(bxi_dmf.x*(sc_m & 0x0F), bxi_dmf.y*(sc_m >> 4));
+#endif // FAST_FP16_AVAILABLE
+
+#ifdef INT8_MMA_AVAILABLE
+ x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + kqsx] = x_dm_ik;
+#else
+ x_dm[i*(WARP_SIZE + 1) + kqsx] = x_dm_ik;
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q2_K_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, mmq_y);
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+ float2 y_df[mmq_x/nwarps];
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ y_df[j0/nwarps] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);
+ }
+
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ if (k01 < WARP_SIZE/2) {
+ constexpr int ns = 2;
+ sum[j0/nwarps*mmq_y/WARP_SIZE + i0/WARP_SIZE] += vec_dot_q2_K_q8_1_impl_mmq<ns>(
+ &x_qs[i*(2*WARP_SIZE + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],
+ &x_dm[i*(WARP_SIZE + 1) + k0/4], k01 < WARP_SIZE/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,
+ &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);
+ } else {
+ constexpr int ns = 1;
+ sum[j0/nwarps*mmq_y/WARP_SIZE + i0/WARP_SIZE] += vec_dot_q2_K_q8_1_impl_mmq<ns>(
+ &x_qs[i*(2*WARP_SIZE + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],
+ &x_dm[i*(WARP_SIZE + 1) + k0/4], k01 < WARP_SIZE/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,
+ &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);
+ }
+ }
+ }
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q2_K_q8_1_mma(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+#ifdef INT8_MMA_AVAILABLE
+
+ typedef mma_int_A_I16K4 mma_A;
+ typedef mma_int_A_I16K8 mma_A_K8;
+ typedef mma_int_B_J8K4 mma_B;
+ typedef mma_int_C_I16J8 mma_C;
+
+ constexpr int granularity = mmq_get_granularity_device(mmq_x);
+ constexpr int rows_per_warp = 2 * granularity;
+ constexpr int ntx = rows_per_warp/mma_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (mma_B::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + WARP_SIZE*2;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+ const int i0 = (threadIdx.y / ntx) * (ntx*mma_A::I);
+
+ mma_A A[ntx][8];
+ float dA[ntx][mma_C::ne/2][8];
+ float mA[ntx][mma_C::ne/2][8];
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QI8_1) {
+ const int k0 = k00 + k01;
+
+ ((mma_A_K8 *) A[n])[k01/QI8_1].load(x_qs + (i0 + n*mma_A::I)*MMQ_MMA_TILE_X_K_Q2_K + k0, MMQ_MMA_TILE_X_K_Q2_K);
+ }
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int i = i0 + n*mma_C::I + mma_C::get_i(2*l);
+
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QI8_1/2) {
+ const int k0 = k00 + k01;
+
+ const float2 dm = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + k0/(QI8_1/2)]);
+
+ dA[n][l][k01/(QI8_1/2)] = dm.x;
+ mA[n][l][k01/(QI8_1/2)] = dm.y;
+ }
+ }
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += ntx*mma_C::J) {
+ float2 dB[mma_C::ne/2];
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int j = j0 + mma_C::get_j(l);
+
+ dB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);
+ }
+
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QI8_1) {
+ mma_B B[2];
+
+ B[0].load(y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);
+ B[1].load(y_qs + j0*MMQ_TILE_Y_K + (k01 + mma_B::K), MMQ_TILE_Y_K);
+
+ mma_C Cm[2];
+ if (k01 >= WARP_SIZE * 3/4) {
+ mma_A A1;
+ A1.x[0] = 0x01010101;
+ A1.x[1] = 0x01010101;
+ Cm[0].mma_K4(A1, B[0]);
+ Cm[1].mma_K4(A1, B[1]);
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ mma_C Cd[2];
+
+ Cd[0].mma_K4(A[n][k01/4 + 0], B[0]);
+ Cd[1].mma_K4(A[n][k01/4 + 1], B[1]);
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne; ++l) {
+ float tmp = Cd[0].x[l]*dA[n][l/2][k01/4 + 0] + Cd[1].x[l]*dA[n][l/2][k01/4 + 1];
+ if (k01 >= WARP_SIZE * 3/4) {
+ tmp -= Cm[0].x[l]*mA[n][l/2][k01/4 + 0] + Cm[1].x[l]*mA[n][l/2][k01/4 + 1];
+ }
+ sum[(j0/mma_C::J + n)*mma_C::ne + l] += tmp*(k01 < WARP_SIZE/2 ? dB[l%2].x : dB[l%2].y);
+ }
+ }
+ }
+
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE * 3/4; k01 += QI8_1) {
+ float2 sB[mma_C::ne/2];
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int j = j0 + mma_C::get_j(l);
+
+ sB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int l = 0; l < mma_C::ne; ++l) {
+ sum[(j0/mma_C::J + n)*mma_C::ne + l] -= mA[n][l/2][k01/4 + 0]*sB[l%2].x;
+ sum[(j0/mma_C::J + n)*mma_C::ne + l] -= mA[n][l/2][k01/4 + 1]*sB[l%2].y;
+ }
+ }
+ }
+ }
+#else
+ GGML_UNUSED(x); GGML_UNUSED(y); GGML_UNUSED(sum);
+ NO_DEVICE_CODE;
+#endif // INT8_MMA_AVAILABLE
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q3_K(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + WARP_SIZE*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, mmq_y);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+ int * x_sc = (int *) (x_df + txs.dm);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kqsx = threadIdx.x % QI3_K;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * WARP_SIZE/QI3_K) {
+ int i = i0 + threadIdx.y * (WARP_SIZE/QI3_K) + threadIdx.x / QI3_K;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;
+
+ const int x_ql_0 = get_int_b2(bxi->qs, kqsx);
+ const int x_qh_0 = get_int_b2(bxi->hmask, kqsx % (QI3_K/2)) >> (4 * (kqsx / (QI3_K/2)));
+
+#pragma unroll
+ for (int l = 0; l < QR3_K; ++l) {
+ const int k = (kqsx/8)*32 + l*8 + kqsx % 8;
+
+ const int x_ql_k = (x_ql_0 >> (2*l)) & 0x03030303;
+ const int x_qh_k = ((x_qh_0 >> l) << 2) & 0x04040404;
+
+ const int x_qs_k = __vsubss4(x_ql_k | x_qh_k, 0x04040404);
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + k] = x_qs_k;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + k] = x_qs_k;
+#endif // INT8_MMA_AVAILABLE
+ }
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps*8) {
+ int i = i0 + threadIdx.y*8 + threadIdx.x/(WARP_SIZE/8);
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;
+
+ const int ksc = threadIdx.x % (WARP_SIZE/8);
+
+ const int ksc_low = ksc % (QI3_K/8);
+ const int shift_low = 4 * (ksc / (QI3_K/8));
+ const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;
+
+ const int ksc_high = QI3_K/8;
+ const int shift_high = 2 * ksc;
+ const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;
+
+ const int sc = __vsubss4(sc_low | sc_high, 0x20202020);
+
+#ifdef INT8_MMA_AVAILABLE
+ const int8_t * sc8 = (const int8_t *) ≻
+ const float d = bxi->d;
+
+#pragma unroll
+ for (int l = 0; l < sizeof(int); ++l) {
+ x_df[i*MMQ_MMA_TILE_X_K_Q3_K + sizeof(int)*(threadIdx.x % (WARP_SIZE/8)) + l] = d*sc8[l];
+ }
+#else
+ x_sc[i*(WARP_SIZE/8) + i/8 + threadIdx.x % (WARP_SIZE/8)] = sc;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+#ifndef INT8_MMA_AVAILABLE
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps*WARP_SIZE) {
+ int i = (i0 + threadIdx.y*WARP_SIZE + threadIdx.x) % mmq_y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;
+
+ x_df[i] = bxi->d;
+ }
+#endif // INT8_MMA_AVAILABLE
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q3_K_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, mmq_y);
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + txs.qs;
+ const int * x_sc = (const int *) x_df + txs.dm;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ const int8_t * scales = ((const int8_t *) (x_sc + i*(WARP_SIZE/8) + i/8)) + k0/4;
+
+ sum[j0/nwarps*mmq_y/WARP_SIZE + i0/WARP_SIZE] += vec_dot_q3_K_q8_1_impl_mmq(
+ &x_qs[i*(2*WARP_SIZE + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], scales,
+ x_df[i], y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+static __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, const int ksc) {
+ // scale arrangement after the following two lines:
+ // - ksc == 0: sc0, sc1, sc2, sc3
+ // - ksc == 1: sc4, sc5, sc6, sc7
+ // - ksc == 2: m0, m1, m2, m3
+ // - ksc == 3: m4, m5, m6, m7
+ return ((scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F) | // lower 4 bits
+ ((scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030); // upper 2 bits
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_K(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + 2*WARP_SIZE);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, mmq_y);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + txs.qs);
+ int * x_sc = (int *) (x_dm + txs.dm);
+#endif // INT8_MMA_AVAILABLE
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;
+ const int qs0 = get_int_b4(bxi->qs, threadIdx.x);
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 16*(threadIdx.x/8) + threadIdx.x % 8 + 0] = (qs0 >> 0) & 0x0F0F0F0F;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 16*(threadIdx.x/8) + threadIdx.x % 8 + 8] = (qs0 >> 4) & 0x0F0F0F0F;
+#else
+ x_qs[i*(WARP_SIZE + 1) + threadIdx.x] = qs0;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+#ifdef INT8_MMA_AVAILABLE
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps*16) {
+ int i = (i0 + threadIdx.y*16 + threadIdx.x/(WARP_SIZE/16)) % mmq_y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;
+
+ const int * scales = (const int *) bxi->scales;
+ const int ksc = threadIdx.x % (WARP_SIZE/16);
+
+ const int sc32 = unpack_scales_q45_K(scales, ksc + 0);
+ const int m32 = unpack_scales_q45_K(scales, ksc + 2);
+
+ const uint8_t * sc8 = (const uint8_t *) &sc32;
+ const uint8_t * m8 = (const uint8_t *) &m32;
+
+ const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);
+
+#pragma unroll
+ for (int l = 0; l < sizeof(int); ++l) {
+ x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);
+ }
+ }
+
+#else
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps*QI4_K) {
+ int i = (i0 + threadIdx.y*QI4_K + threadIdx.x) % mmq_y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;
+
+ x_dm[i] = bxi->dm;
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
+ int i = (i0 + threadIdx.y * 8 + threadIdx.x / (WARP_SIZE/8)) % mmq_y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride + (threadIdx.x % (WARP_SIZE/8)) / (QI4_K/8);
+
+ const int * scales = (const int *) bxi->scales;
+
+ const int ksc = threadIdx.x % (WARP_SIZE/8);
+ const int scales8 = unpack_scales_q45_K(scales, ksc);
+
+ x_sc[i*(WARP_SIZE/8) + i/8 + ksc] = scales8;
+ }
+#endif // INT8_MMA_AVAILABLE
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q4_K_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, mmq_y);
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;
+ const int * x_sc = (const int *) x_dm + txs.dm;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QR4_K*VDR_Q4_K_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ const uint8_t * sc = (const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k0/32] + 2*(k01/16);
+
+ sum[j0/nwarps*mmq_y/WARP_SIZE + i0/WARP_SIZE] += vec_dot_q4_K_q8_1_impl_mmq(
+ &x_qs[i*(WARP_SIZE + 1) + k0/2], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,
+ x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_K(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + WARP_SIZE*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, mmq_y);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + txs.qs);
+ int * x_sc = (int *) (x_dm + txs.dm);
+#endif // INT8_MMA_AVAILABLE
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
+ const int ky = QR5_K*threadIdx.x;
+
+ const int ql = get_int_b4(bxi->qs, threadIdx.x);
+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;
+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;
+
+ const int qh = get_int_b4(bxi->qh, threadIdx.x % (QI5_K/4));
+ const int qh0 = ((qh >> (2 * (threadIdx.x / (QI5_K/4)) + 0)) << 4) & 0x10101010;
+ const int qh1 = ((qh >> (2 * (threadIdx.x / (QI5_K/4)) + 1)) << 4) & 0x10101010;
+
+ const int kq0 = ky - ky % (QI5_K/2) + threadIdx.x % (QI5_K/4) + 0;
+ const int kq1 = ky - ky % (QI5_K/2) + threadIdx.x % (QI5_K/4) + QI5_K/4;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kq0] = ql0 | qh0;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kq1] = ql1 | qh1;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + kq0] = ql0 | qh0;
+ x_qs[i*(2*WARP_SIZE + 1) + kq1] = ql1 | qh1;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+#ifdef INT8_MMA_AVAILABLE
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps*16) {
+ int i = (i0 + threadIdx.y*16 + threadIdx.x/(WARP_SIZE/16)) % mmq_y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
+
+ const int * scales = (const int *) bxi->scales;
+ const int ksc = threadIdx.x % (WARP_SIZE/16);
+
+ const int sc32 = unpack_scales_q45_K(scales, ksc + 0);
+ const int m32 = unpack_scales_q45_K(scales, ksc + 2);
+
+ const uint8_t * sc8 = (const uint8_t *) &sc32;
+ const uint8_t * m8 = (const uint8_t *) &m32;
+
+ const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);
+
+#pragma unroll
+ for (int l = 0; l < sizeof(int); ++l) {
+ x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);
+ }
+ }
+
+#else
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps*QI5_K) {
+ int i = (i0 + threadIdx.y*QI5_K + threadIdx.x) % mmq_y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
+
+ x_dm[i] = bxi->dm;
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps*8) {
+ int i = (i0 + threadIdx.y*8 + threadIdx.x/(WARP_SIZE/8)) % mmq_y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
+
+ const int * scales = (const int *) bxi->scales;
+
+ const int ksc = threadIdx.x % (WARP_SIZE/8);
+ const int scales8 = unpack_scales_q45_K(scales, ksc);
+
+ x_sc[i*(WARP_SIZE/8) + i/8 + ksc] = scales8;
+ }
+#endif // INT8_MMA_AVAILABLE
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q5_K_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, mmq_y);
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;
+ const int * x_sc = (const int *) x_dm + txs.dm;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QR5_K*VDR_Q5_K_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k00/32]) + 2*(k01/16);
+
+ sum[j0/nwarps*mmq_y/WARP_SIZE + i0/WARP_SIZE] += vec_dot_q5_K_q8_1_impl_mmq(
+ &x_qs[i*(QR5_K*WARP_SIZE + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,
+ x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q6_K(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + WARP_SIZE*2);
+ int * x_sc = (int *) (x_df + WARP_SIZE/QI6_K);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, mmq_y);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+ int * x_sc = (int *) (x_df + txs.dm);
+#endif // INT8_MMA_AVAILABLE
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;
+
+ const int ql = get_int_b2(bxi->ql, threadIdx.x);
+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;
+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;
+
+ const int qh = get_int_b2(bxi->qh, (QI6_K/4) * (threadIdx.x / (QI6_K/2)) + threadIdx.x % (QI6_K/4));
+ const int qh0 = ((qh >> ((threadIdx.x & 0x08) >> 2)) << 4) & 0x30303030;
+ const int qh1 = (qh >> ((threadIdx.x & 0x08) >> 2)) & 0x30303030;
+
+ const int kq0 = 2*threadIdx.x - threadIdx.x % (QI6_K/2) + 0;
+ const int kq1 = 2*threadIdx.x - threadIdx.x % (QI6_K/2) + QI6_K/2;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q6_K + kq0] = __vsubss4(ql0 | qh0, 0x20202020);
+ x_qs[i*MMQ_MMA_TILE_X_K_Q6_K + kq1] = __vsubss4(ql1 | qh1, 0x20202020);
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);
+ x_qs[i*(2*WARP_SIZE + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int blocks_per_tile_x_row = WARP_SIZE / QI6_K; // == 1 if QK_K == 256
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row; // == 0 if QK_K == 256
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI6_K) {
+ int i = (i0 + threadIdx.y * QI6_K + threadIdx.x / blocks_per_tile_x_row) % mmq_y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride + kbxd;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_df[i*MMQ_MMA_TILE_X_K_Q6_K + kbxd] = bxi->d;
+#else
+ x_df[i*(WARP_SIZE/QI6_K) + i/QI6_K + kbxd] = bxi->d;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
+ int i = (i0 + threadIdx.y * 8 + threadIdx.x / (WARP_SIZE/8)) % mmq_y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride + (threadIdx.x % (WARP_SIZE/8)) / 4;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_sc[i*MMQ_MMA_TILE_X_K_Q6_K + threadIdx.x % (WARP_SIZE/8)] = get_int_b2(bxi->scales, threadIdx.x % (QI6_K/8));
+#else
+ x_sc[i*(WARP_SIZE/8) + i/8 + threadIdx.x % (WARP_SIZE/8)] = get_int_b2(bxi->scales, threadIdx.x % (QI6_K/8));
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q6_K_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, mmq_y);
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + txs.qs;
+ const int * x_sc = (const int *) x_df + txs.dm;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += QR6_K*VDR_Q6_K_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ const int8_t * sc = ((const int8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k0/16]);
+
+ sum[j0/nwarps*mmq_y/WARP_SIZE + i0/WARP_SIZE] += vec_dot_q6_K_q8_1_impl_mmq(
+ &x_qs[i*(QR6_K*WARP_SIZE + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc,
+ x_df[i*(WARP_SIZE/QI6_K) + i/QI6_K], &y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <int mmq_x, int mmq_y, int nwarps>
+static __device__ __forceinline__ void vec_dot_q6_K_q8_1_mma(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int & k00) {
+#ifdef INT8_MMA_AVAILABLE
+
+ typedef mma_int_A_I16K4 mma_A;
+ typedef mma_int_B_J8K4 mma_B;
+ typedef mma_int_C_I16J8 mma_C;
+
+ constexpr int granularity = mmq_get_granularity_device(mmq_x);
+ constexpr int rows_per_warp = 2 * granularity;
+ constexpr int ntx = rows_per_warp/mma_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (mma_B::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + WARP_SIZE*2;
+ const int * x_sc = (const int *) x_df + WARP_SIZE/QI6_K;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+ const int i0 = (threadIdx.y / ntx) * (ntx*mma_A::I);
+
+ mma_A A[ntx][8];
+ int scA[ntx][mma_C::ne/2][8];
+ float dA[ntx][mma_C::ne/2];
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += 8) {
+ const int k0 = k00 + k01;
+
+ A[n][k01/4 + 0].load(x_qs + (i0 + n*mma_A::I)*MMQ_MMA_TILE_X_K_Q6_K + (k0 + 0), MMQ_MMA_TILE_X_K_Q6_K);
+ A[n][k01/4 + 1].load(x_qs + (i0 + n*mma_A::I)*MMQ_MMA_TILE_X_K_Q6_K + (k0 + mma_A::K), MMQ_MMA_TILE_X_K_Q6_K);
+ }
+
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += 16) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int i = i0 + n*mma_C::I + mma_C::get_i(2*l);
+
+ const int sc_packed = x_sc[i*MMQ_MMA_TILE_X_K_Q6_K + k0/16];
+ const int8_t * sc = (const int8_t *) &sc_packed;
+
+#pragma unroll
+ for (int ksc = 0; ksc < sizeof(int); ++ksc) {
+ scA[n][l][k01/4 + ksc] = sc[ksc];
+ }
+ }
+ }
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int i = i0 + n*mma_C::I + mma_C::get_i(2*l);
+
+ dA[n][l] = x_df[i*MMQ_MMA_TILE_X_K_Q6_K];
+ }
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += ntx*mma_C::J) {
+ float tmp[ntx][mma_C::ne] = {{0.0f}};
+
+#pragma unroll
+ for (int k01 = 0; k01 < WARP_SIZE; k01 += 8) {
+ mma_B B[2];
+ float dB[mma_C::ne/2];
+
+ B[0].load(y_qs + j0*MMQ_TILE_Y_K + 0 + k01, MMQ_TILE_Y_K);
+ B[1].load(y_qs + j0*MMQ_TILE_Y_K + mma_B::K + k01, MMQ_TILE_Y_K);
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne/2; ++l) {
+ const int j = j0 + mma_C::get_j(l);
+
+ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ mma_C C[2];
+ C[0].mma_K4(A[n][k01/4 + 0], B[0]);
+ C[1].mma_K4(A[n][k01/4 + 1], B[1]);
+
+#pragma unroll
+ for (int l = 0; l < mma_C::ne; ++l) {
+ tmp[n][l] += (C[0].x[l]*scA[n][l/2][k01/4 + 0] + C[1].x[l]*scA[n][l/2][k01/4 + 1])*dB[l%2];
+ }
+ }
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int l = 0; l < mma_C::ne; ++l) {
+ sum[(j0/mma_C::J + n)*mma_C::ne + l] += tmp[n][l]*dA[n][l/2];
+ }
+ }
+ }
+#else
+ GGML_UNUSED(x); GGML_UNUSED(y); GGML_UNUSED(sum);
+ NO_DEVICE_CODE;
+#endif // INT8_MMA_AVAILABLE
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_iq4_nl(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + WARP_SIZE*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_NL, mmq_y);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kbx = threadIdx.x / QI4_NL;
+ const int kqsx = threadIdx.x % QI4_NL;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbx;
+
+ const int aux_q4 = get_int_b2(bxi->qs, kqsx);
+ const int2 v = get_int_from_table_16(aux_q4);
+ const int k0 = 8 * (threadIdx.x / 4) + threadIdx.x % 4;
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 0] = v.x;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 4] = v.y;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + k0 + 0] = v.x;
+ x_qs[i*(2*WARP_SIZE + 1) + k0 + 4] = v.y;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int blocks_per_tile_x_row = WARP_SIZE / QI4_NL;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_NL) {
+ int i = i0 + threadIdx.y * QI4_NL + threadIdx.x / blocks_per_tile_x_row;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbxd;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = __half2float(bxi->d);
+#else
+ x_df[i*(WARP_SIZE/4) + i/4 + kbxd] = __half2float(bxi->d);
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_iq2_xxs(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + WARP_SIZE*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XXS, mmq_y);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kqsx = threadIdx.x % (QI2_XXS/2);
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * WARP_SIZE/(QI2_XXS/2)) {
+ int i = i0 + threadIdx.y*(2*WARP_SIZE/QI2_XXS) + threadIdx.x/(QI2_XXS/2);
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_iq2_xxs * bxi = (const block_iq2_xxs *) x + kbx0 + i*stride;
+
+ const int q2 = get_int_b2(bxi->qs, 2*kqsx+0);
+ const uint8_t * aux8 = (const uint8_t *) &q2;
+ const uint32_t aux32 = get_int_b2(bxi->qs, 2*kqsx+1);
+
+#pragma unroll
+ for (int l = 0; l < QR2_XXS; ++l) {
+ const int * grid_pos = (const int *) (iq2xxs_grid + aux8[l]);
+ const int signs_packed = ksigns_iq2xs[(aux32 >> (7*l)) & 0x7F];
+
+ const int signs0 = __vcmpne4(((signs_packed & 0x03) << 7) | ((signs_packed & 0x0C) << 21), 0x00000000);
+ const int grid0 = __vsub4(grid_pos[0] ^ signs0, signs0);
+
+ const int signs1 = __vcmpne4(((signs_packed & 0x30) << 3) | ((signs_packed & 0xC0) << 17), 0x00000000);
+ const int grid1 = __vsub4(grid_pos[1] ^ signs1, signs1);
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 0)] = grid0;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 1)] = grid1;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l + 0)] = grid0;
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l + 1)] = grid1;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int ls = aux32 >> 28;
+ const float d = bxi->d;
+#ifdef INT8_MMA_AVAILABLE
+ x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = (ls*d + d/2)/4;
+#else
+ x_df[i*(WARP_SIZE/4) + i/4 + kqsx] = (ls*d + d/2)/4;
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_iq2_xs(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + WARP_SIZE*2);
+#else
+ constexpr tile_x_sizes txs = MMQ_DP4A_TXS_Q8_0_16;
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kqsx = threadIdx.x % (QI2_XS/2);
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * WARP_SIZE/(QI2_XS/2)) {
+ int i = i0 + threadIdx.y*(2*WARP_SIZE/QI2_XS) + threadIdx.x/(QI2_XS/2);
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_iq2_xs * bxi = (const block_iq2_xs *) x + kbx0 + i*stride;
+
+ const int2 q2_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));
+ const uint16_t * q2 = (const uint16_t *) &q2_packed;
+
+ #pragma unroll
+ for (int l = 0; l < QR2_XS; ++l) {
+ const uint32_t * grid_pos = (const uint32_t *)(iq2xs_grid + (q2[l] & 0x000001FF));
+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));
+
+ const int grid_l = __vsub4(grid_pos[0] ^ signs[0], signs[0]);
+ const int grid_h = __vsub4(grid_pos[1] ^ signs[1], signs[1]);
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 1)] = grid_h;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l + 1)] = grid_h;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int ls = bxi->scales[kqsx];
+ const float d = bxi->d;
+#ifdef INT8_MMA_AVAILABLE
+ x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;
+ x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;
+#else
+ x_df[i*(2*WARP_SIZE*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;
+ x_df[i*(2*WARP_SIZE*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_iq2_s(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + WARP_SIZE*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_S, mmq_y);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kqsx = threadIdx.x % (QI2_S/2);
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * WARP_SIZE/(QI2_S/2)) {
+ int i = i0 + threadIdx.y*(2*WARP_SIZE/QI2_S) + threadIdx.x/(QI2_S/2);
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_iq2_s * bxi = (const block_iq2_s *) x + kbx0 + i*stride;
+
+ const int qs_packed = get_int_b2(bxi->qs, kqsx);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bxi->qh[kqsx];
+
+ const int signs_packed_32 = get_int_b2(bxi->qs, QK_K/32 + kqsx);
+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
+
+#pragma unroll
+ for (int l = 0; l < QR2_S; ++l) {
+ const int * grid_pos = (const int *)(iq2s_grid + (qs[l] | ((qh << (8-2*l)) & 0x300)));
+
+ const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);
+ const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);
+
+ const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);
+ const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 1)] = grid_h;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l + 1)] = grid_h;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int ls = bxi->scales[kqsx];
+ const float d = bxi->d;
+#ifdef INT8_MMA_AVAILABLE
+ x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;
+ x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;
+#else
+ x_df[i*(2*WARP_SIZE*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;
+ x_df[i*(2*WARP_SIZE*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_iq3_xxs(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + WARP_SIZE*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_XXS, mmq_y);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kqsx = threadIdx.x % (QI3_XXS/2);
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * WARP_SIZE/(QI3_XXS/2)) {
+ int i = i0 + threadIdx.y*(2*WARP_SIZE/QI3_XXS) + threadIdx.x/(QI3_XXS/2);
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_iq3_xxs * bxi = (const block_iq3_xxs *) x + kbx0 + i*stride;
+
+ const int2 q3_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));
+ const uint8_t * q3 = (const uint8_t *) &q3_packed;
+ const uint32_t aux32 = get_int_b2(bxi->qs, QK_K/16 + kqsx);
+
+#pragma unroll
+ for (int l = 0; l < QR3_XXS; ++l) {
+ const int2 grid_pos = make_int2(iq3xxs_grid[q3[2*l+0]], iq3xxs_grid[q3[2*l+1]]);
+
+ const int * signs = (const int *)(ksigns64 + ((aux32 >> (7*l)) & 0x7F));
+
+ const int grid_l = __vsub4(grid_pos.x ^ signs[0], signs[0]);
+ const int grid_h = __vsub4(grid_pos.y ^ signs[1], signs[1]);
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 1)] = grid_h;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l + 1)] = grid_h;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int ls = aux32 >> 28;
+ const float d = bxi->d;
+#ifdef INT8_MMA_AVAILABLE
+ x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = (ls*d + d/2)/2;
+#else
+ x_df[i*(WARP_SIZE/4) + i/4 + kqsx] = (ls*d + d/2)/2;
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_iq3_s(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + WARP_SIZE*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, mmq_y);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kqsx = threadIdx.x % (QI3_S/2);
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * WARP_SIZE/(QI3_S/2)) {
+ int i = i0 + threadIdx.y*(2*WARP_SIZE/QI3_S) + threadIdx.x/(QI3_S/2);
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_iq3_s * bxi = (const block_iq3_s *) x + kbx0 + i*stride;
+
+ const int2 qs_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bxi->qh[kqsx];
+
+ const int signs_packed_32 = get_int_b2(bxi->signs, kqsx);
+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
+
+#pragma unroll
+ for (int l = 0; l < QR3_S; ++l) {
+ const int2 grid_pos = make_int2(
+ iq3s_grid[qs[2*l+0] | ((qh << (8 - 2*l)) & 0x100)],
+ iq3s_grid[qs[2*l+1] | ((qh << (7 - 2*l)) & 0x100)]);
+
+ const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);
+ const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);
+
+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);
+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l+0)] = grid_l;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l+1)] = grid_h;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l+0)] = grid_l;
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l+1)] = grid_h;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const int ls = 1 + 2*((bxi->scales[kqsx/2] >> (((2*kqsx) << 1) & 0x04)) & 0x0F);
+ const float d = bxi->d;
+#ifdef INT8_MMA_AVAILABLE
+ x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = ls*d;
+#else
+ x_df[i*(WARP_SIZE/4) + i/4 + kqsx] = ls*d;
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_iq1_s(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ half2 * x_ds = (half2 *) (x_qs + WARP_SIZE*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, mmq_y);
+ int * x_qs = (int *) x_tile;
+ half2 * x_ds = (half2 *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kqsx = threadIdx.x % QI1_S;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * WARP_SIZE/QI1_S) {
+ int i = i0 + threadIdx.y*(WARP_SIZE/QI1_S) + threadIdx.x/QI1_S;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_iq1_s * bxi = (const block_iq1_s *) x + kbx0 + i*stride;
+
+ const int qs_packed = get_int_b2(bxi->qs, kqsx);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bxi->qh[kqsx];
+
+ #pragma unroll
+ for (int l = 0; l < QR1_S/2; ++l) {
+ const int grid = iq1s_grid_gpu[qs[l] | (((qh >> (3*l)) & 0x07) << 8)];
+
+ const int grid0 = (grid >> 0) & 0x0F0F0F0F;
+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;
+
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 8*kqsx + (2*l+0)] = grid0;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 8*kqsx + (2*l+1)] = grid1;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l+0)] = grid0;
+ x_qs[i*(2*WARP_SIZE + 1) + 8*kqsx + (2*l+1)] = grid1;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+ const float d1q = __half2float(bxi->d) * (((qh >> 11) & 0x0E) + 1);
+ const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);
+
+#ifdef INT8_MMA_AVAILABLE
+ x_ds[i*MMQ_MMA_TILE_X_K_Q8_1 + kqsx] = make_half2(d1q, d1q*delta);
+#else
+ x_ds[i*(WARP_SIZE/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta);
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_iq4_xs(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int & kbx0, const int & i_max, const int & stride) {
+
+#ifdef INT8_MMA_AVAILABLE
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + WARP_SIZE*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_XS, mmq_y);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // INT8_MMA_AVAILABLE
+
+ const int kbx = 0; // threadIdx.x / QI4_XS
+ const int kqsx = threadIdx.x; // threadIdx.x % QI4_XS
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride + kbx;
+
+ const int aux_q4 = get_int_b4(bxi->qs, kqsx);
+ const int2 v = get_int_from_table_16(aux_q4);
+ const int k0 = 8 * (threadIdx.x / 4) + threadIdx.x % 4;
+#ifdef INT8_MMA_AVAILABLE
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 0] = v.x;
+ x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 4] = v.y;
+#else
+ x_qs[i*(2*WARP_SIZE + 1) + k0 + 0] = v.x;
+ x_qs[i*(2*WARP_SIZE + 1) + k0 + 4] = v.y;
+#endif // INT8_MMA_AVAILABLE
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
+ int i = i0 + threadIdx.y * 4 + threadIdx.x / (WARP_SIZE/4);
+
+ if (need_check) {
+ i = min(i, i_max);
+ }
+
+ const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;
+
+ const float d = __half2float(bxi->d);
+
+ const int ls = ((bxi->scales_l[(threadIdx.x % 8)/2] >> (4*(threadIdx.x % 2))) & 0x0F)
+ | (((bxi->scales_h >> (2*(threadIdx.x % 8))) & 0x03) << 4);
+
+#ifdef INT8_MMA_AVAILABLE
+ x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + threadIdx.x % 8] = d * (ls - 32);
+#else
+ x_df[i*(WARP_SIZE/4) + i/4 + threadIdx.x % 8] = d * (ls - 32);
+#endif // INT8_MMA_AVAILABLE
+ }
+}
+
+template<int mmq_x, int mmq_y, int nwarps, bool need_check>
+static __device__ __forceinline__ void mmq_write_back_dp4a(
+ const float * __restrict__ sum, float * __restrict__ dst, const int & stride, const int & i_max, const int & j_max) {
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ if (j > j_max) {
+ return;
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ if (need_check && i > i_max) {
+ continue;
+ }
+
+ dst[j*stride + i] = sum[(j0/nwarps) * (mmq_y/WARP_SIZE) + i0/WARP_SIZE];
+ }
+ }
+}
+
+template<int mmq_x, int mmq_y, int nwarps, bool need_check>
+static __device__ __forceinline__ void mmq_write_back_mma(
+ const float * __restrict__ sum, float * __restrict__ dst, const int & stride, const int & i_max, const int & j_max) {
+
+ typedef mma_int_C_I16J8 mma_C;
+
+ constexpr int granularity = mmq_get_granularity_device(mmq_x);
+ constexpr int rows_per_warp = 2 * granularity;
+ constexpr int ntx = rows_per_warp/mma_C::I; // Number of x minitiles per warp.
+
+ const int i0 = (threadIdx.y / ntx) * (ntx*mma_C::I);
+#ifdef INT8_MMA_AVAILABLE
+ static_assert(nwarps*mma_C::I == mmq_y, "nwarps*mma_C::I != mmq_y");
+#endif // INT8_MMA_AVAILABLE
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += ntx*mma_C::J) {
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int l = 0; l < mma_C::ne; ++l) {
+ const int j = j0 + (threadIdx.y % ntx) * mma_C::J + mma_C::get_j(l);
+
+ if (j > j_max) {
+ continue;
+ }
+
+ const int i = i0 + n*mma_C::I + mma_C::get_i(l);
+
+ if (need_check && i > i_max) {
+ continue;
+ }
+
+ dst[j*stride + i] = sum[(j0/mma_C::J + n)*mma_C::ne + l];
+ }
+ }
+ }
+}
+
+// -------------------------------------------------------------------------------------------------------------------------------------
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check, ggml_type type>
+struct mmq_type_traits;
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_Q4_0> {
+ static constexpr int vdr = VDR_Q4_0_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_0<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, nwarps, MMQ_Q8_1_DS_LAYOUT_DS4>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_0_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_Q4_1> {
+ static constexpr int vdr = VDR_Q4_1_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_1<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma<mmq_x, mmq_y, nwarps>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_1_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_Q5_0> {
+ static constexpr int vdr = VDR_Q5_0_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_0<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, nwarps, MMQ_Q8_1_DS_LAYOUT_D4>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_Q5_1> {
+ static constexpr int vdr = VDR_Q5_1_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_1<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma<mmq_x, mmq_y, nwarps>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_1_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_Q8_0> {
+ static constexpr int vdr = VDR_Q8_0_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_q8_0<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, nwarps, MMQ_Q8_1_DS_LAYOUT_D4>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_Q2_K> {
+ static constexpr int vdr = VDR_Q2_K_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_q2_K<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q2_K_q8_1_mma<mmq_x, mmq_y, nwarps>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q2_K_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_Q3_K> {
+ static constexpr int vdr = VDR_Q3_K_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_q3_K<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma<mmq_x, mmq_y, nwarps>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q3_K_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_Q4_K> {
+ static constexpr int vdr = VDR_Q4_K_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_K<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma<mmq_x, mmq_y, nwarps>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_K_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_Q5_K> {
+ static constexpr int vdr = VDR_Q5_K_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_K<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma<mmq_x, mmq_y, nwarps>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q5_K_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_Q6_K> {
+ static constexpr int vdr = VDR_Q6_K_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_q6_K<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q6_K_q8_1_mma<mmq_x, mmq_y, nwarps>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q6_K_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_IQ2_XXS> {
+ static constexpr int vdr = VDR_IQ2_XXS_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_xxs<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, nwarps, MMQ_Q8_1_DS_LAYOUT_D4>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_IQ2_XS> {
+ static constexpr int vdr = VDR_IQ2_XS_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_xs<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma<mmq_x, mmq_y, nwarps>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_16_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_IQ2_S> {
+ static constexpr int vdr = VDR_IQ2_S_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_s<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma<mmq_x, mmq_y, nwarps>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_16_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_IQ3_XXS> {
+ static constexpr int vdr = VDR_IQ3_XXS_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq3_xxs<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, nwarps, MMQ_Q8_1_DS_LAYOUT_D4>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_IQ3_S> {
+ static constexpr int vdr = VDR_IQ3_S_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq3_s<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, nwarps, MMQ_Q8_1_DS_LAYOUT_D4>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_IQ1_S> {
+ static constexpr int vdr = VDR_IQ1_S_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq1_s<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma<mmq_x, mmq_y, nwarps>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_1_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_IQ4_NL> {
+ static constexpr int vdr = VDR_IQ4_NL_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq4_nl<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, nwarps, MMQ_Q8_1_DS_LAYOUT_D4>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <int mmq_x, int mmq_y, int nwarps, bool need_check>
+struct mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, GGML_TYPE_IQ4_XS> {
+ static constexpr int vdr = VDR_IQ4_XS_Q8_1_MMQ;
+ static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq4_xs<mmq_y, nwarps, need_check>;
+ static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, nwarps, MMQ_Q8_1_DS_LAYOUT_D4>;
+ static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y, nwarps>;
+};
+
+template <ggml_type type, int mmq_x, int nwarps, bool need_check, bool fixup>
+static __device__ void mul_mat_q_process_tile(
+ const char * __restrict__ x, const char * __restrict__ yc, float * __restrict__ dst, float * __restrict__ tmp_fixup,
+ const int & ne00, const int & ne01, const int & stride01, const int & ne10, const int & ne11, const int & stride11, const int & ne0,
+ const int & it, const int & jt, const int & kb0_start, const int & kb0_stop) {
+
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int mmq_y = get_mmq_y_device();
+ constexpr load_tiles_mmq_t load_tiles = mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, type>::load_tiles;
+
+ extern __shared__ char data_mul_mat_q[];
+ int * tile_y = (int *) data_mul_mat_q;
+ int * tile_x = tile_y + GGML_PAD(mmq_x*(WARP_SIZE + WARP_SIZE/QI8_1), nwarps*WARP_SIZE);
+
+#ifdef INT8_MMA_AVAILABLE
+ constexpr vec_dot_mmq_t vec_dot = mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, type>::vec_dot_mma;
+ constexpr mmq_write_back_t write_back = mmq_write_back_mma<mmq_x, mmq_y, nwarps, need_check>;
+#else
+ constexpr vec_dot_mmq_t vec_dot = mmq_type_traits<mmq_x, mmq_y, nwarps, need_check, type>::vec_dot_dp4a;
+ constexpr mmq_write_back_t write_back = mmq_write_back_dp4a<mmq_x, mmq_y, nwarps, need_check>;
+#endif // INT8_MMA_AVAILABLE
+
+ constexpr int blocks_per_iter = MMQ_ITER_K / qk;
+
+ float sum[mmq_x*mmq_y / (nwarps*WARP_SIZE)] = {0.0f};
+
+ const int tile_x_max_i = ne01 - it*mmq_y - 1;
+ const int tile_y_max_j = ne11 - jt*mmq_x - 1;
+
+ const int * y = (const int *) yc + jt*(mmq_x*sizeof(block_q8_1_mmq)/sizeof(int));
+
+ for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) {
+ load_tiles(x, tile_x, stride01*it*mmq_y + kb0, tile_x_max_i, stride01);
+
+ {
+ const int * by0 = y + stride11*(kb0*(qk*sizeof(block_q8_1_mmq) / (4*QK8_1*sizeof(int))) + 0*sizeof(block_q8_1_mmq)/sizeof(int));
+#pragma unroll
+ for (int l0 = 0; l0 < mmq_x*MMQ_TILE_Y_K; l0 += nwarps*WARP_SIZE) {
+ int l = l0 + threadIdx.y*WARP_SIZE + threadIdx.x;
+
+ tile_y[l] = by0[l];
+ }
+ }
+
+ __syncthreads();
+
+ vec_dot(tile_x, tile_y, sum, 0);
+
+ __syncthreads();
+
+ {
+ const int * by0 = y + stride11*(kb0*(qk*sizeof(block_q8_1_mmq) / (4*QK8_1*sizeof(int))) + 1*sizeof(block_q8_1_mmq)/sizeof(int));
+#pragma unroll
+ for (int l0 = 0; l0 < mmq_x*MMQ_TILE_Y_K; l0 += nwarps*WARP_SIZE) {
+ int l = l0 + threadIdx.y*WARP_SIZE + threadIdx.x;
+
+ tile_y[l] = by0[l];
+ }
+ }
+
+ __syncthreads();
+
+ vec_dot(tile_x, tile_y, sum, WARP_SIZE);
+
+ __syncthreads();
+ }
+
+ if (fixup) {
+ write_back(sum, tmp_fixup + blockIdx.x*(mmq_x*mmq_y), mmq_y, mmq_y, mmq_x);
+ } else {
+ write_back(sum, dst + jt*mmq_x*ne0 + it*mmq_y, ne0, tile_x_max_i, tile_y_max_j);
+ }
+}
+
+
+// The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598
+
+template <ggml_type type, int mmq_x, int nwarps, bool need_check>
+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+#if defined(RDNA3) || defined(RDNA2)
+ __launch_bounds__(WARP_SIZE*nwarps, 2)
+#endif // defined(RDNA3) || defined(RDNA2)
+#else
+#if __CUDA_ARCH__ >= CC_VOLTA
+ __launch_bounds__(WARP_SIZE*nwarps, 1)
+#else
+ __launch_bounds__(WARP_SIZE*nwarps, 2)
+#endif // __CUDA_ARCH__ >= CC_VOLTA
+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+static __global__ void mul_mat_q(
+ const char * __restrict__ x, const char * __restrict__ yc, float * __restrict__ dst, float * __restrict__ tmp_fixup,
+ const int ne00, const int ne01, const int stride01, const int ne10, const int ne11, const int stride11, const int ne0) {
+
+ // Skip unused template specializations for faster compilation:
+ if (mmq_x > get_mmq_x_max_device() || mmq_x % mmq_get_granularity_device(mmq_x) != 0) {
+ NO_DEVICE_CODE;
+ return;
+ }
+
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int mmq_y = get_mmq_y_device();
+
+ // On AMD or old CUDA the performance with stream-k was worse, use conventional tiling instead:
+#if (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) || __CUDA_ARCH__ < CC_VOLTA
+ {
+ constexpr bool fixup = false;
+ mul_mat_q_process_tile<type, mmq_x, nwarps, need_check, fixup>
+ (x, yc, dst, tmp_fixup, ne00, ne01, stride01, ne10, ne11, stride11, ne0,
+ blockIdx.x, blockIdx.y, 0, ne00/qk);
+ return;
+ }
+#endif // (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) || __CUDA_ARCH__ < CC_VOLTA
+
+ const int64_t blocks_per_ne00 = ne00 / qk;
+ constexpr int blocks_per_iter = MMQ_ITER_K / qk;
+
+ const int ntx = (ne11 + mmq_x - 1) / mmq_x; // Number of tiles x
+ const int nty = (ne01 + mmq_y - 1) / mmq_y; // Number of tiles y
+
+ // kbc == k block continuous, current index in continuous ijk space.
+ int64_t kbc = (int64_t) blockIdx.x *blocks_per_ne00*ntx*nty / gridDim.x;
+ int64_t kbc_stop = (int64_t)(blockIdx.x + 1)*blocks_per_ne00*ntx*nty / gridDim.x;
+
+ kbc -= (kbc % blocks_per_ne00) % blocks_per_iter;
+ kbc_stop -= (kbc_stop % blocks_per_ne00) % blocks_per_iter;
+
+ // kb0 == k index when doing the matrix multiplication for an output tile.
+ int kb0_start = kbc % blocks_per_ne00;
+ int kb0_stop = min(blocks_per_ne00, kb0_start + kbc_stop - kbc);
+ while (kbc < kbc_stop && kb0_stop == blocks_per_ne00) {
+ const int jt = kbc / (blocks_per_ne00*nty); // j index of current tile.
+ const int it = (kbc - jt*(blocks_per_ne00*nty)) / blocks_per_ne00; // i index of current tile.
+
+ constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
+ mul_mat_q_process_tile<type, mmq_x, nwarps, need_check, fixup>
+ (x, yc, dst, tmp_fixup, ne00, ne01, stride01, ne10, ne11, stride11, ne0,
+ it, jt, kb0_start, kb0_stop);
+
+ kbc += blocks_per_ne00;
+ kbc -= kbc % blocks_per_ne00;
+
+ kb0_start = 0;
+ kb0_stop = min(blocks_per_ne00, kbc_stop - kbc);
+ }
+
+ if (kbc >= kbc_stop) {
+ return;
+ }
+
+ const int jt = kbc / (blocks_per_ne00*nty);
+ const int it = (kbc - jt*(blocks_per_ne00*nty)) / blocks_per_ne00;
+
+ constexpr bool fixup = true; // Last index writes it data to fixup buffer to avoid data races with other blocks.
+ mul_mat_q_process_tile<type, mmq_x, nwarps, need_check, fixup>
+ (x, yc, dst, tmp_fixup, ne00, ne01, stride01, ne10, ne11, stride11, ne0,
+ it, jt, kb0_start, kb0_stop);
+}
+
+
+template <ggml_type type, int mmq_x, int nwarps, bool need_check>
+static __global__ void mul_mat_q_stream_k_fixup(
+ float * __restrict__ dst, const float * __restrict__ tmp_last_tile, const int ne00, const int ne01, const int ne11, const int ne0, const int block_num_mmq) {
+
+ constexpr int mmq_y = get_mmq_y_device();
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int blocks_per_iter = MMQ_ITER_K / qk;
+ const int64_t blocks_per_ne00 = ne00 / qk;
+
+ float sum[mmq_x*mmq_y / (nwarps*WARP_SIZE)] = {0.0f};
+
+ const int ntx = (ne11 + mmq_x - 1) / mmq_x;
+ const int nty = (ne01 + mmq_y - 1) / mmq_y;
+
+ bool any_fixup = false;
+
+ const int bidx_start = ((blockIdx.y*nty + blockIdx.x) * block_num_mmq) / (gridDim.y*gridDim.x);
+ const int bidx_stop = ((blockIdx.y*nty + blockIdx.x + 1) * block_num_mmq + gridDim.y*gridDim.x - 1) / (gridDim.y*gridDim.x);
+
+ int64_t kbc_0;
+ int64_t kbc_stop_0 = (int64_t) bidx_start*blocks_per_ne00*ntx*nty / block_num_mmq;
+
+ for (int bidx = bidx_start; bidx < bidx_stop; ++bidx) {
+ kbc_0 = kbc_stop_0;
+ kbc_stop_0 = (int64_t) (bidx + 1)*blocks_per_ne00*ntx*nty / block_num_mmq;
+
+ const int64_t kbc = kbc_0 - (kbc_0 % blocks_per_ne00) % blocks_per_iter;
+ const int64_t kbc_stop = kbc_stop_0 - (kbc_stop_0 % blocks_per_ne00) % blocks_per_iter;
+
+ // Skip fixup tile if the MMQ CUDA block never wrote anything to it:
+ if (kbc == kbc_stop || kbc_stop % blocks_per_ne00 == 0) {
+ continue;
+ }
+
+ const int jt = kbc_stop / (blocks_per_ne00*nty);
+ const int it = (kbc_stop - jt*(blocks_per_ne00*nty)) / blocks_per_ne00;
+
+ // Skip fixup tile if it's unrelated to the output tile assigned to this CUDA block:
+ if (it != blockIdx.x || jt != blockIdx.y) {
+ continue;
+ }
+
+ any_fixup = true;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ sum[(j0/nwarps) * (mmq_y/WARP_SIZE) + i0/WARP_SIZE] += tmp_last_tile[bidx*(mmq_x*mmq_y) + j*mmq_y + i];
+ }
+ }
+ }
+
+ if (!any_fixup) {
+ return;
+ }
+
+ dst += blockIdx.y*mmq_x*ne0 + blockIdx.x*mmq_y;
+
+ const int i_max = ne01 - blockIdx.x*mmq_y - 1;
+ const int j_max = ne11 - blockIdx.y*mmq_x - 1;
+
+#pragma unroll
+ for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ if (j > j_max) {
+ return;
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < mmq_y; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+
+ if (need_check && i > i_max) {
+ continue;
+ }
+
+ dst[j*ne0 + i] += sum[(j0/nwarps) * (mmq_y/WARP_SIZE) + i0/WARP_SIZE];
+ }
+ }
+}
+
+struct mmq_args {
+ const char * x; const char * y; float * dst;
+ int64_t ne00; int64_t ne01; int64_t stride01;
+ int64_t ne10; int64_t ne11; int64_t stride11;
+ int64_t ne0;
+};
+
+template<ggml_type type>
+static int mmq_get_shmem(const int mmq_x, const int mmq_y, const int cc) {
+ const tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, mmq_y);
+ const int mmq_tile_x_k = mmq_get_mma_tile_x_k(type);
+ const int shmem_x = int8_mma_available(cc) ? mmq_y*mmq_tile_x_k*sizeof(int) : txs.qs*sizeof(int) + txs.dm*sizeof(half2) + txs.sc*sizeof(int);
+ const int shmem_y = mmq_x*sizeof(block_q8_1_mmq);
+ return shmem_x + GGML_PAD(shmem_y, MMQ_NWARPS*WARP_SIZE*sizeof(int));
+}
+
+template <ggml_type type, int mmq_x>
+static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
+ const int id = ggml_cuda_get_device();
+ const int cc = ggml_cuda_info().devices[id].cc;
+ const int nsm = ggml_cuda_info().devices[id].nsm;
+ const int mmq_y = get_mmq_y_host(cc);
+
+ const dim3 block_dims(WARP_SIZE, MMQ_NWARPS, 1);
+
+ const int shmem = mmq_get_shmem<type>(mmq_x, mmq_y, cc);
+
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+ static bool shmem_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};
+ if (!shmem_limit_raised[id]) {
+ CUDA_CHECK(cudaFuncSetAttribute(mul_mat_q<type, mmq_x, MMQ_NWARPS, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, shmem));
+ CUDA_CHECK(cudaFuncSetAttribute(mul_mat_q<type, mmq_x, MMQ_NWARPS, true>, cudaFuncAttributeMaxDynamicSharedMemorySize, shmem));
+ shmem_limit_raised[id] = true;
+ }
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+
+ const int nty = (args.ne01 + mmq_y - 1) / mmq_y;
+ const int ntx = (args.ne11 + mmq_x - 1) / mmq_x;
+ const dim3 block_nums_xy_tiling(nty, ntx, 1);
+
+ const bool use_stream_k = cc >= CC_VOLTA && cc < CC_OFFSET_AMD;
+ if (!use_stream_k) {
+ if (args.ne01 % mmq_y == 0) {
+ constexpr bool need_check = false;
+ mul_mat_q<type, mmq_x, MMQ_NWARPS, need_check><<<block_nums_xy_tiling, block_dims, shmem, stream>>>
+ (args.x, args.y, args.dst, nullptr, args.ne00, args.ne01, args.stride01, args.ne10, args.ne11, args.stride11, args.ne0);
+ } else {
+ constexpr bool need_check = true;
+ mul_mat_q<type, mmq_x, MMQ_NWARPS, need_check><<<block_nums_xy_tiling, block_dims, shmem, stream>>>
+ (args.x, args.y, args.dst, nullptr, args.ne00, args.ne01, args.stride01, args.ne10, args.ne11, args.stride11, args.ne0);
+ }
+ return;
+ }
+
+ const dim3 block_nums_mmq(nsm, 1, 1);
+
+ ggml_cuda_pool & pool = ctx.pool(id);
+ ggml_cuda_pool_alloc<float> tmp_fixup(pool, block_nums_mmq.x * mmq_x*mmq_y);
+
+ if (args.ne01 % mmq_y == 0) {
+ constexpr bool need_check = false;
+
+ mul_mat_q<type, mmq_x, MMQ_NWARPS, need_check><<<block_nums_mmq, block_dims, shmem, stream>>>
+ (args.x, args.y, args.dst, tmp_fixup.ptr, args.ne00, args.ne01, args.stride01, args.ne10, args.ne11, args.stride11, args.ne0);
+
+ mul_mat_q_stream_k_fixup<type, mmq_x, MMQ_NWARPS, need_check><<<block_nums_xy_tiling, block_dims, 0, stream>>>
+ (args.dst, tmp_fixup.ptr, args.ne00, args.ne01, args.ne11, args.ne0, block_nums_mmq.x);
+ } else {
+ constexpr bool need_check = true;
+
+ mul_mat_q<type, mmq_x, MMQ_NWARPS, need_check><<<block_nums_mmq, block_dims, shmem, stream>>>
+ (args.x, args.y, args.dst, tmp_fixup.ptr, args.ne00, args.ne01, args.stride01, args.ne10, args.ne11, args.stride11, args.ne0);
+
+ mul_mat_q_stream_k_fixup<type, mmq_x, MMQ_NWARPS, need_check><<<block_nums_xy_tiling, block_dims, 0, stream>>>
+ (args.dst, tmp_fixup.ptr, args.ne00, args.ne01, args.ne11, args.ne0, block_nums_mmq.x);
+ }
+}
+
+template <ggml_type type>
+void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
+ const int id = ggml_cuda_get_device();
+ const int nsm = ggml_cuda_info().devices[id].nsm;
+ const int cc = ggml_cuda_info().devices[id].cc;
+ const int smpbo = ggml_cuda_info().devices[id].smpbo;
+
+ const int mmq_x_max = get_mmq_x_max_host(cc);
+ const int mmq_y = get_mmq_y_host(cc);
+ const int block_num_y = (args.ne01 + mmq_y - 1) / mmq_y;
+ const bool use_stream_k = cc >= CC_VOLTA && cc < CC_OFFSET_AMD;
+
+ int mmq_x_best = 0;
+ int nparts_best = INT_MAX;
+
+ for (int mmq_x = 8; mmq_x <= mmq_x_max && nparts_best > 1; mmq_x += 8) {
+ const int granularity = mmq_get_granularity_host(mmq_x, cc);
+
+ if (mmq_x % granularity != 0 || mmq_get_shmem<type>(mmq_x, mmq_y, cc) > smpbo) {
+ continue;
+ }
+
+ const int ntiles_x = (args.ne11 + mmq_x - 1) / mmq_x;
+ const int nwaves_xy_tiling = ntiles_x*block_num_y;
+ const int nparts = use_stream_k ? ntiles_x : nwaves_xy_tiling;
+
+ if (nparts < nparts_best) {
+ mmq_x_best = mmq_x;
+ nparts_best = nparts;
+ }
+ }
+
+ switch (mmq_x_best) {
+ case 8:
+ launch_mul_mat_q<type, 8>(ctx, args, stream);
+ break;
+ case 16:
+ launch_mul_mat_q<type, 16>(ctx, args, stream);
+ break;
+ case 24:
+ launch_mul_mat_q<type, 24>(ctx, args, stream);
+ break;
+ case 32:
+ launch_mul_mat_q<type, 32>(ctx, args, stream);
+ break;
+ case 40:
+ launch_mul_mat_q<type, 40>(ctx, args, stream);
+ break;
+ case 48:
+ launch_mul_mat_q<type, 48>(ctx, args, stream);
+ break;
+ case 56:
+ launch_mul_mat_q<type, 56>(ctx, args, stream);
+ break;
+ case 64:
+ launch_mul_mat_q<type, 64>(ctx, args, stream);
+ break;
+ case 72:
+ launch_mul_mat_q<type, 72>(ctx, args, stream);
+ break;
+ case 80:
+ launch_mul_mat_q<type, 80>(ctx, args, stream);
+ break;
+ case 88:
+ launch_mul_mat_q<type, 88>(ctx, args, stream);
+ break;
+ case 96:
+ launch_mul_mat_q<type, 96>(ctx, args, stream);
+ break;
+ case 104:
+ launch_mul_mat_q<type, 104>(ctx, args, stream);
+ break;
+ case 112:
+ launch_mul_mat_q<type, 112>(ctx, args, stream);
+ break;
+ case 120:
+ launch_mul_mat_q<type, 120>(ctx, args, stream);
+ break;
+ case 128:
+ launch_mul_mat_q<type, 128>(ctx, args, stream);
+ break;
+ default:
+ fprintf(stderr, "mmq_x_best=%d\n", mmq_x_best);
+ GGML_ABORT("fatal error");
+ break;
+ }
+}
+
+#define DECL_MMQ_CASE(type) \
+ template void mul_mat_q_case<type>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \
+
+extern DECL_MMQ_CASE(GGML_TYPE_Q4_0);
+extern DECL_MMQ_CASE(GGML_TYPE_Q4_1);
+extern DECL_MMQ_CASE(GGML_TYPE_Q5_0);
+extern DECL_MMQ_CASE(GGML_TYPE_Q5_1);
+extern DECL_MMQ_CASE(GGML_TYPE_Q8_0);
+extern DECL_MMQ_CASE(GGML_TYPE_Q2_K);
+extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);
+extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);
+extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);
+extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);
+
+// -------------------------------------------------------------------------------------------------------------------------
+
+void ggml_cuda_op_mul_mat_q(
+ ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
+ const int64_t src1_padded_row_size, cudaStream_t stream);
+
+bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11);
diff --git a/llama/ggml-cuda/mmvq.cu b/llama/ggml-cuda/mmvq.cu
new file mode 100644
index 00000000..9c87a602
--- /dev/null
+++ b/llama/ggml-cuda/mmvq.cu
@@ -0,0 +1,451 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "mmvq.cuh"
+#include "vecdotq.cuh"
+
+typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs);
+
+static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) {
+ return type == GGML_TYPE_Q4_0 ? vec_dot_q4_0_q8_1 :
+ type == GGML_TYPE_Q4_1 ? vec_dot_q4_1_q8_1 :
+ type == GGML_TYPE_Q5_0 ? vec_dot_q5_0_q8_1 :
+ type == GGML_TYPE_Q5_1 ? vec_dot_q5_1_q8_1 :
+ type == GGML_TYPE_Q8_0 ? vec_dot_q8_0_q8_1 :
+ type == GGML_TYPE_Q2_K ? vec_dot_q2_K_q8_1 :
+ type == GGML_TYPE_Q3_K ? vec_dot_q3_K_q8_1 :
+ type == GGML_TYPE_Q4_K ? vec_dot_q4_K_q8_1 :
+ type == GGML_TYPE_Q5_K ? vec_dot_q5_K_q8_1 :
+ type == GGML_TYPE_Q6_K ? vec_dot_q6_K_q8_1 :
+ type == GGML_TYPE_IQ2_XXS ? vec_dot_iq2_xxs_q8_1 :
+ type == GGML_TYPE_IQ2_XS ? vec_dot_iq2_xs_q8_1 :
+ type == GGML_TYPE_IQ2_S ? vec_dot_iq2_s_q8_1 :
+ type == GGML_TYPE_IQ3_XXS ? vec_dot_iq3_xxs_q8_1 :
+ type == GGML_TYPE_IQ1_S ? vec_dot_iq1_s_q8_1 :
+ type == GGML_TYPE_IQ1_M ? vec_dot_iq1_m_q8_1 :
+ type == GGML_TYPE_IQ4_NL ? vec_dot_iq4_nl_q8_1 :
+ type == GGML_TYPE_IQ4_XS ? vec_dot_iq4_xs_q8_1 :
+ type == GGML_TYPE_IQ3_S ? vec_dot_iq3_s_q8_1 :
+ nullptr;
+}
+
+static constexpr __device__ int get_vdr_mmvq(ggml_type type) {
+ return type == GGML_TYPE_Q4_0 ? VDR_Q4_0_Q8_1_MMVQ :
+ type == GGML_TYPE_Q4_1 ? VDR_Q4_1_Q8_1_MMVQ :
+ type == GGML_TYPE_Q5_0 ? VDR_Q5_0_Q8_1_MMVQ :
+ type == GGML_TYPE_Q5_1 ? VDR_Q5_1_Q8_1_MMVQ :
+ type == GGML_TYPE_Q8_0 ? VDR_Q8_0_Q8_1_MMVQ :
+ type == GGML_TYPE_Q2_K ? VDR_Q2_K_Q8_1_MMVQ :
+ type == GGML_TYPE_Q3_K ? VDR_Q3_K_Q8_1_MMVQ :
+ type == GGML_TYPE_Q4_K ? VDR_Q4_K_Q8_1_MMVQ :
+ type == GGML_TYPE_Q5_K ? VDR_Q5_K_Q8_1_MMVQ :
+ type == GGML_TYPE_Q6_K ? VDR_Q6_K_Q8_1_MMVQ :
+ type == GGML_TYPE_IQ2_XXS ? VDR_IQ2_XXS_Q8_1_MMVQ :
+ type == GGML_TYPE_IQ2_XS ? VDR_IQ2_XS_Q8_1_MMVQ :
+ type == GGML_TYPE_IQ2_S ? VDR_IQ2_S_Q8_1_MMVQ :
+ type == GGML_TYPE_IQ3_XXS ? VDR_IQ3_XXS_Q8_1_MMVQ :
+ type == GGML_TYPE_IQ3_S ? VDR_IQ3_S_Q8_1_MMVQ :
+ type == GGML_TYPE_IQ4_NL ? VDR_IQ4_NL_Q8_1_MMVQ :
+ type == GGML_TYPE_IQ4_XS ? VDR_IQ4_XS_Q8_1_MMVQ :
+ 1;
+}
+
+template <ggml_type type, int ncols_y>
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+// tell the compiler to use as many registers as it wants, see nwarps definition below
+__launch_bounds__((ncols_y <= 4 ? 4 : 2)*WARP_SIZE, 1)
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+static __global__ void mul_mat_vec_q(
+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int nrows_dst) {
+
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int qi = ggml_cuda_type_traits<type>::qi;
+ constexpr int vdr = get_vdr_mmvq(type);
+
+ constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);
+
+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && (defined(RDNA2) || defined(RDNA3))
+ constexpr int nwarps = 1;
+ constexpr int rows_per_cuda_block = 1;
+#else
+ constexpr int nwarps = ncols_y <= 4 ? 4 : 2;
+ constexpr int rows_per_cuda_block = ncols_y == 1 ? 1 : 2;
+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && !defined(RDNA2) && !defined(RDNA3)
+
+ const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;
+ const int row0 = rows_per_cuda_block*blockIdx.x;
+ const int blocks_per_row_x = ncols_x / qk;
+ const int blocks_per_col_y = nrows_y / QK8_1;
+ constexpr int blocks_per_iter = vdr * nwarps*WARP_SIZE / qi;
+
+// partial sum for each thread
+ float tmp[ncols_y][rows_per_cuda_block] = {0.0f};
+
+ const block_q8_1 * y = (const block_q8_1 *) vy;
+
+ for (int kbx = tid / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {
+ const int kby = kbx * (qk/QK8_1); // y block index that aligns with kbx
+
+ // x block quant index when casting the quants to int
+ const int kqs = vdr * (tid % (qi/vdr));
+
+#pragma unroll
+ for (int j = 0; j < ncols_y; ++j) {
+#pragma unroll
+ for (int i = 0; i < rows_per_cuda_block; ++i) {
+ tmp[j][i] += vec_dot_q_cuda(vx, &y[j*blocks_per_col_y + kby], (row0 + i)*blocks_per_row_x + kbx, kqs);
+ }
+ }
+ }
+
+ __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_y][rows_per_cuda_block][WARP_SIZE];
+ if (threadIdx.y > 0) {
+#pragma unroll
+ for (int j = 0; j < ncols_y; ++j) {
+#pragma unroll
+ for (int i = 0; i < rows_per_cuda_block; ++i) {
+ tmp_shared[threadIdx.y-1][j][i][threadIdx.x] = tmp[j][i];
+ }
+ }
+ }
+ __syncthreads();
+ if (threadIdx.y > 0) {
+ return;
+ }
+
+ // sum up partial sums and write back result
+#pragma unroll
+ for (int j = 0; j < ncols_y; ++j) {
+#pragma unroll
+ for (int i = 0; i < rows_per_cuda_block; ++i) {
+#pragma unroll
+ for (int l = 0; l < nwarps-1; ++l) {
+ tmp[j][i] += tmp_shared[l][j][i][threadIdx.x];
+ }
+ tmp[j][i] = warp_reduce_sum(tmp[j][i]);
+ }
+
+ if (threadIdx.x < rows_per_cuda_block && (rows_per_cuda_block == 1 || row0 + threadIdx.x < nrows_dst)) {
+ dst[j*nrows_dst + row0 + threadIdx.x] = tmp[j][threadIdx.x];
+ }
+ }
+}
+
+template <ggml_type type>
+static void mul_mat_vec_q_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ GGML_ASSERT(ncols_x % ggml_blck_size(type) == 0);
+ GGML_ASSERT(ncols_y <= MMVQ_MAX_BATCH_SIZE);
+
+ int id = ggml_cuda_get_device();
+
+ int64_t nwarps = 1;
+ int64_t rows_per_cuda_block = 1;
+
+ if (ggml_cuda_info().devices[id].cc < CC_RDNA2) { // NVIDIA and AMD older than RDNA2
+ switch(ncols_y) {
+ case 1:
+ nwarps = 4;
+ rows_per_cuda_block = 1;
+ break;
+ case 2:
+ case 3:
+ case 4:
+ nwarps = 4;
+ rows_per_cuda_block = 2;
+ break;
+ case 5:
+ case 6:
+ case 7:
+ case 8:
+ nwarps = 2;
+ rows_per_cuda_block = 2;
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+ }
+ const int64_t nblocks = (nrows_x + rows_per_cuda_block - 1) / rows_per_cuda_block;
+ const dim3 block_nums(nblocks, 1, 1);
+ const dim3 block_dims(WARP_SIZE, nwarps, 1);
+
+ switch (ncols_y) {
+ case 1:
+ mul_mat_vec_q<type, 1><<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);
+ break;
+ case 2:
+ mul_mat_vec_q<type, 2><<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);
+ break;
+ case 3:
+ mul_mat_vec_q<type, 3><<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);
+ break;
+ case 4:
+ mul_mat_vec_q<type, 4><<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);
+ break;
+ case 5:
+ mul_mat_vec_q<type, 5><<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);
+ break;
+ case 6:
+ mul_mat_vec_q<type, 6><<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);
+ break;
+ case 7:
+ mul_mat_vec_q<type, 7><<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);
+ break;
+ case 8:
+ mul_mat_vec_q<type, 8><<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+}
+
+static void mul_mat_vec_q4_0_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_Q4_0>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_q4_1_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_Q4_1>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_q5_0_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_Q5_0>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_q5_1_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_Q5_1>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_q8_0_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_Q8_0>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_q2_K_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_Q2_K>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_q3_K_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_Q3_K>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_q4_K_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_Q4_K>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_q5_K_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_Q5_K>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_q6_K_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_Q6_K>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_iq2_xxs_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_IQ2_XXS>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_iq2_xs_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_IQ2_XS>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_iq2_s_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_IQ2_S>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_iq3_xxs_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_IQ3_XXS>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_iq1_s_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_IQ1_S>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_iq1_m_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_IQ1_M>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_iq4_nl_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_IQ4_NL>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_iq4_xs_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_IQ4_XS>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+static void mul_mat_vec_iq3_s_q8_1_cuda(
+ const void * vx, const void * vy, float * dst,
+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {
+
+ mul_mat_vec_q_cuda<GGML_TYPE_IQ3_S>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst, stream);
+}
+
+void ggml_cuda_op_mul_mat_vec_q(
+ ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
+ const int64_t src1_padded_row_size, cudaStream_t stream) {
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t row_diff = row_high - row_low;
+
+ const int64_t ne10 = src1->ne[0];
+ GGML_ASSERT(ne10 % QK8_1 == 0);
+
+ const int64_t ne0 = dst->ne[0];
+
+ int id = ggml_cuda_get_device();
+
+ // the main device has a larger memory buffer to hold the results from all GPUs
+ // nrows_dst == nrows of the matrix that the kernel writes into
+ const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff;
+
+ switch (src0->type) {
+ case GGML_TYPE_Q4_0:
+ mul_mat_vec_q4_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_Q4_1:
+ mul_mat_vec_q4_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_Q5_0:
+ mul_mat_vec_q5_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_Q5_1:
+ mul_mat_vec_q5_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_Q8_0:
+ mul_mat_vec_q8_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_Q2_K:
+ mul_mat_vec_q2_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_Q3_K:
+ mul_mat_vec_q3_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_Q4_K:
+ mul_mat_vec_q4_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_Q5_K:
+ mul_mat_vec_q5_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_Q6_K:
+ mul_mat_vec_q6_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_IQ2_XXS:
+ mul_mat_vec_iq2_xxs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_IQ2_XS:
+ mul_mat_vec_iq2_xs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_IQ2_S:
+ mul_mat_vec_iq2_s_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_IQ3_XXS:
+ mul_mat_vec_iq3_xxs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_IQ1_S:
+ mul_mat_vec_iq1_s_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_IQ1_M:
+ mul_mat_vec_iq1_m_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_IQ4_NL:
+ mul_mat_vec_iq4_nl_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_IQ4_XS:
+ mul_mat_vec_iq4_xs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ case GGML_TYPE_IQ3_S:
+ mul_mat_vec_iq3_s_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+
+ GGML_UNUSED(src1);
+ GGML_UNUSED(dst);
+ GGML_UNUSED(src1_ddf_i);
+ GGML_UNUSED(src1_ncols);
+ GGML_UNUSED(src1_padded_row_size);
+}
diff --git a/llama/ggml-cuda/mmvq.cuh b/llama/ggml-cuda/mmvq.cuh
new file mode 100644
index 00000000..3da3b904
--- /dev/null
+++ b/llama/ggml-cuda/mmvq.cuh
@@ -0,0 +1,35 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define MMVQ_MAX_BATCH_SIZE 8 // Max. batch size for which to use MMVQ kernels.
+
+void ggml_cuda_op_mul_mat_vec_q(
+ ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
+ const int64_t src1_padded_row_size, cudaStream_t stream);
diff --git a/llama/ggml-cuda/norm.cu b/llama/ggml-cuda/norm.cu
new file mode 100644
index 00000000..1f79c462
--- /dev/null
+++ b/llama/ggml-cuda/norm.cu
@@ -0,0 +1,250 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "norm.cuh"
+
+template <int block_size>
+static __global__ void norm_f32(const float * x, float * dst, const int ncols, const float eps) {
+ const int row = blockIdx.x*blockDim.y + threadIdx.y;
+ const int tid = threadIdx.x;
+
+ float2 mean_var = make_float2(0.f, 0.f);
+
+ for (int col = tid; col < ncols; col += block_size) {
+ const float xi = x[row*ncols + col];
+ mean_var.x += xi;
+ mean_var.y += xi * xi;
+ }
+
+ // sum up partial sums
+ mean_var = warp_reduce_sum(mean_var);
+ if (block_size > WARP_SIZE) {
+ __shared__ float2 s_sum[32];
+ int warp_id = threadIdx.x / WARP_SIZE;
+ int lane_id = threadIdx.x % WARP_SIZE;
+ if (lane_id == 0) {
+ s_sum[warp_id] = mean_var;
+ }
+ __syncthreads();
+ mean_var = s_sum[lane_id];
+ mean_var = warp_reduce_sum(mean_var);
+ }
+
+ const float mean = mean_var.x / ncols;
+ const float var = mean_var.y / ncols - mean * mean;
+ const float inv_std = rsqrtf(var + eps);
+
+ for (int col = tid; col < ncols; col += block_size) {
+ dst[row*ncols + col] = (x[row*ncols + col] - mean) * inv_std;
+ }
+}
+
+template <int block_size>
+static __global__ void group_norm_f32(const float * x, float * dst, const int group_size, const int ne_elements, const float eps) {
+ // blockIdx.x: num_groups idx
+ // threadIdx.x: block_size idx
+ int start = blockIdx.x * group_size;
+ int end = start + group_size;
+
+ start += threadIdx.x;
+
+ if (end >= ne_elements) {
+ end = ne_elements;
+ }
+
+ float tmp = 0.0f; // partial sum for thread in warp
+
+ for (int j = start; j < end; j += block_size) {
+ tmp += x[j];
+ }
+
+ tmp = warp_reduce_sum(tmp);
+ if (block_size > WARP_SIZE) {
+ __shared__ float s_sum[32];
+ int warp_id = threadIdx.x / WARP_SIZE;
+ int lane_id = threadIdx.x % WARP_SIZE;
+ if (lane_id == 0) {
+ s_sum[warp_id] = tmp;
+ }
+ __syncthreads();
+ tmp = s_sum[lane_id];
+ tmp = warp_reduce_sum(tmp);
+ }
+
+ float mean = tmp / group_size;
+ tmp = 0.0f;
+
+ for (int j = start; j < end; j += block_size) {
+ float xi = x[j] - mean;
+ dst[j] = xi;
+ tmp += xi * xi;
+ }
+
+ tmp = warp_reduce_sum(tmp);
+ if (block_size > WARP_SIZE) {
+ __shared__ float s_sum[32];
+ int warp_id = threadIdx.x / WARP_SIZE;
+ int lane_id = threadIdx.x % WARP_SIZE;
+ if (lane_id == 0) {
+ s_sum[warp_id] = tmp;
+ }
+ __syncthreads();
+ tmp = s_sum[lane_id];
+ tmp = warp_reduce_sum(tmp);
+ }
+
+ float variance = tmp / group_size;
+ float scale = rsqrtf(variance + eps);
+ for (int j = start; j < end; j += block_size) {
+ dst[j] *= scale;
+ }
+}
+
+template <int block_size>
+static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, const float eps) {
+ const int row = blockIdx.x*blockDim.y + threadIdx.y;
+ const int tid = threadIdx.x;
+
+ float tmp = 0.0f; // partial sum for thread in warp
+
+ for (int col = tid; col < ncols; col += block_size) {
+ const float xi = x[row*ncols + col];
+ tmp += xi * xi;
+ }
+
+ // sum up partial sums
+ tmp = warp_reduce_sum(tmp);
+ if (block_size > WARP_SIZE) {
+ __shared__ float s_sum[32];
+ int warp_id = threadIdx.x / WARP_SIZE;
+ int lane_id = threadIdx.x % WARP_SIZE;
+ if (lane_id == 0) {
+ s_sum[warp_id] = tmp;
+ }
+ __syncthreads();
+ tmp = s_sum[lane_id];
+ tmp = warp_reduce_sum(tmp);
+ }
+
+ const float mean = tmp / ncols;
+ const float scale = rsqrtf(mean + eps);
+
+ for (int col = tid; col < ncols; col += block_size) {
+ dst[row*ncols + col] = scale * x[row*ncols + col];
+ }
+}
+
+static void norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {
+ GGML_ASSERT(ncols % WARP_SIZE == 0);
+ if (ncols < 1024) {
+ const dim3 block_dims(WARP_SIZE, 1, 1);
+ norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
+ } else {
+ const dim3 block_dims(1024, 1, 1);
+ norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
+ }
+}
+
+static void group_norm_f32_cuda(const float * x, float * dst, const int num_groups, const float eps, const int group_size, const int ne_elements, cudaStream_t stream) {
+ if (group_size < 1024) {
+ const dim3 block_dims(WARP_SIZE, 1, 1);
+ group_norm_f32<WARP_SIZE><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);
+ } else {
+ const dim3 block_dims(1024, 1, 1);
+ group_norm_f32<1024><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);
+ }
+}
+
+static void rms_norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {
+ GGML_ASSERT(ncols % WARP_SIZE == 0);
+ if (ncols < 1024) {
+ const dim3 block_dims(WARP_SIZE, 1, 1);
+ rms_norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
+ } else {
+ const dim3 block_dims(1024, 1, 1);
+ rms_norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
+ }
+}
+
+void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t nrows = ggml_nrows(src0);
+
+ float eps;
+ memcpy(&eps, dst->op_params, sizeof(float));
+
+ norm_f32_cuda(src0_d, dst_d, ne00, nrows, eps, stream);
+}
+
+void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ int num_groups = dst->op_params[0];
+
+ float eps;
+ memcpy(&eps, dst->op_params + 1, sizeof(float));
+
+ int group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups);
+ group_norm_f32_cuda(src0_d, dst_d, num_groups * src0->ne[3], eps, group_size, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t nrows = ggml_nrows(src0);
+
+ float eps;
+ memcpy(&eps, dst->op_params, sizeof(float));
+
+ rms_norm_f32_cuda(src0_d, dst_d, ne00, nrows, eps, stream);
+}
diff --git a/llama/ggml-cuda/norm.cuh b/llama/ggml-cuda/norm.cuh
new file mode 100644
index 00000000..c7975033
--- /dev/null
+++ b/llama/ggml-cuda/norm.cuh
@@ -0,0 +1,33 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/pad.cu b/llama/ggml-cuda/pad.cu
new file mode 100644
index 00000000..dd121289
--- /dev/null
+++ b/llama/ggml-cuda/pad.cu
@@ -0,0 +1,75 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "pad.cuh"
+
+static __global__ void pad_f32(const float * x, float * dst, const int ne0, const int ne00, const int ne01, const int ne02, const int ne03) {
+ // blockIdx.z: idx of ne2*ne3, aka ne02*ne03
+ // blockIdx.y: idx of ne1
+ // blockIDx.x: idx of ne0 / BLOCK_SIZE
+ int nidx = threadIdx.x + blockIdx.x * blockDim.x;
+ if (nidx >= ne0) {
+ return;
+ }
+
+ // operation
+ int offset_dst =
+ nidx +
+ blockIdx.y * ne0 +
+ blockIdx.z * ne0 * gridDim.y;
+ if (nidx < ne00 && blockIdx.y < ne01 && blockIdx.z < ne02*ne03) {
+ int offset_src =
+ nidx +
+ blockIdx.y * ne00 +
+ blockIdx.z * ne00 * ne01;
+ dst[offset_dst] = x[offset_src];
+ } else {
+ dst[offset_dst] = 0.0f;
+ }
+}
+
+static void pad_f32_cuda(const float * x, float * dst,
+ const int ne00, const int ne01, const int ne02, const int ne03,
+ const int ne0, const int ne1, const int ne2, const int ne3, cudaStream_t stream) {
+ int num_blocks = (ne0 + CUDA_PAD_BLOCK_SIZE - 1) / CUDA_PAD_BLOCK_SIZE;
+ dim3 gridDim(num_blocks, ne1, ne2*ne3);
+ pad_f32<<<gridDim, CUDA_PAD_BLOCK_SIZE, 0, stream>>>(x, dst, ne0, ne00, ne01, ne02, ne03);
+}
+
+void ggml_cuda_op_pad(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
+ GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors
+
+ pad_f32_cuda(src0_d, dst_d,
+ src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
+ dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], stream);
+}
diff --git a/llama/ggml-cuda/pad.cuh b/llama/ggml-cuda/pad.cuh
new file mode 100644
index 00000000..afb93f25
--- /dev/null
+++ b/llama/ggml-cuda/pad.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_PAD_BLOCK_SIZE 256
+
+void ggml_cuda_op_pad(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/pool2d.cu b/llama/ggml-cuda/pool2d.cu
new file mode 100644
index 00000000..54f6ca5c
--- /dev/null
+++ b/llama/ggml-cuda/pool2d.cu
@@ -0,0 +1,120 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "pool2d.cuh"
+
+template <typename Ti, typename To>
+static __global__ void pool2d_nchw_kernel(
+ const int ih, const int iw, const int oh, const int ow,
+ const int kh, const int kw, const int sh, const int sw,
+ const int ph, const int pw, const int parallel_elements,
+ const Ti* src, To* dst, const enum ggml_op_pool op) {
+ int idx = threadIdx.x + blockIdx.x * blockDim.x;
+ if (idx >= parallel_elements) {
+ return;
+ }
+
+ const int I_HW = ih * iw;
+ const int O_HW = oh * ow;
+ const int nc = idx / O_HW;
+ const int cur_oh = idx % O_HW / ow;
+ const int cur_ow = idx % O_HW % ow;
+ const Ti* i_ptr = src + nc * I_HW;
+ To* o_ptr = dst + nc * O_HW;
+ const int start_h = cur_oh * sh - ph;
+ const int bh = max(0, start_h);
+ const int eh = min(ih, start_h + kh);
+ const int start_w = cur_ow * sw - pw;
+ const int bw = max(0, start_w);
+ const int ew = min(iw, start_w + kw);
+ const To scale = 1. / (kh * kw);
+ To res = 0;
+
+ switch (op) {
+ case GGML_OP_POOL_AVG: res = 0; break;
+ case GGML_OP_POOL_MAX: res = -FLT_MAX; break;
+ default: assert(false);
+ }
+
+ for (int i = bh; i < eh; i += 1) {
+ for (int j = bw; j < ew; j += 1) {
+#if __CUDA_ARCH__ >= 350
+ Ti cur = __ldg(i_ptr + i * iw + j);
+#else
+ Ti cur = i_ptr[i * iw + j];
+#endif
+ switch (op) {
+ case GGML_OP_POOL_AVG: res += cur * scale; break;
+ case GGML_OP_POOL_MAX: res = max(res, (To)cur); break;
+ default: assert(false);
+ }
+ }
+ }
+ o_ptr[cur_oh * ow + cur_ow] = res;
+}
+
+static void pool2d_nchw_kernel_f32_f32_cuda(
+ const int ih, const int iw, const int oh, const int ow,
+ const int kh, const int kw, const int sh, const int sw,
+ const int ph, const int pw, const int parallel_elements,
+ const float * src, float * dst, const enum ggml_op_pool op,
+ cudaStream_t stream) {
+
+ const int num_blocks = (parallel_elements + CUDA_POOL2D_BLOCK_SIZE - 1) / CUDA_POOL2D_BLOCK_SIZE;
+ dim3 block_nums(num_blocks);
+ pool2d_nchw_kernel<<<block_nums, CUDA_POOL2D_BLOCK_SIZE, 0, stream>>>(ih, iw, oh, ow, kh, kw, sh, sw, ph, pw, parallel_elements, src, dst, op);
+}
+
+void ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ const int32_t * opts = (const int32_t *)dst->op_params;
+ enum ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);
+ const int k0 = opts[1];
+ const int k1 = opts[2];
+ const int s0 = opts[3];
+ const int s1 = opts[4];
+ const int p0 = opts[5];
+ const int p1 = opts[6];
+
+ const int64_t IH = src0->ne[1];
+ const int64_t IW = src0->ne[0];
+
+ const int64_t N = dst->ne[3];
+ const int64_t OC = dst->ne[2];
+ const int64_t OH = dst->ne[1];
+ const int64_t OW = dst->ne[0];
+
+ const int parallel_elements = N * OC * OH * OW;
+
+ pool2d_nchw_kernel_f32_f32_cuda(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, parallel_elements, src0_d, dst_d, op, stream);
+}
diff --git a/llama/ggml-cuda/pool2d.cuh b/llama/ggml-cuda/pool2d.cuh
new file mode 100644
index 00000000..e058cfd0
--- /dev/null
+++ b/llama/ggml-cuda/pool2d.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_POOL2D_BLOCK_SIZE 256
+
+void ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/quantize.cu b/llama/ggml-cuda/quantize.cu
new file mode 100644
index 00000000..fff2d49e
--- /dev/null
+++ b/llama/ggml-cuda/quantize.cu
@@ -0,0 +1,195 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "quantize.cuh"
+#include <cstdint>
+
+static __global__ void quantize_q8_1(const float * __restrict__ x, void * __restrict__ vy, const int64_t kx, const int64_t kx0_padded) {
+ const int64_t ix0 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (ix0 >= kx0_padded) {
+ return;
+ }
+
+ const int64_t ix1 = blockIdx.y;
+
+ const int64_t i_padded = ix1*kx0_padded + ix0;
+
+ block_q8_1 * y = (block_q8_1 *) vy;
+
+ const int64_t ib = i_padded / QK8_1; // block index
+ const int64_t iqs = i_padded % QK8_1; // quant index
+
+ const float xi = ix0 < kx ? x[ix1*kx + ix0] : 0.0f;
+ float amax = fabsf(xi);
+ float sum = xi;
+
+ amax = warp_reduce_max(amax);
+ sum = warp_reduce_sum(sum);
+
+ const float d = amax / 127;
+ const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
+
+ y[ib].qs[iqs] = q;
+
+ if (iqs > 0) {
+ return;
+ }
+
+ reinterpret_cast<half&>(y[ib].ds.x) = d;
+ reinterpret_cast<half&>(y[ib].ds.y) = sum;
+}
+
+template <mmq_q8_1_ds_layout ds_layout>
+static __global__ void quantize_mmq_q8_1(
+ const float * __restrict__ x, void * __restrict__ vy, const int64_t kx0, const int64_t kx1, const int64_t kx0_padded) {
+
+ constexpr int vals_per_scale = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 64 : 32;
+ constexpr int vals_per_sum = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 16 : 32;
+
+ const int64_t ix0 = ((int64_t)blockDim.x*blockIdx.x + threadIdx.x)*4;
+
+ if (ix0 >= kx0_padded) {
+ return;
+ }
+
+ const float4 * x4 = (const float4 *) x;
+
+ const int64_t ix1 = kx1*blockIdx.z + blockIdx.y;
+
+ block_q8_1_mmq * y = (block_q8_1_mmq *) vy;
+
+ const int64_t ib0 = blockIdx.z*((int64_t)gridDim.y*gridDim.x*blockDim.x/QK8_1); // first block of channel
+ const int64_t ib = ib0 + (ix0 / (4*QK8_1))*kx1 + blockIdx.y; // block index in channel
+ const int64_t iqs = ix0 % (4*QK8_1); // quant index in block
+
+ // Load 4 floats per thread and calculate max. abs. value between them:
+ const float4 xi = ix0 < kx0 ? x4[(ix1*kx0 + ix0)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+ float amax = fabsf(xi.x);
+ amax = fmaxf(amax, fabsf(xi.y));
+ amax = fmaxf(amax, fabsf(xi.z));
+ amax = fmaxf(amax, fabsf(xi.w));
+
+ // Exchange max. abs. value between vals_per_scale/4 threads.
+#pragma unroll
+ for (int mask = vals_per_scale/8; mask > 0; mask >>= 1) {
+ amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, mask, WARP_SIZE));
+ }
+
+ float sum;
+ if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {
+ sum = xi.x + xi.y + xi.z + xi.w;
+
+ // Exchange calculate sum across vals_per_sum/4 threads.
+#pragma unroll
+ for (int mask = vals_per_sum/8; mask > 0; mask >>= 1) {
+ sum += __shfl_xor_sync(0xFFFFFFFF, sum, mask, WARP_SIZE);
+ }
+ }
+
+ const float d_inv = 127.0f / amax;
+ char4 q;
+ q.x = roundf(xi.x*d_inv);
+ q.y = roundf(xi.y*d_inv);
+ q.z = roundf(xi.z*d_inv);
+ q.w = roundf(xi.w*d_inv);
+
+ // Write back 4 int8 values as a single 32 bit value for better memroy bandwidth:
+ char4 * yqs4 = (char4 *) y[ib].qs;
+ yqs4[iqs/4] = q;
+
+ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) {
+ if (iqs % 16 != 0 || iqs >= 96) {
+ return;
+ }
+
+ y[ib].d2s6[2 + iqs/16] = sum;
+
+ if (iqs % 64 != 0) {
+ return;
+ }
+
+ const float d = 1.0f / d_inv;
+
+ y[ib].d2s6[iqs/64] = d;
+
+ return;
+ }
+
+ if (iqs % 32 != 0) {
+ return;
+ }
+
+ const float d = 1.0f / d_inv;
+
+ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) {
+ y[ib].ds4[iqs/32] = make_half2(d, sum);
+ } else {
+ y[ib].d4[iqs/32] = d;
+ }
+}
+
+void quantize_row_q8_1_cuda(
+ const float * x, void * vy, const int64_t kx0, const int64_t kx1, const int64_t channels,
+ const int64_t kx0_padded, const ggml_type type_x, cudaStream_t stream) {
+
+ GGML_ASSERT(kx0_padded % QK8_1 == 0);
+
+ const int64_t block_num_x = (kx0_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
+ const dim3 num_blocks(block_num_x, kx1*channels, 1);
+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1);
+ quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(x, vy, kx0, kx0_padded);
+
+ GGML_UNUSED(type_x);
+}
+
+void quantize_mmq_q8_1_cuda(
+ const float * x, void * vy, const int64_t kx0, const int64_t kx1, const int64_t channels,
+ const int64_t kx0_padded, const ggml_type type_x, cudaStream_t stream) {
+
+ GGML_ASSERT(kx0_padded % (4*QK8_1) == 0);
+
+ const int64_t block_num_x = (kx0_padded + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ);
+ const dim3 num_blocks(block_num_x, kx1, channels);
+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
+ switch (mmq_get_q8_1_ds_layout(type_x)) {
+ case MMQ_Q8_1_DS_LAYOUT_D4:
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4>
+ <<<num_blocks, block_size, 0, stream>>>(x, vy, kx0, kx1, kx0_padded);
+ break;
+ case MMQ_Q8_1_DS_LAYOUT_DS4:
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4>
+ <<<num_blocks, block_size, 0, stream>>>(x, vy, kx0, kx1, kx0_padded);
+ break;
+ case MMQ_Q8_1_DS_LAYOUT_D2S6:
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6>
+ <<<num_blocks, block_size, 0, stream>>>(x, vy, kx0, kx1, kx0_padded);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ break;
+ }
+}
diff --git a/llama/ggml-cuda/quantize.cuh b/llama/ggml-cuda/quantize.cuh
new file mode 100644
index 00000000..32296287
--- /dev/null
+++ b/llama/ggml-cuda/quantize.cuh
@@ -0,0 +1,50 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "common.cuh"
+#include "mmq.cuh"
+
+#include <cstdint>
+
+#define CUDA_QUANTIZE_BLOCK_SIZE 256
+#define CUDA_QUANTIZE_BLOCK_SIZE_MMQ 128
+
+static_assert(MATRIX_ROW_PADDING % CUDA_QUANTIZE_BLOCK_SIZE == 0, "Risk of out-of-bounds access.");
+static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access.");
+
+typedef void (*quantize_cuda_t)(
+ const float * x, void * vy, const int64_t kx0, const int64_t kx1, const int64_t channels, const int64_t kx0_padded,
+ const ggml_type type_x, cudaStream_t stream);
+
+void quantize_row_q8_1_cuda(
+ const float * x, void * vy, const int64_t kx0, const int64_t kx1, const int64_t channels, const int64_t kx0_padded,
+ const ggml_type type_x, cudaStream_t stream);
+
+void quantize_mmq_q8_1_cuda(
+ const float * x, void * vy, const int64_t kx0, const int64_t kx1, const int64_t channels, const int64_t kx0_padded,
+ const ggml_type type_x, cudaStream_t stream);
diff --git a/llama/ggml-cuda/rope.cu b/llama/ggml-cuda/rope.cu
new file mode 100644
index 00000000..4a50372c
--- /dev/null
+++ b/llama/ggml-cuda/rope.cu
@@ -0,0 +1,297 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rope.cuh"
+
+struct rope_corr_dims {
+ float v[2];
+};
+
+static __device__ float rope_yarn_ramp(const float low, const float high, const int i0) {
+ const float y = (i0 / 2 - low) / max(0.001f, high - low);
+ return 1.0f - min(1.0f, max(0.0f, y));
+}
+
+// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn
+// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.
+static __device__ void rope_yarn(
+ float theta_extrap, float freq_scale, rope_corr_dims corr_dims, int64_t i0, float ext_factor, float mscale,
+ float * cos_theta, float * sin_theta) {
+ // Get n-d rotational scaling corrected for extrapolation
+ float theta_interp = freq_scale * theta_extrap;
+ float theta = theta_interp;
+ if (ext_factor != 0.0f) {
+ float ramp_mix = rope_yarn_ramp(corr_dims.v[0], corr_dims.v[1], i0) * ext_factor;
+ theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;
+
+ // Get n-d magnitude scaling corrected for interpolation
+ mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);
+ }
+ *cos_theta = cosf(theta) * mscale;
+ *sin_theta = sinf(theta) * mscale;
+}
+
+template<typename T, bool has_ff>
+static __global__ void rope_norm(
+ const T * x, T * dst, int ne0, int n_dims, const int32_t * pos, float freq_scale, int p_delta_rows,
+ float ext_factor, float attn_factor, rope_corr_dims corr_dims, float theta_scale, const float * freq_factors) {
+ const int i0 = 2*(blockDim.y*blockIdx.y + threadIdx.y);
+
+ if (i0 >= ne0) {
+ return;
+ }
+
+ const int row = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i0 >= n_dims) {
+ const int i = row*ne0 + i0;
+
+ dst[i + 0] = x[i + 0];
+ dst[i + 1] = x[i + 1];
+
+ return;
+ }
+
+ const int i = row*ne0 + i0;
+ const int i2 = row/p_delta_rows;
+
+ const float theta_base = pos[i2]*powf(theta_scale, i0/2.0f);
+
+ const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f;
+
+ float cos_theta;
+ float sin_theta;
+
+ rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta);
+
+ const float x0 = x[i + 0];
+ const float x1 = x[i + 1];
+
+ dst[i + 0] = x0*cos_theta - x1*sin_theta;
+ dst[i + 1] = x0*sin_theta + x1*cos_theta;
+}
+
+template<typename T, bool has_ff>
+static __global__ void rope_neox(
+ const T * x, T * dst, int ne0, int n_dims, const int32_t * pos, float freq_scale, int p_delta_rows,
+ float ext_factor, float attn_factor, rope_corr_dims corr_dims, float theta_scale, const float * freq_factors) {
+ const int i0 = 2*(blockDim.y*blockIdx.y + threadIdx.y);
+
+ if (i0 >= ne0) {
+ return;
+ }
+
+ const int row = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i0 >= n_dims) {
+ const int i = row*ne0 + i0;
+
+ dst[i + 0] = x[i + 0];
+ dst[i + 1] = x[i + 1];
+
+ return;
+ }
+
+ const int i = row*ne0 + i0/2;
+ const int i2 = row/p_delta_rows;
+
+ const float theta_base = pos[i2]*powf(theta_scale, i0/2.0f);
+
+ const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f;
+
+ float cos_theta;
+ float sin_theta;
+
+ rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta);
+
+ const float x0 = x[i + 0];
+ const float x1 = x[i + n_dims/2];
+
+ dst[i + 0] = x0*cos_theta - x1*sin_theta;
+ dst[i + n_dims/2] = x0*sin_theta + x1*cos_theta;
+}
+
+template<typename T>
+static void rope_norm_cuda(
+ const T * x, T * dst, int ne0, int n_dims, int nr, const int32_t * pos, float freq_scale, int p_delta_rows,
+ float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, const float * freq_factors, cudaStream_t stream) {
+ GGML_ASSERT(ne0 % 2 == 0);
+ const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);
+ const int n_blocks_x = (ne0 + 2*CUDA_ROPE_BLOCK_SIZE - 1) / (2*CUDA_ROPE_BLOCK_SIZE);
+ const dim3 block_nums(nr, n_blocks_x, 1);
+
+ const float theta_scale = powf(freq_base, -2.0f/n_dims);
+
+ if (freq_factors == nullptr) {
+ rope_norm<T, false><<<block_nums, block_dims, 0, stream>>>(
+ x, dst, ne0, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,
+ theta_scale, freq_factors
+ );
+ } else {
+ rope_norm<T, true><<<block_nums, block_dims, 0, stream>>>(
+ x, dst, ne0, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,
+ theta_scale, freq_factors
+ );
+ }
+}
+
+template<typename T>
+static void rope_neox_cuda(
+ const T * x, T * dst, int ne0, int n_dims, int nr, const int32_t * pos, float freq_scale, int p_delta_rows,
+ float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, const float * freq_factors, cudaStream_t stream) {
+ GGML_ASSERT(ne0 % 2 == 0);
+ const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);
+ const int n_blocks_x = (ne0 + 2*CUDA_ROPE_BLOCK_SIZE - 1) / (2*CUDA_ROPE_BLOCK_SIZE);
+ const dim3 block_nums(nr, n_blocks_x, 1);
+
+ const float theta_scale = powf(freq_base, -2.0f/n_dims);
+
+ if (freq_factors == nullptr) {
+ rope_neox<T, false><<<block_nums, block_dims, 0, stream>>>(
+ x, dst, ne0, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,
+ theta_scale, freq_factors
+ );
+ } else {
+ rope_neox<T, true><<<block_nums, block_dims, 0, stream>>>(
+ x, dst, ne0, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,
+ theta_scale, freq_factors
+ );
+ }
+}
+
+static void rope_norm_cuda_f16(
+ const half * x, half * dst, int ne0, int n_dims, int nr, const int32_t * pos, float freq_scale, int p_delta_rows,
+ float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, const float * freq_factors, cudaStream_t stream) {
+
+ rope_norm_cuda<half>(x, dst, ne0, n_dims, nr, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, stream);
+}
+
+static void rope_norm_cuda_f32(
+ const float * x, float * dst, int ne0, int n_dims, int nr, const int32_t * pos, float freq_scale, int p_delta_rows,
+ float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, const float * freq_factors, cudaStream_t stream) {
+
+ rope_norm_cuda<float>(x, dst, ne0, n_dims, nr, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, stream);
+}
+
+static void rope_neox_cuda_f16(
+ const half * x, half * dst, int ne0, int n_dims, int nr, const int32_t * pos, float freq_scale, int p_delta_rows,
+ float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, const float * freq_factors, cudaStream_t stream) {
+
+ rope_neox_cuda<half>(x, dst, ne0, n_dims, nr, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, stream);
+}
+
+static void rope_neox_cuda_f32(
+ const float * x, float * dst, int ne0, int n_dims, int nr, const int32_t * pos, float freq_scale, int p_delta_rows,
+ float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, const float * freq_factors, cudaStream_t stream
+) {
+
+ rope_neox_cuda<float>(x, dst, ne0, n_dims, nr, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, stream);
+}
+
+void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ const ggml_tensor * src2 = dst->src[2];
+
+ const float * src0_d = (const float *)src0->data;
+ const float * src1_d = (const float *)src1->data;
+
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
+ GGML_ASSERT(src0->type == dst->type);
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t ne01 = src0->ne[1];
+ const int64_t nr = ggml_nrows(src0);
+
+ //const int n_past = ((int32_t *) dst->op_params)[0];
+ const int n_dims = ((int32_t *) dst->op_params)[1];
+ const int mode = ((int32_t *) dst->op_params)[2];
+ //const int n_ctx = ((int32_t *) dst->op_params)[3];
+ const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
+
+ // RoPE alteration for extended context
+ float freq_base;
+ float freq_scale;
+ float ext_factor;
+ float attn_factor;
+ float beta_fast;
+ float beta_slow;
+
+ memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
+ memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
+ memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
+ memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
+ memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
+ memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
+
+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
+
+ const int32_t * pos = (const int32_t *) src1_d;
+
+ const float * freq_factors = nullptr;
+ if (src2 != nullptr) {
+ freq_factors = (const float *) src2->data;
+ }
+
+ rope_corr_dims corr_dims;
+ ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims.v);
+
+ // compute
+ if (is_neox) {
+ if (src0->type == GGML_TYPE_F32) {
+ rope_neox_cuda_f32(
+ (const float *)src0_d, (float *)dst_d, ne00, n_dims, nr, pos, freq_scale, ne01, freq_base, ext_factor,
+ attn_factor, corr_dims, freq_factors, stream
+ );
+ } else if (src0->type == GGML_TYPE_F16) {
+ rope_neox_cuda_f16(
+ (const half *)src0_d, (half *)dst_d, ne00, n_dims, nr, pos, freq_scale, ne01, freq_base, ext_factor,
+ attn_factor, corr_dims, freq_factors, stream
+ );
+ } else {
+ GGML_ABORT("fatal error");
+ }
+ } else {
+ if (src0->type == GGML_TYPE_F32) {
+ rope_norm_cuda_f32(
+ (const float *)src0_d, (float *)dst_d, ne00, n_dims, nr, pos, freq_scale, ne01, freq_base, ext_factor,
+ attn_factor, corr_dims, freq_factors, stream
+ );
+ } else if (src0->type == GGML_TYPE_F16) {
+ rope_norm_cuda_f16(
+ (const half *)src0_d, (half *)dst_d, ne00, n_dims, nr, pos, freq_scale, ne01, freq_base, ext_factor,
+ attn_factor, corr_dims, freq_factors, stream
+ );
+ } else {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
diff --git a/llama/ggml-cuda/rope.cuh b/llama/ggml-cuda/rope.cuh
new file mode 100644
index 00000000..a0cd2f14
--- /dev/null
+++ b/llama/ggml-cuda/rope.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_ROPE_BLOCK_SIZE 256
+
+void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/scale.cu b/llama/ggml-cuda/scale.cu
new file mode 100644
index 00000000..46498c41
--- /dev/null
+++ b/llama/ggml-cuda/scale.cu
@@ -0,0 +1,57 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "scale.cuh"
+
+static __global__ void scale_f32(const float * x, float * dst, const float scale, const int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+
+ dst[i] = scale * x[i];
+}
+
+static void scale_f32_cuda(const float * x, float * dst, const float scale, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;
+ scale_f32<<<num_blocks, CUDA_SCALE_BLOCK_SIZE, 0, stream>>>(x, dst, scale, k);
+}
+
+void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ float scale;
+ memcpy(&scale, dst->op_params, sizeof(float));
+
+ scale_f32_cuda(src0_d, dst_d, scale, ggml_nelements(src0), stream);
+}
diff --git a/llama/ggml-cuda/scale.cuh b/llama/ggml-cuda/scale.cuh
new file mode 100644
index 00000000..7aa65385
--- /dev/null
+++ b/llama/ggml-cuda/scale.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_SCALE_BLOCK_SIZE 256
+
+void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/softmax.cu b/llama/ggml-cuda/softmax.cu
new file mode 100644
index 00000000..97fcf81b
--- /dev/null
+++ b/llama/ggml-cuda/softmax.cu
@@ -0,0 +1,232 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+#include "softmax.cuh"
+
+template <typename T>
+static __device__ __forceinline__ float t2f32(T val) {
+ return (float) val;
+}
+
+template <>
+__device__ float __forceinline__ t2f32<half>(half val) {
+ return __half2float(val);
+}
+
+template <bool vals_smem, int ncols_template, int block_size_template, typename T>
+static __global__ void soft_max_f32(const float * x, const T * mask, float * dst, const int ncols_par, const int nrows_y, const float scale, const float max_bias, const float m0, const float m1, uint32_t n_head_log2) {
+ const int ncols = ncols_template == 0 ? ncols_par : ncols_template;
+
+ const int tid = threadIdx.x;
+ const int rowx = blockIdx.x;
+ const int rowy = rowx % nrows_y; // broadcast the mask in the row dimension
+
+ const int block_size = block_size_template == 0 ? blockDim.x : block_size_template;
+
+ const int warp_id = threadIdx.x / WARP_SIZE;
+ const int lane_id = threadIdx.x % WARP_SIZE;
+
+ const float slope = get_alibi_slope(max_bias, rowx/nrows_y, n_head_log2, m0, m1);
+
+ extern __shared__ float data_soft_max_f32[];
+ float * buf_iw = data_soft_max_f32; // shared memory buffer for inter-warp communication
+ // shared memory buffer to cache values between iterations:
+ float * vals = vals_smem ? buf_iw + WARP_SIZE : dst + (int64_t)rowx*ncols;
+
+ float max_val = -INFINITY;
+
+#pragma unroll
+ for (int col0 = 0; col0 < ncols; col0 += block_size) {
+ const int col = col0 + tid;
+
+ if (ncols_template == 0 && col >= ncols) {
+ break;
+ }
+
+ const int64_t ix = (int64_t)rowx*ncols + col;
+ const int64_t iy = (int64_t)rowy*ncols + col;
+
+ const float val = x[ix]*scale + (mask ? slope*t2f32(mask[iy]) : 0.0f);
+
+ vals[col] = val;
+ max_val = max(max_val, val);
+ }
+
+ // find the max value in the block
+ max_val = warp_reduce_max(max_val);
+ if (block_size > WARP_SIZE) {
+ if (warp_id == 0) {
+ buf_iw[lane_id] = -INFINITY;
+ }
+ __syncthreads();
+
+ if (lane_id == 0) {
+ buf_iw[warp_id] = max_val;
+ }
+ __syncthreads();
+
+ max_val = buf_iw[lane_id];
+ max_val = warp_reduce_max(max_val);
+ }
+
+ float tmp = 0.0f; // partial sum
+
+#pragma unroll
+ for (int col0 = 0; col0 < ncols; col0 += block_size) {
+ const int col = col0 + tid;
+
+ if (ncols_template == 0 && col >= ncols) {
+ break;
+ }
+
+ const float val = expf(vals[col] - max_val);
+ tmp += val;
+ vals[col] = val;
+ }
+
+ // find the sum of exps in the block
+ tmp = warp_reduce_sum(tmp);
+ if (block_size > WARP_SIZE) {
+ __syncthreads();
+ if (warp_id == 0) {
+ buf_iw[lane_id] = 0.0f;
+ }
+ __syncthreads();
+
+ if (lane_id == 0) {
+ buf_iw[warp_id] = tmp;
+ }
+ __syncthreads();
+
+ tmp = buf_iw[lane_id];
+ tmp = warp_reduce_sum(tmp);
+ }
+
+ const float inv_sum = 1.0f / tmp;
+
+#pragma unroll
+ for (int col0 = 0; col0 < ncols; col0 += block_size) {
+ const int col = col0 + tid;
+
+ if (ncols_template == 0 && col >= ncols) {
+ return;
+ }
+
+ const int64_t idst = (int64_t)rowx*ncols + col;
+ dst[idst] = vals[col] * inv_sum;
+ }
+}
+
+template<typename T>
+static void soft_max_f32_cuda(const float * x, const T * mask, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, const float max_bias, cudaStream_t stream) {
+ int nth = WARP_SIZE;
+ while (nth < ncols_x && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2;
+ const dim3 block_dims(nth, 1, 1);
+ const dim3 block_nums(nrows_x, 1, 1);
+ const size_t shmem = (GGML_PAD(ncols_x, WARP_SIZE) + WARP_SIZE)*sizeof(float);
+ static_assert(CUDA_SOFT_MAX_BLOCK_SIZE == 1024, "These values need to be adjusted.");
+
+ const uint32_t n_head = nrows_x/nrows_y;
+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));
+
+ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
+
+ // FIXME: this limit could be raised by ~2-4x on Ampere or newer
+ if (shmem < ggml_cuda_info().devices[ggml_cuda_get_device()].smpb) {
+ switch (ncols_x) {
+ case 32:
+ soft_max_f32<true, 32, 32><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);
+ break;
+ case 64:
+ soft_max_f32<true, 64, 64><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);
+ break;
+ case 128:
+ soft_max_f32<true, 128, 128><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);
+ break;
+ case 256:
+ soft_max_f32<true, 256, 256><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);
+ break;
+ case 512:
+ soft_max_f32<true, 512, 512><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);
+ break;
+ case 1024:
+ soft_max_f32<true, 1024, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);
+ break;
+ case 2048:
+ soft_max_f32<true, 2048, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);
+ break;
+ case 4096:
+ soft_max_f32<true, 4096, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);
+ break;
+ default:
+ soft_max_f32<true, 0, 0><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);
+ break;
+ }
+ } else {
+ const size_t shmem_low = WARP_SIZE*sizeof(float);
+ soft_max_f32<false, 0, 0><<<block_nums, block_dims, shmem_low, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);
+ }
+}
+
+void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+
+ const float * src0_d = (const float *)src0->data;
+ const void * src1_d = src1 ? (const void *)src1->data : nullptr;
+
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t nrows_x = ggml_nrows(src0);
+ const int64_t nrows_y = src0->ne[1];
+
+ float scale = 1.0f;
+ float max_bias = 0.0f;
+
+ memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));
+ memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));
+
+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);
+
+ if (use_f16) {
+ const half * src1_dd = (const half *)src1_d;
+
+ soft_max_f32_cuda(src0_d, src1_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);
+ } else {
+ const float * src1_dd = (const float *)src1_d;
+
+ soft_max_f32_cuda(src0_d, src1_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);
+ }
+}
diff --git a/llama/ggml-cuda/softmax.cuh b/llama/ggml-cuda/softmax.cuh
new file mode 100644
index 00000000..3c408dc6
--- /dev/null
+++ b/llama/ggml-cuda/softmax.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_SOFT_MAX_BLOCK_SIZE 1024
+
+void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/sumrows.cu b/llama/ggml-cuda/sumrows.cu
new file mode 100644
index 00000000..819103b1
--- /dev/null
+++ b/llama/ggml-cuda/sumrows.cu
@@ -0,0 +1,65 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "sumrows.cuh"
+
+static __global__ void k_sum_rows_f32(const float * x, float * dst, const int ncols) {
+ const int row = blockIdx.x;
+ const int col = threadIdx.x;
+
+ float sum = 0.0f;
+ for (int i = col; i < ncols; i += blockDim.x) {
+ sum += x[row * ncols + i];
+ }
+
+ sum = warp_reduce_sum(sum);
+
+ if (col == 0) {
+ dst[row] = sum;
+ }
+}
+
+void sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
+ const dim3 block_dims(WARP_SIZE, 1, 1);
+ const dim3 block_nums(nrows, 1, 1);
+ k_sum_rows_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);
+}
+
+void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ const int64_t ncols = src0->ne[0];
+ const int64_t nrows = ggml_nrows(src0);
+
+ sum_rows_f32_cuda(src0_d, dst_d, ncols, nrows, stream);
+}
diff --git a/llama/ggml-cuda/sumrows.cuh b/llama/ggml-cuda/sumrows.cuh
new file mode 100644
index 00000000..db47343f
--- /dev/null
+++ b/llama/ggml-cuda/sumrows.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+void sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream);
+
+void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu
new file mode 100644
index 00000000..ac7f1476
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu
new file mode 100644
index 00000000..810f9df7
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu
new file mode 100644
index 00000000..411d9f8f
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu
new file mode 100644
index 00000000..64386f63
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu
new file mode 100644
index 00000000..346a5859
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu
new file mode 100644
index 00000000..22ff28fe
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu
new file mode 100644
index 00000000..bf5757a3
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu
new file mode 100644
index 00000000..50c950d8
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu
new file mode 100644
index 00000000..c1a1d4ff
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu
new file mode 100644
index 00000000..134a5f5d
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu
new file mode 100644
index 00000000..738fb97f
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu
new file mode 100644
index 00000000..406abd32
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu
new file mode 100644
index 00000000..fb3d9019
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu
new file mode 100644
index 00000000..0b587c21
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu
new file mode 100644
index 00000000..41287725
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu
new file mode 100644
index 00000000..67ef8ae4
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu
new file mode 100644
index 00000000..38d0e15c
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu
new file mode 100644
index 00000000..bf3c378f
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu
new file mode 100644
index 00000000..4b0bf994
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu
new file mode 100644
index 00000000..347a03d8
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu
new file mode 100644
index 00000000..b2b6e74f
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu
new file mode 100644
index 00000000..8db57b23
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu
new file mode 100644
index 00000000..f46dec7d
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu
new file mode 100644
index 00000000..f01b195c
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu
new file mode 100644
index 00000000..0267b25a
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu
new file mode 100644
index 00000000..f96463a2
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu
new file mode 100644
index 00000000..7b769933
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu
new file mode 100644
index 00000000..78b49309
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu
new file mode 100644
index 00000000..01f7eac0
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu
new file mode 100644
index 00000000..4d357674
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu
new file mode 100644
index 00000000..fd9ff44b
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu
new file mode 100644
index 00000000..63ef4c2a
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu
new file mode 100644
index 00000000..625ee2b5
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu
new file mode 100644
index 00000000..b0969686
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu
new file mode 100644
index 00000000..c77b22ba
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu
new file mode 100644
index 00000000..881243e8
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu
new file mode 100644
index 00000000..803673f9
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu
new file mode 100644
index 00000000..062e8e22
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(64, GGML_TYPE_F16, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu
new file mode 100644
index 00000000..9c552c80
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(64, GGML_TYPE_F16, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu
new file mode 100644
index 00000000..ef9a2bc5
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(64, GGML_TYPE_F16, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu
new file mode 100644
index 00000000..fc989011
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(64, GGML_TYPE_F16, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu
new file mode 100644
index 00000000..42c4755e
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(64, GGML_TYPE_F16, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu
new file mode 100644
index 00000000..70d69826
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f16.cuh"
+
+DECL_FATTN_VEC_F16_CASE(64, GGML_TYPE_F16, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu
new file mode 100644
index 00000000..ea8f8aa4
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu
new file mode 100644
index 00000000..8f504656
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu
new file mode 100644
index 00000000..ab860347
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu
new file mode 100644
index 00000000..77bff98e
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu
new file mode 100644
index 00000000..235714f0
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu
new file mode 100644
index 00000000..6bf113ef
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu
new file mode 100644
index 00000000..23e0a8c0
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu
new file mode 100644
index 00000000..4a77ace9
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu
new file mode 100644
index 00000000..02f804e3
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu
new file mode 100644
index 00000000..309a6be6
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu
new file mode 100644
index 00000000..241795bd
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu
new file mode 100644
index 00000000..50166446
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu
new file mode 100644
index 00000000..b3affe61
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu
new file mode 100644
index 00000000..a0e65a0c
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu
new file mode 100644
index 00000000..ecaa5d17
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu
new file mode 100644
index 00000000..d6b289c6
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu
new file mode 100644
index 00000000..36a2a3d4
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu
new file mode 100644
index 00000000..96ab4e8c
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu
new file mode 100644
index 00000000..ae224afe
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu
new file mode 100644
index 00000000..1ac462ca
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu
new file mode 100644
index 00000000..74d21b53
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu
new file mode 100644
index 00000000..4f146c76
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu
new file mode 100644
index 00000000..6a4170bf
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu
new file mode 100644
index 00000000..b57023ae
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu
new file mode 100644
index 00000000..e84fab88
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu
new file mode 100644
index 00000000..3a78463f
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu
new file mode 100644
index 00000000..75654de9
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu
new file mode 100644
index 00000000..83fe95c1
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu
new file mode 100644
index 00000000..13d7e745
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu
new file mode 100644
index 00000000..59d87603
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu
new file mode 100644
index 00000000..de9f5f40
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu
new file mode 100644
index 00000000..4fc0dd24
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu
new file mode 100644
index 00000000..84251d1e
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu
new file mode 100644
index 00000000..36f8c3ac
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu
new file mode 100644
index 00000000..384fc92f
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu
new file mode 100644
index 00000000..fb47bc9f
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu
new file mode 100644
index 00000000..8887b421
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu
new file mode 100644
index 00000000..19e4e297
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(64, GGML_TYPE_F16, GGML_TYPE_F16);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu
new file mode 100644
index 00000000..e565fdb4
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(64, GGML_TYPE_F16, GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu
new file mode 100644
index 00000000..dcb661ff
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(64, GGML_TYPE_F16, GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu
new file mode 100644
index 00000000..9a9e6abb
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(64, GGML_TYPE_F16, GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu
new file mode 100644
index 00000000..0e21fd0a
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(64, GGML_TYPE_F16, GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu
new file mode 100644
index 00000000..f699f56b
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec-f32.cuh"
+
+DECL_FATTN_VEC_F32_CASE(64, GGML_TYPE_F16, GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu b/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu
new file mode 100644
index 00000000..9b6a210c
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu
@@ -0,0 +1,36 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-wmma-f16.cuh"
+
+DECL_FATTN_WMMA_F16_CASE(64, 16, float);
+DECL_FATTN_WMMA_F16_CASE(80, 16, float);
+DECL_FATTN_WMMA_F16_CASE(96, 16, float);
+DECL_FATTN_WMMA_F16_CASE(112, 16, float);
+DECL_FATTN_WMMA_F16_CASE(128, 16, float);
+DECL_FATTN_WMMA_F16_CASE(256, 16, float);
diff --git a/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu b/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu
new file mode 100644
index 00000000..74e78f9e
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu
@@ -0,0 +1,35 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-wmma-f16.cuh"
+
+DECL_FATTN_WMMA_F16_CASE(64, 32, float);
+DECL_FATTN_WMMA_F16_CASE(80, 32, float);
+DECL_FATTN_WMMA_F16_CASE(96, 32, float);
+DECL_FATTN_WMMA_F16_CASE(112, 32, float);
+DECL_FATTN_WMMA_F16_CASE(128, 32, float);
diff --git a/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu b/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu
new file mode 100644
index 00000000..8fc18cef
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu
@@ -0,0 +1,36 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-wmma-f16.cuh"
+
+DECL_FATTN_WMMA_F16_CASE(64, 16, half);
+DECL_FATTN_WMMA_F16_CASE(80, 16, half);
+DECL_FATTN_WMMA_F16_CASE(96, 16, half);
+DECL_FATTN_WMMA_F16_CASE(112, 16, half);
+DECL_FATTN_WMMA_F16_CASE(128, 16, half);
+DECL_FATTN_WMMA_F16_CASE(256, 16, half);
diff --git a/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu b/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu
new file mode 100644
index 00000000..d877e1d2
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu
@@ -0,0 +1,36 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-wmma-f16.cuh"
+
+DECL_FATTN_WMMA_F16_CASE(64, 32, half);
+DECL_FATTN_WMMA_F16_CASE(80, 32, half);
+DECL_FATTN_WMMA_F16_CASE(96, 32, half);
+DECL_FATTN_WMMA_F16_CASE(112, 32, half);
+DECL_FATTN_WMMA_F16_CASE(128, 32, half);
+DECL_FATTN_WMMA_F16_CASE(256, 32, half);
diff --git a/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu b/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu
new file mode 100644
index 00000000..987915a4
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu
@@ -0,0 +1,34 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-wmma-f16.cuh"
+
+DECL_FATTN_WMMA_F16_CASE(64, 8, half);
+DECL_FATTN_WMMA_F16_CASE(96, 8, half);
+DECL_FATTN_WMMA_F16_CASE(128, 8, half);
+DECL_FATTN_WMMA_F16_CASE(256, 8, half);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-iq1_s.cu b/llama/ggml-cuda/template-instances/mmq-instance-iq1_s.cu
new file mode 100644
index 00000000..03486aaf
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-iq1_s.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-iq2_s.cu b/llama/ggml-cuda/template-instances/mmq-instance-iq2_s.cu
new file mode 100644
index 00000000..ba63cbac
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-iq2_s.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ2_S);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu b/llama/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu
new file mode 100644
index 00000000..3c1283c7
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu b/llama/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu
new file mode 100644
index 00000000..a2344a5d
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-iq3_s.cu b/llama/ggml-cuda/template-instances/mmq-instance-iq3_s.cu
new file mode 100644
index 00000000..197b619f
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-iq3_s.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ3_S);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu b/llama/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu
new file mode 100644
index 00000000..8edd4ad9
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu b/llama/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu
new file mode 100644
index 00000000..a1c3902f
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu b/llama/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu
new file mode 100644
index 00000000..c421db27
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-q2_k.cu b/llama/ggml-cuda/template-instances/mmq-instance-q2_k.cu
new file mode 100644
index 00000000..726a07a9
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-q2_k.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q2_K);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-q3_k.cu b/llama/ggml-cuda/template-instances/mmq-instance-q3_k.cu
new file mode 100644
index 00000000..e0781f05
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-q3_k.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q3_K);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-q4_0.cu b/llama/ggml-cuda/template-instances/mmq-instance-q4_0.cu
new file mode 100644
index 00000000..e6d79a89
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-q4_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q4_0);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-q4_1.cu b/llama/ggml-cuda/template-instances/mmq-instance-q4_1.cu
new file mode 100644
index 00000000..0c60bc56
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-q4_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q4_1);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-q4_k.cu b/llama/ggml-cuda/template-instances/mmq-instance-q4_k.cu
new file mode 100644
index 00000000..acd1f1f9
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-q4_k.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q4_K);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-q5_0.cu b/llama/ggml-cuda/template-instances/mmq-instance-q5_0.cu
new file mode 100644
index 00000000..c301011e
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-q5_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q5_0);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-q5_1.cu b/llama/ggml-cuda/template-instances/mmq-instance-q5_1.cu
new file mode 100644
index 00000000..15eb7d24
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-q5_1.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q5_1);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-q5_k.cu b/llama/ggml-cuda/template-instances/mmq-instance-q5_k.cu
new file mode 100644
index 00000000..2cc91773
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-q5_k.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q5_K);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-q6_k.cu b/llama/ggml-cuda/template-instances/mmq-instance-q6_k.cu
new file mode 100644
index 00000000..ccf7e83c
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-q6_k.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q6_K);
diff --git a/llama/ggml-cuda/template-instances/mmq-instance-q8_0.cu b/llama/ggml-cuda/template-instances/mmq-instance-q8_0.cu
new file mode 100644
index 00000000..276505ba
--- /dev/null
+++ b/llama/ggml-cuda/template-instances/mmq-instance-q8_0.cu
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q8_0);
diff --git a/llama/ggml-cuda/tsembd.cu b/llama/ggml-cuda/tsembd.cu
new file mode 100644
index 00000000..1b6cfa28
--- /dev/null
+++ b/llama/ggml-cuda/tsembd.cu
@@ -0,0 +1,73 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "tsembd.cuh"
+
+static __global__ void timestep_embedding_f32(const float * timesteps, float * dst, const int nb1, const int dim, const int max_period) {
+ // blockIDx.y: idx of timesteps->ne[0]
+ // blockIDx.x: idx of ((dim + 1) / 2) / BLOCK_SIZE
+ int i = blockIdx.y;
+ int j = threadIdx.x + blockIdx.x * blockDim.x;
+ float * embed_data = (float *)((char *)dst + i*nb1);
+
+ if (dim % 2 != 0 && j == ((dim + 1) / 2)) {
+ embed_data[dim] = 0.f;
+ }
+
+ int half = dim / 2;
+ if (j >= half) {
+ return;
+ }
+
+ float timestep = timesteps[i];
+ float freq = (float)expf(-logf(max_period) * j / half);
+ float arg = timestep * freq;
+ embed_data[j] = cosf(arg);
+ embed_data[j + half] = sinf(arg);
+}
+
+static void timestep_embedding_f32_cuda(const float * x, float * dst, const int ne00, const int nb1,
+ const int dim, const int max_period, cudaStream_t stream) {
+ int half_ceil = (dim + 1) / 2;
+ int num_blocks = (half_ceil + CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE - 1) / CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE;
+ dim3 gridDim(num_blocks, ne00, 1);
+ timestep_embedding_f32<<<gridDim, CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE, 0, stream>>>(x, dst, nb1, dim, max_period);
+}
+
+void ggml_cuda_op_timestep_embedding(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
+
+ const int dim = dst->op_params[0];
+ const int max_period = dst->op_params[1];
+
+ timestep_embedding_f32_cuda(src0_d, dst_d, src0->ne[0], dst->nb[1], dim, max_period, stream);
+}
diff --git a/llama/ggml-cuda/tsembd.cuh b/llama/ggml-cuda/tsembd.cuh
new file mode 100644
index 00000000..f5eec9b8
--- /dev/null
+++ b/llama/ggml-cuda/tsembd.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE 256
+
+void ggml_cuda_op_timestep_embedding(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/unary.cu b/llama/ggml-cuda/unary.cu
new file mode 100644
index 00000000..79ba0e57
--- /dev/null
+++ b/llama/ggml-cuda/unary.cu
@@ -0,0 +1,396 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "unary.cuh"
+
+static __global__ void gelu_f32(const float * x, float * dst, const int k) {
+ const float GELU_COEF_A = 0.044715f;
+ const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+
+ float xi = x[i];
+ dst[i] = 0.5f*xi*(1.0f + tanhf(SQRT_2_OVER_PI*xi*(1.0f + GELU_COEF_A*xi*xi)));
+}
+
+static __global__ void gelu_quick_f32(const float * x, float * dst, int k) {
+ const float GELU_QUICK_COEF = -1.702f;
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+ if (i >= k) {
+ return;
+ }
+ dst[i] = x[i] * (1.0f / (1.0f + expf(GELU_QUICK_COEF * x[i])));
+}
+
+static __global__ void silu_f32(const float * x, float * dst, const int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+ dst[i] = x[i] / (1.0f + expf(-x[i]));
+}
+
+static __global__ void tanh_f32(const float * x, float * dst, int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+ if (i >= k) {
+ return;
+ }
+ dst[i] = tanhf(x[i]);
+}
+
+static __global__ void relu_f32(const float * x, float * dst, const int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+ dst[i] = fmaxf(x[i], 0);
+}
+
+static __global__ void sigmoid_f32(const float * x, float * dst, const int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+ dst[i] = 1.0f / (1.0f + expf(-x[i]));
+}
+
+static __global__ void hardsigmoid_f32(const float * x, float * dst, const int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+ dst[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f));
+}
+
+static __global__ void hardswish_f32(const float * x, float * dst, const int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+ dst[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f));
+}
+
+static __global__ void leaky_relu_f32(const float * x, float * dst, const int k, const float negative_slope) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+ if (i >= k) {
+ return;
+ }
+ dst[i] = fmaxf(x[i], 0) + fminf(x[i], 0.0f) * negative_slope;
+}
+
+static __global__ void sqr_f32(const float * x, float * dst, const int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+ dst[i] = x[i] * x[i];
+}
+
+static __global__ void sqrt_f32(const float * x, float * dst, const int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+ dst[i] = sqrtf(x[i]);
+}
+
+static __global__ void sin_f32(const float * x, float * dst, const int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+ dst[i] = sinf(x[i]);
+}
+
+static __global__ void cos_f32(const float * x, float * dst, const int k) {
+ const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+ dst[i] = cosf(x[i]);
+}
+
+static void gelu_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_GELU_BLOCK_SIZE - 1) / CUDA_GELU_BLOCK_SIZE;
+ gelu_f32<<<num_blocks, CUDA_GELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+static void gelu_quick_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_GELU_BLOCK_SIZE - 1) / CUDA_GELU_BLOCK_SIZE;
+ gelu_quick_f32<<<num_blocks, CUDA_GELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+static void silu_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_SILU_BLOCK_SIZE - 1) / CUDA_SILU_BLOCK_SIZE;
+ silu_f32<<<num_blocks, CUDA_SILU_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+static void tanh_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_TANH_BLOCK_SIZE - 1) / CUDA_TANH_BLOCK_SIZE;
+ tanh_f32<<<num_blocks, CUDA_TANH_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+static void relu_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_RELU_BLOCK_SIZE - 1) / CUDA_RELU_BLOCK_SIZE;
+ relu_f32<<<num_blocks, CUDA_RELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+static void sigmoid_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_SIGMOID_BLOCK_SIZE - 1) / CUDA_SIGMOID_BLOCK_SIZE;
+ sigmoid_f32<<<num_blocks, CUDA_SIGMOID_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+static void hardsigmoid_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_HARDSIGMOID_BLOCK_SIZE - 1) / CUDA_HARDSIGMOID_BLOCK_SIZE;
+ hardsigmoid_f32<<<num_blocks, CUDA_HARDSIGMOID_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+static void hardswish_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_HARDSWISH_BLOCK_SIZE - 1) / CUDA_HARDSWISH_BLOCK_SIZE;
+ hardswish_f32<<<num_blocks, CUDA_HARDSWISH_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+static void leaky_relu_f32_cuda(const float * x, float * dst, const int k, const float negative_slope, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_RELU_BLOCK_SIZE - 1) / CUDA_RELU_BLOCK_SIZE;
+ leaky_relu_f32<<<num_blocks, CUDA_RELU_BLOCK_SIZE, 0, stream>>>(x, dst, k, negative_slope);
+}
+
+static void sqr_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_SQR_BLOCK_SIZE - 1) / CUDA_SQR_BLOCK_SIZE;
+ sqr_f32<<<num_blocks, CUDA_SQR_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+static void sqrt_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_SQRT_BLOCK_SIZE - 1) / CUDA_SQRT_BLOCK_SIZE;
+ sqrt_f32<<<num_blocks, CUDA_SQRT_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+static void sin_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_SIN_BLOCK_SIZE - 1) / CUDA_SIN_BLOCK_SIZE;
+ sin_f32<<<num_blocks, CUDA_SIN_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+static void cos_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_COS_BLOCK_SIZE - 1) / CUDA_COS_BLOCK_SIZE;
+ cos_f32<<<num_blocks, CUDA_COS_BLOCK_SIZE, 0, stream>>>(x, dst, k);
+}
+
+void ggml_cuda_op_gelu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ gelu_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_silu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ silu_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_gelu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ gelu_quick_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_tanh(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ tanh_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ relu_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_sigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ sigmoid_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_hardsigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ hardsigmoid_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_hardswish(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ hardswish_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_leaky_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ float negative_slope;
+ memcpy(&negative_slope, dst->op_params, sizeof(float));
+
+ leaky_relu_f32_cuda(src0_d, dst_d, ggml_nelements(src0), negative_slope, stream);
+}
+
+void ggml_cuda_op_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ sqr_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_sqrt(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ sqrt_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_sin(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ sin_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
+
+void ggml_cuda_op_cos(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ cos_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);
+}
diff --git a/llama/ggml-cuda/unary.cuh b/llama/ggml-cuda/unary.cuh
new file mode 100644
index 00000000..d60e47e4
--- /dev/null
+++ b/llama/ggml-cuda/unary.cuh
@@ -0,0 +1,65 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_GELU_BLOCK_SIZE 256
+#define CUDA_SILU_BLOCK_SIZE 256
+#define CUDA_TANH_BLOCK_SIZE 256
+#define CUDA_RELU_BLOCK_SIZE 256
+#define CUDA_SIGMOID_BLOCK_SIZE 256
+#define CUDA_HARDSIGMOID_BLOCK_SIZE 256
+#define CUDA_HARDSWISH_BLOCK_SIZE 256
+#define CUDA_SQR_BLOCK_SIZE 256
+#define CUDA_SQRT_BLOCK_SIZE 256
+#define CUDA_SIN_BLOCK_SIZE 256
+#define CUDA_COS_BLOCK_SIZE 256
+
+void ggml_cuda_op_gelu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_silu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_gelu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_tanh(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_sigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_hardsigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_hardswish(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_leaky_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_sqrt(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_sin(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_cos(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/upscale.cu b/llama/ggml-cuda/upscale.cu
new file mode 100644
index 00000000..d2f4ce00
--- /dev/null
+++ b/llama/ggml-cuda/upscale.cu
@@ -0,0 +1,77 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "upscale.cuh"
+
+static __global__ void upscale_f32(const float * x, float * dst,
+ const int nb00, const int nb01, const int nb02, const int nb03,
+ const int ne10, const int ne11, const int ne12, const int ne13,
+ const float sf0, const float sf1, const float sf2, const float sf3) {
+ int index = threadIdx.x + blockIdx.x * blockDim.x;
+ if (index >= ne10 * ne11 * ne12 * ne13) {
+ return;
+ }
+
+ int i10 = index % ne10;
+ int i11 = (index / ne10) % ne11;
+ int i12 = (index / (ne10 * ne11)) % ne12;
+ int i13 = (index / (ne10 * ne11 * ne12)) % ne13;
+
+ int i00 = i10 / sf0;
+ int i01 = i11 / sf1;
+ int i02 = i12 / sf2;
+ int i03 = i13 / sf3;
+
+ dst[index] = *(float *)((char *)x + i03 * nb03 + i02 * nb02 + i01 * nb01 + i00 * nb00);
+}
+
+static void upscale_f32_cuda(const float * x, float * dst,
+ const int nb00, const int nb01, const int nb02, const int nb03,
+ const int ne10, const int ne11, const int ne12, const int ne13,
+ const float sf0, const float sf1, const float sf2, const float sf3,
+ cudaStream_t stream) {
+ int dst_size = ne10 * ne11 * ne12 * ne13;
+ int num_blocks = (dst_size + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE;
+
+ upscale_f32<<<num_blocks, CUDA_UPSCALE_BLOCK_SIZE,0,stream>>>(x, dst, nb00, nb01, nb02, nb03, ne10, ne11, ne12, ne13, sf0, sf1, sf2, sf3);
+}
+
+void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const float * src0_d = (const float *)src0->data;
+ float * dst_d = (float *)dst->data;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ const float sf0 = (float)dst->ne[0]/src0->ne[0];
+ const float sf1 = (float)dst->ne[1]/src0->ne[1];
+ const float sf2 = (float)dst->ne[2]/src0->ne[2];
+ const float sf3 = (float)dst->ne[3]/src0->ne[3];
+
+ upscale_f32_cuda(src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], sf0, sf1, sf2, sf3, stream);
+}
diff --git a/llama/ggml-cuda/upscale.cuh b/llama/ggml-cuda/upscale.cuh
new file mode 100644
index 00000000..cc4e4aa1
--- /dev/null
+++ b/llama/ggml-cuda/upscale.cuh
@@ -0,0 +1,31 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+
+#define CUDA_UPSCALE_BLOCK_SIZE 256
+
+void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/llama/ggml-cuda/vecdotq.cuh b/llama/ggml-cuda/vecdotq.cuh
new file mode 100644
index 00000000..cc69f02c
--- /dev/null
+++ b/llama/ggml-cuda/vecdotq.cuh
@@ -0,0 +1,1159 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "common.cuh"
+#include <cstdint>
+
+static __device__ __forceinline__ int get_int_b2(const void * x, const int & i32) {
+ const uint16_t * x16 = (const uint16_t *) x; // assume at least 2 byte alignment
+
+ int x32 = x16[2*i32 + 0] << 0;
+ x32 |= x16[2*i32 + 1] << 16;
+
+ return x32;
+}
+
+static __device__ __forceinline__ int get_int_b4(const void * x, const int & i32) {
+ return ((const int *) x)[i32]; // assume at least 4 byte alignment
+}
+
+// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called
+// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q
+
+#define VDR_Q4_0_Q8_1_MMVQ 2
+#define VDR_Q4_0_Q8_1_MMQ 4
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(
+ const int * v, const int * u, const float & d4, const half2 & ds8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
+
+ // SIMD dot product of quantized values
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);
+ }
+
+ const float2 ds8f = __half22float2(ds8);
+
+ // second part effectively subtracts 8 from each quant value
+ return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);
+}
+
+#define VDR_Q4_1_Q8_1_MMVQ 2
+#define VDR_Q4_1_Q8_1_MMQ 4
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(
+ const int * v, const int * u, const half2 & dm4, const half2 & ds8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
+
+ // SIMD dot product of quantized values
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);
+ }
+
+#ifdef GGML_CUDA_F16
+ const float2 tmp = __half22float2(__hmul2(dm4, ds8));
+ const float d4d8 = tmp.x;
+ const float m4s8 = tmp.y;
+#else
+ const float2 dm4f = __half22float2(dm4);
+ const float2 ds8f = __half22float2(ds8);
+ const float d4d8 = dm4f.x * ds8f.x;
+ const float m4s8 = dm4f.y * ds8f.y;
+#endif // GGML_CUDA_F16
+
+ // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it
+ return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));
+}
+
+#define VDR_Q5_0_Q8_1_MMVQ 2
+#define VDR_Q5_0_Q8_1_MMQ 4
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(
+ const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
+
+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
+ }
+
+ const float2 ds8f = __half22float2(ds8);
+
+ // second part effectively subtracts 16 from each quant value
+ return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);
+}
+
+#define VDR_Q5_1_Q8_1_MMVQ 2
+#define VDR_Q5_1_Q8_1_MMQ 4
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(
+ const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
+
+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
+ }
+
+#ifdef GGML_CUDA_F16
+ const float2 tmp = __half22float2(__hmul2(dm5, ds8));
+ const float d5d8 = tmp.x;
+ const float m5s8 = tmp.y;
+#else
+ const float2 dm5f = __half22float2(dm5);
+ const float2 ds8f = __half22float2(ds8);
+ const float d5d8 = dm5f.x * ds8f.x;
+ const float m5s8 = dm5f.y * ds8f.y;
+#endif // GGML_CUDA_F16
+
+ // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it
+ return sumi*d5d8 + m5s8 / (QI5_1 / vdr);
+}
+
+#define VDR_Q8_0_Q8_1_MMVQ 2
+#define VDR_Q8_0_Q8_1_MMQ 8
+
+template <typename T, int vdr> static __device__ __forceinline__ T vec_dot_q8_0_q8_1_impl(
+ const int * v, const int * u, const T & d8_0, const T & d8_1) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ // SIMD dot product of quantized values
+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
+ }
+
+ return d8_0*d8_1 * ((T) sumi);
+}
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(
+ const int * v, const int * u, const half2 & dm8, const half2 & ds8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ // SIMD dot product of quantized values
+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
+ }
+
+#ifdef GGML_CUDA_F16
+ const float2 tmp = __half22float2(__hmul2(dm8, ds8));
+ const float d8d8 = tmp.x;
+ const float m8s8 = tmp.y;
+#else
+ const float2 dm8f = __half22float2(dm8);
+ const float2 ds8f = __half22float2(ds8);
+ const float d8d8 = dm8f.x * ds8f.x;
+ const float m8s8 = dm8f.y * ds8f.y;
+#endif // GGML_CUDA_F16
+
+ // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it
+ return sumi*d8d8 + m8s8 / (QI8_1 / vdr);
+}
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_16_q8_1_impl(
+ const int * v, const int * u, const float * d8_0, const float & d8_1) {
+
+ float sumf = 0.0f;
+
+#pragma unroll
+ for (int i0 = 0; i0 < vdr; i0 += QI8_0/2) {
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = i0; i < i0 + QI8_0/2; ++i) {
+ // SIMD dot product of quantized values
+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
+ }
+
+ sumf += d8_0[i0/(QI8_0/2)]*sumi;
+ }
+
+ return d8_1*sumf;
+}
+
+#define VDR_Q2_K_Q8_1_MMVQ 1
+#define VDR_Q2_K_Q8_1_MMQ 4
+
+// contiguous v/x values
+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
+ const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
+ const half2 & dm2, const float * __restrict__ d8) {
+
+ float sumf_d = 0.0f;
+ float sumf_m = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR2_K; ++i) {
+ const int sc = scales[2*i];
+
+ const int vi = (v >> (2*i)) & 0x03030303;
+
+ sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product
+
+ // fill int with 4x m
+ int m = sc >> 4;
+ m |= m << 8;
+ m |= m << 16;
+ sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values
+ }
+
+ const float2 dm2f = __half22float2(dm2);
+
+ return dm2f.x*sumf_d - dm2f.y*sumf_m;
+}
+
+// contiguous v/x + u/y values
+template <int ns8>
+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
+ const int * __restrict__ v, const int * __restrict__ u, const half2 * dm2, const float & d8, const half2 * s8) {
+
+ float sumf = 0.0f;
+ float sumf_d8 = 0.0f;
+
+#pragma unroll
+ for (int i0 = 0; i0 < QR2_K*VDR_Q2_K_Q8_1_MMQ; i0 += QI8_1) {
+ const float2 dm2f0 = __half22float2(dm2[i0/(QI8_1/2) + 0]);
+ int sumi_d0 = 0;
+
+ const float2 dm2f1 = __half22float2(dm2[i0/(QI8_1/2) + 1]);
+ int sumi_d1 = 0;
+
+#pragma unroll
+ for (int i = i0; i < i0 + QI8_1/2; ++i) {
+ sumi_d0 = ggml_cuda_dp4a(v[i], u[i], sumi_d0);
+ }
+ sumf_d8 += dm2f0.x * sumi_d0;
+
+#pragma unroll
+ for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {
+ sumi_d1 = ggml_cuda_dp4a(v[i], u[i], sumi_d1);
+ }
+ sumf_d8 += dm2f1.x * sumi_d1;
+
+ if (i0/QI8_1 < ns8) {
+ const float2 s8f = __half22float2(s8[i0/QI8_1]);
+ sumf -= dm2f0.y*s8f.x;
+ sumf -= dm2f1.y*s8f.y;
+ } else {
+ int sumi_m0 = 0;
+#pragma unroll
+ for (int i = i0; i < i0 + QI8_1/2; ++i) {
+ sumi_m0 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m0);
+ }
+ sumf_d8 -= dm2f0.y * sumi_m0;
+
+ int sumi_m1 = 0;
+#pragma unroll
+ for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {
+ sumi_m1 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m1);
+ }
+ sumf_d8 -= dm2f1.y * sumi_m1;
+ }
+ }
+
+ return sumf + d8*sumf_d8;
+}
+
+#define VDR_Q3_K_Q8_1_MMVQ 1
+#define VDR_Q3_K_Q8_1_MMQ 2
+
+// contiguous v/x values
+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
+ const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,
+ const int & scale_offset, const float & d3, const float * __restrict__ d8) {
+
+ float sumf = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR3_K; ++i) {
+ const int isc = scale_offset + 2*i;
+
+ const int isc_low = isc % (QK_K/32);
+ const int sc_shift_low = 4 * (isc / (QK_K/32));
+ const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;
+
+ const int isc_high = isc % (QK_K/64);
+ const int sc_shift_high = 2 * (isc / (QK_K/64));
+ const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;
+
+ const int sc = (sc_low | sc_high) - 32;
+
+ const int vil = (vl >> (2*i)) & 0x03030303;
+
+ const int vih = ((vh >> i) << 2) & 0x04040404;
+
+ const int vi = __vsubss4(vil, vih);
+
+ sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
+ }
+
+ return d3 * sumf;
+}
+
+// contiguous v/x + u/y values
+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,
+ const float & d3, const float & d8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {
+ int sumi_sc = 0;
+
+#pragma unroll
+ for (int i = i0; i < i0 + QI8_1/2; ++i) {
+ sumi_sc = ggml_cuda_dp4a(v[i], u[i], sumi_sc); // SIMD dot product
+ }
+
+ sumi += sumi_sc * scales[i0 / (QI8_1/2)];
+ }
+
+ return d3*d8 * sumi;
+}
+
+#define VDR_Q4_K_Q8_1_MMVQ 2
+#define VDR_Q4_K_Q8_1_MMQ 8
+
+// contiguous v/x values
+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
+ const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {
+
+ float sumf_d = 0.0f;
+ float sumf_m = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR4_K; ++i) {
+ const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;
+ const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;
+
+ const int dot1 = ggml_cuda_dp4a(v1i, u[2*i+1], ggml_cuda_dp4a(v0i, u[2*i+0], 0)); // SIMD dot product
+ const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+1], ggml_cuda_dp4a(0x01010101, u[2*i+0], 0)); // sum of u
+
+ sumf_d += d8[i] * (dot1 * sc[i]);
+ sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values
+ }
+
+ const float2 dm4f = __half22float2(dm4);
+
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
+}
+
+// contiguous v/x + u/y values
+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
+
+ float sumf_d = 0.0f;
+ float sumf_m = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {
+ int sumi_d = 0;
+
+#pragma unroll
+ for (int j = 0; j < QI8_1; ++j) {
+ sumi_d = ggml_cuda_dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product
+ }
+
+ const float2 ds8f = __half22float2(ds8[i]);
+
+ sumf_d += ds8f.x * (sc[i] * sumi_d);
+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
+ }
+
+ const float2 dm4f = __half22float2(dm4);
+
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
+}
+
+#define VDR_Q5_K_Q8_1_MMVQ 2
+#define VDR_Q5_K_Q8_1_MMQ 8
+
+// contiguous v/x values
+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
+ const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,
+ const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {
+
+ float sumf_d = 0.0f;
+ float sumf_m = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR5_K; ++i) {
+ const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;
+ const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;
+
+ const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;
+ const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;
+
+ const int v0i = vl0i | vh0i;
+ const int v1i = vl1i | vh1i;
+
+ const int dot1 = ggml_cuda_dp4a(v0i, u[2*i+0], ggml_cuda_dp4a(v1i, u[2*i+1], 0)); // SIMD dot product
+ const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+0], ggml_cuda_dp4a(0x01010101, u[2*i+1], 0)); // sum of u
+
+ sumf_d += d8[i] * (dot1 * sc[i]);
+ sumf_m += d8[i] * (dot2 * m[i]);
+
+ }
+
+ const float2 dm5f = __half22float2(dm5);
+
+ return dm5f.x*sumf_d - dm5f.y*sumf_m;
+}
+
+// contiguous v/x + u/y values
+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
+
+ float sumf_d = 0.0f;
+ float sumf_m = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {
+ int sumi_d = 0;
+
+#pragma unroll
+ for (int j = 0; j < QI8_1; ++j) {
+ sumi_d = ggml_cuda_dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product
+ }
+
+ const float2 ds8f = __half22float2(ds8[i]);
+
+ sumf_d += ds8f.x * (sc[i] * sumi_d);
+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
+ }
+
+ const float2 dm4f = __half22float2(dm4);
+
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
+}
+
+#define VDR_Q6_K_Q8_1_MMVQ 1
+#define VDR_Q6_K_Q8_1_MMQ 8
+
+// contiguous v/x values
+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
+ const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
+ const float & d, const float * __restrict__ d8) {
+
+ float sumf = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR6_K; ++i) {
+ const int sc = scales[4*i];
+
+ const int vil = (vl >> (4*i)) & 0x0F0F0F0F;
+
+ const int vih = ((vh >> (4*i)) << 4) & 0x30303030;
+
+ const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32
+
+ sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
+ }
+
+ return d*sumf;
+}
+
+// contiguous v/x + u/y values
+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(
+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,
+ const float & d6, const float * __restrict__ d8) {
+
+ float sumf_d = 0.0f;
+
+ const int sc_packed = get_int_b4(sc, 0);
+ const int8_t * sc_reg = (const int8_t *) &sc_packed;
+
+#pragma unroll
+ for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {
+ int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale
+
+#pragma unroll
+ for (int i = i0; i < i0 + 2; ++i) {
+ sumi_d.x = ggml_cuda_dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product
+ sumi_d.x = ggml_cuda_dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product
+
+ sumi_d.y = ggml_cuda_dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product
+ sumi_d.y = ggml_cuda_dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product
+ }
+
+ sumf_d += d8[i0/4] * (sc_reg[i0/2+0]*sumi_d.x + sc_reg[i0/2+1]*sumi_d.y);
+ }
+
+ return d6 * sumf_d;
+}
+
+static __device__ __forceinline__ float vec_dot_q4_0_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq + kbx;
+
+ int v[VDR_Q4_0_Q8_1_MMVQ];
+ int u[2*VDR_Q4_0_Q8_1_MMVQ];
+
+#pragma unroll
+ for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {
+ v[i] = get_int_b2(bq4_0->qs, iqs + i);
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_0);
+ }
+
+ return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, bq4_0->d, bq8_1->ds);
+}
+
+
+static __device__ __forceinline__ float vec_dot_q4_1_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq + kbx;
+
+ int v[VDR_Q4_1_Q8_1_MMVQ];
+ int u[2*VDR_Q4_1_Q8_1_MMVQ];
+
+#pragma unroll
+ for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {
+ v[i] = get_int_b4(bq4_1->qs, iqs + i);
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_1);
+ }
+
+ return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);
+}
+
+static __device__ __forceinline__ float vec_dot_q5_0_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq + kbx;
+
+ int vl[VDR_Q5_0_Q8_1_MMVQ];
+ int vh[VDR_Q5_0_Q8_1_MMVQ];
+ int u[2*VDR_Q5_0_Q8_1_MMVQ];
+
+#pragma unroll
+ for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {
+ vl[i] = get_int_b2(bq5_0->qs, iqs + i);
+ vh[i] = get_int_b2(bq5_0->qh, 0) >> (4 * (iqs + i));
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_0);
+ }
+
+ return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, bq5_0->d, bq8_1->ds);
+}
+
+static __device__ __forceinline__ float vec_dot_q5_1_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq + kbx;
+
+ int vl[VDR_Q5_1_Q8_1_MMVQ];
+ int vh[VDR_Q5_1_Q8_1_MMVQ];
+ int u[2*VDR_Q5_1_Q8_1_MMVQ];
+
+#pragma unroll
+ for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {
+ vl[i] = get_int_b4(bq5_1->qs, iqs + i);
+ vh[i] = get_int_b4(bq5_1->qh, 0) >> (4 * (iqs + i));
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_1);
+ }
+
+ return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);
+}
+
+static __device__ __forceinline__ float vec_dot_q8_0_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq + kbx;
+
+ int v[VDR_Q8_0_Q8_1_MMVQ];
+ int u[VDR_Q8_0_Q8_1_MMVQ];
+
+#pragma unroll
+ for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {
+ v[i] = get_int_b2(bq8_0->qs, iqs + i);
+ u[i] = get_int_b4(bq8_1->qs, iqs + i);
+ }
+
+ return vec_dot_q8_0_q8_1_impl<float, VDR_Q8_0_Q8_1_MMVQ>(v, u, bq8_0->d, __low2half(bq8_1->ds));
+}
+
+static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx;
+
+ const int bq8_offset = QR2_K * (iqs / QI8_1);
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
+
+ const uint8_t * scales = bq2_K->scales + scale_offset;
+
+ const int v = get_int_b4(bq2_K->qs, iqs);
+ int u[QR2_K];
+ float d8[QR2_K];
+
+#pragma unroll
+ for (int i = 0; i < QR2_K; ++ i) {
+ u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
+ }
+
+ return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);
+}
+
+static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx;
+
+ const int bq8_offset = QR3_K * (iqs / (QI3_K/2));
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
+
+ const float d = bq3_K->d;
+
+ const int vl = get_int_b2(bq3_K->qs, iqs);
+
+ // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
+ const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;
+
+ int u[QR3_K];
+ float d8[QR3_K];
+
+#pragma unroll
+ for (int i = 0; i < QR3_K; ++i) {
+ u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
+ }
+
+ return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);
+}
+
+static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q4_K * bq4_K = (const block_q4_K *) vbq + kbx;
+
+ int v[2];
+ int u[2*QR4_K];
+ float d8[QR4_K];
+
+ // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6
+ const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));
+
+ // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12
+ // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44
+ // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76
+ // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108
+
+ const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
+ v[0] = q4[0];
+ v[1] = q4[4];
+
+ const uint16_t * scales = (const uint16_t *)bq4_K->scales;
+ uint16_t aux[2];
+ const int j = bq8_offset/2;
+ if (j < 2) {
+ aux[0] = scales[j+0] & 0x3f3f;
+ aux[1] = scales[j+2] & 0x3f3f;
+ } else {
+ aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
+ aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
+ }
+ const uint8_t * sc = (const uint8_t *)aux;
+ const uint8_t * m = sc + 2;
+
+ for (int i = 0; i < QR4_K; ++i) {
+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
+ d8[i] = __low2float(bq8i->ds);
+
+ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
+ u[2*i+0] = q8[0];
+ u[2*i+1] = q8[4];
+ }
+
+ return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);
+}
+
+static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q5_K * bq5_K = (const block_q5_K *) vbq + kbx;
+
+ int vl[2];
+ int vh[2];
+ int u[2*QR5_K];
+ float d8[QR5_K];
+
+ const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));
+ const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
+ const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));
+
+ vl[0] = ql[0];
+ vl[1] = ql[4];
+
+ vh[0] = qh[0] >> bq8_offset;
+ vh[1] = qh[4] >> bq8_offset;
+
+ const uint16_t * scales = (const uint16_t *)bq5_K->scales;
+ uint16_t aux[2];
+ const int j = bq8_offset/2;
+ if (j < 2) {
+ aux[0] = scales[j+0] & 0x3f3f;
+ aux[1] = scales[j+2] & 0x3f3f;
+ } else {
+ aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
+ aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
+ }
+ const uint8_t * sc = (const uint8_t *)aux;
+ const uint8_t * m = sc + 2;
+
+#pragma unroll
+ for (int i = 0; i < QR5_K; ++i) {
+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
+ d8[i] = __low2float(bq8i->ds);
+
+ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
+ u[2*i+0] = q8[0];
+ u[2*i+1] = q8[4];
+ }
+
+ return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);
+}
+
+static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q6_K * bq6_K = (const block_q6_K *) vbq + kbx;
+
+ const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);
+ const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);
+ const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));
+
+ const int vl = get_int_b2(bq6_K->ql, iqs);
+ const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;
+
+ const int8_t * scales = bq6_K->scales + scale_offset;
+
+ int u[QR6_K];
+ float d8[QR6_K];
+
+#pragma unroll
+ for (int i = 0; i < QR6_K; ++i) {
+ u[i] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);
+ d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);
+ }
+
+ return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8);
+}
+
+#define VDR_IQ2_XXS_Q8_1_MMVQ 2
+#define VDR_IQ2_XXS_Q8_1_MMQ 2
+
+static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq + kbx;
+
+ const int q2 = get_int_b2(bq2->qs, iqs);
+ const uint8_t * aux8 = (const uint8_t *) &q2;
+ const uint32_t aux32 = get_int_b2(bq2->qs, iqs + 1);
+
+ int sumi = 0;
+#pragma unroll
+ for (int k0 = 0; k0 < 8; k0 += 2) {
+ const int * grid_pos = (const int *) (iq2xxs_grid + aux8[k0/2]);
+ const int signs_packed = ksigns_iq2xs[(aux32 >> (7*k0/2)) & 0x7F];
+
+ const int signs0 = __vcmpne4(((signs_packed & 0x03) << 7) | ((signs_packed & 0x0C) << 21), 0x00000000);
+ const int grid0 = __vsub4(grid_pos[0] ^ signs0, signs0);
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, k0 + 0);
+ sumi = ggml_cuda_dp4a(grid0, u0, sumi);
+
+ const int signs1 = __vcmpne4(((signs_packed & 0x30) << 3) | ((signs_packed & 0xC0) << 17), 0x00000000);
+ const int grid1 = __vsub4(grid_pos[1] ^ signs1, signs1);
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, k0 + 1);
+ sumi = ggml_cuda_dp4a(grid1, u1, sumi);
+ }
+
+ const int ls = aux32 >> 28;
+ sumi = (ls*sumi + sumi/2)/4;
+ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
+ return d * sumi;
+}
+
+#define VDR_IQ2_XS_Q8_1_MMVQ 2
+#define VDR_IQ2_XS_Q8_1_MMQ 2
+
+static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq + kbx;
+
+ const int2 q2_packed = make_int2(get_int_b2(bq2->qs, iqs + 0), get_int_b2(bq2->qs, iqs + 1));
+ const uint16_t * q2 = (const uint16_t *) &q2_packed;
+ const int ls0 = bq2->scales[iqs/2] & 0x0F;
+ const int ls1 = bq2->scales[iqs/2] >> 4;
+
+ int sumi0 = 0;
+ int sumi1 = 0;
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const uint32_t * grid_pos = (const uint32_t *)(iq2xs_grid + (q2[l0/2] & 0x000001FF));
+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l0/2] >> 9));
+
+ const int grid_l = __vsub4(grid_pos[0] ^ signs[0], signs[0]);
+ const int grid_h = __vsub4(grid_pos[1] ^ signs[1], signs[1]);
+
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
+
+ if (l0 < 4) {
+ sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);
+ sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);
+ } else {
+ sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);
+ sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);
+ }
+ }
+ const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;
+ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
+ return d * sumi;
+}
+
+#define VDR_IQ2_S_Q8_1_MMVQ 2
+#define VDR_IQ2_S_Q8_1_MMQ 2
+
+static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq2_s * bq2 = (const block_iq2_s *) vbq + kbx;
+
+ const int qs_packed = get_int_b2(bq2->qs, iqs/2);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bq2->qh[iqs/2];
+
+ const int signs_packed_32 = get_int_b2(bq2->qs, QK_K/32 + iqs/2);
+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
+
+ const int ls0 = bq2->scales[iqs/2] & 0x0F;
+ const int ls1 = bq2->scales[iqs/2] >> 4;
+
+ int sumi0 = 0;
+ int sumi1 = 0;
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const int * grid_pos = (const int *)(iq2s_grid + (qs[l0/2] | ((qh << (8-l0)) & 0x300)));
+
+ const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);
+ const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);
+
+ const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);
+ const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);
+
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
+
+ if (l0 < 4) {
+ sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);
+ sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);
+ } else {
+ sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);
+ sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);
+ }
+ }
+ const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;
+
+ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
+ return d * sumi;
+}
+
+#define VDR_IQ3_XXS_Q8_1_MMVQ 2
+#define VDR_IQ3_XXS_Q8_1_MMQ 2
+
+static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq3_xxs * bq3 = (const block_iq3_xxs *) vbq + kbx;
+
+ const int2 q3_packed = make_int2(get_int_b2(bq3->qs, iqs), get_int_b2(bq3->qs, iqs+1));
+ const uint8_t * q3 = (const uint8_t *) &q3_packed;
+ const uint32_t aux32 = get_int_b2(bq3->qs, QK_K/16 + iqs/2);
+
+ int sumi = 0;
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const int2 grid_pos = make_int2(iq3xxs_grid[q3[l0 + 0]], iq3xxs_grid[q3[l0 + 1]]);
+
+ const int * signs = (const int *)(ksigns64 + ((aux32 >> (7*l0/2)) & 0x7F));
+
+ const int grid_l = __vsub4(grid_pos.x ^ signs[0], signs[0]);
+ const int grid_h = __vsub4(grid_pos.y ^ signs[1], signs[1]);
+
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
+
+ sumi = ggml_cuda_dp4a(grid_l, u0, sumi);
+ sumi = ggml_cuda_dp4a(grid_h, u1, sumi);
+ }
+
+ const int ls = aux32 >> 28;
+ sumi = (ls*sumi + sumi/2)/2;
+ const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);
+ return d * sumi;
+}
+
+#define VDR_IQ3_S_Q8_1_MMVQ 2
+#define VDR_IQ3_S_Q8_1_MMQ 2
+
+// TODO: don't use lookup table for signs
+static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq3_s * bq3 = (const block_iq3_s *) vbq + kbx;
+
+ const int2 qs_packed = make_int2(get_int_b2(bq3->qs, iqs + 0), get_int_b2(bq3->qs, iqs + 1));
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bq3->qh[iqs/2];
+
+ const int signs_packed_32 = get_int_b2(bq3->signs, iqs/2);
+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
+
+ int sumi = 0;
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const int2 grid_pos = make_int2(
+ iq3s_grid[qs[l0 + 0] | ((qh << (8 - l0)) & 0x100)],
+ iq3s_grid[qs[l0 + 1] | ((qh << (7 - l0)) & 0x100)]);
+
+ const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);
+ const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);
+
+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);
+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);
+
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
+
+ sumi = ggml_cuda_dp4a(grid_l, u0, sumi);
+ sumi = ggml_cuda_dp4a(grid_h, u1, sumi);
+ }
+
+ sumi *= 1 + 2*((bq3->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F);
+
+ const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);
+ return d * sumi;
+}
+
+#define VDR_IQ1_S_Q8_1_MMVQ 1
+#define VDR_IQ1_S_Q8_1_MMQ 1
+
+static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+ const block_iq1_s * bq1 = (const block_iq1_s *) vbq + kbx;
+
+ const int qs_packed = get_int_b2(bq1->qs, iqs);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bq1->qh[iqs];
+
+ int sumi = 0;
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const int grid = iq1s_grid_gpu[qs[l0/2] | (((qh >> 3*(l0/2)) & 0x07) << 8)];
+
+ const int grid0 = (grid >> 0) & 0x0F0F0F0F;
+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;
+
+ const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
+ const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
+
+ sumi = ggml_cuda_dp4a(grid0, u0, sumi);
+ sumi = ggml_cuda_dp4a(grid1, u1, sumi);
+ }
+
+ const float d1q = __half2float(bq1->d) * (((qh >> 11) & 0x0E) + 1);
+ const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);
+ const float2 ds = __half22float2(bq8_1[iqs].ds);
+ return d1q * (ds.x*sumi + ds.y*delta);
+}
+
+#define VDR_IQ1_M_Q8_1_MMVQ 1
+#define VDR_IQ1_M_Q8_1_MMQ 1
+
+static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq1_m * bq1 = (const block_iq1_m *) vbq + kbx;
+
+ const int qs_packed = get_int_b4(bq1->qs, iqs);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ int sumi[2] = {0};
+ float sumf[2] = {0.0f};
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const int qhl = bq1->qh[2*iqs + l0/4] >> (4 * ((l0/2) % 2));
+
+ const int grid = iq1s_grid_gpu[qs[l0/2] | ((qhl & 0x07) << 8)];
+
+ const int grid0 = (grid >> 0) & 0x0F0F0F0F;
+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;
+
+ const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
+ const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
+
+ sumi[l0/4] = ggml_cuda_dp4a(grid0, u0, sumi[l0/4]);
+ sumi[l0/4] = ggml_cuda_dp4a(grid1, u1, sumi[l0/4]);
+
+ const float delta = -1.0f + IQ1M_DELTA - (qhl & 0x08) * (2.0f*IQ1M_DELTA/0x08);
+ int sumy = 0;
+ sumy = ggml_cuda_dp4a(u0, 0x01010101, sumy);
+ sumy = ggml_cuda_dp4a(u1, 0x01010101, sumy);
+ sumf[l0/4] += delta*sumy;
+ }
+
+ const uint16_t * sc = (const uint16_t *) bq1->scales;
+
+ iq1m_scale_t scale;
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);
+ const float d = __half2float(scale.f16) * __low2float(bq8_1[iqs].ds);
+
+ const int tmp = sc[iqs/2] >> (6*(iqs%2));
+ const int sc0 = 2*((tmp >> 0) & 0x07) + 1;
+ const int sc1 = 2*((tmp >> 3) & 0x07) + 1;
+ return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1);
+}
+
+static __device__ __forceinline__ int2 get_int_from_table_16(const int & q4) {
+ const int q0_32 = (q4 >> 0) & 0x0F0F0F0F;
+ const int8_t * q0_8 = (const int8_t *) &q0_32;
+ const char4 val0_8 = make_char4(
+ kvalues_iq4nl[q0_8[0]], kvalues_iq4nl[q0_8[1]], kvalues_iq4nl[q0_8[2]], kvalues_iq4nl[q0_8[3]]);
+
+ const int q1_32 = (q4 >> 4) & 0x0F0F0F0F;
+ const int8_t * q1_8 = (const int8_t *) &q1_32;
+ const char4 val1_8 = make_char4(
+ kvalues_iq4nl[q1_8[0]], kvalues_iq4nl[q1_8[1]], kvalues_iq4nl[q1_8[2]], kvalues_iq4nl[q1_8[3]]);
+
+ return make_int2(*((const int *) &val0_8), *((const int *) &val1_8));
+}
+
+#define VDR_IQ4_NL_Q8_1_MMVQ 2
+#define VDR_IQ4_NL_Q8_1_MMQ 4
+
+static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq4_nl * bq4 = (const block_iq4_nl *) vbq + kbx;
+
+ const int * q8 = (const int *) bq8_1->qs + iqs;
+
+ int sumi = 0;
+#pragma unroll
+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {
+ const int aux_q4 = get_int_b2(bq4->qs, iqs + l);
+ const int2 v = get_int_from_table_16(aux_q4);
+
+ sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi);
+ sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi);
+ }
+
+ const float d = __half2float(bq4->d) * __low2float(bq8_1->ds);
+ return d * sumi;
+}
+
+#define VDR_IQ4_XS_Q8_1_MMVQ 4
+#define VDR_IQ4_XS_Q8_1_MMQ 4
+
+static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq + kbx;
+
+ int sumi = 0;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ const int aux_q4 = get_int_b4(bq4->qs, iqs + j);
+ const int2 v = get_int_from_table_16(aux_q4);
+
+ const int u0 = get_int_b4(bq8_1[iqs/4].qs, j + 0);
+ const int u1 = get_int_b4(bq8_1[iqs/4].qs, j + 4);
+
+ sumi = ggml_cuda_dp4a(v.x, u0, sumi);
+ sumi = ggml_cuda_dp4a(v.y, u1, sumi);
+ }
+
+ const int ls = ((bq4->scales_l[iqs/8] >> (iqs & 0x04)) & 0x0F) | (((bq4->scales_h >> (iqs/2)) & 0x03) << 4);
+ sumi *= ls - 32;
+
+ const float d = __half2float(bq4->d) * __low2float(bq8_1[iqs/4].ds);
+ return d * sumi;
+}
diff --git a/llama/ggml-cuda/vendors/cuda.h b/llama/ggml-cuda/vendors/cuda.h
new file mode 100644
index 00000000..20b0af80
--- /dev/null
+++ b/llama/ggml-cuda/vendors/cuda.h
@@ -0,0 +1,40 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include <cuda_runtime.h>
+#include <cuda.h>
+#include <cublas_v2.h>
+#include <cuda_fp16.h>
+
+#if CUDART_VERSION < 11020
+#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED
+#define CUBLAS_TF32_TENSOR_OP_MATH CUBLAS_TENSOR_OP_MATH
+#define CUBLAS_COMPUTE_16F CUDA_R_16F
+#define CUBLAS_COMPUTE_32F CUDA_R_32F
+#define cublasComputeType_t cudaDataType_t
+#endif // CUDART_VERSION < 11020
diff --git a/llama/ggml-cuda/vendors/hip.h b/llama/ggml-cuda/vendors/hip.h
new file mode 100644
index 00000000..f96a6140
--- /dev/null
+++ b/llama/ggml-cuda/vendors/hip.h
@@ -0,0 +1,203 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include <hip/hip_runtime.h>
+#include <hipblas/hipblas.h>
+#include <hip/hip_fp16.h>
+#ifdef __HIP_PLATFORM_AMD__
+// for rocblas_initialize()
+#include "rocblas/rocblas.h"
+#endif // __HIP_PLATFORM_AMD__
+#define CUBLAS_COMPUTE_16F HIPBLAS_R_16F
+#define CUBLAS_COMPUTE_32F HIPBLAS_R_32F
+#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_R_32F
+#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT
+#define CUBLAS_GEMM_DEFAULT_TENSOR_OP HIPBLAS_GEMM_DEFAULT
+#define CUBLAS_OP_N HIPBLAS_OP_N
+#define CUBLAS_OP_T HIPBLAS_OP_T
+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS
+#define CUBLAS_TF32_TENSOR_OP_MATH 0
+#define CUDA_R_16F HIPBLAS_R_16F
+#define CUDA_R_32F HIPBLAS_R_32F
+#define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)
+#define cublasComputeType_t hipblasDatatype_t //deprecated, new hipblasComputeType_t not in 5.6
+#define cublasCreate hipblasCreate
+#define cublasDestroy hipblasDestroy
+#define cublasGemmEx hipblasGemmEx
+#define cublasGemmBatchedEx hipblasGemmBatchedEx
+#define cublasGemmStridedBatchedEx hipblasGemmStridedBatchedEx
+#define cublasHandle_t hipblasHandle_t
+#define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS
+#define cublasSetStream hipblasSetStream
+#define cublasSgemm hipblasSgemm
+#define cublasStatus_t hipblasStatus_t
+#define cudaDataType_t hipblasDatatype_t //deprecated, new hipblasDatatype not in 5.6
+#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer
+#define cudaDeviceDisablePeerAccess hipDeviceDisablePeerAccess
+#define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccess
+#define cudaDeviceProp hipDeviceProp_t
+#define cudaDeviceSynchronize hipDeviceSynchronize
+#define cudaError_t hipError_t
+#define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled
+#define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled
+#define cudaEventCreateWithFlags hipEventCreateWithFlags
+#define cudaEventDisableTiming hipEventDisableTiming
+#define cudaEventRecord hipEventRecord
+#define cudaEventSynchronize hipEventSynchronize
+#define cudaEvent_t hipEvent_t
+#define cudaEventDestroy hipEventDestroy
+#define cudaFree hipFree
+#define cudaFreeHost hipHostFree
+#define cudaGetDevice hipGetDevice
+#define cudaGetDeviceCount hipGetDeviceCount
+#define cudaGetDeviceProperties hipGetDeviceProperties
+#define cudaGetErrorString hipGetErrorString
+#define cudaGetLastError hipGetLastError
+#define cudaHostRegister hipHostRegister
+#define cudaHostRegisterPortable hipHostRegisterPortable
+#define cudaHostRegisterReadOnly hipHostRegisterReadOnly
+#define cudaHostUnregister hipHostUnregister
+#define cudaLaunchHostFunc hipLaunchHostFunc
+#define cudaMalloc hipMalloc
+#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size, hipHostMallocDefault)
+#define cudaMemcpy hipMemcpy
+#define cudaMemcpyAsync hipMemcpyAsync
+#define cudaMemcpyPeerAsync hipMemcpyPeerAsync
+#define cudaMemcpy2DAsync hipMemcpy2DAsync
+#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice
+#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost
+#define cudaMemcpyHostToDevice hipMemcpyHostToDevice
+#define cudaMemcpyKind hipMemcpyKind
+#define cudaMemset hipMemset
+#define cudaMemsetAsync hipMemsetAsync
+#define cudaMemGetInfo hipMemGetInfo
+#define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize
+#define cudaSetDevice hipSetDevice
+#define cudaStreamCreateWithFlags hipStreamCreateWithFlags
+#define cudaStreamDestroy hipStreamDestroy
+#define cudaStreamFireAndForget hipStreamFireAndForget
+#define cudaStreamNonBlocking hipStreamNonBlocking
+#define cudaStreamPerThread hipStreamPerThread
+#define cudaStreamSynchronize hipStreamSynchronize
+#define cudaStreamWaitEvent(stream, event, flags) hipStreamWaitEvent(stream, event, flags)
+#define cudaStream_t hipStream_t
+#define cudaSuccess hipSuccess
+#define __trap() do { abort(); __builtin_unreachable(); } while(0)
+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS
+#define CUBLAS_STATUS_NOT_INITIALIZED HIPBLAS_STATUS_NOT_INITIALIZED
+#define CUBLAS_STATUS_ALLOC_FAILED HIPBLAS_STATUS_ALLOC_FAILED
+#define CUBLAS_STATUS_INVALID_VALUE HIPBLAS_STATUS_INVALID_VALUE
+#define CUBLAS_STATUS_ARCH_MISMATCH HIPBLAS_STATUS_ARCH_MISMATCH
+#define CUBLAS_STATUS_MAPPING_ERROR HIPBLAS_STATUS_MAPPING_ERROR
+#define CUBLAS_STATUS_EXECUTION_FAILED HIPBLAS_STATUS_EXECUTION_FAILED
+#define CUBLAS_STATUS_INTERNAL_ERROR HIPBLAS_STATUS_INTERNAL_ERROR
+#define CUBLAS_STATUS_NOT_SUPPORTED HIPBLAS_STATUS_NOT_SUPPORTED
+
+#define __CUDA_ARCH__ 1300
+
+#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || \
+ defined(__gfx1150__) || defined(__gfx1151__)
+#define RDNA3
+#endif
+
+#if defined(__gfx1030__) || defined(__gfx1031__) || defined(__gfx1032__) || defined(__gfx1033__) || \
+ defined(__gfx1034__) || defined(__gfx1035__) || defined(__gfx1036__) || defined(__gfx1037__)
+#define RDNA2
+#endif
+
+#if defined(__gfx1010__) || defined(__gfx1012__)
+#define RDNA1
+#endif
+
+#ifndef __has_builtin
+ #define __has_builtin(x) 0
+#endif
+
+typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));
+typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4)));
+static __device__ __forceinline__ int __vsubss4(const int a, const int b) {
+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
+#if __has_builtin(__builtin_elementwise_sub_sat)
+ const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);
+ return reinterpret_cast<const int &>(c);
+#else
+ int8x4_t c;
+ int16_t tmp;
+#pragma unroll
+ for (int i = 0; i < 4; i++) {
+ tmp = va[i] - vb[i];
+ if(tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();
+ if(tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();
+ c[i] = tmp;
+ }
+ return reinterpret_cast<int &>(c);
+#endif // __has_builtin(__builtin_elementwise_sub_sat)
+}
+
+static __device__ __forceinline__ int __vsub4(const int a, const int b) {
+ return __vsubss4(a, b);
+}
+
+static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigned int b) {
+ const uint8x4_t& va = reinterpret_cast<const uint8x4_t&>(a);
+ const uint8x4_t& vb = reinterpret_cast<const uint8x4_t&>(b);
+ unsigned int c;
+ uint8x4_t& vc = reinterpret_cast<uint8x4_t&>(c);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ vc[i] = va[i] == vb[i] ? 0xff : 0x00;
+ }
+ return c;
+}
+
+static __device__ __forceinline__ unsigned int __vcmpne4(unsigned int a, unsigned int b) {
+ const uint8x4_t& va = reinterpret_cast<const uint8x4_t&>(a);
+ const uint8x4_t& vb = reinterpret_cast<const uint8x4_t&>(b);
+ unsigned int c;
+ uint8x4_t& vc = reinterpret_cast<uint8x4_t&>(c);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ vc[i] = va[i] == vb[i] ? 0x00 : 0xff;
+ }
+ return c;
+}
+
+#if defined(__HIP_PLATFORM_AMD__) && HIP_VERSION < 50600000
+// __shfl_xor() for half2 was added in ROCm 5.6
+static __device__ __forceinline__ half2 __shfl_xor(half2 var, int laneMask, int width) {
+ typedef union half2_b32 {
+ half2 val;
+ int b32;
+ } half2_b32_t;
+ half2_b32_t tmp;
+ tmp.val = var;
+ tmp.b32 = __shfl_xor(tmp.b32, laneMask, width);
+ return tmp.val;
+}
+#endif // defined(__HIP_PLATFORM_AMD__) && HIP_VERSION < 50600000
diff --git a/llama/ggml-cuda/vendors/musa.h b/llama/ggml-cuda/vendors/musa.h
new file mode 100644
index 00000000..c0dc04d7
--- /dev/null
+++ b/llama/ggml-cuda/vendors/musa.h
@@ -0,0 +1,197 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include <musa_runtime.h>
+#include <musa.h>
+#include <mublas.h>
+#include <musa_fp16.h>
+#define CUBLAS_COMPUTE_16F CUDA_R_16F
+#define CUBLAS_COMPUTE_32F CUDA_R_32F
+#define CUBLAS_COMPUTE_32F_FAST_16F MUBLAS_COMPUTE_32F_FAST_16F
+#define CUBLAS_GEMM_DEFAULT MUBLAS_GEMM_DEFAULT
+#define CUBLAS_GEMM_DEFAULT_TENSOR_OP MUBLAS_GEMM_DEFAULT
+#define CUBLAS_OP_N MUBLAS_OP_N
+#define CUBLAS_OP_T MUBLAS_OP_T
+#define CUBLAS_STATUS_SUCCESS MUBLAS_STATUS_SUCCESS
+#define CUBLAS_TF32_TENSOR_OP_MATH MUBLAS_MATH_MODE_DEFAULT
+#define CUDA_R_16F MUSA_R_16F
+#define CUDA_R_32F MUSA_R_32F
+#define cublasComputeType_t cudaDataType_t
+#define cublasCreate mublasCreate
+#define cublasDestroy mublasDestroy
+#define cublasGemmEx mublasGemmEx
+#define cublasGemmBatchedEx mublasGemmBatchedEx
+#define cublasGemmStridedBatchedEx mublasGemmStridedBatchedEx
+#define cublasHandle_t mublasHandle_t
+#define cublasSetMathMode mublasSetMathMode
+#define cublasSetStream mublasSetStream
+#define cublasSgemm mublasSgemm
+#define cublasStatus_t mublasStatus_t
+#define cublasGetStatusString mublasStatus_to_string
+#define cudaDataType_t musaDataType_t
+#define cudaDeviceCanAccessPeer musaDeviceCanAccessPeer
+#define cudaDeviceDisablePeerAccess musaDeviceDisablePeerAccess
+#define cudaDeviceEnablePeerAccess musaDeviceEnablePeerAccess
+#define cudaDeviceProp musaDeviceProp
+#define cudaDeviceSynchronize musaDeviceSynchronize
+#define cudaError_t musaError_t
+#define cudaErrorPeerAccessAlreadyEnabled musaErrorPeerAccessAlreadyEnabled
+#define cudaErrorPeerAccessNotEnabled musaErrorPeerAccessNotEnabled
+#define cudaEventCreateWithFlags musaEventCreateWithFlags
+#define cudaEventDisableTiming musaEventDisableTiming
+#define cudaEventRecord musaEventRecord
+#define cudaEventSynchronize musaEventSynchronize
+#define cudaEvent_t musaEvent_t
+#define cudaEventDestroy musaEventDestroy
+#define cudaFree musaFree
+#define cudaFreeHost musaFreeHost
+#define cudaGetDevice musaGetDevice
+#define cudaGetDeviceCount musaGetDeviceCount
+#define cudaGetDeviceProperties musaGetDeviceProperties
+#define cudaGetErrorString musaGetErrorString
+#define cudaGetLastError musaGetLastError
+#define cudaHostRegister musaHostRegister
+#define cudaHostRegisterPortable musaHostRegisterPortable
+#define cudaHostRegisterReadOnly musaHostRegisterReadOnly
+#define cudaHostUnregister musaHostUnregister
+#define cudaLaunchHostFunc musaLaunchHostFunc
+#define cudaMalloc musaMalloc
+#define cudaMallocHost musaMallocHost
+#define cudaMemcpy musaMemcpy
+#define cudaMemcpyAsync musaMemcpyAsync
+#define cudaMemcpyPeerAsync musaMemcpyPeerAsync
+#define cudaMemcpy2DAsync musaMemcpy2DAsync
+#define cudaMemcpyDeviceToDevice musaMemcpyDeviceToDevice
+#define cudaMemcpyDeviceToHost musaMemcpyDeviceToHost
+#define cudaMemcpyHostToDevice musaMemcpyHostToDevice
+#define cudaMemcpyKind musaMemcpyKind
+#define cudaMemset musaMemset
+#define cudaMemsetAsync musaMemsetAsync
+#define cudaMemGetInfo musaMemGetInfo
+#define cudaOccupancyMaxPotentialBlockSize musaOccupancyMaxPotentialBlockSize
+#define cudaSetDevice musaSetDevice
+#define cudaStreamCreateWithFlags musaStreamCreateWithFlags
+#define cudaStreamDestroy musaStreamDestroy
+#define cudaStreamFireAndForget musaStreamFireAndForget
+#define cudaStreamNonBlocking musaStreamNonBlocking
+#define cudaStreamPerThread musaStreamPerThread
+#define cudaStreamSynchronize musaStreamSynchronize
+#define cudaStreamWaitEvent musaStreamWaitEvent
+#define cudaStream_t musaStream_t
+#define cudaSuccess musaSuccess
+
+// Additional mappings for MUSA virtual memory pool
+#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED MU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED
+#define CU_MEM_ACCESS_FLAGS_PROT_READWRITE MU_MEM_ACCESS_FLAGS_PROT_READWRITE
+#define CU_MEM_ALLOC_GRANULARITY_RECOMMENDED MU_MEM_ALLOC_GRANULARITY_RECOMMENDED
+#define CU_MEM_ALLOCATION_TYPE_PINNED MU_MEM_ALLOCATION_TYPE_PINNED
+#define CU_MEM_LOCATION_TYPE_DEVICE MU_MEM_LOCATION_TYPE_DEVICE
+#define CUdevice MUdevice
+#define CUdeviceptr MUdeviceptr
+#define CUmemAccessDesc MUmemAccessDesc
+#define CUmemAllocationProp MUmemAllocationProp
+#define CUmemGenericAllocationHandle MUmemGenericAllocationHandle
+#define cuDeviceGet muDeviceGet
+#define cuDeviceGetAttribute muDeviceGetAttribute
+#define cuMemAddressFree muMemAddressFree
+#define cuMemAddressReserve muMemAddressReserve
+#define cuMemCreate muMemCreate
+#define cuMemGetAllocationGranularity muMemGetAllocationGranularity
+#define cuMemMap muMemMap
+#define cuMemRelease muMemRelease
+#define cuMemSetAccess muMemSetAccess
+#define cuMemUnmap muMemUnmap
+#define cudaFuncAttributeMaxDynamicSharedMemorySize musaFuncAttributeMaxDynamicSharedMemorySize
+#define cudaFuncSetAttribute musaFuncSetAttribute
+#define cudaMemcpy3DPeerParms musaMemcpy3DPeerParms
+#define make_cudaExtent make_musaExtent
+#define make_cudaPitchedPtr make_musaPitchedPtr
+
+// Additional mappings for MUSA graphs
+#define CUDA_SUCCESS MUSA_SUCCESS
+#define CUresult MUresult
+#define cuGetErrorString muGetErrorString
+#define cudaErrorGraphExecUpdateFailure musaErrorGraphExecUpdateFailure
+#define cudaErrorInvalidDeviceFunction musaErrorInvalidDeviceFunction
+#define cudaGraphDestroy musaGraphDestroy
+#define cudaGraphExecDestroy musaGraphExecDestroy
+#define cudaGraphExec_t musaGraphExec_t
+#define cudaGraphExecUpdate musaGraphExecUpdate
+#define cudaGraphExecUpdateResultInfo musaGraphExecUpdateResult
+#define cudaGraphGetNodes musaGraphGetNodes
+#define cudaGraphInstantiate musaGraphInstantiate
+#define cudaGraphKernelNodeGetParams musaGraphKernelNodeGetParams
+#define cudaGraphKernelNodeSetParams musaGraphKernelNodeSetParams
+#define cudaGraphLaunch musaGraphLaunch
+#define cudaGraphNodeGetType musaGraphNodeGetType
+#define cudaGraphNode_t musaGraphNode_t
+#define cudaGraphNodeType musaGraphNodeType
+#define cudaGraphNodeTypeKernel musaGraphNodeTypeKernel
+#define cudaGraph_t musaGraph_t
+#define cudaKernelNodeParams musaKernelNodeParams
+#define cudaStreamCaptureModeRelaxed musaStreamCaptureModeRelaxed
+#define cudaStreamEndCapture musaStreamEndCapture
+
+// XXX: Clang builtins mapping
+#define __vsub4 __vsub4_musa
+#define __vcmpeq4 __vcmpeq4_musa
+#define __vcmpne4 __vcmpne4_musa
+
+#ifndef __has_builtin
+ #define __has_builtin(x) 0
+#endif
+
+typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4)));
+
+static __device__ __forceinline__ int __vsub4_musa(const int a, const int b) {
+ return __vsubss4(a, b);
+}
+
+static __device__ __forceinline__ unsigned int __vcmpeq4_musa(unsigned int a, unsigned int b) {
+ const uint8x4_t& va = reinterpret_cast<const uint8x4_t&>(a);
+ const uint8x4_t& vb = reinterpret_cast<const uint8x4_t&>(b);
+ unsigned int c;
+ uint8x4_t& vc = reinterpret_cast<uint8x4_t&>(c);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ vc[i] = va[i] == vb[i] ? 0xff : 0x00;
+ }
+ return c;
+}
+
+static __device__ __forceinline__ unsigned int __vcmpne4_musa(unsigned int a, unsigned int b) {
+ const uint8x4_t& va = reinterpret_cast<const uint8x4_t&>(a);
+ const uint8x4_t& vb = reinterpret_cast<const uint8x4_t&>(b);
+ unsigned int c;
+ uint8x4_t& vc = reinterpret_cast<uint8x4_t&>(c);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ vc[i] = va[i] == vb[i] ? 0x00 : 0xff;
+ }
+ return c;
+}
diff --git a/llama/ggml-impl.h b/llama/ggml-impl.h
new file mode 100644
index 00000000..2d6cb470
--- /dev/null
+++ b/llama/ggml-impl.h
@@ -0,0 +1,779 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "ggml.h"
+
+// GGML internal header
+
+#include <assert.h>
+#include <stdlib.h> // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/
+#include <stddef.h>
+#include <stdbool.h>
+#include <string.h> // memcpy
+#include <math.h> // fabsf
+
+#undef MIN
+#undef MAX
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+#if defined(_MSC_VER)
+
+#define m512bh(p) p
+#define m512i(p) p
+
+#else
+
+#define m512bh(p) (__m512bh)(p)
+#define m512i(p) (__m512i)(p)
+
+#endif
+
+/**
+ * Converts brain16 to float32.
+ *
+ * The bfloat16 floating point format has the following structure:
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌──┴───┐┌─┴───┐
+ * 0b0000000000000000 brain16
+ *
+ * Since bf16 has the same number of exponent bits as a 32bit float,
+ * encoding and decoding numbers becomes relatively straightforward.
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌──┴───┐┌─┴───────────────────┐
+ * 0b00000000000000000000000000000000 IEEE binary32
+ *
+ * For comparison, the standard fp16 format has fewer exponent bits.
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌─┴─┐┌─┴──────┐
+ * 0b0000000000000000 IEEE binary16
+ *
+ * @see IEEE 754-2008
+ */
+static inline float ggml_compute_bf16_to_fp32(ggml_bf16_t h) {
+ union {
+ float f;
+ uint32_t i;
+ } u;
+ u.i = (uint32_t)h.bits << 16;
+ return u.f;
+}
+
+/**
+ * Converts float32 to brain16.
+ *
+ * This is binary identical with Google Brain float conversion.
+ * Floats shall round to nearest even, and NANs shall be quiet.
+ * Subnormals aren't flushed to zero, except perhaps when used.
+ * This code should vectorize nicely if using modern compilers.
+ */
+static inline ggml_bf16_t ggml_compute_fp32_to_bf16(float s) {
+ ggml_bf16_t h;
+ union {
+ float f;
+ uint32_t i;
+ } u;
+ u.f = s;
+ if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */
+ h.bits = (u.i >> 16) | 64; /* force to quiet */
+ return h;
+ }
+ h.bits = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16;
+ return h;
+}
+
+#define GGML_FP32_TO_BF16(x) ggml_compute_fp32_to_bf16(x)
+#define GGML_BF16_TO_FP32(x) ggml_compute_bf16_to_fp32(x)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// static_assert should be a #define, but if it's not,
+// fall back to the _Static_assert C11 keyword.
+// if C99 - static_assert is noop
+// ref: https://stackoverflow.com/a/53923785/4039976
+#ifndef __cplusplus
+#ifndef static_assert
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)
+#define static_assert(cond, msg) _Static_assert(cond, msg)
+#else
+#define static_assert(cond, msg) struct global_scope_noop_trick
+#endif
+#endif
+#endif
+
+// __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512
+#if defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__))
+#ifndef __FMA__
+#define __FMA__
+#endif
+#ifndef __F16C__
+#define __F16C__
+#endif
+#endif
+
+// __SSE3__ and __SSSE3__ are not defined in MSVC, but SSE3/SSSE3 are present when AVX/AVX2/AVX512 are available
+#if defined(_MSC_VER) && (defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__))
+#ifndef __SSE3__
+#define __SSE3__
+#endif
+#ifndef __SSSE3__
+#define __SSSE3__
+#endif
+#endif
+
+#if defined(__ARM_FEATURE_SVE)
+#include <arm_sve.h>
+#include <sys/prctl.h>
+#endif
+
+// 16-bit float
+// on Arm, we use __fp16
+// on x86, we use uint16_t
+#if defined(__ARM_NEON)
+
+// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:
+//
+// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/
+//
+#include <arm_neon.h>
+
+#ifdef _MSC_VER
+
+typedef uint16_t ggml_fp16_internal_t;
+
+#define ggml_vld1q_u32(w,x,y,z) { ((w) + ((uint64_t)(x) << 32)), ((y) + ((uint64_t)(z) << 32)) }
+
+#else
+
+typedef __fp16 ggml_fp16_internal_t;
+
+#define ggml_vld1q_u32(w,x,y,z) { (w), (x), (y), (z) }
+
+#endif // _MSC_VER
+
+#if !defined(__aarch64__)
+
+// 32-bit ARM compatibility
+
+// vaddvq_s16
+// vpaddq_s16
+// vpaddq_s32
+// vaddvq_s32
+// vaddvq_f32
+// vmaxvq_f32
+// vcvtnq_s32_f32
+// vzip1_u8
+// vzip2_u8
+
+inline static int32_t vaddvq_s16(int16x8_t v) {
+ return
+ (int32_t)vgetq_lane_s16(v, 0) + (int32_t)vgetq_lane_s16(v, 1) +
+ (int32_t)vgetq_lane_s16(v, 2) + (int32_t)vgetq_lane_s16(v, 3) +
+ (int32_t)vgetq_lane_s16(v, 4) + (int32_t)vgetq_lane_s16(v, 5) +
+ (int32_t)vgetq_lane_s16(v, 6) + (int32_t)vgetq_lane_s16(v, 7);
+}
+
+inline static int16x8_t vpaddq_s16(int16x8_t a, int16x8_t b) {
+ int16x4_t a0 = vpadd_s16(vget_low_s16(a), vget_high_s16(a));
+ int16x4_t b0 = vpadd_s16(vget_low_s16(b), vget_high_s16(b));
+ return vcombine_s16(a0, b0);
+}
+
+inline static int32x4_t vpaddq_s32(int32x4_t a, int32x4_t b) {
+ int32x2_t a0 = vpadd_s32(vget_low_s32(a), vget_high_s32(a));
+ int32x2_t b0 = vpadd_s32(vget_low_s32(b), vget_high_s32(b));
+ return vcombine_s32(a0, b0);
+}
+
+inline static int32_t vaddvq_s32(int32x4_t v) {
+ return vgetq_lane_s32(v, 0) + vgetq_lane_s32(v, 1) + vgetq_lane_s32(v, 2) + vgetq_lane_s32(v, 3);
+}
+
+inline static float vaddvq_f32(float32x4_t v) {
+ return vgetq_lane_f32(v, 0) + vgetq_lane_f32(v, 1) + vgetq_lane_f32(v, 2) + vgetq_lane_f32(v, 3);
+}
+
+inline static float vmaxvq_f32(float32x4_t v) {
+ return
+ MAX(MAX(vgetq_lane_f32(v, 0), vgetq_lane_f32(v, 1)),
+ MAX(vgetq_lane_f32(v, 2), vgetq_lane_f32(v, 3)));
+}
+
+inline static int32x4_t vcvtnq_s32_f32(float32x4_t v) {
+ int32x4_t res;
+
+ res[0] = roundf(vgetq_lane_f32(v, 0));
+ res[1] = roundf(vgetq_lane_f32(v, 1));
+ res[2] = roundf(vgetq_lane_f32(v, 2));
+ res[3] = roundf(vgetq_lane_f32(v, 3));
+
+ return res;
+}
+
+inline static uint8x8_t vzip1_u8(uint8x8_t a, uint8x8_t b) {
+ uint8x8_t res;
+
+ res[0] = a[0]; res[1] = b[0];
+ res[2] = a[1]; res[3] = b[1];
+ res[4] = a[2]; res[5] = b[2];
+ res[6] = a[3]; res[7] = b[3];
+
+ return res;
+}
+
+inline static uint8x8_t vzip2_u8(uint8x8_t a, uint8x8_t b) {
+ uint8x8_t res;
+
+ res[0] = a[4]; res[1] = b[4];
+ res[2] = a[5]; res[3] = b[5];
+ res[4] = a[6]; res[5] = b[6];
+ res[6] = a[7]; res[7] = b[7];
+
+ return res;
+}
+
+// vld1q_s16_x2
+// vld1q_u8_x2
+// vld1q_u8_x4
+// vld1q_s8_x2
+// vld1q_s8_x4
+// TODO: double-check these work correctly
+
+typedef struct ggml_int16x8x2_t {
+ int16x8_t val[2];
+} ggml_int16x8x2_t;
+
+inline static ggml_int16x8x2_t ggml_vld1q_s16_x2(const int16_t * ptr) {
+ ggml_int16x8x2_t res;
+
+ res.val[0] = vld1q_s16(ptr + 0);
+ res.val[1] = vld1q_s16(ptr + 8);
+
+ return res;
+}
+
+typedef struct ggml_uint8x16x2_t {
+ uint8x16_t val[2];
+} ggml_uint8x16x2_t;
+
+inline static ggml_uint8x16x2_t ggml_vld1q_u8_x2(const uint8_t * ptr) {
+ ggml_uint8x16x2_t res;
+
+ res.val[0] = vld1q_u8(ptr + 0);
+ res.val[1] = vld1q_u8(ptr + 16);
+
+ return res;
+}
+
+typedef struct ggml_uint8x16x4_t {
+ uint8x16_t val[4];
+} ggml_uint8x16x4_t;
+
+inline static ggml_uint8x16x4_t ggml_vld1q_u8_x4(const uint8_t * ptr) {
+ ggml_uint8x16x4_t res;
+
+ res.val[0] = vld1q_u8(ptr + 0);
+ res.val[1] = vld1q_u8(ptr + 16);
+ res.val[2] = vld1q_u8(ptr + 32);
+ res.val[3] = vld1q_u8(ptr + 48);
+
+ return res;
+}
+
+typedef struct ggml_int8x16x2_t {
+ int8x16_t val[2];
+} ggml_int8x16x2_t;
+
+inline static ggml_int8x16x2_t ggml_vld1q_s8_x2(const int8_t * ptr) {
+ ggml_int8x16x2_t res;
+
+ res.val[0] = vld1q_s8(ptr + 0);
+ res.val[1] = vld1q_s8(ptr + 16);
+
+ return res;
+}
+
+typedef struct ggml_int8x16x4_t {
+ int8x16_t val[4];
+} ggml_int8x16x4_t;
+
+inline static ggml_int8x16x4_t ggml_vld1q_s8_x4(const int8_t * ptr) {
+ ggml_int8x16x4_t res;
+
+ res.val[0] = vld1q_s8(ptr + 0);
+ res.val[1] = vld1q_s8(ptr + 16);
+ res.val[2] = vld1q_s8(ptr + 32);
+ res.val[3] = vld1q_s8(ptr + 48);
+
+ return res;
+}
+
+// NOTE: not tested
+inline static int8x16_t ggml_vqtbl1q_s8(int8x16_t a, uint8x16_t b) {
+ int8x16_t res;
+
+ res[ 0] = a[b[ 0]];
+ res[ 1] = a[b[ 1]];
+ res[ 2] = a[b[ 2]];
+ res[ 3] = a[b[ 3]];
+ res[ 4] = a[b[ 4]];
+ res[ 5] = a[b[ 5]];
+ res[ 6] = a[b[ 6]];
+ res[ 7] = a[b[ 7]];
+ res[ 8] = a[b[ 8]];
+ res[ 9] = a[b[ 9]];
+ res[10] = a[b[10]];
+ res[11] = a[b[11]];
+ res[12] = a[b[12]];
+ res[13] = a[b[13]];
+ res[14] = a[b[14]];
+ res[15] = a[b[15]];
+
+ return res;
+}
+
+// NOTE: not tested
+inline static uint8x16_t ggml_vqtbl1q_u8(uint8x16_t a, uint8x16_t b) {
+ uint8x16_t res;
+
+ res[ 0] = a[b[ 0]];
+ res[ 1] = a[b[ 1]];
+ res[ 2] = a[b[ 2]];
+ res[ 3] = a[b[ 3]];
+ res[ 4] = a[b[ 4]];
+ res[ 5] = a[b[ 5]];
+ res[ 6] = a[b[ 6]];
+ res[ 7] = a[b[ 7]];
+ res[ 8] = a[b[ 8]];
+ res[ 9] = a[b[ 9]];
+ res[10] = a[b[10]];
+ res[11] = a[b[11]];
+ res[12] = a[b[12]];
+ res[13] = a[b[13]];
+ res[14] = a[b[14]];
+ res[15] = a[b[15]];
+
+ return res;
+}
+
+#else
+
+#define ggml_int16x8x2_t int16x8x2_t
+#define ggml_uint8x16x2_t uint8x16x2_t
+#define ggml_uint8x16x4_t uint8x16x4_t
+#define ggml_int8x16x2_t int8x16x2_t
+#define ggml_int8x16x4_t int8x16x4_t
+
+#define ggml_vld1q_s16_x2 vld1q_s16_x2
+#define ggml_vld1q_u8_x2 vld1q_u8_x2
+#define ggml_vld1q_u8_x4 vld1q_u8_x4
+#define ggml_vld1q_s8_x2 vld1q_s8_x2
+#define ggml_vld1q_s8_x4 vld1q_s8_x4
+#define ggml_vqtbl1q_s8 vqtbl1q_s8
+#define ggml_vqtbl1q_u8 vqtbl1q_u8
+
+#endif // !defined(__aarch64__)
+
+#if !defined(__ARM_FEATURE_DOTPROD)
+
+inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) {
+ const int16x8_t p0 = vmull_s8(vget_low_s8 (a), vget_low_s8 (b));
+ const int16x8_t p1 = vmull_s8(vget_high_s8(a), vget_high_s8(b));
+
+ return vaddq_s32(acc, vaddq_s32(vpaddlq_s16(p0), vpaddlq_s16(p1)));
+}
+
+#else
+
+#define ggml_vdotq_s32(a, b, c) vdotq_s32(a, b, c)
+
+#endif // !defined(__ARM_FEATURE_DOTPROD)
+
+#endif // defined(__ARM_NEON)
+
+#if defined(__ARM_NEON) && !defined(_MSC_VER)
+
+#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)
+#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)
+
+#define GGML_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)
+
+static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {
+ ggml_fp16_internal_t tmp;
+ memcpy(&tmp, &h, sizeof(ggml_fp16_t));
+ return (float)tmp;
+}
+
+static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {
+ ggml_fp16_t res;
+ ggml_fp16_internal_t tmp = f;
+ memcpy(&res, &tmp, sizeof(ggml_fp16_t));
+ return res;
+}
+
+#else
+
+#ifdef __wasm_simd128__
+#include <wasm_simd128.h>
+#else
+#ifdef __POWER9_VECTOR__
+#include <altivec.h>
+#undef bool
+#define bool _Bool
+#else
+#if defined(_MSC_VER) || defined(__MINGW32__)
+#include <intrin.h>
+#else
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__) || defined(__SSE3__) || defined(__SSE__)
+#if !defined(__riscv)
+#include <immintrin.h>
+#endif
+#endif
+#endif
+#endif
+#endif
+
+#ifdef __riscv_v_intrinsic
+#include <riscv_vector.h>
+#endif
+
+#if defined(__loongarch64)
+#if defined(__loongarch_asx)
+#include <lasxintrin.h>
+#endif
+#if defined(__loongarch_sx)
+#include <lsxintrin.h>
+#endif
+#endif
+
+#if defined(__loongarch_asx)
+
+typedef union {
+ int32_t i;
+ float f;
+} ft_union;
+
+/* float type data load instructions */
+static __m128 __lsx_vreplfr2vr_s(float val) {
+ ft_union fi_tmpval = {.f = val};
+ return (__m128)__lsx_vreplgr2vr_w(fi_tmpval.i);
+}
+
+static __m256 __lasx_xvreplfr2vr_s(float val) {
+ ft_union fi_tmpval = {.f = val};
+ return (__m256)__lasx_xvreplgr2vr_w(fi_tmpval.i);
+}
+#endif
+
+#ifdef __F16C__
+
+#ifdef _MSC_VER
+#define GGML_COMPUTE_FP16_TO_FP32(x) _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x)))
+#define GGML_COMPUTE_FP32_TO_FP16(x) _mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(x), 0), 0)
+#else
+#define GGML_COMPUTE_FP16_TO_FP32(x) _cvtsh_ss(x)
+#define GGML_COMPUTE_FP32_TO_FP16(x) _cvtss_sh(x, 0)
+#endif
+
+#elif defined(__POWER9_VECTOR__)
+
+#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)
+#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)
+/* the inline asm below is about 12% faster than the lookup method */
+#define GGML_FP16_TO_FP32(x) GGML_COMPUTE_FP16_TO_FP32(x)
+#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)
+
+static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {
+ register float f;
+ register double d;
+ __asm__(
+ "mtfprd %0,%2\n"
+ "xscvhpdp %0,%0\n"
+ "frsp %1,%0\n" :
+ /* temp */ "=d"(d),
+ /* out */ "=f"(f):
+ /* in */ "r"(h));
+ return f;
+}
+
+static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {
+ register double d;
+ register ggml_fp16_t r;
+ __asm__( /* xscvdphp can work on double or single precision */
+ "xscvdphp %0,%2\n"
+ "mffprd %1,%0\n" :
+ /* temp */ "=d"(d),
+ /* out */ "=r"(r):
+ /* in */ "f"(f));
+ return r;
+}
+
+#else
+
+// FP16 <-> FP32
+// ref: https://github.com/Maratyszcza/FP16
+
+static inline float fp32_from_bits(uint32_t w) {
+ union {
+ uint32_t as_bits;
+ float as_value;
+ } fp32;
+ fp32.as_bits = w;
+ return fp32.as_value;
+}
+
+static inline uint32_t fp32_to_bits(float f) {
+ union {
+ float as_value;
+ uint32_t as_bits;
+ } fp32;
+ fp32.as_value = f;
+ return fp32.as_bits;
+}
+
+static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {
+ const uint32_t w = (uint32_t) h << 16;
+ const uint32_t sign = w & UINT32_C(0x80000000);
+ const uint32_t two_w = w + w;
+
+ const uint32_t exp_offset = UINT32_C(0xE0) << 23;
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)
+ const float exp_scale = 0x1.0p-112f;
+#else
+ const float exp_scale = fp32_from_bits(UINT32_C(0x7800000));
+#endif
+ const float normalized_value = fp32_from_bits((two_w >> 4) + exp_offset) * exp_scale;
+
+ const uint32_t magic_mask = UINT32_C(126) << 23;
+ const float magic_bias = 0.5f;
+ const float denormalized_value = fp32_from_bits((two_w >> 17) | magic_mask) - magic_bias;
+
+ const uint32_t denormalized_cutoff = UINT32_C(1) << 27;
+ const uint32_t result = sign |
+ (two_w < denormalized_cutoff ? fp32_to_bits(denormalized_value) : fp32_to_bits(normalized_value));
+ return fp32_from_bits(result);
+}
+
+static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)
+ const float scale_to_inf = 0x1.0p+112f;
+ const float scale_to_zero = 0x1.0p-110f;
+#else
+ const float scale_to_inf = fp32_from_bits(UINT32_C(0x77800000));
+ const float scale_to_zero = fp32_from_bits(UINT32_C(0x08800000));
+#endif
+ float base = (fabsf(f) * scale_to_inf) * scale_to_zero;
+
+ const uint32_t w = fp32_to_bits(f);
+ const uint32_t shl1_w = w + w;
+ const uint32_t sign = w & UINT32_C(0x80000000);
+ uint32_t bias = shl1_w & UINT32_C(0xFF000000);
+ if (bias < UINT32_C(0x71000000)) {
+ bias = UINT32_C(0x71000000);
+ }
+
+ base = fp32_from_bits((bias >> 1) + UINT32_C(0x07800000)) + base;
+ const uint32_t bits = fp32_to_bits(base);
+ const uint32_t exp_bits = (bits >> 13) & UINT32_C(0x00007C00);
+ const uint32_t mantissa_bits = bits & UINT32_C(0x00000FFF);
+ const uint32_t nonsign = exp_bits + mantissa_bits;
+ return (sign >> 16) | (shl1_w > UINT32_C(0xFF000000) ? UINT16_C(0x7E00) : nonsign);
+}
+
+#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)
+#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)
+
+#endif // __F16C__
+
+#endif // defined(__ARM_NEON) && (!defined(__MSC_VER)
+
+#ifdef __ARM_FEATURE_SVE
+#include <arm_sve.h>
+#endif // __ARM_FEATURE_SVE
+
+// precomputed f32 table for f16 (256 KB)
+// defined in ggml.c, initialized in ggml_init()
+extern float ggml_table_f32_f16[1 << 16];
+
+// On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32,
+// so we define GGML_FP16_TO_FP32 and GGML_FP32_TO_FP16 elsewhere for NEON.
+// This is also true for POWER9.
+#if !defined(GGML_FP16_TO_FP32)
+inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) {
+ uint16_t s;
+ memcpy(&s, &f, sizeof(uint16_t));
+ return ggml_table_f32_f16[s];
+}
+
+#define GGML_FP16_TO_FP32(x) ggml_lookup_fp16_to_fp32(x)
+#endif
+
+#if !defined(GGML_FP32_TO_FP16)
+#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)
+#endif
+
+// bitset
+
+static_assert(sizeof(ggml_bitset_t) == 4, "bitset_t constants must be updated");
+#define BITSET_SHR 5 // log2(sizeof(ggml_bitset_t)*8)
+#define BITSET_MASK (sizeof(ggml_bitset_t)*8 - 1)
+
+static size_t ggml_bitset_size(size_t n) {
+ return (n + BITSET_MASK) >> BITSET_SHR;
+}
+
+static inline bool ggml_bitset_get(const ggml_bitset_t * bitset, size_t i) {
+ return !!(bitset[i >> BITSET_SHR] & (1u << (i & BITSET_MASK)));
+}
+
+static inline void ggml_bitset_set(ggml_bitset_t * bitset, size_t i) {
+ bitset[i >> BITSET_SHR] |= (1u << (i & BITSET_MASK));
+}
+
+static inline void ggml_bitset_clear(ggml_bitset_t * bitset, size_t i) {
+ bitset[i >> BITSET_SHR] &= ~(1u << (i & BITSET_MASK));
+}
+
+// hash set
+
+#define GGML_HASHSET_FULL ((size_t)-1)
+#define GGML_HASHSET_ALREADY_EXISTS ((size_t)-2)
+
+struct ggml_hash_set ggml_hash_set_new(size_t size);
+void ggml_hash_set_free(struct ggml_hash_set * hash_set);
+
+// returns the minimum size for a hash set that can hold min_sz elements
+size_t ggml_hash_size(size_t min_sz);
+
+// remove all elements from the hash set
+void ggml_hash_set_reset(struct ggml_hash_set * hash_set);
+
+// returns true if key is in the hash set
+static bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key);
+
+// returns GGML_HASHSET_FULL if table is full, otherwise the current index of the key or where it should be inserted
+static size_t ggml_hash_find(const struct ggml_hash_set * hash_set, struct ggml_tensor * key);
+
+// returns GGML_HASHSET_ALREADY_EXISTS if key already exists, index otherwise, asserts if table is full
+static size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key);
+
+// return index, asserts if table is full
+static size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key);
+
+// hash function for ggml_tensor
+static inline size_t ggml_hash(const struct ggml_tensor * p) {
+ // the last 4 bits are always zero due to alignment
+ return (size_t)(uintptr_t)p >> 4;
+}
+
+static size_t ggml_hash_find(const struct ggml_hash_set * hash_set, struct ggml_tensor * key) {
+ size_t h = ggml_hash(key) % hash_set->size;
+
+ // linear probing
+ size_t i = h;
+ while (ggml_bitset_get(hash_set->used, i) && hash_set->keys[i] != key) {
+ i = (i + 1) % hash_set->size;
+ if (i == h) {
+ // visited all hash table entries -> not found
+ return GGML_HASHSET_FULL;
+ }
+ }
+ return i;
+}
+
+static bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key) {
+ size_t i = ggml_hash_find(hash_set, key);
+ return i != GGML_HASHSET_FULL && ggml_bitset_get(hash_set->used, i);
+}
+
+static size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) {
+ size_t h = ggml_hash(key) % hash_set->size;
+
+ // linear probing
+ size_t i = h;
+ do {
+ if (!ggml_bitset_get(hash_set->used, i)) {
+ ggml_bitset_set(hash_set->used, i);
+ hash_set->keys[i] = key;
+ return i;
+ }
+ if (hash_set->keys[i] == key) {
+ return GGML_HASHSET_ALREADY_EXISTS;
+ }
+ i = (i + 1) % hash_set->size;
+ } while (i != h);
+
+ // visited all hash table entries -> not found
+ GGML_ABORT("fatal error");
+}
+
+static size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) {
+ size_t h = ggml_hash(key) % hash_set->size;
+
+ // linear probing
+ size_t i = h;
+ do {
+ if (!ggml_bitset_get(hash_set->used, i)) {
+ ggml_bitset_set(hash_set->used, i);
+ hash_set->keys[i] = key;
+ return i;
+ }
+ if (hash_set->keys[i] == key) {
+ return i;
+ }
+ i = (i + 1) % hash_set->size;
+ } while (i != h);
+
+ // visited all hash table entries -> not found
+ GGML_ABORT("fatal error");
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama/ggml-metal.h b/llama/ggml-metal.h
new file mode 100644
index 00000000..f8a9c3cb
--- /dev/null
+++ b/llama/ggml-metal.h
@@ -0,0 +1,93 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// An interface allowing to compute ggml_cgraph with Metal
+//
+// This is a fully functional interface that extends ggml with GPU support for Apple devices.
+// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, etc.)
+//
+// How it works?
+//
+// As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this
+// interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you
+// use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.)
+//
+// You only need to make sure that all memory buffers that you used during the graph creation
+// are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is
+// used during the graph evaluation to determine the arguments of the compute kernels.
+//
+// Synchronization between device and host memory (for example for input and output tensors)
+// is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions.
+//
+
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#include <stddef.h>
+#include <stdbool.h>
+
+// max memory buffers that can be mapped to the device
+#define GGML_METAL_MAX_BUFFERS 64
+
+struct ggml_tensor;
+struct ggml_cgraph;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//
+// backend API
+// user-code should use only these functions
+//
+
+GGML_API void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data);
+
+GGML_API ggml_backend_t ggml_backend_metal_init(void);
+
+GGML_API bool ggml_backend_is_metal(ggml_backend_t backend);
+
+GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size);
+
+GGML_API void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb);
+
+GGML_API void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data);
+
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);
+
+// helper to check if the device supports a specific family
+// ideally, the user code should be doing these checks
+// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
+GGML_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family);
+
+// capture all command buffers committed the next time `ggml_backend_graph_compute` is called
+GGML_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama/ggml-metal.metal b/llama/ggml-metal.metal
new file mode 100644
index 00000000..95f65668
--- /dev/null
+++ b/llama/ggml-metal.metal
@@ -0,0 +1,6400 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define GGML_COMMON_DECL_METAL
+#define GGML_COMMON_IMPL_METAL
+#include "ggml-common.h"
+
+#include <metal_stdlib>
+
+using namespace metal;
+
+#define MAX(x, y) ((x) > (y) ? (x) : (y))
+#define MIN(x, y) ((x) < (y) ? (x) : (y))
+#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }
+
+#define N_SIMDWIDTH 32 // assuming SIMD group size is 32
+
+enum ggml_sort_order {
+ GGML_SORT_ORDER_ASC,
+ GGML_SORT_ORDER_DESC,
+};
+
+// general-purpose kernel for addition, subtraction, multiplication and division of two tensors
+// pros: works for non-contiguous tensors, supports broadcast across all dims
+// cons: not very efficient
+kernel void kernel_add(
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ constant int64_t & offs,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = tgpig.z;
+ const int64_t i02 = tgpig.y;
+ const int64_t i01 = tgpig.x;
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+
+ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01 + offs;
+ device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;
+ device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1 + offs;
+
+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {
+ const int i10 = i0 % ne10;
+ *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) + *((device float *)(src1_ptr + i10*nb10));
+ }
+}
+
+kernel void kernel_sub(
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ constant int64_t & offs,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = tgpig.z;
+ const int64_t i02 = tgpig.y;
+ const int64_t i01 = tgpig.x;
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+
+ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01 + offs;
+ device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;
+ device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1 + offs;
+
+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {
+ const int i10 = i0 % ne10;
+ *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) - *((device float *)(src1_ptr + i10*nb10));
+ }
+}
+
+kernel void kernel_mul(
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = tgpig.z;
+ const int64_t i02 = tgpig.y;
+ const int64_t i01 = tgpig.x;
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+
+ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;
+ device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;
+ device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;
+
+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {
+ const int i10 = i0 % ne10;
+ *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) * *((device float *)(src1_ptr + i10*nb10));
+ }
+}
+
+kernel void kernel_div(
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = tgpig.z;
+ const int64_t i02 = tgpig.y;
+ const int64_t i01 = tgpig.x;
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+
+ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;
+ device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;
+ device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;
+
+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {
+ const int i10 = i0 % ne10;
+ *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) / *((device float *)(src1_ptr + i10*nb10));
+ }
+}
+
+template<typename T>
+kernel void kernel_repeat(
+ device const char * src0,
+ device char * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i3 = tgpig.z;
+ const int64_t i2 = tgpig.y;
+ const int64_t i1 = tgpig.x;
+
+ const int64_t i03 = i3 % ne03;
+ const int64_t i02 = i2 % ne02;
+ const int64_t i01 = i1 % ne01;
+
+ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;
+ device char * dst_ptr = dst + i3*nb3 + i2*nb2 + i1*nb1 ;
+
+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {
+ const int i00 = i0 % ne00;
+ *((device T *)(dst_ptr + i0*nb0)) = *((device T *)(src0_ptr + i00*nb00));
+ }
+}
+
+typedef decltype(kernel_repeat<float>) kernel_repeat_t;
+
+template [[host_name("kernel_repeat_f32")]] kernel kernel_repeat_t kernel_repeat<float>;
+template [[host_name("kernel_repeat_f16")]] kernel kernel_repeat_t kernel_repeat<half>;
+template [[host_name("kernel_repeat_i32")]] kernel kernel_repeat_t kernel_repeat<int>;
+template [[host_name("kernel_repeat_i16")]] kernel kernel_repeat_t kernel_repeat<short>;
+
+// assumption: src1 is a row
+// broadcast src1 into src0
+kernel void kernel_add_row(
+ device const float4 * src0,
+ device const float4 * src1,
+ device float4 * dst,
+ constant uint64_t & nb [[buffer(28)]],
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = src0[tpig] + src1[tpig % nb];
+}
+
+kernel void kernel_sub_row(
+ device const float4 * src0,
+ device const float4 * src1,
+ device float4 * dst,
+ constant uint64_t & nb [[buffer(28)]],
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = src0[tpig] - src1[tpig % nb];
+}
+
+kernel void kernel_mul_row(
+ device const float4 * src0,
+ device const float4 * src1,
+ device float4 * dst,
+ constant uint64_t & nb [[buffer(28)]],
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = src0[tpig] * src1[tpig % nb];
+}
+
+kernel void kernel_div_row(
+ device const float4 * src0,
+ device const float4 * src1,
+ device float4 * dst,
+ constant uint64_t & nb [[buffer(28)]],
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = src0[tpig] / src1[tpig % nb];
+}
+
+kernel void kernel_scale(
+ device const float * src0,
+ device float * dst,
+ constant float & scale,
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = src0[tpig] * scale;
+}
+
+kernel void kernel_scale_4(
+ device const float4 * src0,
+ device float4 * dst,
+ constant float & scale,
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = src0[tpig] * scale;
+}
+
+kernel void kernel_clamp(
+ device const float * src0,
+ device float * dst,
+ constant float & min,
+ constant float & max,
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = src0[tpig] < min ? min : (src0[tpig] > max ? max : src0[tpig]);
+}
+
+kernel void kernel_relu(
+ device const float * src0,
+ device float * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = max(0.0f, src0[tpig]);
+}
+
+kernel void kernel_sigmoid(
+ device const float * src0,
+ device float * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = 1.0f / (1.0f + exp(-src0[tpig]));
+}
+
+kernel void kernel_tanh(
+ device const float * src0,
+ device float * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ device const float & x = src0[tpig];
+ dst[tpig] = precise::tanh(x);
+}
+
+constant float GELU_COEF_A = 0.044715f;
+constant float GELU_QUICK_COEF = -1.702f;
+constant float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
+
+kernel void kernel_gelu(
+ device const float * src0,
+ device float * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ device const float & x = src0[tpig];
+
+ dst[tpig] = 0.5f*x*(1.0f + precise::tanh(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));
+}
+
+kernel void kernel_gelu_4(
+ device const float4 * src0,
+ device float4 * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ device const float4 & x = src0[tpig];
+
+ // BEWARE !!!
+ // Simply using "tanh" instead of "precise::tanh" will sometimes results in NaNs!
+ // This was observed with Falcon 7B and 40B models
+ //
+ dst[tpig] = 0.5f*x*(1.0f + precise::tanh(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));
+}
+
+kernel void kernel_gelu_quick(
+ device const float * src0,
+ device float * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ device const float & x = src0[tpig];
+
+ dst[tpig] = x*(1.0f/(1.0f+exp(GELU_QUICK_COEF*x)));
+}
+
+kernel void kernel_gelu_quick_4(
+ device const float4 * src0,
+ device float4 * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ device const float4 & x = src0[tpig];
+
+ dst[tpig] = x*(1.0f/(1.0f+exp(GELU_QUICK_COEF*x)));
+}
+
+kernel void kernel_silu(
+ device const float * src0,
+ device float * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ device const float & x = src0[tpig];
+ dst[tpig] = x / (1.0f + exp(-x));
+}
+
+kernel void kernel_silu_4(
+ device const float4 * src0,
+ device float4 * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ device const float4 & x = src0[tpig];
+ dst[tpig] = x / (1.0f + exp(-x));
+}
+
+kernel void kernel_sqr(
+ device const float * src0,
+ device float * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = src0[tpig] * src0[tpig];
+}
+
+kernel void kernel_sqrt(
+ device const float * src0,
+ device float * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = sqrt(src0[tpig]);
+}
+
+kernel void kernel_sin(
+ device const float * src0,
+ device float * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = sin(src0[tpig]);
+}
+
+kernel void kernel_cos(
+ device const float * src0,
+ device float * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = cos(src0[tpig]);
+}
+
+kernel void kernel_sum_rows(
+ device const float * src0,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tpig[[thread_position_in_grid]]) {
+ int64_t i3 = tpig.z;
+ int64_t i2 = tpig.y;
+ int64_t i1 = tpig.x;
+
+ if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {
+ return;
+ }
+
+ device const float * src_row = (device const float *) ((device const char *) src0 + i1*nb01 + i2*nb02 + i3*nb03);
+ device float * dst_row = (device float *) ((device char *) dst + i1*nb1 + i2*nb2 + i3*nb3);
+
+ float row_sum = 0;
+
+ for (int64_t i0 = 0; i0 < ne00; i0++) {
+ row_sum += src_row[i0];
+ }
+
+ dst_row[0] = row_sum;
+}
+
+template<typename T>
+kernel void kernel_soft_max(
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant float & scale,
+ constant float & max_bias,
+ constant float & m0,
+ constant float & m1,
+ constant uint32_t & n_head_log2,
+ threadgroup float * buf [[threadgroup(0)]],
+ uint tgpig[[threadgroup_position_in_grid]],
+ uint tpitg[[thread_position_in_threadgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = (tgpig) / (ne02*ne01);
+ const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;
+ const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);
+
+ device const float * psrc0 = (device const float *) src0 + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);
+ device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*ne00 : nullptr;
+ device float * pdst = (device float *) dst + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);
+
+ float slope = 1.0f;
+
+ // ALiBi
+ if (max_bias > 0.0f) {
+ const int64_t h = i02;
+
+ const float base = h < n_head_log2 ? m0 : m1;
+ const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;
+
+ slope = pow(base, exp);
+ }
+
+ // parallel max
+ float lmax = -INFINITY;
+
+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {
+ lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f));
+ }
+
+ // find the max value in the block
+ float max_val = simd_max(lmax);
+ if (ntg > N_SIMDWIDTH) {
+ if (sgitg == 0) {
+ buf[tiisg] = -INFINITY;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (tiisg == 0) {
+ buf[sgitg] = max_val;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ max_val = buf[tiisg];
+ max_val = simd_max(max_val);
+ }
+
+ // parallel sum
+ float lsum = 0.0f;
+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {
+ const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f)) - max_val);
+ lsum += exp_psrc0;
+ pdst[i00] = exp_psrc0;
+ }
+
+ // This barrier fixes a failing test
+ // ref: https://github.com/ggerganov/ggml/pull/621#discussion_r1425156335
+ threadgroup_barrier(mem_flags::mem_none);
+
+ float sum = simd_sum(lsum);
+
+ if (ntg > N_SIMDWIDTH) {
+ if (sgitg == 0) {
+ buf[tiisg] = 0.0f;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (tiisg == 0) {
+ buf[sgitg] = sum;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ sum = buf[tiisg];
+ sum = simd_sum(sum);
+ }
+
+ const float inv_sum = 1.0f/sum;
+
+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {
+ pdst[i00] *= inv_sum;
+ }
+}
+
+template<typename T>
+kernel void kernel_soft_max_4(
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant float & scale,
+ constant float & max_bias,
+ constant float & m0,
+ constant float & m1,
+ constant uint32_t & n_head_log2,
+ threadgroup float * buf [[threadgroup(0)]],
+ uint tgpig[[threadgroup_position_in_grid]],
+ uint tpitg[[thread_position_in_threadgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = (tgpig) / (ne02*ne01);
+ const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;
+ const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);
+
+ device const float4 * psrc4 = (device const float4 *) src0 + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00)/4;
+ device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*ne00/4 : nullptr;
+ device float4 * pdst4 = (device float4 *) dst + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00)/4;
+
+ float slope = 1.0f;
+
+ if (max_bias > 0.0f) {
+ const int64_t h = i02;
+
+ const float base = h < n_head_log2 ? m0 : m1;
+ const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;
+
+ slope = pow(base, exp);
+ }
+
+ // parallel max
+ float4 lmax4 = -INFINITY;
+
+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {
+ lmax4 = fmax(lmax4, psrc4[i00]*scale + (float4)((pmask ? slope*pmask[i00] : 0.0f)));
+ }
+
+ const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));
+
+ float max_val = simd_max(lmax);
+ if (ntg > N_SIMDWIDTH) {
+ if (sgitg == 0) {
+ buf[tiisg] = -INFINITY;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (tiisg == 0) {
+ buf[sgitg] = max_val;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ max_val = buf[tiisg];
+ max_val = simd_max(max_val);
+ }
+
+ // parallel sum
+ float4 lsum4 = 0.0f;
+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {
+ const float4 exp_psrc4 = exp((psrc4[i00]*scale + (float4)((pmask ? slope*pmask[i00] : 0.0f))) - max_val);
+ lsum4 += exp_psrc4;
+ pdst4[i00] = exp_psrc4;
+ }
+
+ const float lsum = lsum4[0] + lsum4[1] + lsum4[2] + lsum4[3];
+
+ // This barrier fixes a failing test
+ // ref: https://github.com/ggerganov/ggml/pull/621#discussion_r1425156335
+ threadgroup_barrier(mem_flags::mem_none);
+
+ float sum = simd_sum(lsum);
+
+ if (ntg > N_SIMDWIDTH) {
+ if (sgitg == 0) {
+ buf[tiisg] = 0.0f;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (tiisg == 0) {
+ buf[sgitg] = sum;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ sum = buf[tiisg];
+ sum = simd_sum(sum);
+ }
+
+ const float inv_sum = 1.0f/sum;
+
+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {
+ pdst4[i00] *= inv_sum;
+ }
+}
+
+typedef decltype(kernel_soft_max<float>) kernel_soft_max_t;
+typedef decltype(kernel_soft_max_4<float4>) kernel_soft_max_4_t;
+
+template [[host_name("kernel_soft_max_f16")]] kernel kernel_soft_max_t kernel_soft_max<half>;
+template [[host_name("kernel_soft_max_f32")]] kernel kernel_soft_max_t kernel_soft_max<float>;
+template [[host_name("kernel_soft_max_f16_4")]] kernel kernel_soft_max_4_t kernel_soft_max_4<half4>;
+template [[host_name("kernel_soft_max_f32_4")]] kernel kernel_soft_max_4_t kernel_soft_max_4<float4>;
+
+kernel void kernel_diag_mask_inf(
+ device const float * src0,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int & n_past,
+ uint3 tpig[[thread_position_in_grid]]) {
+ const int64_t i02 = tpig[2];
+ const int64_t i01 = tpig[1];
+ const int64_t i00 = tpig[0];
+
+ if (i00 > n_past + i01) {
+ dst[i02*ne01*ne00 + i01*ne00 + i00] = -INFINITY;
+ } else {
+ dst[i02*ne01*ne00 + i01*ne00 + i00] = src0[i02*ne01*ne00 + i01*ne00 + i00];
+ }
+}
+
+kernel void kernel_diag_mask_inf_8(
+ device const float4 * src0,
+ device float4 * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int & n_past,
+ uint3 tpig[[thread_position_in_grid]]) {
+
+ const int64_t i = 2*tpig[0];
+
+ dst[i+0] = src0[i+0];
+ dst[i+1] = src0[i+1];
+ int64_t i4 = 4*i;
+ const int64_t i02 = i4/(ne00*ne01); i4 -= i02*ne00*ne01;
+ const int64_t i01 = i4/(ne00); i4 -= i01*ne00;
+ const int64_t i00 = i4;
+ for (int k = 3; k >= 0; --k) {
+ if (i00 + 4 + k <= n_past + i01) {
+ break;
+ }
+ dst[i+1][k] = -INFINITY;
+ if (i00 + k > n_past + i01) {
+ dst[i][k] = -INFINITY;
+ }
+ }
+}
+
+// ref: ggml.c:ggml_compute_forward_ssm_conv_f32
+// TODO: optimize
+kernel void kernel_ssm_conv_f32(
+ device const void * src0,
+ device const void * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t ir = tgpig.x;
+ const int64_t i2 = tgpig.y;
+ const int64_t i3 = tgpig.z;
+
+ const int64_t nc = ne10;
+ const int64_t ncs = ne00;
+ const int64_t nr = ne01;
+ const int64_t n_t = ne1;
+ const int64_t n_s = ne2;
+
+ device const float * s = (device const float *) ((device const char *) src0 + ir*nb01 + i2*nb00 + i3*nb02);
+ device const float * c = (device const float *) ((device const char *) src1 + ir*nb11);
+ device float * x = (device float *) ((device char *) dst + ir*nb0 + i2*nb1 + i3*nb2);
+
+ float sumf = 0.0f;
+
+ for (int64_t i0 = 0; i0 < nc; ++i0) {
+ sumf += s[i0] * c[i0];
+ }
+
+ x[0] = sumf;
+}
+
+// ref: ggml.c:ggml_compute_forward_ssm_scan_f32
+// TODO: optimize
+kernel void kernel_ssm_scan_f32(
+ device const void * src0,
+ device const void * src1,
+ device const void * src2,
+ device const void * src3,
+ device const void * src4,
+ device const void * src5,
+ device float * dst,
+ constant int64_t & d_state,
+ constant int64_t & d_inner,
+ constant int64_t & n_seq_tokens,
+ constant int64_t & n_seqs,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant uint64_t & nb20,
+ constant uint64_t & nb21,
+ constant uint64_t & nb22,
+ constant uint64_t & nb30,
+ constant uint64_t & nb31,
+ constant uint64_t & nb40,
+ constant uint64_t & nb41,
+ constant uint64_t & nb42,
+ constant uint64_t & nb50,
+ constant uint64_t & nb51,
+ constant uint64_t & nb52,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t ir = tgpig.x;
+ const int64_t i3 = tgpig.y;
+
+ const int64_t nc = d_state;
+ const int64_t nr = d_inner;
+ const int64_t n_t = n_seq_tokens;
+ const int64_t n_s = n_seqs;
+
+ for (int64_t i2 = 0; i2 < n_t; ++i2) {
+ device const float * s0 = (device const float *) ((device const char *) src0 + ir*nb01 + i3*nb02);
+ device const float * x = (device const float *) ((device const char *) src1 + ir*nb10 + i2*nb11 + i3*nb12);
+ device const float * dt = (device const float *) ((device const char *) src2 + ir*nb20 + i2*nb21 + i3*nb22);
+ device const float * A = (device const float *) ((device const char *) src3 + ir*nb31);
+ device const float * B = (device const float *) ((device const char *) src4 + i2*nb41 + i3*nb42);
+ device const float * C = (device const float *) ((device const char *) src5 + i2*nb51 + i3*nb52);
+ device float * y = (device float *) ((device char *) dst + ir*nb10 + i2*nb11 + i3*nb12); // TODO: do not use src1 strides
+ device float * s = (device float *) ((device char *) dst + ir*nb01 + i3*nb02 + nb13);
+
+ if (i2 > 0) {
+ s0 = s;
+ }
+
+ // i1 == 0
+ float dt_soft_plus = dt[0] <= 20.0f ? log(1.0f + exp(dt[0])) : dt[0];
+ float x_dt = x[0] * dt_soft_plus;
+ float sumf = 0.0f;
+
+ for (int64_t i0 = 0; i0 < nc; ++i0) {
+ int64_t i = i0;
+ float state = (s0[i] * exp(dt_soft_plus * A[i])) + (B[i0] * x_dt);
+ sumf += state * C[i0];
+ s[i] = state;
+ }
+
+ y[0] = sumf;
+ }
+}
+
+kernel void kernel_norm(
+ device const void * src0,
+ device float * dst,
+ constant int64_t & ne00,
+ constant uint64_t & nb01,
+ constant float & eps,
+ threadgroup float * sum [[threadgroup(0)]],
+ uint tgpig[[threadgroup_position_in_grid]],
+ uint tpitg[[thread_position_in_threadgroup]],
+ uint ntg[[threads_per_threadgroup]]) {
+ device const float * x = (device const float *) ((device const char *) src0 + tgpig*nb01);
+ // MEAN
+ // parallel sum
+ sum[tpitg] = 0.0f;
+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {
+ sum[tpitg] += x[i00];
+ }
+ // reduce
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (uint i = ntg/2; i > 0; i /= 2) {
+ if (tpitg < i) {
+ sum[tpitg] += sum[tpitg + i];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ const float mean = sum[0] / ne00;
+
+ // recenter and VARIANCE
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ device float * y = dst + tgpig*ne00;
+ sum[tpitg] = 0.0f;
+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {
+ y[i00] = x[i00] - mean;
+ sum[tpitg] += y[i00] * y[i00];
+ }
+
+ // reduce
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (uint i = ntg/2; i > 0; i /= 2) {
+ if (tpitg < i) {
+ sum[tpitg] += sum[tpitg + i];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ const float variance = sum[0] / ne00;
+
+ const float scale = 1.0f/sqrt(variance + eps);
+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {
+ y[i00] = y[i00] * scale;
+ }
+}
+
+kernel void kernel_rms_norm(
+ device const void * src0,
+ device float * dst,
+ constant int64_t & ne00,
+ constant uint64_t & nb01,
+ constant float & eps,
+ threadgroup float * buf [[threadgroup(0)]],
+ uint tgpig[[threadgroup_position_in_grid]],
+ uint tpitg[[thread_position_in_threadgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint ntg[[threads_per_threadgroup]]) {
+ device const float4 * x = (device const float4 *) ((device const char *) src0 + tgpig*nb01);
+
+ float4 sumf = 0;
+ float all_sum = 0;
+
+ // parallel sum
+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {
+ sumf += x[i00] * x[i00];
+ }
+ all_sum = sumf[0] + sumf[1] + sumf[2] + sumf[3];
+ all_sum = simd_sum(all_sum);
+ if (ntg > N_SIMDWIDTH) {
+ if (sgitg == 0) {
+ buf[tiisg] = 0.0f;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (tiisg == 0) {
+ buf[sgitg] = all_sum;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ all_sum = buf[tiisg];
+ all_sum = simd_sum(all_sum);
+ }
+
+ const float mean = all_sum/ne00;
+ const float scale = 1.0f/sqrt(mean + eps);
+
+ device float4 * y = (device float4 *) (dst + tgpig*ne00);
+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {
+ y[i00] = x[i00] * scale;
+ }
+}
+
+kernel void kernel_group_norm(
+ device const float * src0,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int32_t & n_groups,
+ constant float & eps,
+ threadgroup float * buf [[threadgroup(0)]],
+ uint tgpig[[threadgroup_position_in_grid]],
+ uint tpitg[[thread_position_in_threadgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint ntg[[threads_per_threadgroup]]) {
+ const int64_t ne = ne00*ne01*ne02;
+ const int64_t gs = ne00*ne01*((ne02 + n_groups - 1) / n_groups);
+
+ int start = tgpig * gs;
+ int end = start + gs;
+
+ start += tpitg;
+
+ if (end >= ne) {
+ end = ne;
+ }
+
+ float tmp = 0.0f; // partial sum for thread in warp
+
+ for (int j = start; j < end; j += ntg) {
+ tmp += src0[j];
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ tmp = simd_sum(tmp);
+ if (ntg > N_SIMDWIDTH) {
+ if (sgitg == 0) {
+ buf[tiisg] = 0.0f;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (tiisg == 0) {
+ buf[sgitg] = tmp;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ tmp = buf[tiisg];
+ tmp = simd_sum(tmp);
+ }
+
+ const float mean = tmp / gs;
+ tmp = 0.0f;
+
+ for (int j = start; j < end; j += ntg) {
+ float xi = src0[j] - mean;
+ dst[j] = xi;
+ tmp += xi * xi;
+ }
+
+ tmp = simd_sum(tmp);
+ if (ntg > N_SIMDWIDTH) {
+ if (sgitg == 0) {
+ buf[tiisg] = 0.0f;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (tiisg == 0) {
+ buf[sgitg] = tmp;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ tmp = buf[tiisg];
+ tmp = simd_sum(tmp);
+ }
+
+ const float variance = tmp / gs;
+ const float scale = 1.0f/sqrt(variance + eps);
+ for (int j = start; j < end; j += ntg) {
+ dst[j] *= scale;
+ }
+}
+
+// function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i])
+// il indicates where the q4 quants begin (0 or QK4_0/4)
+// we assume that the yl's have been multiplied with the appropriate scale factor
+// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)
+inline float block_q_n_dot_y(device const block_q4_0 * qb_curr, float sumy, thread float * yl, int il) {
+ float d = qb_curr->d;
+
+ float2 acc = 0.f;
+
+ device const uint16_t * qs = ((device const uint16_t *)qb_curr + 1 + il/2);
+
+ for (int i = 0; i < 8; i+=2) {
+ acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F)
+ + yl[i + 1] * (qs[i / 2] & 0x0F00);
+ acc[1] += yl[i + 8] * (qs[i / 2] & 0x00F0)
+ + yl[i + 9] * (qs[i / 2] & 0xF000);
+ }
+ return d * (sumy * -8.f + acc[0] + acc[1]);
+}
+
+// function for calculate inner product between half a q4_1 block and 16 floats (yl), sumy is SUM(yl[i])
+// il indicates where the q4 quants begin (0 or QK4_0/4)
+// we assume that the yl's have been multiplied with the appropriate scale factor
+// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)
+inline float block_q_n_dot_y(device const block_q4_1 * qb_curr, float sumy, thread float * yl, int il) {
+ float d = qb_curr->d;
+ float m = qb_curr->m;
+
+ float2 acc = 0.f;
+
+ device const uint16_t * qs = ((device const uint16_t *)qb_curr + 2 + il/2);
+
+ for (int i = 0; i < 8; i+=2) {
+ acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F)
+ + yl[i + 1] * (qs[i / 2] & 0x0F00);
+ acc[1] += yl[i + 8] * (qs[i / 2] & 0x00F0)
+ + yl[i + 9] * (qs[i / 2] & 0xF000);
+ }
+ return d * (acc[0] + acc[1]) + sumy * m;
+}
+
+// function for calculate inner product between half a q5_0 block and 16 floats (yl), sumy is SUM(yl[i])
+// il indicates where the q5 quants begin (0 or QK5_0/4)
+// we assume that the yl's have been multiplied with the appropriate scale factor
+// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)
+inline float block_q_n_dot_y(device const block_q5_0 * qb_curr, float sumy, thread float * yl, int il) {
+ float d = qb_curr->d;
+
+ float2 acc = 0.f;
+
+ device const uint16_t * qs = ((device const uint16_t *)qb_curr + 3 + il/2);
+ const uint32_t qh = *((device const uint32_t *)qb_curr->qh);
+
+ for (int i = 0; i < 8; i+=2) {
+ acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010))
+ + yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000));
+ acc[1] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100))
+ + yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000));
+ }
+ return d * (sumy * -16.f + acc[0] + acc[1]);
+}
+
+// function for calculate inner product between half a q5_1 block and 16 floats (yl), sumy is SUM(yl[i])
+// il indicates where the q5 quants begin (0 or QK5_1/4)
+// we assume that the yl's have been multiplied with the appropriate scale factor
+// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)
+inline float block_q_n_dot_y(device const block_q5_1 * qb_curr, float sumy, thread float * yl, int il) {
+ float d = qb_curr->d;
+ float m = qb_curr->m;
+
+ float2 acc = 0.f;
+
+ device const uint16_t * qs = ((device const uint16_t *)qb_curr + 4 + il/2);
+ const uint32_t qh = *((device const uint32_t *)qb_curr->qh);
+
+ for (int i = 0; i < 8; i+=2) {
+ acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010))
+ + yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000));
+ acc[1] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100))
+ + yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000));
+ }
+ return d * (acc[0] + acc[1]) + sumy * m;
+}
+
+// putting them in the kernel cause a significant performance penalty
+#define N_DST 4 // each SIMD group works on 4 rows
+#define N_SIMDGROUP 2 // number of SIMD groups in a thread group
+//Note: This is a template, but strictly speaking it only applies to
+// quantizations where the block size is 32. It also does not
+// guard against the number of rows not being divisible by
+// N_DST, so this is another explicit assumption of the implementation.
+template<typename block_q_type, int nr, int nsg, int nw>
+void mul_vec_q_n_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig, uint tiisg, uint sgitg) {
+ const int nb = ne00/QK4_0;
+
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * nsg + sgitg) * nr;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_q_type * x = (device const block_q_type *) src0 + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[16]; // src1 vector cache
+ float sumf[nr] = {0.f};
+
+ const int ix = (tiisg/2);
+ const int il = (tiisg%2)*8;
+
+ device const float * yb = y + ix * QK4_0 + il;
+
+ // each thread in a SIMD group deals with half a block.
+ for (int ib = ix; ib < nb; ib += nw/2) {
+ float sumy = 0;
+ for (int i = 0; i < 8; i += 2) {
+ sumy += yb[i] + yb[i+1];
+ yl[i+0] = yb[i+ 0];
+ yl[i+1] = yb[i+ 1]/256.f;
+
+ sumy += yb[i+16] + yb[i+17];
+ yl[i+8] = yb[i+16]/16.f;
+ yl[i+9] = yb[i+17]/4096.f;
+ }
+
+ for (int row = 0; row < nr; row++) {
+ sumf[row] += block_q_n_dot_y(x+ib+row*nb, sumy, yl, il);
+ }
+
+ yb += QK4_0 * 16;
+ }
+
+ for (int row = 0; row < nr; ++row) {
+ const float tot = simd_sum(sumf[row]);
+ if (tiisg == 0 && first_row + row < ne01) {
+ dst[im*ne0*ne1 + r1*ne0 + first_row + row] = tot;
+ }
+ }
+}
+
+kernel void kernel_mul_mv_q4_0_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+ mul_vec_q_n_f32_impl<block_q4_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,nullptr,tgpig,tiisg,sgitg);
+}
+
+kernel void kernel_mul_mv_q4_1_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+ mul_vec_q_n_f32_impl<block_q4_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,nullptr,tgpig,tiisg,sgitg);
+}
+
+kernel void kernel_mul_mv_q5_0_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+ mul_vec_q_n_f32_impl<block_q5_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,nullptr,tgpig,tiisg,sgitg);
+}
+
+kernel void kernel_mul_mv_q5_1_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+ mul_vec_q_n_f32_impl<block_q5_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,nullptr,tgpig,tiisg,sgitg);
+}
+
+
+#define NB_Q8_0 8
+
+void kernel_mul_mv_q8_0_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+ const int nr = N_DST;
+ const int nsg = N_SIMDGROUP;
+ const int nw = N_SIMDWIDTH;
+
+ const int nb = ne00/QK8_0;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * nsg + sgitg) * nr;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_q8_0 * x = (device const block_q8_0 *) src0 + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[NB_Q8_0];
+ float sumf[nr]={0.f};
+
+ const int ix = tiisg/4;
+ const int il = tiisg%4;
+
+ device const float * yb = y + ix * QK8_0 + NB_Q8_0*il;
+
+ // each thread in a SIMD group deals with NB_Q8_0 quants at a time
+ for (int ib = ix; ib < nb; ib += nw/4) {
+ for (int i = 0; i < NB_Q8_0; ++i) {
+ yl[i] = yb[i];
+ }
+
+ for (int row = 0; row < nr; row++) {
+ device const int8_t * qs = x[ib+row*nb].qs + NB_Q8_0*il;
+ float sumq = 0.f;
+ for (int iq = 0; iq < NB_Q8_0; ++iq) {
+ sumq += qs[iq] * yl[iq];
+ }
+ sumf[row] += sumq*x[ib+row*nb].d;
+ }
+
+ yb += NB_Q8_0 * nw;
+ }
+
+ for (int row = 0; row < nr; ++row) {
+ const float tot = simd_sum(sumf[row]);
+ if (tiisg == 0 && first_row + row < ne01) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_q8_0_f32")]]
+kernel void kernel_mul_mv_q8_0_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+ kernel_mul_mv_q8_0_f32_impl(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,nullptr,tgpig,tiisg,sgitg);
+}
+
+#define N_MV_T_T 4
+
+template<typename T0, typename T04, typename T1, typename T14>
+void kernel_mul_mv_impl(
+ device const char * src0,
+ device const char * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ uint64_t nb00,
+ uint64_t nb01,
+ uint64_t nb02,
+ int64_t ne10,
+ int64_t ne11,
+ int64_t ne12,
+ uint64_t nb10,
+ uint64_t nb11,
+ uint64_t nb12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ uint3 tgpig,
+ uint tiisg) {
+ const int64_t r0 = tgpig.x;
+ const int64_t rb = tgpig.y*N_MV_T_T;
+ const int64_t im = tgpig.z;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;
+
+ device const T0 * x = (device const T0 *) (src0 + offset0);
+
+ if (ne00 < 128) {
+ for (int row = 0; row < N_MV_T_T; ++row) {
+ int r1 = rb + row;
+ if (r1 >= ne11) {
+ break;
+ }
+
+ device const T1 * y = (device const T1 *) (src1 + r1*nb11 + im*nb12);
+
+ float sumf = 0;
+ for (int i = tiisg; i < ne00; i += 32) {
+ sumf += (T0) x[i] * (T1) y[i];
+ }
+
+ float all_sum = simd_sum(sumf);
+ if (tiisg == 0) {
+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;
+ }
+ }
+ } else {
+ device const T04 * x4 = (device const T04 *) x;
+ for (int row = 0; row < N_MV_T_T; ++row) {
+ int r1 = rb + row;
+ if (r1 >= ne11) {
+ break;
+ }
+
+ device const T1 * y = (device const T1 *) (src1 + r1*nb11 + im*nb12);
+ device const T14 * y4 = (device const T14 *) y;
+
+ float sumf = 0;
+ for (int i = tiisg; i < ne00/4; i += 32) {
+ for (int k = 0; k < 4; ++k) sumf += (float) (x4[i][k] * y4[i][k]);
+ }
+
+ float all_sum = simd_sum(sumf);
+ if (tiisg == 0) {
+ for (int i = 4*(ne00/4); i < ne00; ++i) all_sum += (float) (x[i] * y[i]);
+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;
+ }
+ }
+ }
+}
+
+template<typename T0, typename T04, typename T1, typename T14>
+kernel void kernel_mul_mv(
+ device const char * src0,
+ device const char * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]]) {
+ kernel_mul_mv_impl<T0, T04, T1, T14>(
+ src0,
+ src1,
+ dst,
+ ne00,
+ ne01,
+ ne02,
+ nb00,
+ nb01,
+ nb02,
+ ne10,
+ ne11,
+ ne12,
+ nb10,
+ nb11,
+ nb12,
+ ne0,
+ ne1,
+ r2,
+ r3,
+ tgpig,
+ tiisg);
+}
+
+typedef decltype(kernel_mul_mv<half, half4, half, half4>) mul_mv_t;
+
+template [[host_name("kernel_mul_mv_f32_f32")]] kernel mul_mv_t kernel_mul_mv<float, float4, float, float4>;
+template [[host_name("kernel_mul_mv_f16_f32")]] kernel mul_mv_t kernel_mul_mv<half, half4, float, float4>;
+template [[host_name("kernel_mul_mv_f16_f16")]] kernel mul_mv_t kernel_mul_mv<half, half4, half, half4>;
+
+template<typename T, typename T4>
+kernel void kernel_mul_mv_1row(
+ device const char * src0,
+ device const char * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]]) {
+
+ const int64_t r0 = tgpig.x;
+ const int64_t r1 = tgpig.y;
+ const int64_t im = tgpig.z;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;
+
+ device const T * x = (device const T *) (src0 + offset0);
+ device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);
+
+ float sumf = 0;
+ if (ne00 < 128) {
+ for (int i = tiisg; i < ne00; i += 32) {
+ sumf += (float) x[i] * (float) y[i];
+ }
+ float all_sum = simd_sum(sumf);
+ if (tiisg == 0) {
+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;
+ }
+ } else {
+ device const T4 * x4 = (device const T4 *) x;
+ device const float4 * y4 = (device const float4 *) y;
+
+ for (int i = tiisg; i < ne00/4; i += 32) {
+ for (int k = 0; k < 4; ++k) sumf += (float) (x4[i][k] * y4[i][k]);
+ }
+
+ float all_sum = simd_sum(sumf);
+
+ if (tiisg == 0) {
+ for (int i = 4*(ne00/4); i < ne00; ++i) all_sum += (float) (x[i] * y[i]);
+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;
+ }
+ }
+}
+
+typedef decltype(kernel_mul_mv_1row<half, half4>) mul_mv_1row_t;
+
+template [[host_name("kernel_mul_mv_f16_f32_1row")]] kernel mul_mv_1row_t kernel_mul_mv_1row<half, half4>;
+
+// Assumes row size (ne00) is a multiple of 4
+template<typename T, typename T4>
+kernel void kernel_mul_mv_l4(
+ device const char * src0,
+ device const char * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]]) {
+
+ const int nrows = ne11;
+ const int64_t r0 = tgpig.x;
+ const int64_t im = tgpig.z;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;
+
+ device const T4 * x4 = (device const T4 *) (src0 + offset0);
+
+ for (int r1 = 0; r1 < nrows; ++r1) {
+ device const float4 * y4 = (device const float4 *) (src1 + r1*nb11 + im*nb12);
+
+ float sumf = 0;
+ for (int i = tiisg; i < ne00/4; i += 32) {
+ for (int k = 0; k < 4; ++k) sumf += (float) (x4[i][k] * y4[i][k]);
+ }
+
+ float all_sum = simd_sum(sumf);
+ if (tiisg == 0) {
+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;
+ }
+ }
+}
+
+typedef decltype(kernel_mul_mv_l4<half, half4>) mul_mv_l4_t;
+
+template [[host_name("kernel_mul_mv_f16_f32_l4")]] kernel mul_mv_l4_t kernel_mul_mv_l4<half, half4>;
+
+static float rope_yarn_ramp(const float low, const float high, const int i0) {
+ const float y = (i0 / 2 - low) / max(0.001f, high - low);
+ return 1.0f - min(1.0f, max(0.0f, y));
+}
+
+// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn
+// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.
+static void rope_yarn(
+ float theta_extrap, float freq_scale, float corr_dims[2], int64_t i0, float ext_factor, float mscale,
+ thread float * cos_theta, thread float * sin_theta) {
+ // Get n-d rotational scaling corrected for extrapolation
+ float theta_interp = freq_scale * theta_extrap;
+ float theta = theta_interp;
+ if (ext_factor != 0.0f) {
+ float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor;
+ theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;
+
+ // Get n-d magnitude scaling corrected for interpolation
+ mscale *= 1.0f + 0.1f * log(1.0f / freq_scale);
+ }
+ *cos_theta = cos(theta) * mscale;
+ *sin_theta = sin(theta) * mscale;
+}
+
+// Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get
+// `corr_fac(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`
+static float rope_yarn_corr_factor(int n_dims, int n_ctx_orig, float n_rot, float base) {
+ return n_dims * log(n_ctx_orig / (n_rot * 2 * M_PI_F)) / (2 * log(base));
+}
+
+static void rope_yarn_corr_dims(
+ int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]
+) {
+ // start and end correction dims
+ dims[0] = max(0.0f, floor(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_fast, freq_base)));
+ dims[1] = min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_slow, freq_base)));
+}
+
+template<typename T>
+kernel void kernel_rope_norm(
+ device const void * src0,
+ device const int32_t * src1,
+ device const float * src2,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ constant int & n_past,
+ constant int & n_dims,
+ constant int & n_ctx_orig,
+ constant float & freq_base,
+ constant float & freq_scale,
+ constant float & ext_factor,
+ constant float & attn_factor,
+ constant float & beta_fast,
+ constant float & beta_slow,
+ uint tiitg[[thread_index_in_threadgroup]],
+ uint3 tptg[[threads_per_threadgroup]],
+ uint3 tgpig[[threadgroup_position_in_grid]]) {
+ const int64_t i3 = tgpig[2];
+ const int64_t i2 = tgpig[1];
+ const int64_t i1 = tgpig[0];
+
+ float corr_dims[2];
+ rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
+
+ device const int32_t * pos = src1;
+
+ const float theta_base = (float) pos[i2];
+ const float inv_ndims = -1.f/n_dims;
+
+ float cos_theta;
+ float sin_theta;
+
+ for (int64_t i0 = 2*tiitg; i0 < ne0; i0 += 2*tptg.x) {
+ if (i0 < n_dims) {
+ const int64_t ic = i0/2;
+
+ const float theta = theta_base * pow(freq_base, inv_ndims*i0);
+
+ const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
+
+ rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta);
+
+ device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+ device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ const float x0 = src[0];
+ const float x1 = src[1];
+
+ dst_data[0] = x0*cos_theta - x1*sin_theta;
+ dst_data[1] = x0*sin_theta + x1*cos_theta;
+ } else {
+ device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+ device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ dst_data[0] = src[0];
+ dst_data[1] = src[1];
+ }
+ }
+}
+
+template<typename T>
+kernel void kernel_rope_neox(
+ device const void * src0,
+ device const int32_t * src1,
+ device const float * src2,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ constant int & n_past,
+ constant int & n_dims,
+ constant int & n_ctx_orig,
+ constant float & freq_base,
+ constant float & freq_scale,
+ constant float & ext_factor,
+ constant float & attn_factor,
+ constant float & beta_fast,
+ constant float & beta_slow,
+ uint tiitg[[thread_index_in_threadgroup]],
+ uint3 tptg[[threads_per_threadgroup]],
+ uint3 tgpig[[threadgroup_position_in_grid]]) {
+ const int64_t i3 = tgpig[2];
+ const int64_t i2 = tgpig[1];
+ const int64_t i1 = tgpig[0];
+
+ float corr_dims[2];
+ rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
+
+ device const int32_t * pos = src1;
+
+ const float theta_base = (float) pos[i2];
+ const float inv_ndims = -1.f/n_dims;
+
+ float cos_theta;
+ float sin_theta;
+
+ for (int64_t i0 = 2*tiitg; i0 < ne0; i0 += 2*tptg.x) {
+ if (i0 < n_dims) {
+ const int64_t ic = i0/2;
+
+ const float theta = theta_base * pow(freq_base, inv_ndims*i0);
+
+ const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
+
+ rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta);
+
+ device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
+ device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);
+
+ const float x0 = src[0];
+ const float x1 = src[n_dims/2];
+
+ dst_data[0] = x0*cos_theta - x1*sin_theta;
+ dst_data[n_dims/2] = x0*sin_theta + x1*cos_theta;
+ } else {
+ device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+ device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ dst_data[0] = src[0];
+ dst_data[1] = src[1];
+ }
+ }
+}
+
+typedef decltype(kernel_rope_norm<float>) kernel_rope_norm_t;
+typedef decltype(kernel_rope_neox<float>) kernel_rope_neox_t;
+
+template [[host_name("kernel_rope_norm_f32")]] kernel kernel_rope_norm_t kernel_rope_norm<float>;
+template [[host_name("kernel_rope_norm_f16")]] kernel kernel_rope_norm_t kernel_rope_norm<half>;
+
+template [[host_name("kernel_rope_neox_f32")]] kernel kernel_rope_neox_t kernel_rope_neox<float>;
+template [[host_name("kernel_rope_neox_f16")]] kernel kernel_rope_neox_t kernel_rope_neox<half>;
+
+typedef void (im2col_t)(
+ device const float * x,
+ device char * dst,
+ constant int32_t & ofs0,
+ constant int32_t & ofs1,
+ constant int32_t & IW,
+ constant int32_t & IH,
+ constant int32_t & CHW,
+ constant int32_t & s0,
+ constant int32_t & s1,
+ constant int32_t & p0,
+ constant int32_t & p1,
+ constant int32_t & d0,
+ constant int32_t & d1,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tgpg[[threadgroups_per_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]);
+
+template <typename T>
+kernel void kernel_im2col(
+ device const float * x,
+ device char * dst,
+ constant int32_t & ofs0,
+ constant int32_t & ofs1,
+ constant int32_t & IW,
+ constant int32_t & IH,
+ constant int32_t & CHW,
+ constant int32_t & s0,
+ constant int32_t & s1,
+ constant int32_t & p0,
+ constant int32_t & p1,
+ constant int32_t & d0,
+ constant int32_t & d1,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tgpg[[threadgroups_per_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int32_t iiw = tgpig[2] * s0 + tpitg[2] * d0 - p0;
+ const int32_t iih = tgpig[1] * s1 + tpitg[1] * d1 - p1;
+
+ const int32_t offset_dst =
+ (tpitg[0] * tgpg[1] * tgpg[2] + tgpig[1] * tgpg[2] + tgpig[2]) * CHW +
+ (tgpig[0] * (ntg[1] * ntg[2]) + tpitg[1] * ntg[2] + tpitg[2]);
+
+ device T * pdst = (device T *) (dst);
+
+ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
+ pdst[offset_dst] = 0.0f;
+ } else {
+ const int32_t offset_src = tpitg[0] * ofs0 + tgpig[0] * ofs1;
+ pdst[offset_dst] = x[offset_src + iih * IW + iiw];
+ }
+}
+
+template [[host_name("kernel_im2col_f32")]] kernel im2col_t kernel_im2col<float>;
+template [[host_name("kernel_im2col_f16")]] kernel im2col_t kernel_im2col<half>;
+
+kernel void kernel_upscale_f32(
+ device const char * src0,
+ device char * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ constant float & sf0,
+ constant float & sf1,
+ constant float & sf2,
+ constant float & sf3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+
+ const int64_t i3 = tgpig.z;
+ const int64_t i2 = tgpig.y;
+ const int64_t i1 = tgpig.x;
+
+ const int64_t i03 = i3/sf3;
+ const int64_t i02 = i2/sf2;
+ const int64_t i01 = i1/sf1;
+
+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {
+ const int64_t i00 = i0/sf0;
+
+ device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);
+ device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ dst_ptr[0] = src0_ptr[0];
+ }
+}
+
+kernel void kernel_pad_f32(
+ device const char * src0,
+ device char * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+
+ const int64_t i3 = tgpig.z;
+ const int64_t i2 = tgpig.y;
+ const int64_t i1 = tgpig.x;
+
+ const int64_t i03 = i3;
+ const int64_t i02 = i2;
+ const int64_t i01 = i1;
+
+ device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);
+ device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1);
+
+ if (i1 < ne01 && i2 < ne02 && i3 < ne03) {
+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {
+ if (i0 < ne00) {
+ dst_ptr[i0] = src0_ptr[i0];
+ } else {
+ dst_ptr[i0] = 0.0f;
+ }
+ }
+
+ return;
+ }
+
+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {
+ dst_ptr[i0] = 0.0f;
+ }
+}
+
+kernel void kernel_arange_f32(
+ device char * dst,
+ constant int64_t & ne0,
+ constant float & start,
+ constant float & step,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+
+ device float * dst_ptr = (device float *) dst;
+
+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {
+ dst_ptr[i0] = start + step * i0;
+ }
+}
+
+kernel void kernel_timestep_embedding_f32(
+ device const char * src0,
+ device char * dst,
+ constant uint64_t & nb1,
+ constant int & dim,
+ constant int & max_period,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+
+ int i = tgpig.x;
+ device float * embed_data = (device float *)(dst + i*nb1);
+
+ int half_ = dim / 2;
+ for (int j = tpitg.x; j < half_; j += ntg.x) {
+ float timestep = ((device float *)src0)[i];
+ float freq = (float)exp(-log((float)max_period) * j / half_);
+ float arg = timestep * freq;
+ embed_data[j ] = cos(arg);
+ embed_data[j + half_] = sin(arg);
+ }
+
+ if (dim % 2 != 0 && tpitg.x == 0) {
+ embed_data[dim] = 0.f;
+ }
+}
+
+// bitonic sort implementation following the CUDA kernels as reference
+typedef void (argsort_t)(
+ device const float * x,
+ device int32_t * dst,
+ constant int64_t & ncols,
+ constant int64_t & ncols_pad,
+ threadgroup int32_t * shared_values [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]]);
+
+template<ggml_sort_order order>
+kernel void kernel_argsort_f32_i32(
+ device const float * x,
+ device int32_t * dst,
+ constant int64_t & ncols,
+ constant int64_t & ncols_pad,
+ threadgroup int32_t * shared_values [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]]) {
+ // bitonic sort
+ int col = tpitg[0];
+ int row = tgpig[1];
+
+ if (col >= ncols_pad) return;
+
+ device const float * x_row = x + row * ncols;
+ threadgroup int32_t * dst_row = shared_values;
+
+ // initialize indices
+ dst_row[col] = col;
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ for (int k = 2; k <= ncols_pad; k *= 2) {
+ for (int j = k / 2; j > 0; j /= 2) {
+ int ixj = col ^ j;
+ if (ixj > col) {
+ if ((col & k) == 0) {
+ if (dst_row[col] >= ncols ||
+ (dst_row[ixj] < ncols && (order == GGML_SORT_ORDER_ASC ?
+ x_row[dst_row[col]] > x_row[dst_row[ixj]] :
+ x_row[dst_row[col]] < x_row[dst_row[ixj]]))
+ ) {
+ SWAP(dst_row[col], dst_row[ixj]);
+ }
+ } else {
+ if (dst_row[ixj] >= ncols ||
+ (dst_row[col] < ncols && (order == GGML_SORT_ORDER_ASC ?
+ x_row[dst_row[col]] < x_row[dst_row[ixj]] :
+ x_row[dst_row[col]] > x_row[dst_row[ixj]]))
+ ) {
+ SWAP(dst_row[col], dst_row[ixj]);
+ }
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ }
+
+ // copy the result to dst without the padding
+ if (col < ncols) {
+ dst[row * ncols + col] = dst_row[col];
+ }
+}
+
+template [[host_name("kernel_argsort_f32_i32_asc")]] kernel argsort_t kernel_argsort_f32_i32<GGML_SORT_ORDER_ASC>;
+template [[host_name("kernel_argsort_f32_i32_desc")]] kernel argsort_t kernel_argsort_f32_i32<GGML_SORT_ORDER_DESC>;
+
+kernel void kernel_leaky_relu_f32(
+ device const float * src0,
+ device float * dst,
+ constant float & slope,
+ uint tpig[[thread_position_in_grid]]) {
+ dst[tpig] = src0[tpig] > 0.0f ? src0[tpig] : src0[tpig] * slope;
+}
+
+typedef void (flash_attn_ext_f16_t)(
+ device const char * q,
+ device const char * k,
+ device const char * v,
+ device const char * mask,
+ device float * dst,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant uint64_t & nb21,
+ constant uint64_t & nb22,
+ constant uint64_t & nb23,
+ constant uint64_t & nb31,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant float & scale,
+ constant float & max_bias,
+ constant float & m0,
+ constant float & m1,
+ constant uint32_t & n_head_log2,
+ constant float & logit_softcap,
+ threadgroup half * shared,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]]);
+
+// ref: https://arxiv.org/pdf/2307.08691.pdf
+template<int64_t D, int64_t Q = 8, int64_t C = 32> // head size, queries per threadgroup, cache items per threadgroup
+kernel void kernel_flash_attn_ext_f16(
+ device const char * q,
+ device const char * k,
+ device const char * v,
+ device const char * mask,
+ device float * dst,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant uint64_t & nb21,
+ constant uint64_t & nb22,
+ constant uint64_t & nb23,
+ constant uint64_t & nb31,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant float & scale,
+ constant float & max_bias,
+ constant float & m0,
+ constant float & m1,
+ constant uint32_t & n_head_log2,
+ constant float & logit_softcap,
+ threadgroup half * shared [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+ const short nsg = ntg.y; // number of simdgroups
+
+ const short iq3 = tgpig[2];
+ const short iq2 = tgpig[1];
+ const short iq1 = tgpig[0]*Q;
+
+ const short D4 = D/4;
+ const short D8 = D/8;
+ //const short Q8 = Q/8;
+ const short NW = N_SIMDWIDTH;
+ const short SH = (C + Q); // shared memory per simdgroup in (half)
+
+ const short T = D + 2*nsg*SH; // shared memory size per query in (half)
+ const short TF = T/2; // shared memory size per query in (float)
+ const short T4 = T/4; // shared memory size per query in (half4)
+
+ threadgroup half * sq = (threadgroup half *) (shared + 0*D); // holds the query data
+ threadgroup half4 * sq4 = (threadgroup half4 *) (shared + 0*D); // same as above but in half4
+ threadgroup float * ss = (threadgroup float *) (shared + 2*sgitg*SH + 1*D); // scratch buffer for attention and diagonal matrix
+
+ // store the result for all queries in local memory in 8x8 matrices (the O matrix from the paper)
+ simdgroup_half8x8 lo[D8];
+
+ // load heads from Q to shared memory
+ for (short j = sgitg; j < Q; j += nsg) {
+ device const float4 * q4 = (device const float4 *) ((device const char *) q + ((iq1 + j)*nb01 + iq2*nb02 + iq3*nb03));
+
+ for (short i = tiisg; i < D4; i += NW) {
+ if (iq1 + j < ne01) {
+ sq4[j*T4 + i] = (half4) q4[i];
+ } else {
+ sq4[j*T4 + i] = 0.0h;
+ }
+ }
+ }
+
+ // zero out lo
+ for (short i = 0; i < D8; ++i) {
+ lo[i] = make_filled_simdgroup_matrix<half, 8>(0.0h);
+ }
+
+ // zero out shared memory SH
+ for (short j = 0; j < Q; ++j) {
+ for (short i = tiisg; i < SH; i += NW) {
+ ss[j*TF + i] = 0.0f;
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ {
+ float S[Q] = { [0 ... Q-1] = 0.0h };
+ float M[Q] = { [0 ... Q-1] = -FLT_MAX/2 };
+
+ // assume K and V are same shape
+ const short ne22 = ne12;
+ const short ne23 = ne13;
+
+ // broadcast
+ const short rk2 = ne02/ne12;
+ const short rk3 = ne03/ne13;
+
+ const short rv2 = ne02/ne22;
+ const short rv3 = ne03/ne23;
+
+ // k indices
+ const short ik2 = iq2/rk2;
+ const short ik3 = iq3/rk3;
+
+ // v indices
+ const short iv2 = iq2/rv2;
+ const short iv3 = iq3/rv3;
+
+ // load the queries from shared memory into local memory
+ simdgroup_half8x8 mq[D8];
+
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_load(mq[i], sq + i*8, T);
+ }
+
+ // pointer to the mask
+ device const half * mp = (device const half *) (mask + iq1*nb31);
+
+ float slope = 1.0f;
+
+ // ALiBi
+ if (max_bias > 0.0f) {
+ const uint32_t h = iq2;
+
+ const float base = h < n_head_log2 ? m0 : m1;
+ const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;
+
+ slope = pow(base, exph);
+ }
+
+ // loop over the KV cache
+ // each simdgroup handles blocks of Q rows and C columns
+ for (int ic0 = 0; ic0 < ne11; ic0 += C*nsg) {
+ const int ic = ic0 + C*sgitg;
+ if (ic >= ne11) {
+ break;
+ }
+
+ // Q*K^T
+ {
+ for (short cc = 0; cc < C/8; ++cc) {
+ simdgroup_float8x8 mqk = make_filled_simdgroup_matrix<float, 8>(0.h);
+
+ device const half * pk = (device const half *) ((device const char *) k + ((ic + 8*cc)*nb11 + ik2*nb12 + ik3*nb13));
+
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_half8x8 mk;
+ simdgroup_load(mk, pk + i*8, nb11/sizeof(half), 0, true); // transpose
+
+ simdgroup_multiply_accumulate(mqk, mq[i], mk, mqk);
+ }
+
+ simdgroup_store(mqk, ss + 8*cc, TF, 0, false);
+ }
+ }
+
+ // used to detect blocks full of -INF
+ float smax = -INFINITY;
+
+ // online softmax
+ {
+ float ms[Q];
+
+ for (short j = 0; j < Q; ++j) {
+ const float m = M[j];
+
+ // scale and apply the logitcap / mask
+ float s = ss[j*TF + tiisg]*scale;
+
+ if (logit_softcap != 0.0f) {
+ s = logit_softcap*precise::tanh(s);
+ }
+
+ if (mask != q) {
+ // mqk = mqk + mask*slope
+ s += slope*mp[ic + j*nb31/sizeof(half) + tiisg];
+ }
+
+ smax = simd_max(max(smax, s));
+ M[j] = simd_max(max(M[j], s));
+
+ ms[j] = exp(m - M[j]);
+ const float vs = exp(s - M[j]);
+
+ S[j] = S[j]*ms[j] + simd_sum(vs);
+
+ // the P matrix from the paper (Q rows, C columns)
+ ss[j*TF + tiisg] = vs;
+ }
+
+ // create a QxQ diagonal matrix for rescaling the output
+ if (tiisg < Q) {
+ ss[tiisg*TF + C + tiisg] = ms[tiisg];
+ }
+ }
+
+ // skip -INF blocks
+ if (smax == -INFINITY) {
+ continue;
+ }
+
+ // O = diag(ms)*O
+ {
+ simdgroup_float8x8 mm;
+ simdgroup_load(mm, ss + C, TF, 0, false);
+
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_multiply(lo[i], mm, lo[i]);
+ }
+ }
+
+ // O = O + (Q*K^T)*V
+ {
+ for (short cc = 0; cc < C/8; ++cc) {
+ device const half * pv = (device const half *) ((device const char *) v + ((ic + 8*cc)*nb21 + iv2*nb22 + iv3*nb23));
+
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_half8x8 mk;
+ simdgroup_load(mk, pv + i*8, nb21/sizeof(half), 0, false);
+
+ simdgroup_float8x8 mv;
+ simdgroup_load(mv, ss + 8*cc, TF, 0, false);
+
+ simdgroup_multiply_accumulate(lo[i], mv, mk, lo[i]);
+ }
+ }
+ }
+ }
+
+ // these are needed for reducing the results from the simdgroups (reuse the ss buffer)
+ for (short j = 0; j < Q; ++j) {
+ if (tiisg == 0) {
+ ss[j*TF + 0] = S[j];
+ ss[j*TF + 1] = M[j];
+ }
+ }
+ }
+
+ // reduce the warps sequentially
+ for (short sg = 1; sg < nsg; ++sg) {
+ float S = { 0.0h };
+ float M = { -FLT_MAX/2 };
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // each simdgroup stores its output to shared memory, reusing sq
+ if (sgitg == sg) {
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_store(lo[i], sq + i*8, T, 0, false);
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // the first simdgroup accumulates the results from the other simdgroups
+ if (sgitg == 0) {
+ for (short j = 0; j < Q; ++j) {
+ const float S0 = ss[j*TF + 0];
+ const float S1 = ss[j*TF + sg*SH + 0];
+
+ const float M0 = ss[j*TF + 1];
+ const float M1 = ss[j*TF + sg*SH + 1];
+
+ M = max(M0, M1);
+
+ const float ms0 = exp(M0 - M);
+ const float ms1 = exp(M1 - M);
+
+ S = S0*ms0 + S1*ms1;
+
+ if (tiisg == 0) {
+ ss[j*TF + 0] = S;
+ ss[j*TF + 1] = M;
+
+ ss[j*TF + C + j ] = ms0;
+ ss[j*TF + C + j + sg*SH] = ms1;
+ }
+ }
+
+ // O_0 = diag(ms0)*O_0 + diag(ms1)*O_1
+ {
+ simdgroup_half8x8 t;
+ simdgroup_float8x8 ms0;
+ simdgroup_float8x8 ms1;
+
+ simdgroup_load(ms0, ss + C, TF, 0, false);
+ simdgroup_load(ms1, ss + C + sg*SH, TF, 0, false);
+
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_load (t, sq + i*8, T, 0, false);
+ simdgroup_multiply(t, ms1, t);
+
+ simdgroup_multiply_accumulate(lo[i], ms0, lo[i], t);
+ }
+ }
+ }
+ }
+
+ // store result to shared memory (reuse sq)
+ if (sgitg == 0) {
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_store(lo[i], sq + i*8, T, 0, false);
+ }
+ }
+
+ device float4 * dst4 = (device float4 *) dst;
+
+ // final rescale with 1/S and store to global memory
+ if (sgitg == 0) {
+ for (short j = 0; j < Q && iq1 + j < ne01; ++j) {
+ const float S = ss[j*TF + 0];
+
+ for (short i = tiisg; i < D4; i += NW) {
+ dst4[(iq3*ne2*ne1 + iq2 + (iq1 + j)*ne1)*D4 + i] = (float4) sq4[j*T4 + i]/S;
+ }
+ }
+ }
+}
+
+template [[host_name("kernel_flash_attn_ext_f16_h64" )]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<64>;
+template [[host_name("kernel_flash_attn_ext_f16_h80" )]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<80>;
+template [[host_name("kernel_flash_attn_ext_f16_h96" )]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<96>;
+template [[host_name("kernel_flash_attn_ext_f16_h112")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<112>;
+template [[host_name("kernel_flash_attn_ext_f16_h128")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<128>;
+//template [[host_name("kernel_flash_attn_ext_f16_h256")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<256>;
+
+template<int64_t D, int64_t Q = 1, int64_t C = 32> // head size, queries per threadgroup, cache items per threadgroup
+kernel void kernel_flash_attn_ext_vec_f16(
+ device const char * q,
+ device const char * k,
+ device const char * v,
+ device const char * mask,
+ device float * dst,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant uint64_t & nb21,
+ constant uint64_t & nb22,
+ constant uint64_t & nb23,
+ constant uint64_t & nb31,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant float & scale,
+ constant float & max_bias,
+ constant float & m0,
+ constant float & m1,
+ constant uint32_t & n_head_log2,
+ constant float & logit_softcap,
+ threadgroup half * shared [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+ const short nsg = ntg.y; // number of simdgroups
+
+ const short iq3 = tgpig[2];
+ const short iq2 = tgpig[1];
+ const short iq1 = tgpig[0];
+
+ const short D4 = D/4;
+ const short NW = N_SIMDWIDTH;
+ const short SH = (C + Q); // shared memory per simdgroup in (half)
+
+ const short T = D + 2*nsg*SH; // shared memory size per query in (half)
+
+ float slope = 1.0f;
+
+ // ALiBi
+ if (max_bias > 0.0f) {
+ const uint32_t h = iq2;
+
+ const float base = h < n_head_log2 ? m0 : m1;
+ const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;
+
+ slope = pow(base, exp);
+ }
+
+ //threadgroup half * sq = (threadgroup half *) (shared + 0*D); // holds the query data
+ threadgroup half4 * sq4 = (threadgroup half4 *) (shared + 0*D); // same as above but in half4
+ threadgroup float * ss = (threadgroup float *) (shared + 2*sgitg*SH + 1*D); // scratch buffer for attention and diagonal matrix
+ threadgroup float4 * ss4 = (threadgroup float4 *) (shared + 2*sgitg*SH + 1*D); // same as above but in half4
+ threadgroup half4 * sr4 = (threadgroup half4 *) (shared + sgitg*D + 1*T); // scratch buffer for the results
+
+ // store the result for all queries in local memory in 8x8 matrices (the O matrix from the paper)
+ half4 lo[D4/NW];
+
+ // load heads from Q to shared memory
+ device const float4 * q4 = (device const float4 *) ((device const char *) q + (iq1*nb01 + iq2*nb02 + iq3*nb03));
+
+ for (short i = tiisg; i < D4; i += NW) {
+ if (iq1 < ne01) {
+ sq4[i] = (half4) q4[i];
+ } else {
+ sq4[i] = 0.0h;
+ }
+ }
+
+ // zero out lo
+ for (short i = tiisg; i < D4; i += NW) {
+ lo[i/NW] = 0.0h;
+ }
+
+ // zero out shared memory SH
+ for (short i = tiisg; i < SH/4; i += NW) {
+ ss4[i] = 0.0h;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ {
+ float S = { 0.0h };
+ float M = { -FLT_MAX/2 };
+
+ // assume K and V are same shape
+ const short ne22 = ne12;
+ const short ne23 = ne13;
+
+ // broadcast
+ const short rk2 = ne02/ne12;
+ const short rk3 = ne03/ne13;
+
+ const short rv2 = ne02/ne22;
+ const short rv3 = ne03/ne23;
+
+ // k indices
+ const short ik2 = iq2 / rk2;
+ const short ik3 = iq3 / rk3;
+
+ // v indices
+ const short iv2 = iq2 / rv2;
+ const short iv3 = iq3 / rv3;
+
+ // load the queries from shared memory into local memory
+ half4 mq[D4];
+
+ for (short ii = 0; ii < D4; ii += NW) {
+ short i = ii + tiisg;
+ mq[i] = sq4[i];
+ }
+
+ // pointer to the mask
+ device const half4 * mp4 = (device const half4 *) (mask + iq1*nb31);
+
+ // loop over the KV cache
+ // each simdgroup handles blocks of Q rows and C columns
+ for (int ic0 = 0; ic0 < ne11; ic0 += C*nsg) {
+ const int ic = ic0 + C*sgitg;
+ if (ic >= ne11) {
+ break;
+ }
+
+ // Q*K^T
+ {
+#pragma unroll
+ for (short cc = 0; cc < C/4; ++cc) {
+ float4 mqk = { 0.0h };
+
+ device const half4 * pk4 = (device const half4 *) ((device const char *) k + ((ic + 4*cc)*nb11 + ik2*nb12 + ik3*nb13));
+
+#pragma unroll
+ for (short ii = 0; ii < D4; ii += NW) {
+ const short i = ii + tiisg;
+
+ half4x4 mk;
+ mk[0] = pk4[i + 0*(nb11/8)];
+ mk[1] = pk4[i + 1*(nb11/8)];
+ mk[2] = pk4[i + 2*(nb11/8)];
+ mk[3] = pk4[i + 3*(nb11/8)];
+
+ mqk += (float4) (mq[i] * mk);
+ }
+
+ // reduce the results from the threads in the simdgroup
+ mqk += simd_shuffle_down(mqk, 16);
+ mqk += simd_shuffle_down(mqk, 8);
+ mqk += simd_shuffle_down(mqk, 4);
+ mqk += simd_shuffle_down(mqk, 2);
+ mqk += simd_shuffle_down(mqk, 1);
+
+ // mqk = mqk*scale + mask*slope
+ if (tiisg == 0) {
+ mqk *= scale;
+
+ if (logit_softcap != 0.0f) {
+ mqk = logit_softcap*precise::tanh(mqk);
+ }
+
+ mqk += (mask != q) ? ((float4) mp4[ic/4 + cc])*slope : (float4) 0.0f;
+
+ ss4[cc] = mqk;
+ }
+ }
+ }
+
+ // online softmax
+ {
+ const short p = tiisg;
+
+ const float m = M;
+ const float s = ss[p];
+
+ M = simd_max(max(M, s));
+
+ const float ms = exp(m - M);
+ const float vs = exp(s - M);
+
+ S = S*ms + simd_sum(vs);
+
+ // the P matrix from the paper (Q rows, C columns)
+ ss[p] = vs;
+
+ // O = diag(ms)*O
+#pragma unroll
+ for (short ii = 0; ii < D4; ii += NW) {
+ const short i = ii + tiisg;
+ lo[i/NW] *= ms;
+ }
+ }
+
+ // O = O + (Q*K^T)*V
+ {
+#pragma unroll
+ for (short cc = 0; cc < C/4; ++cc) {
+ device const half4 * pv4 = (device const half4 *) ((device const char *) v + ((ic + 4*cc)*nb21 + iv2*nb22 + iv3*nb23));
+
+#pragma unroll
+ for (short ii = 0; ii < D4; ii += NW) {
+ const short i = ii + tiisg;
+
+ lo[i/NW] += pv4[i + 0*(nb21/8)] * ss[4*cc + 0];
+ lo[i/NW] += pv4[i + 1*(nb21/8)] * ss[4*cc + 1];
+ lo[i/NW] += pv4[i + 2*(nb21/8)] * ss[4*cc + 2];
+ lo[i/NW] += pv4[i + 3*(nb21/8)] * ss[4*cc + 3];
+ }
+ }
+ }
+
+ }
+
+ // these are needed for reducing the results from the simdgroups (reuse the ss buffer)
+ if (tiisg == 0) {
+ ss[0] = S;
+ ss[1] = M;
+ }
+ }
+
+ // store results to shared memory
+ for (short ii = 0; ii < D4; ii += NW) {
+ short i = ii + tiisg;
+ sr4[i] = lo[ii/NW];
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // parallel reduce
+ for (short r = nsg/2; r > 0; r >>= 1) {
+ if (sgitg < r) {
+ const float S0 = ss[ 0];
+ const float S1 = ss[r*SH + 0];
+
+ const float M0 = ss[ 1];
+ const float M1 = ss[r*SH + 1];
+
+ const float M = max(M0, M1);
+
+ const float ms0 = exp(M0 - M);
+ const float ms1 = exp(M1 - M);
+
+ const float S = S0*ms0 + S1*ms1;
+
+ if (tiisg == 0) {
+ ss[0] = S;
+ ss[1] = M;
+ }
+
+ // O_0 = diag(ms0)*O_0 + diag(ms1)*O_1
+ for (short ii = 0; ii < D4; ii += NW) {
+ short i = ii + tiisg;
+ sr4[i] = sr4[i]*ms0 + sr4[i + r*D4]*ms1;
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ device float4 * dst4 = (device float4 *) dst;
+
+ // final rescale with 1/S and store to global memory
+ if (sgitg == 0) {
+ const float S = ss[0];
+
+ for (short ii = 0; ii < D4; ii += NW) {
+ short i = ii + tiisg;
+ dst4[(iq3*ne2*ne1 + iq2 + (iq1)*ne1)*D4 + i] = (float4) sr4[i]/S;
+ }
+ }
+}
+
+template [[host_name("kernel_flash_attn_ext_vec_f16_h128")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_vec_f16<128>;
+//template [[host_name("kernel_flash_attn_ext_vec_f16_h256")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_vec_f16<256>;
+
+template<typename T0, typename T1>
+kernel void kernel_cpy(
+ device const void * src0,
+ device void * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = tgpig[2];
+ const int64_t i02 = tgpig[1];
+ const int64_t i01 = tgpig[0];
+
+ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
+
+ const int64_t i3 = n / (ne2*ne1*ne0);
+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);
+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;
+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);
+
+ device T1 * dst_data = (device T1 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {
+ device const T0 * src = (device T0 *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);
+ dst_data[i00] = (T1) src[0];
+ }
+}
+
+typedef decltype(kernel_cpy<float, float>) kernel_cpy_t;
+
+template [[host_name("kernel_cpy_f32_f32")]] kernel kernel_cpy_t kernel_cpy<float, float>;
+template [[host_name("kernel_cpy_f32_f16")]] kernel kernel_cpy_t kernel_cpy<float, half>;
+template [[host_name("kernel_cpy_f16_f16")]] kernel kernel_cpy_t kernel_cpy<half, half>;
+template [[host_name("kernel_cpy_f16_f32")]] kernel kernel_cpy_t kernel_cpy<half, float>;
+
+kernel void kernel_cpy_f32_q8_0(
+ device const float * src0,
+ device void * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = tgpig[2];
+ const int64_t i02 = tgpig[1];
+ const int64_t i01 = tgpig[0];
+
+ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
+
+ const int64_t i3 = n / (ne2*ne1*ne0);
+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);
+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;
+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK8_0;
+
+ device block_q8_0 * dst_data = (device block_q8_0 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ for (int64_t i00 = tpitg.x*QK8_0; i00 < ne00; i00 += ntg.x*QK8_0) {
+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);
+
+ float amax = 0.0f; // absolute max
+
+ for (int j = 0; j < QK8_0; j++) {
+ const float v = src[j];
+ amax = MAX(amax, fabs(v));
+ }
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ dst_data[i00/QK8_0].d = d;
+
+ for (int j = 0; j < QK8_0; ++j) {
+ const float x0 = src[j]*id;
+
+ dst_data[i00/QK8_0].qs[j] = round(x0);
+ }
+ }
+}
+
+kernel void kernel_cpy_f32_q4_0(
+ device const float * src0,
+ device void * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = tgpig[2];
+ const int64_t i02 = tgpig[1];
+ const int64_t i01 = tgpig[0];
+
+ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
+
+ const int64_t i3 = n / (ne2*ne1*ne0);
+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);
+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;
+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK4_0;
+
+ device block_q4_0 * dst_data = (device block_q4_0 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ for (int64_t i00 = tpitg.x*QK4_0; i00 < ne00; i00 += ntg.x*QK4_0) {
+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);
+
+ float amax = 0.0f; // absolute max
+ float max = 0.0f;
+
+ for (int j = 0; j < QK4_0; j++) {
+ const float v = src[j];
+ if (amax < fabs(v)) {
+ amax = fabs(v);
+ max = v;
+ }
+ }
+
+ const float d = max / -8;
+ const float id = d ? 1.0f/d : 0.0f;
+
+ dst_data[i00/QK4_0].d = d;
+
+ for (int j = 0; j < QK4_0/2; ++j) {
+ const float x0 = src[0 + j]*id;
+ const float x1 = src[QK4_0/2 + j]*id;
+
+ const uint8_t xi0 = MIN(15, (int8_t)(x0 + 8.5f));
+ const uint8_t xi1 = MIN(15, (int8_t)(x1 + 8.5f));
+
+ dst_data[i00/QK4_0].qs[j] = xi0;
+ dst_data[i00/QK4_0].qs[j] |= xi1 << 4;
+ }
+ }
+}
+
+kernel void kernel_cpy_f32_q4_1(
+ device const float * src0,
+ device void * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = tgpig[2];
+ const int64_t i02 = tgpig[1];
+ const int64_t i01 = tgpig[0];
+
+ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
+
+ const int64_t i3 = n / (ne2*ne1*ne0);
+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);
+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;
+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK4_1;
+
+ device block_q4_1 * dst_data = (device block_q4_1 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ for (int64_t i00 = tpitg.x*QK4_1; i00 < ne00; i00 += ntg.x*QK4_1) {
+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);
+
+ float min = FLT_MAX;
+ float max = -FLT_MAX;
+
+ for (int j = 0; j < QK4_1; j++) {
+ const float v = src[j];
+ if (min > v) min = v;
+ if (max < v) max = v;
+ }
+
+ const float d = (max - min) / ((1 << 4) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ dst_data[i00/QK4_1].d = d;
+ dst_data[i00/QK4_1].m = min;
+
+ for (int j = 0; j < QK4_1/2; ++j) {
+ const float x0 = (src[0 + j] - min)*id;
+ const float x1 = (src[QK4_1/2 + j] - min)*id;
+
+ const uint8_t xi0 = MIN(15, (int8_t)(x0 + 0.5f));
+ const uint8_t xi1 = MIN(15, (int8_t)(x1 + 0.5f));
+
+ dst_data[i00/QK4_1].qs[j] = xi0;
+ dst_data[i00/QK4_1].qs[j] |= xi1 << 4;
+ }
+ }
+}
+
+kernel void kernel_cpy_f32_q5_0(
+ device const float * src0,
+ device void * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = tgpig[2];
+ const int64_t i02 = tgpig[1];
+ const int64_t i01 = tgpig[0];
+
+ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
+
+ const int64_t i3 = n / (ne2*ne1*ne0);
+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);
+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;
+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK5_0;
+
+ device block_q5_0 * dst_data = (device block_q5_0 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ for (int64_t i00 = tpitg.x*QK5_0; i00 < ne00; i00 += ntg.x*QK5_0) {
+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);
+
+ float amax = 0.0f; // absolute max
+ float max = 0.0f;
+
+ for (int j = 0; j < QK5_0; j++) {
+ const float v = src[j];
+ if (amax < fabs(v)) {
+ amax = fabs(v);
+ max = v;
+ }
+ }
+
+ const float d = max / -16;
+ const float id = d ? 1.0f/d : 0.0f;
+
+ dst_data[i00/QK5_0].d = d;
+
+ uint32_t qh = 0;
+ for (int j = 0; j < QK5_0/2; ++j) {
+ const float x0 = src[0 + j]*id;
+ const float x1 = src[QK5_0/2 + j]*id;
+
+ const uint8_t xi0 = MIN(31, (int8_t)(x0 + 16.5f));
+ const uint8_t xi1 = MIN(31, (int8_t)(x1 + 16.5f));
+
+ dst_data[i00/QK5_0].qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);
+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);
+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);
+ }
+ thread const uint8_t * qh8 = (thread const uint8_t *)&qh;
+ for (int j = 0; j < 4; ++j) {
+ dst_data[i00/QK5_0].qh[j] = qh8[j];
+ }
+ }
+}
+
+kernel void kernel_cpy_f32_q5_1(
+ device const float * src0,
+ device void * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = tgpig[2];
+ const int64_t i02 = tgpig[1];
+ const int64_t i01 = tgpig[0];
+
+ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
+
+ const int64_t i3 = n / (ne2*ne1*ne0);
+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);
+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;
+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK5_1;
+
+ device block_q5_1 * dst_data = (device block_q5_1 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ for (int64_t i00 = tpitg.x*QK5_1; i00 < ne00; i00 += ntg.x*QK5_1) {
+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);
+
+ float max = src[0];
+ float min = src[0];
+
+ for (int j = 1; j < QK5_1; j++) {
+ const float v = src[j];
+ min = v < min ? v : min;
+ max = v > max ? v : max;
+ }
+
+ const float d = (max - min) / 31;
+ const float id = d ? 1.0f/d : 0.0f;
+
+ dst_data[i00/QK5_1].d = d;
+ dst_data[i00/QK5_1].m = min;
+
+ uint32_t qh = 0;
+ for (int j = 0; j < QK5_1/2; ++j) {
+ const float x0 = (src[0 + j] - min)*id;
+ const float x1 = (src[QK5_1/2 + j] - min)*id;
+
+ const uint8_t xi0 = (uint8_t)(x0 + 0.5f);
+ const uint8_t xi1 = (uint8_t)(x1 + 0.5f);
+
+ dst_data[i00/QK5_1].qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);
+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);
+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2);
+ }
+ thread const uint8_t * qh8 = (thread const uint8_t *)&qh;
+ for (int j = 0; j < 4; ++j) {
+ dst_data[i00/QK5_1].qh[j] = qh8[j];
+ }
+ }
+}
+
+static inline int best_index_int8(int n, constant float * val, float x) {
+ if (x <= val[0]) return 0;
+ if (x >= val[n-1]) return n-1;
+ int ml = 0, mu = n-1;
+ while (mu-ml > 1) {
+ int mav = (ml+mu)/2;
+ if (x < val[mav]) mu = mav; else ml = mav;
+ }
+ return x - val[mu-1] < val[mu] - x ? mu-1 : mu;
+}
+
+constexpr constant static float kvalues_iq4nl_f[16] = {
+ -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f
+};
+
+kernel void kernel_cpy_f32_iq4_nl(
+ device const float * src0,
+ device void * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+ const int64_t i03 = tgpig[2];
+ const int64_t i02 = tgpig[1];
+ const int64_t i01 = tgpig[0];
+
+ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
+
+ const int64_t i3 = n / (ne2*ne1*ne0);
+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);
+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;
+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK4_NL;
+
+ device block_iq4_nl * dst_data = (device block_iq4_nl *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ for (int64_t i00 = tpitg.x*QK4_NL; i00 < ne00; i00 += ntg.x*QK4_NL) {
+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);
+
+ float amax = 0.0f; // absolute max
+ float max = 0.0f;
+
+ for (int j = 0; j < QK4_0; j++) {
+ const float v = src[j];
+ if (amax < fabs(v)) {
+ amax = fabs(v);
+ max = v;
+ }
+ }
+
+ const float d = max / kvalues_iq4nl_f[0];
+ const float id = d ? 1.0f/d : 0.0f;
+
+ float sumqx = 0, sumq2 = 0;
+ for (int j = 0; j < QK4_NL/2; ++j) {
+ const float x0 = src[0 + j]*id;
+ const float x1 = src[QK4_NL/2 + j]*id;
+
+ const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl_f, x0);
+ const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl_f, x1);
+
+ dst_data[i00/QK4_NL].qs[j] = xi0 | (xi1 << 4);
+
+ const float v0 = kvalues_iq4nl_f[xi0];
+ const float v1 = kvalues_iq4nl_f[xi1];
+ const float w0 = src[0 + j]*src[0 + j];
+ const float w1 = src[QK4_NL/2 + j]*src[QK4_NL/2 + j];
+ sumqx += w0*v0*src[j] + w1*v1*src[QK4_NL/2 + j];
+ sumq2 += w0*v0*v0 + w1*v1*v1;
+
+ }
+
+ dst_data[i00/QK4_NL].d = sumq2 > 0 ? sumqx/sumq2 : d;
+
+ }
+}
+
+kernel void kernel_concat(
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant uint64_t & nb0,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ constant uint64_t & nb3,
+ constant int32_t & dim,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]]) {
+
+ const int64_t i3 = tgpig.z;
+ const int64_t i2 = tgpig.y;
+ const int64_t i1 = tgpig.x;
+
+ int64_t o[4] = {0, 0, 0, 0};
+ o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));
+
+ device const float * x;
+
+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {
+ if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
+ x = (device const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);
+ } else {
+ x = (device const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
+ }
+
+ device float * y = (device float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ *y = *x;
+ }
+}
+
+void kernel_mul_mv_q2_K_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const int nb = ne00/QK_K;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;
+ const int ib_row = first_row * nb;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_q2_K * x = (device const block_q2_K *) src0 + ib_row + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[32];
+ float sumf[N_DST]={0.f}, all_sum;
+
+ const int step = sizeof(block_q2_K) * nb;
+
+ const int ix = tiisg/8; // 0...3
+ const int it = tiisg%8; // 0...7
+ const int iq = it/4; // 0 or 1
+ const int ir = it%4; // 0...3
+ const int is = (8*ir)/16;// 0 or 1
+
+ device const float * y4 = y + ix * QK_K + 128 * iq + 8 * ir;
+
+ for (int ib = ix; ib < nb; ib += 4) {
+
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
+ for (int i = 0; i < 8; ++i) {
+ yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0];
+ yl[i+ 8] = y4[i+32]; sumy[1] += yl[i+ 8];
+ yl[i+16] = y4[i+64]; sumy[2] += yl[i+16];
+ yl[i+24] = y4[i+96]; sumy[3] += yl[i+24];
+ }
+
+ device const uint8_t * sc = (device const uint8_t *)x[ib].scales + 8*iq + is;
+ device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;
+ device const half * dh = &x[ib].d;
+
+ for (int row = 0; row < N_DST; row++) {
+
+ float4 acc1 = {0.f, 0.f, 0.f, 0.f};
+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};
+ for (int i = 0; i < 8; i += 2) {
+ acc1[0] += yl[i+ 0] * (qs[i/2] & 0x0003);
+ acc2[0] += yl[i+ 1] * (qs[i/2] & 0x0300);
+ acc1[1] += yl[i+ 8] * (qs[i/2] & 0x000c);
+ acc2[1] += yl[i+ 9] * (qs[i/2] & 0x0c00);
+ acc1[2] += yl[i+16] * (qs[i/2] & 0x0030);
+ acc2[2] += yl[i+17] * (qs[i/2] & 0x3000);
+ acc1[3] += yl[i+24] * (qs[i/2] & 0x00c0);
+ acc2[3] += yl[i+25] * (qs[i/2] & 0xc000);
+ }
+ float dall = dh[0];
+ float dmin = dh[1] * 1.f/16.f;
+ sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc2[0]) * (sc[0] & 0xF) * 1.f/ 1.f +
+ (acc1[1] + 1.f/256.f * acc2[1]) * (sc[2] & 0xF) * 1.f/ 4.f +
+ (acc1[2] + 1.f/256.f * acc2[2]) * (sc[4] & 0xF) * 1.f/16.f +
+ (acc1[3] + 1.f/256.f * acc2[3]) * (sc[6] & 0xF) * 1.f/64.f) -
+ dmin * (sumy[0] * (sc[0] & 0xF0) + sumy[1] * (sc[2] & 0xF0) + sumy[2] * (sc[4] & 0xF0) + sumy[3] * (sc[6] & 0xF0));
+
+ qs += step/2;
+ sc += step;
+ dh += step/2;
+ }
+
+ y4 += 4 * QK_K;
+ }
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_q2_K_f32")]]
+kernel void kernel_mul_mv_q2_K_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_q2_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, nullptr, tgpig, tiisg, sgitg);
+}
+
+void kernel_mul_mv_q3_K_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const int nb = ne00/QK_K;
+
+ const int64_t r0 = tgpig.x;
+ const int64_t r1 = tgpig.y;
+ const int64_t im = tgpig.z;
+
+ const int first_row = (r0 * N_SIMDGROUP + sgitg) * 2;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_q3_K * x = (device const block_q3_K *) src0 + first_row*nb + offset0;
+ device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[32];
+
+ //const uint16_t kmask1 = 0x3030;
+ //const uint16_t kmask2 = 0x0f0f;
+
+ const int tid = tiisg/4;
+ const int ix = tiisg%4;
+ const int ip = tid/4; // 0 or 1
+ const int il = 2*((tid%4)/2); // 0 or 2
+ const int ir = tid%2;
+ const int n = 8;
+ const int l0 = n*ir;
+
+ // One would think that the Metal compiler would figure out that ip and il can only have
+ // 4 possible states, and optimize accordingly. Well, no. It needs help, and we do it
+ // with these two tales.
+ //
+ // Possible masks for the high bit
+ const ushort4 mm[4] = {{0x0001, 0x0100, 0x0002, 0x0200}, // ip = 0, il = 0
+ {0x0004, 0x0400, 0x0008, 0x0800}, // ip = 0, il = 2
+ {0x0010, 0x1000, 0x0020, 0x2000}, // ip = 1, il = 0
+ {0x0040, 0x4000, 0x0080, 0x8000}}; // ip = 1, il = 2
+
+ // Possible masks for the low 2 bits
+ const int4 qm[2] = {{0x0003, 0x0300, 0x000c, 0x0c00}, {0x0030, 0x3000, 0x00c0, 0xc000}};
+
+ const ushort4 hm = mm[2*ip + il/2];
+
+ const int shift = 2*il;
+ const float v1 = il == 0 ? 4.f : 64.f;
+ const float v2 = 4.f * v1;
+
+ const uint16_t s_shift1 = 4*ip;
+ const uint16_t s_shift2 = s_shift1 + il;
+
+ const int q_offset = 32*ip + l0;
+ const int y_offset = 128*ip + 32*il + l0;
+
+ const int step = sizeof(block_q3_K) * nb / 2;
+
+ device const float * y1 = yy + ix*QK_K + y_offset;
+
+ uint32_t scales32, aux32;
+ thread uint16_t * scales16 = (thread uint16_t *)&scales32;
+ thread const int8_t * scales = (thread const int8_t *)&scales32;
+
+ float sumf1[2] = {0.f};
+ float sumf2[2] = {0.f};
+ for (int i = ix; i < nb; i += 4) {
+
+ for (int l = 0; l < 8; ++l) {
+ yl[l+ 0] = y1[l+ 0];
+ yl[l+ 8] = y1[l+16];
+ yl[l+16] = y1[l+32];
+ yl[l+24] = y1[l+48];
+ }
+
+ device const uint16_t * q = (device const uint16_t *)(x[i].qs + q_offset);
+ device const uint16_t * h = (device const uint16_t *)(x[i].hmask + l0);
+ device const uint16_t * a = (device const uint16_t *)(x[i].scales);
+ device const half * dh = &x[i].d;
+
+ for (int row = 0; row < 2; ++row) {
+
+ const float d_all = (float)dh[0];
+
+ scales16[0] = a[4];
+ scales16[1] = a[5];
+ aux32 = ((scales32 >> s_shift2) << 4) & 0x30303030;
+ scales16[0] = a[il+0];
+ scales16[1] = a[il+1];
+ scales32 = ((scales32 >> s_shift1) & 0x0f0f0f0f) | aux32;
+
+ float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0;
+ for (int l = 0; l < n; l += 2) {
+ const int32_t qs = q[l/2];
+ s1 += yl[l+0] * (qs & qm[il/2][0]);
+ s2 += yl[l+1] * (qs & qm[il/2][1]);
+ s3 += ((h[l/2] & hm[0]) ? 0.f : yl[l+0]) + ((h[l/2] & hm[1]) ? 0.f : yl[l+1]);
+ s4 += yl[l+16] * (qs & qm[il/2][2]);
+ s5 += yl[l+17] * (qs & qm[il/2][3]);
+ s6 += ((h[l/2] & hm[2]) ? 0.f : yl[l+16]) + ((h[l/2] & hm[3]) ? 0.f : yl[l+17]);
+ }
+ float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
+ float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
+ sumf1[row] += d1 * (scales[0] - 32);
+ sumf2[row] += d2 * (scales[2] - 32);
+
+ s1 = s2 = s3 = s4 = s5 = s6 = 0;
+ for (int l = 0; l < n; l += 2) {
+ const int32_t qs = q[l/2+8];
+ s1 += yl[l+8] * (qs & qm[il/2][0]);
+ s2 += yl[l+9] * (qs & qm[il/2][1]);
+ s3 += ((h[l/2+8] & hm[0]) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm[1]) ? 0.f : yl[l+9]);
+ s4 += yl[l+24] * (qs & qm[il/2][2]);
+ s5 += yl[l+25] * (qs & qm[il/2][3]);
+ s6 += ((h[l/2+8] & hm[2]) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm[3]) ? 0.f : yl[l+25]);
+ }
+ d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
+ d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
+ sumf1[row] += d1 * (scales[1] - 32);
+ sumf2[row] += d2 * (scales[3] - 32);
+
+ q += step;
+ h += step;
+ a += step;
+ dh += step;
+
+ }
+
+ y1 += 4 * QK_K;
+
+ }
+
+ for (int row = 0; row < 2; ++row) {
+ const float sumf = (sumf1[row] + 0.25f * sumf2[row]) / (1 << shift);
+ sumf1[row] = simd_sum(sumf);
+ }
+ if (tiisg == 0) {
+ for (int row = 0; row < 2; ++row) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = sumf1[row];
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_q3_K_f32")]]
+kernel void kernel_mul_mv_q3_K_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_q3_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, nullptr, tgpig, tiisg, sgitg);
+}
+
+void kernel_mul_mv_q4_K_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const uint16_t kmask1 = 0x3f3f;
+ const uint16_t kmask2 = 0x0f0f;
+ const uint16_t kmask3 = 0xc0c0;
+
+ const int ix = tiisg/8; // 0...3
+ const int it = tiisg%8; // 0...7
+ const int iq = it/4; // 0 or 1
+ const int ir = it%4; // 0...3
+
+ const int nb = ne00/QK_K;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+ //const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;
+ const int first_row = r0 * N_DST;
+ const int ib_row = first_row * nb;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_q4_K * x = (device const block_q4_K *) src0 + ib_row + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[16];
+ float yh[16];
+ float sumf[N_DST]={0.f}, all_sum;
+
+ const int step = sizeof(block_q4_K) * nb / 2;
+
+ device const float * y4 = y + ix * QK_K + 64 * iq + 8 * ir;
+
+ uint16_t sc16[4];
+ thread const uint8_t * sc8 = (thread const uint8_t *)sc16;
+
+ for (int ib = ix; ib < nb; ib += 4) {
+
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
+ for (int i = 0; i < 8; ++i) {
+ yl[i+0] = y4[i+ 0]; sumy[0] += yl[i+0];
+ yl[i+8] = y4[i+ 32]; sumy[1] += yl[i+8];
+ yh[i+0] = y4[i+128]; sumy[2] += yh[i+0];
+ yh[i+8] = y4[i+160]; sumy[3] += yh[i+8];
+ }
+
+ device const uint16_t * sc = (device const uint16_t *)x[ib].scales + iq;
+ device const uint16_t * q1 = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;
+ device const half * dh = &x[ib].d;
+
+ for (int row = 0; row < N_DST; row++) {
+
+ sc16[0] = sc[0] & kmask1;
+ sc16[1] = sc[2] & kmask1;
+ sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2);
+ sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2);
+
+ device const uint16_t * q2 = q1 + 32;
+
+ float4 acc1 = {0.f, 0.f, 0.f, 0.f};
+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};
+ for (int i = 0; i < 8; i += 2) {
+ acc1[0] += yl[i+0] * (q1[i/2] & 0x000F);
+ acc1[1] += yl[i+1] * (q1[i/2] & 0x0F00);
+ acc1[2] += yl[i+8] * (q1[i/2] & 0x00F0);
+ acc1[3] += yl[i+9] * (q1[i/2] & 0xF000);
+ acc2[0] += yh[i+0] * (q2[i/2] & 0x000F);
+ acc2[1] += yh[i+1] * (q2[i/2] & 0x0F00);
+ acc2[2] += yh[i+8] * (q2[i/2] & 0x00F0);
+ acc2[3] += yh[i+9] * (q2[i/2] & 0xF000);
+ }
+
+ float dall = dh[0];
+ float dmin = dh[1];
+ sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc1[1]) * sc8[0] +
+ (acc1[2] + 1.f/256.f * acc1[3]) * sc8[1] * 1.f/16.f +
+ (acc2[0] + 1.f/256.f * acc2[1]) * sc8[4] +
+ (acc2[2] + 1.f/256.f * acc2[3]) * sc8[5] * 1.f/16.f) -
+ dmin * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]);
+
+ q1 += step;
+ sc += step;
+ dh += step;
+ }
+
+ y4 += 4 * QK_K;
+ }
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_q4_K_f32")]]
+kernel void kernel_mul_mv_q4_K_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_q4_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, nullptr, tgpig, tiisg, sgitg);
+}
+
+void kernel_mul_mv_q5_K_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const int nb = ne00/QK_K;
+
+ const int64_t r0 = tgpig.x;
+ const int64_t r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * N_SIMDGROUP + sgitg) * 2;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_q5_K * x = (device const block_q5_K *) src0 + first_row*nb + offset0;
+ device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float sumf[2]={0.f};
+
+ const int step = sizeof(block_q5_K) * nb;
+
+ float yl[16], yh[16];
+
+ const uint16_t kmask1 = 0x3f3f;
+ const uint16_t kmask2 = 0x0f0f;
+ const uint16_t kmask3 = 0xc0c0;
+
+ const int tid = tiisg/4;
+ const int ix = tiisg%4;
+ const int iq = tid/4;
+ const int ir = tid%4;
+ const int n = 8;
+
+ const int l0 = n*ir;
+ const int q_offset = 32*iq + l0;
+ const int y_offset = 64*iq + l0;
+
+ const uint8_t hm1 = 1u << (2*iq);
+ const uint8_t hm2 = hm1 << 1;
+ const uint8_t hm3 = hm1 << 4;
+ const uint8_t hm4 = hm2 << 4;
+
+ uint16_t sc16[4];
+ thread const uint8_t * sc8 = (thread const uint8_t *)sc16;
+
+ device const float * y1 = yy + ix*QK_K + y_offset;
+
+ for (int i = ix; i < nb; i += 4) {
+
+ device const uint8_t * q1 = x[i].qs + q_offset;
+ device const uint8_t * qh = x[i].qh + l0;
+ device const half * dh = &x[i].d;
+ device const uint16_t * a = (device const uint16_t *)x[i].scales + iq;
+
+ device const float * y2 = y1 + 128;
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
+ for (int l = 0; l < 8; ++l) {
+ yl[l+0] = y1[l+ 0]; sumy[0] += yl[l+0];
+ yl[l+8] = y1[l+32]; sumy[1] += yl[l+8];
+ yh[l+0] = y2[l+ 0]; sumy[2] += yh[l+0];
+ yh[l+8] = y2[l+32]; sumy[3] += yh[l+8];
+ }
+
+ for (int row = 0; row < 2; ++row) {
+
+ device const uint8_t * q2 = q1 + 64;
+
+ sc16[0] = a[0] & kmask1;
+ sc16[1] = a[2] & kmask1;
+ sc16[2] = ((a[4] >> 0) & kmask2) | ((a[0] & kmask3) >> 2);
+ sc16[3] = ((a[4] >> 4) & kmask2) | ((a[2] & kmask3) >> 2);
+
+ float4 acc1 = {0.f};
+ float4 acc2 = {0.f};
+ for (int l = 0; l < n; ++l) {
+ uint8_t h = qh[l];
+ acc1[0] += yl[l+0] * (q1[l] & 0x0F);
+ acc1[1] += yl[l+8] * (q1[l] & 0xF0);
+ acc1[2] += yh[l+0] * (q2[l] & 0x0F);
+ acc1[3] += yh[l+8] * (q2[l] & 0xF0);
+ acc2[0] += h & hm1 ? yl[l+0] : 0.f;
+ acc2[1] += h & hm2 ? yl[l+8] : 0.f;
+ acc2[2] += h & hm3 ? yh[l+0] : 0.f;
+ acc2[3] += h & hm4 ? yh[l+8] : 0.f;
+ }
+ const float dall = dh[0];
+ const float dmin = dh[1];
+ sumf[row] += dall * (sc8[0] * (acc1[0] + 16.f*acc2[0]) +
+ sc8[1] * (acc1[1]/16.f + 16.f*acc2[1]) +
+ sc8[4] * (acc1[2] + 16.f*acc2[2]) +
+ sc8[5] * (acc1[3]/16.f + 16.f*acc2[3])) -
+ dmin * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]);
+
+ q1 += step;
+ qh += step;
+ dh += step/2;
+ a += step/2;
+
+ }
+
+ y1 += 4 * QK_K;
+
+ }
+
+ for (int row = 0; row < 2; ++row) {
+ const float tot = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_q5_K_f32")]]
+kernel void kernel_mul_mv_q5_K_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_q5_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, nullptr, tgpig, tiisg, sgitg);
+}
+
+void kernel_mul_mv_q6_K_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const uint8_t kmask1 = 0x03;
+ const uint8_t kmask2 = 0x0C;
+ const uint8_t kmask3 = 0x30;
+ const uint8_t kmask4 = 0xC0;
+
+ const int nb = ne00/QK_K;
+
+ const int64_t r0 = tgpig.x;
+ const int64_t r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int row = 2 * r0 + sgitg;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_q6_K * x = (device const block_q6_K *) src0 + row * nb + offset0;
+ device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float sumf = 0;
+
+ const int tid = tiisg/2;
+ const int ix = tiisg%2;
+ const int ip = tid/8; // 0 or 1
+ const int il = tid%8;
+ const int n = 4;
+ const int l0 = n*il;
+ const int is = 8*ip + l0/16;
+
+ const int y_offset = 128*ip + l0;
+ const int q_offset_l = 64*ip + l0;
+ const int q_offset_h = 32*ip + l0;
+
+ for (int i = ix; i < nb; i += 2) {
+
+ device const uint8_t * q1 = x[i].ql + q_offset_l;
+ device const uint8_t * q2 = q1 + 32;
+ device const uint8_t * qh = x[i].qh + q_offset_h;
+ device const int8_t * sc = x[i].scales + is;
+
+ device const float * y = yy + i * QK_K + y_offset;
+
+ const float dall = x[i].d;
+
+ float4 sums = {0.f, 0.f, 0.f, 0.f};
+ for (int l = 0; l < n; ++l) {
+ sums[0] += y[l+ 0] * ((int8_t)((q1[l] & 0xF) | ((qh[l] & kmask1) << 4)) - 32);
+ sums[1] += y[l+32] * ((int8_t)((q2[l] & 0xF) | ((qh[l] & kmask2) << 2)) - 32);
+ sums[2] += y[l+64] * ((int8_t)((q1[l] >> 4) | ((qh[l] & kmask3) << 0)) - 32);
+ sums[3] += y[l+96] * ((int8_t)((q2[l] >> 4) | ((qh[l] & kmask4) >> 2)) - 32);
+ }
+
+ sumf += dall * (sums[0] * sc[0] + sums[1] * sc[2] + sums[2] * sc[4] + sums[3] * sc[6]);
+
+ }
+
+ const float tot = simd_sum(sumf);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + row] = tot;
+ }
+}
+
+[[host_name("kernel_mul_mv_q6_K_f32")]]
+kernel void kernel_mul_mv_q6_K_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_q6_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, nullptr, tgpig, tiisg, sgitg);
+}
+
+// ======================= "True" 2-bit
+
+void kernel_mul_mv_iq2_xxs_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const int nb = ne00/QK_K;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;
+ const int ib_row = first_row * nb;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_iq2_xxs * x = (device const block_iq2_xxs *) src0 + ib_row + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[32];
+ float sumf[N_DST]={0.f}, all_sum;
+
+ const int nb32 = nb * (QK_K / 32);
+
+ threadgroup uint64_t * values = (threadgroup uint64_t *)shared_values;
+ threadgroup uint8_t * shared_signs = (threadgroup uint8_t *)(values + 256);
+ {
+ int nval = 4;
+ int pos = (32*sgitg + tiisg)*nval;
+ for (int i = 0; i < nval; ++i) values[pos + i] = iq2xxs_grid[pos + i];
+ nval = 2;
+ pos = (32*sgitg + tiisg)*nval;
+ for (int i = 0; i < nval; ++i) shared_signs[pos+i] = ksigns_iq2xs[pos+i];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ const int ix = tiisg;
+
+ device const float * y4 = y + 32 * ix;
+
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
+
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ }
+
+ const int ibl = ib32 / (QK_K / 32);
+ const int ib = ib32 % (QK_K / 32);
+
+ device const block_iq2_xxs * xr = x + ibl;
+ device const uint16_t * q2 = xr->qs + 4 * ib;
+ device const half * dh = &xr->d;
+
+ for (int row = 0; row < N_DST; row++) {
+
+ const float db = dh[0];
+ device const uint8_t * aux8 = (device const uint8_t *)q2;
+ const uint32_t aux32 = q2[2] | (q2[3] << 16);
+ const float d = db * (0.5f + (aux32 >> 28));
+
+ float sum = 0;
+ for (int l = 0; l < 4; ++l) {
+ const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(values + aux8[l]);
+ const uint8_t signs = shared_signs[(aux32 >> 7*l) & 127];
+ for (int j = 0; j < 8; ++j) {
+ sum += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
+ }
+ }
+ sumf[row] += d * sum;
+
+ dh += nb*sizeof(block_iq2_xxs)/2;
+ q2 += nb*sizeof(block_iq2_xxs)/2;
+ }
+
+ y4 += 32 * 32;
+ }
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum * 0.25f;
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_iq2_xxs_f32")]]
+kernel void kernel_mul_mv_iq2_xxs_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ threadgroup int8_t * shared_values [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_iq2_xxs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);
+}
+
+void kernel_mul_mv_iq2_xs_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const int nb = ne00/QK_K;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;
+ const int ib_row = first_row * nb;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_iq2_xs * x = (device const block_iq2_xs *) src0 + ib_row + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[32];
+ float sumf[N_DST]={0.f}, all_sum;
+
+ const int nb32 = nb * (QK_K / 32);
+
+ threadgroup uint64_t * values = (threadgroup uint64_t *)shared_values;
+ threadgroup uint8_t * shared_signs = (threadgroup uint8_t *)(values + 512);
+ {
+ int nval = 8;
+ int pos = (32*sgitg + tiisg)*nval;
+ for (int i = 0; i < nval; ++i) values[pos + i] = iq2xs_grid[pos + i];
+ nval = 2;
+ pos = (32*sgitg + tiisg)*nval;
+ for (int i = 0; i < nval; ++i) shared_signs[pos+i] = ksigns_iq2xs[pos+i];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ const int ix = tiisg;
+
+ device const float * y4 = y + 32 * ix;
+
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
+
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ }
+
+ const int ibl = ib32 / (QK_K / 32);
+ const int ib = ib32 % (QK_K / 32);
+
+ device const block_iq2_xs * xr = x + ibl;
+ device const uint16_t * q2 = xr->qs + 4 * ib;
+ device const uint8_t * sc = xr->scales + ib;
+ device const half * dh = &xr->d;
+
+ for (int row = 0; row < N_DST; row++) {
+
+ const float db = dh[0];
+ const uint8_t ls1 = sc[0] & 0xf;
+ const uint8_t ls2 = sc[0] >> 4;
+ const float d1 = db * (0.5f + ls1);
+ const float d2 = db * (0.5f + ls2);
+
+ float sum1 = 0, sum2 = 0;
+ for (int l = 0; l < 2; ++l) {
+ const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(values + (q2[l] & 511));
+ const uint8_t signs = shared_signs[(q2[l] >> 9)];
+ for (int j = 0; j < 8; ++j) {
+ sum1 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
+ }
+ }
+ for (int l = 2; l < 4; ++l) {
+ const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(values + (q2[l] & 511));
+ const uint8_t signs = shared_signs[(q2[l] >> 9)];
+ for (int j = 0; j < 8; ++j) {
+ sum2 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
+ }
+ }
+ sumf[row] += d1 * sum1 + d2 * sum2;
+
+ dh += nb*sizeof(block_iq2_xs)/2;
+ q2 += nb*sizeof(block_iq2_xs)/2;
+ sc += nb*sizeof(block_iq2_xs);
+ }
+
+ y4 += 32 * 32;
+ }
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum * 0.25f;
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_iq2_xs_f32")]]
+kernel void kernel_mul_mv_iq2_xs_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ threadgroup int8_t * shared_values [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_iq2_xs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);
+}
+
+void kernel_mul_mv_iq3_xxs_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const int nb = ne00/QK_K;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;
+ const int ib_row = first_row * nb;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_iq3_xxs * x = (device const block_iq3_xxs *) src0 + ib_row + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[32];
+ float sumf[N_DST]={0.f}, all_sum;
+
+ const int nb32 = nb * (QK_K / 32);
+
+ threadgroup uint32_t * values = (threadgroup uint32_t *)shared_values;
+ threadgroup uint8_t * shared_signs = (threadgroup uint8_t *)(values + 256);
+ {
+ int nval = 4;
+ int pos = (32*sgitg + tiisg)*nval;
+ for (int i = 0; i < nval; ++i) values[pos + i] = iq3xxs_grid[pos + i];
+ nval = 2;
+ pos = (32*sgitg + tiisg)*nval;
+ for (int i = 0; i < nval; ++i) shared_signs[pos+i] = ksigns_iq2xs[pos+i];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ const int ix = tiisg;
+
+ device const float * y4 = y + 32 * ix;
+
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
+
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ }
+
+ const int ibl = ib32 / (QK_K / 32);
+ const int ib = ib32 % (QK_K / 32);
+
+ device const block_iq3_xxs * xr = x + ibl;
+ device const uint8_t * q3 = xr->qs + 8 * ib;
+ device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib;
+ device const half * dh = &xr->d;
+
+ for (int row = 0; row < N_DST; row++) {
+
+ const float db = dh[0];
+ const uint32_t aux32 = gas[0] | (gas[1] << 16);
+ const float d = db * (0.5f + (aux32 >> 28));
+
+ float2 sum = {0};
+ for (int l = 0; l < 4; ++l) {
+ const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(values + q3[2*l+0]);
+ const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(values + q3[2*l+1]);
+ const uint8_t signs = shared_signs[(aux32 >> 7*l) & 127];
+ for (int j = 0; j < 4; ++j) {
+ sum[0] += yl[8*l + j + 0] * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
+ sum[1] += yl[8*l + j + 4] * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
+ }
+ }
+ sumf[row] += d * (sum[0] + sum[1]);
+
+ dh += nb*sizeof(block_iq3_xxs)/2;
+ q3 += nb*sizeof(block_iq3_xxs);
+ gas += nb*sizeof(block_iq3_xxs)/2;
+ }
+
+ y4 += 32 * 32;
+ }
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum * 0.5f;
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_iq3_xxs_f32")]]
+kernel void kernel_mul_mv_iq3_xxs_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ threadgroup int8_t * shared_values [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_iq3_xxs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);
+}
+
+void kernel_mul_mv_iq3_s_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const int nb = ne00/QK_K;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;
+ const int ib_row = first_row * nb;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_iq3_s * x = (device const block_iq3_s *) src0 + ib_row + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[32];
+ float sumf[N_DST]={0.f}, all_sum;
+
+ const int nb32 = nb * (QK_K / 32);
+
+ threadgroup uint32_t * values = (threadgroup uint32_t *)shared_values;
+ {
+ int nval = 8;
+ int pos = (32*sgitg + tiisg)*nval;
+ for (int i = 0; i < nval; ++i) values[pos + i] = iq3s_grid[pos + i];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ const int ix = tiisg;
+
+ device const float * y4 = y + 32 * ix;
+
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
+
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ }
+
+ const int ibl = ib32 / (QK_K / 32);
+ const int ib = ib32 % (QK_K / 32);
+
+ device const block_iq3_s * xr = x + ibl;
+ device const uint8_t * qs = xr->qs + 8 * ib;
+ device const uint8_t * qh = xr->qh + ib;
+ device const uint8_t * sc = xr->scales + (ib/2);
+ device const uint8_t * signs = xr->signs + 4 * ib;
+ device const half * dh = &xr->d;
+
+ for (int row = 0; row < N_DST; row++) {
+
+ const float db = dh[0];
+ const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf));
+
+ float2 sum = {0};
+ for (int l = 0; l < 4; ++l) {
+ const threadgroup uint32_t * table1 = qh[0] & kmask_iq2xs[2*l+0] ? values + 256 : values;
+ const threadgroup uint32_t * table2 = qh[0] & kmask_iq2xs[2*l+1] ? values + 256 : values;
+ const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(table1 + qs[2*l+0]);
+ const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(table2 + qs[2*l+1]);
+ for (int j = 0; j < 4; ++j) {
+ sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l] & kmask_iq2xs[j+0]);
+ sum[1] += yl[8*l + j + 4] * grid2[j] * select(1, -1, signs[l] & kmask_iq2xs[j+4]);
+ }
+ }
+ sumf[row] += d * (sum[0] + sum[1]);
+
+ dh += nb*sizeof(block_iq3_s)/2;
+ qs += nb*sizeof(block_iq3_s);
+ qh += nb*sizeof(block_iq3_s);
+ sc += nb*sizeof(block_iq3_s);
+ signs += nb*sizeof(block_iq3_s);
+ }
+
+ y4 += 32 * 32;
+ }
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_iq3_s_f32")]]
+kernel void kernel_mul_mv_iq3_s_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ threadgroup int8_t * shared_values [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_iq3_s_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);
+}
+
+void kernel_mul_mv_iq2_s_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const int nb = ne00/QK_K;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;
+ const int ib_row = first_row * nb;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ device const block_iq2_s * x = (device const block_iq2_s *) src0 + ib_row + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[32];
+ float sumf[N_DST]={0.f}, all_sum;
+
+ const int nb32 = nb * (QK_K / 32);
+
+ //threadgroup uint64_t * values = (threadgroup uint64_t *)shared_values;
+ //{
+ // int nval = 32;
+ // int pos = (32*sgitg + tiisg)*nval;
+ // for (int i = 0; i < nval; ++i) values[pos + i] = iq2s_grid[pos + i];
+ // threadgroup_barrier(mem_flags::mem_threadgroup);
+ //}
+
+ const int ix = tiisg;
+
+ device const float * y4 = y + 32 * ix;
+
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
+
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ }
+
+ const int ibl = ib32 / (QK_K / 32);
+ const int ib = ib32 % (QK_K / 32);
+
+ device const block_iq2_s * xr = x + ibl;
+ device const uint8_t * qs = xr->qs + 4 * ib;
+ device const uint8_t * qh = xr->qh + ib;
+ device const uint8_t * sc = xr->scales + ib;
+ device const uint8_t * signs = qs + QK_K/8;
+ device const half * dh = &xr->d;
+
+ for (int row = 0; row < N_DST; row++) {
+
+ const float db = dh[0];
+ const float d1 = db * (0.5f + (sc[0] & 0xf));
+ const float d2 = db * (0.5f + (sc[0] >> 4));
+
+ float2 sum = {0};
+ for (int l = 0; l < 2; ++l) {
+ //const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(values + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300)));
+ //const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(values + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300)));
+ constant uint8_t * grid1 = (constant uint8_t *)(iq2s_grid + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300)));
+ constant uint8_t * grid2 = (constant uint8_t *)(iq2s_grid + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300)));
+ for (int j = 0; j < 8; ++j) {
+ sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l+0] & kmask_iq2xs[j]);
+ sum[1] += yl[8*l + j + 16] * grid2[j] * select(1, -1, signs[l+2] & kmask_iq2xs[j]);
+ }
+ }
+ sumf[row] += d1 * sum[0] + d2 * sum[1];
+
+ dh += nb*sizeof(block_iq2_s)/2;
+ qs += nb*sizeof(block_iq2_s);
+ qh += nb*sizeof(block_iq2_s);
+ sc += nb*sizeof(block_iq2_s);
+ signs += nb*sizeof(block_iq2_s);
+ }
+
+ y4 += 32 * 32;
+ }
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum * 0.25f;
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_iq2_s_f32")]]
+kernel void kernel_mul_mv_iq2_s_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ threadgroup int8_t * shared_values [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_iq2_s_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);
+}
+
+void kernel_mul_mv_iq1_s_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_value,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const int nb = ne00/QK_K;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;
+ const int ib_row = first_row * nb;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+ device const block_iq1_s * x = (device const block_iq1_s *) src0 + ib_row + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[32];
+ float sumf[N_DST]={0.f}, all_sum;
+
+ const int nb32 = nb * (QK_K / 32);
+
+ const int ix = tiisg;
+
+ device const float * y4 = y + 32 * ix;
+
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
+
+ float sumy = 0;
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ sumy += yl[i];
+ }
+
+ const int ibl = ib32 / (QK_K / 32);
+ const int ib = ib32 % (QK_K / 32);
+
+ device const block_iq1_s * xr = x + ibl;
+ device const uint8_t * qs = xr->qs + 4 * ib;
+ device const uint16_t * qh = xr->qh + ib;
+ device const half * dh = &xr->d;
+
+ for (int row = 0; row < N_DST; row++) {
+
+ constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700)));
+ constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700)));
+ constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700)));
+ constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[0] >> 1) & 0x700)));
+
+ float sum = 0;
+ for (int j = 0; j < 4; ++j) {
+ sum += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4)
+ + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4)
+ + yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4)
+ + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4);
+ }
+ sumf[row] += (float)dh[0] * (sum + sumy * (qh[0] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA)) * (2*((qh[0] >> 12) & 7) + 1);
+
+ dh += nb*sizeof(block_iq1_s)/2;
+ qs += nb*sizeof(block_iq1_s);
+ qh += nb*sizeof(block_iq1_s)/2;
+ }
+
+ y4 += 32 * 32;
+ }
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;
+ }
+ }
+}
+
+void kernel_mul_mv_iq1_m_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_value,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ const int nb = ne00/QK_K;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;
+ const int ib_row = first_row * nb;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+ device const block_iq1_m * x = (device const block_iq1_m *) src0 + ib_row + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ float yl[32];
+ float sumf[N_DST]={0.f}, all_sum;
+
+ const int nb32 = nb * (QK_K / 32);
+
+ const int ix = tiisg;
+
+ device const float * y4 = y + 32 * ix;
+
+ iq1m_scale_t scale;
+
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
+
+ float4 sumy = {0.f};
+ for (int i = 0; i < 8; ++i) {
+ yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0];
+ yl[i+ 8] = y4[i+ 8]; sumy[1] += yl[i+ 8];
+ yl[i+16] = y4[i+16]; sumy[2] += yl[i+16];
+ yl[i+24] = y4[i+24]; sumy[3] += yl[i+24];
+ }
+
+ const int ibl = ib32 / (QK_K / 32);
+ const int ib = ib32 % (QK_K / 32);
+
+ device const block_iq1_m * xr = x + ibl;
+ device const uint8_t * qs = xr->qs + 4 * ib;
+ device const uint8_t * qh = xr->qh + 2 * ib;
+ device const uint16_t * sc = (device const uint16_t *)xr->scales;
+
+ for (int row = 0; row < N_DST; row++) {
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+
+ constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700)));
+ constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 4) & 0x700)));
+ constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[1] << 8) & 0x700)));
+ constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[1] << 4) & 0x700)));
+
+ float2 sum = {0.f};
+ for (int j = 0; j < 4; ++j) {
+ sum[0] += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4)
+ + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4);
+ sum[1] += yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4)
+ + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4);
+ }
+ const float delta1 = sumy[0] * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[1] * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA);
+ const float delta2 = sumy[2] * (qh[1] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[3] * (qh[1] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA);
+
+ sumf[row] += (float)scale.f16 * ((sum[0] + delta1) * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 7) + 1) +
+ (sum[1] + delta2) * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 7) + 1));
+
+ sc += nb*sizeof(block_iq1_m)/2;
+ qs += nb*sizeof(block_iq1_m);
+ qh += nb*sizeof(block_iq1_m);
+ }
+
+ y4 += 32 * 32;
+ }
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;
+ }
+ }
+}
+
+void kernel_mul_mv_iq4_nl_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values_i8,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ threadgroup float * shared_values = (threadgroup float *)shared_values_i8;
+ const int nb = ne00/QK4_NL;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+ const int first_row = (r0 * 2 + sgitg) * 2;
+ const int ib_row = first_row * nb;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+ device const block_iq4_nl * x = (device const block_iq4_nl *) src0 + ib_row + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ const int ix = tiisg/2; // 0...15
+ const int it = tiisg%2; // 0 or 1
+
+ shared_values[tiisg] = kvalues_iq4nl_f[tiisg%16];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ float4 yl[4];
+ float sumf[2]={0.f}, all_sum;
+
+ device const float * yb = y + ix * QK4_NL + it * 8;
+
+ uint32_t aux32[2];
+ thread const uint8_t * q8 = (thread const uint8_t *)aux32;
+
+ float4 qf1, qf2;
+
+ for (int ib = ix; ib < nb; ib += 16) {
+
+ device const float4 * y4 = (device const float4 *)yb;
+ yl[0] = y4[0]; yl[1] = y4[4]; yl[2] = y4[1]; yl[3] = y4[5];
+
+ for (int row = 0; row < 2 && first_row + row < ne01; ++row) {
+
+ device const block_iq4_nl & xb = x[row*nb + ib];
+ device const uint16_t * q4 = (device const uint16_t *)(xb.qs + 8*it);
+
+ float4 acc1 = {0.f}, acc2 = {0.f};
+
+ aux32[0] = q4[0] | (q4[1] << 16);
+ aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f;
+ aux32[0] &= 0x0f0f0f0f;
+ qf1 = {shared_values[q8[0]], shared_values[q8[1]], shared_values[q8[2]], shared_values[q8[3]]};
+ qf2 = {shared_values[q8[4]], shared_values[q8[5]], shared_values[q8[6]], shared_values[q8[7]]};
+ acc1 += yl[0] * qf1;
+ acc2 += yl[1] * qf2;
+
+ aux32[0] = q4[2] | (q4[3] << 16);
+ aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f;
+ aux32[0] &= 0x0f0f0f0f;
+ qf1 = {shared_values[q8[0]], shared_values[q8[1]], shared_values[q8[2]], shared_values[q8[3]]};
+ qf2 = {shared_values[q8[4]], shared_values[q8[5]], shared_values[q8[6]], shared_values[q8[7]]};
+ acc1 += yl[2] * qf1;
+ acc2 += yl[3] * qf2;
+
+ acc1 += acc2;
+
+ sumf[row] += (float)xb.d * (acc1[0] + acc1[1] + acc1[2] + acc1[3]);
+
+ }
+
+ yb += 16 * QK4_NL;
+ }
+
+ for (int row = 0; row < 2 && first_row + row < ne01; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;
+ }
+ }
+}
+
+void kernel_mul_mv_iq4_xs_f32_impl(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values_i8,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg) {
+
+ threadgroup float * shared_values = (threadgroup float *)shared_values_i8;
+ const int nb = ne00/QK_K;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+ const int first_row = (r0 * 2 + sgitg) * 2;
+ const int ib_row = first_row * nb;
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+ device const block_iq4_xs * x = (device const block_iq4_xs *) src0 + ib_row + offset0;
+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;
+
+ const int ix = tiisg/16; // 0 or 1
+ const int it = tiisg%16; // 0...15
+ const int ib = it/2;
+ const int il = it%2;
+
+ shared_values[tiisg] = kvalues_iq4nl_f[tiisg%16];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ float4 yl[4];
+ float sumf[2]={0.f}, all_sum;
+
+ device const float * yb = y + ix * QK_K + ib * 32 + il * 8;
+
+ uint32_t aux32[2];
+ thread const uint8_t * q8 = (thread const uint8_t *)aux32;
+
+ float4 qf1, qf2;
+
+ for (int ibl = ix; ibl < nb; ibl += 2) {
+
+ device const float4 * y4 = (device const float4 *)yb;
+ yl[0] = y4[0]; yl[1] = y4[4]; yl[2] = y4[1]; yl[3] = y4[5];
+
+ for (int row = 0; row < 2; ++row) {
+
+ device const block_iq4_xs & xb = x[row*nb + ibl];
+ device const uint32_t * q4 = (device const uint32_t *)(xb.qs + 16*ib + 8*il);
+
+ float4 acc1 = {0.f}, acc2 = {0.f};
+
+ aux32[0] = q4[0] & 0x0f0f0f0f;
+ aux32[1] = (q4[0] >> 4) & 0x0f0f0f0f;
+ qf1 = {shared_values[q8[0]], shared_values[q8[1]], shared_values[q8[2]], shared_values[q8[3]]};
+ qf2 = {shared_values[q8[4]], shared_values[q8[5]], shared_values[q8[6]], shared_values[q8[7]]};
+ acc1 += yl[0] * qf1;
+ acc2 += yl[1] * qf2;
+
+ aux32[0] = q4[1] & 0x0f0f0f0f;
+ aux32[1] = (q4[1] >> 4) & 0x0f0f0f0f;
+ qf1 = {shared_values[q8[0]], shared_values[q8[1]], shared_values[q8[2]], shared_values[q8[3]]};
+ qf2 = {shared_values[q8[4]], shared_values[q8[5]], shared_values[q8[6]], shared_values[q8[7]]};
+ acc1 += yl[2] * qf1;
+ acc2 += yl[3] * qf2;
+
+ acc1 += acc2;
+
+ const int ls = (((xb.scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((xb.scales_h >> 2*ib) & 3) << 4)) - 32;
+ sumf[row] += (float)xb.d * ls * (acc1[0] + acc1[1] + acc1[2] + acc1[3]);
+
+ }
+
+ yb += 2 * QK_K;
+ }
+
+ for (int row = 0; row < 2; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_iq1_s_f32")]]
+kernel void kernel_mul_mv_iq1_s_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_iq1_s_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, nullptr, tgpig, tiisg, sgitg);
+}
+
+[[host_name("kernel_mul_mv_iq1_m_f32")]]
+kernel void kernel_mul_mv_iq1_m_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_iq1_m_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, nullptr, tgpig, tiisg, sgitg);
+}
+
+[[host_name("kernel_mul_mv_iq4_nl_f32")]]
+kernel void kernel_mul_mv_iq4_nl_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ threadgroup int8_t * shared_values [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_iq4_nl_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);
+}
+
+[[host_name("kernel_mul_mv_iq4_xs_f32")]]
+kernel void kernel_mul_mv_iq4_xs_f32(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ threadgroup int8_t * shared_values [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_iq4_xs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);
+}
+
+//============================= templates and their specializations =============================
+
+// NOTE: this is not dequantizing - we are simply fitting the template
+template <typename type4x4>
+void dequantize_f32(device const float4x4 * src, short il, thread type4x4 & reg) {
+ float4x4 temp = *(((device float4x4 *)src));
+ for (int i = 0; i < 16; i++){
+ reg[i/4][i%4] = temp[i/4][i%4];
+ }
+}
+
+template <typename type4x4>
+void dequantize_f16(device const half4x4 * src, short il, thread type4x4 & reg) {
+ half4x4 temp = *(((device half4x4 *)src));
+ for (int i = 0; i < 16; i++){
+ reg[i/4][i%4] = temp[i/4][i%4];
+ }
+}
+
+template <typename type4x4>
+void dequantize_q4_0(device const block_q4_0 *xb, short il, thread type4x4 & reg) {
+ device const uint16_t * qs = ((device const uint16_t *)xb + 1);
+ const float d1 = il ? (xb->d / 16.h) : xb->d;
+ const float d2 = d1 / 256.f;
+ const float md = -8.h * xb->d;
+ const ushort mask0 = il ? 0x00F0 : 0x000F;
+ const ushort mask1 = mask0 << 8;
+
+ for (int i=0;i<8;i++) {
+ reg[i/2][2*(i%2)+0] = d1 * (qs[i] & mask0) + md;
+ reg[i/2][2*(i%2)+1] = d2 * (qs[i] & mask1) + md;
+ }
+}
+
+template <typename type4x4>
+void dequantize_q4_1(device const block_q4_1 *xb, short il, thread type4x4 & reg) {
+ device const uint16_t * qs = ((device const uint16_t *)xb + 2);
+ const float d1 = il ? (xb->d / 16.h) : xb->d;
+ const float d2 = d1 / 256.f;
+ const float m = xb->m;
+ const ushort mask0 = il ? 0x00F0 : 0x000F;
+ const ushort mask1 = mask0 << 8;
+
+ for (int i=0;i<8;i++) {
+ reg[i/2][2*(i%2)+0] = ((qs[i] & mask0) * d1) + m;
+ reg[i/2][2*(i%2)+1] = ((qs[i] & mask1) * d2) + m;
+ }
+}
+
+template <typename type4x4>
+void dequantize_q5_0(device const block_q5_0 *xb, short il, thread type4x4 & reg) {
+ device const uint16_t * qs = ((device const uint16_t *)xb + 3);
+ const float d = xb->d;
+ const float md = -16.h * xb->d;
+ const ushort mask = il ? 0x00F0 : 0x000F;
+
+ const uint32_t qh = *((device const uint32_t *)xb->qh);
+
+ const int x_mv = il ? 4 : 0;
+
+ const int gh_mv = il ? 12 : 0;
+ const int gh_bk = il ? 0 : 4;
+
+ for (int i = 0; i < 8; i++) {
+ // extract the 5-th bits for x0 and x1
+ const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10;
+ const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10;
+
+ // combine the 4-bits from qs with the 5th bit
+ const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0);
+ const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1);
+
+ reg[i/2][2*(i%2)+0] = d * x0 + md;
+ reg[i/2][2*(i%2)+1] = d * x1 + md;
+ }
+}
+
+template <typename type4x4>
+void dequantize_q5_1(device const block_q5_1 *xb, short il, thread type4x4 & reg) {
+ device const uint16_t * qs = ((device const uint16_t *)xb + 4);
+ const float d = xb->d;
+ const float m = xb->m;
+ const ushort mask = il ? 0x00F0 : 0x000F;
+
+ const uint32_t qh = *((device const uint32_t *)xb->qh);
+
+ const int x_mv = il ? 4 : 0;
+
+ const int gh_mv = il ? 12 : 0;
+ const int gh_bk = il ? 0 : 4;
+
+ for (int i = 0; i < 8; i++) {
+ // extract the 5-th bits for x0 and x1
+ const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10;
+ const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10;
+
+ // combine the 4-bits from qs with the 5th bit
+ const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0);
+ const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1);
+
+ reg[i/2][2*(i%2)+0] = d * x0 + m;
+ reg[i/2][2*(i%2)+1] = d * x1 + m;
+ }
+}
+
+template <typename type4x4>
+void dequantize_q8_0(device const block_q8_0 *xb, short il, thread type4x4 & reg) {
+ device const int8_t * qs = ((device const int8_t *)xb->qs);
+ const half d = xb->d;
+
+ for (int i = 0; i < 16; i++) {
+ reg[i/4][i%4] = (qs[i + 16*il] * d);
+ }
+}
+
+template <typename type4x4>
+void dequantize_q2_K(device const block_q2_K *xb, short il, thread type4x4 & reg) {
+ const float d = xb->d;
+ const float min = xb->dmin;
+ device const uint8_t * q = (device const uint8_t *)xb->qs;
+ float dl, ml;
+ uint8_t sc = xb->scales[il];
+
+ q = q + 32*(il/8) + 16*(il&1);
+ il = (il/2)%4;
+
+ half coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h);
+ uchar mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3);
+ dl = d * (sc & 0xF) * coef, ml = min * (sc >> 4);
+ for (int i = 0; i < 16; ++i) {
+ reg[i/4][i%4] = dl * (q[i] & mask) - ml;
+ }
+}
+
+template <typename type4x4>
+void dequantize_q3_K(device const block_q3_K *xb, short il, thread type4x4 & reg) {
+ const half d_all = xb->d;
+ device const uint8_t * q = (device const uint8_t *)xb->qs;
+ device const uint8_t * h = (device const uint8_t *)xb->hmask;
+ device const int8_t * scales = (device const int8_t *)xb->scales;
+
+ q = q + 32 * (il/8) + 16 * (il&1);
+ h = h + 16 * (il&1);
+ uint8_t m = 1 << (il/2);
+ uint16_t kmask1 = (il/4)>1 ? ((il/4)>2 ? 192 : 48) : \
+ ((il/4)>0 ? 12 : 3);
+ uint16_t kmask2 = il/8 ? 0xF0 : 0x0F;
+ uint16_t scale_2 = scales[il%8], scale_1 = scales[8 + il%4];
+ int16_t dl_int = (il/4)&1 ? (scale_2&kmask2) | ((scale_1&kmask1) << 2)
+ : (scale_2&kmask2) | ((scale_1&kmask1) << 4);
+ float dl = il<8 ? d_all * (dl_int - 32.f) : d_all * (dl_int / 16.f - 32.f);
+ const float ml = 4.f * dl;
+
+ il = (il/2) & 3;
+ const half coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h);
+ const uint8_t mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3);
+ dl *= coef;
+
+ for (int i = 0; i < 16; ++i) {
+ reg[i/4][i%4] = dl * (q[i] & mask) - (h[i] & m ? 0 : ml);
+ }
+}
+
+static inline uchar2 get_scale_min_k4_just2(int j, int k, device const uchar * q) {
+ return j < 4 ? uchar2{uchar(q[j+0+k] & 63), uchar(q[j+4+k] & 63)}
+ : uchar2{uchar((q[j+4+k] & 0xF) | ((q[j-4+k] & 0xc0) >> 2)), uchar((q[j+4+k] >> 4) | ((q[j-0+k] & 0xc0) >> 2))};
+}
+
+template <typename type4x4>
+void dequantize_q4_K(device const block_q4_K *xb, short il, thread type4x4 & reg) {
+ device const uchar * q = xb->qs;
+
+ short is = (il/4) * 2;
+ q = q + (il/4) * 32 + 16 * (il&1);
+ il = il & 3;
+ const uchar2 sc = get_scale_min_k4_just2(is, il/2, xb->scales);
+ const float d = il < 2 ? xb->d : xb->d / 16.h;
+ const float min = xb->dmin;
+ const float dl = d * sc[0];
+ const float ml = min * sc[1];
+
+ const ushort mask = il<2 ? 0x0F : 0xF0;
+ for (int i = 0; i < 16; ++i) {
+ reg[i/4][i%4] = dl * (q[i] & mask) - ml;
+ }
+}
+
+template <typename type4x4>
+void dequantize_q5_K(device const block_q5_K *xb, short il, thread type4x4 & reg) {
+ device const uint8_t * q = xb->qs;
+ device const uint8_t * qh = xb->qh;
+
+ short is = (il/4) * 2;
+ q = q + 32 * (il/4) + 16 * (il&1);
+ qh = qh + 16 * (il&1);
+ uint8_t ul = 1 << (il/2);
+ il = il & 3;
+ const uchar2 sc = get_scale_min_k4_just2(is, il/2, xb->scales);
+ const float d = il < 2 ? xb->d : xb->d / 16.f;
+ const float min = xb->dmin;
+ const float dl = d * sc[0];
+ const float ml = min * sc[1];
+
+ const ushort mask = il<2 ? 0x0F : 0xF0;
+ const float qh_val = il<2 ? 16.f : 256.f;
+ for (int i = 0; i < 16; ++i) {
+ reg[i/4][i%4] = dl * ((q[i] & mask) + (qh[i] & ul ? qh_val : 0)) - ml;
+ }
+}
+
+template <typename type4x4>
+void dequantize_q6_K(device const block_q6_K *xb, short il, thread type4x4 & reg) {
+ const half d_all = xb->d;
+ device const uint8_t * ql = (device const uint8_t *)xb->ql;
+ device const uint8_t * qh = (device const uint8_t *)xb->qh;
+ device const int8_t * scales = (device const int8_t *)xb->scales;
+
+ ql = ql + 64*(il/8) + 32*((il/2)&1) + 16*(il&1);
+ qh = qh + 32*(il/8) + 16*(il&1);
+ float sc = scales[(il%2) + 2 * ((il/2))];
+ il = (il/2) & 3;
+
+ const uint16_t kmask1 = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3);
+ const uint16_t kmask2 = il>1 ? 0xF0 : 0x0F;
+ const float coef = il>1 ? 1.f/16.f : 1.f;
+ const float ml = d_all * sc * 32.f;
+ const float dl = d_all * sc * coef;
+ for (int i = 0; i < 16; ++i) {
+ const half q = il&1 ? ((ql[i] & kmask2) | ((qh[i] & kmask1) << 2))
+ : ((ql[i] & kmask2) | ((qh[i] & kmask1) << 4));
+ reg[i/4][i%4] = dl * q - ml;
+ }
+}
+
+template <typename type4x4>
+void dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4x4 & reg) {
+ // il is 0...15 for QK_K = 256 => index of block of 32 is il/2
+ const float d = xb->d;
+ const int ib32 = il/2;
+ il = il%2;
+ // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16
+ // each block of 32 needs 2 uint32_t's for the quants & scale, so 4 uint16_t's.
+ device const uint16_t * q2 = xb->qs + 4*ib32;
+ const uint32_t aux32_g = q2[0] | (q2[1] << 16);
+ const uint32_t aux32_s = q2[2] | (q2[3] << 16);
+ thread const uint8_t * aux8 = (thread const uint8_t *)&aux32_g;
+ const float dl = d * (0.5f + (aux32_s >> 28)) * 0.25f;
+ constant uint8_t * grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+0]);
+ uint8_t signs = ksigns_iq2xs[(aux32_s >> 14*il) & 127];
+ for (int i = 0; i < 8; ++i) {
+ reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);
+ }
+ grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+1]);
+ signs = ksigns_iq2xs[(aux32_s >> (14*il+7)) & 127];
+ for (int i = 0; i < 8; ++i) {
+ reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);
+ }
+}
+
+template <typename type4x4>
+void dequantize_iq2_xs(device const block_iq2_xs * xb, short il, thread type4x4 & reg) {
+ // il is 0...15 for QK_K = 256 => index of block of 32 is il/2
+ const float d = xb->d;
+ const int ib32 = il/2;
+ il = il%2;
+ // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16
+ device const uint16_t * q2 = xb->qs + 4*ib32;
+ const float dl = d * (0.5f + ((xb->scales[ib32] >> 4*il) & 0xf)) * 0.25f;
+ constant uint8_t * grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+0] & 511));
+ uint8_t signs = ksigns_iq2xs[q2[2*il+0] >> 9];
+ for (int i = 0; i < 8; ++i) {
+ reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);
+ }
+ grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+1] & 511));
+ signs = ksigns_iq2xs[q2[2*il+1] >> 9];
+ for (int i = 0; i < 8; ++i) {
+ reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);
+ }
+}
+
+template <typename type4x4>
+void dequantize_iq3_xxs(device const block_iq3_xxs * xb, short il, thread type4x4 & reg) {
+ // il is 0...15 for QK_K = 256 => index of block of 32 is il/2
+ const float d = xb->d;
+ const int ib32 = il/2;
+ il = il%2;
+ // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16
+ device const uint8_t * q3 = xb->qs + 8*ib32;
+ device const uint16_t * gas = (device const uint16_t *)(xb->qs + QK_K/4) + 2*ib32;
+ const uint32_t aux32 = gas[0] | (gas[1] << 16);
+ const float dl = d * (0.5f + (aux32 >> 28)) * 0.5f;
+ constant uint8_t * grid1 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+0]);
+ constant uint8_t * grid2 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+1]);
+ uint8_t signs = ksigns_iq2xs[(aux32 >> 14*il) & 127];
+ for (int i = 0; i < 4; ++i) {
+ reg[0][i] = dl * grid1[i] * (signs & kmask_iq2xs[i+0] ? -1.f : 1.f);
+ reg[1][i] = dl * grid2[i] * (signs & kmask_iq2xs[i+4] ? -1.f : 1.f);
+ }
+ grid1 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+2]);
+ grid2 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+3]);
+ signs = ksigns_iq2xs[(aux32 >> (14*il+7)) & 127];
+ for (int i = 0; i < 4; ++i) {
+ reg[2][i] = dl * grid1[i] * (signs & kmask_iq2xs[i+0] ? -1.f : 1.f);
+ reg[3][i] = dl * grid2[i] * (signs & kmask_iq2xs[i+4] ? -1.f : 1.f);
+ }
+}
+
+template <typename type4x4>
+void dequantize_iq3_s(device const block_iq3_s * xb, short il, thread type4x4 & reg) {
+ // il is 0...15 for QK_K = 256 => index of block of 32 is il/2
+ const float d = xb->d;
+ const int ib32 = il/2;
+ il = il%2;
+ // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16
+ device const uint8_t * qs = xb->qs + 8*ib32;
+ device const uint8_t * signs = xb->signs + 4*ib32 + 2*il;
+ const uint8_t qh = xb->qh[ib32] >> 4*il;
+ const float dl = d * (1 + 2*((xb->scales[ib32/2] >> 4*(ib32%2)) & 0xf));
+ constant uint8_t * grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+0] | ((qh << 8) & 256)));
+ constant uint8_t * grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+1] | ((qh << 7) & 256)));
+ for (int i = 0; i < 4; ++i) {
+ reg[0][i] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i+0]);
+ reg[1][i] = dl * grid2[i] * select(1, -1, signs[0] & kmask_iq2xs[i+4]);
+ }
+ grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+2] | ((qh << 6) & 256)));
+ grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+3] | ((qh << 5) & 256)));
+ for (int i = 0; i < 4; ++i) {
+ reg[2][i] = dl * grid1[i] * select(1, -1, signs[1] & kmask_iq2xs[i+0]);
+ reg[3][i] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i+4]);
+ }
+}
+
+template <typename type4x4>
+void dequantize_iq2_s(device const block_iq2_s * xb, short il, thread type4x4 & reg) {
+ // il is 0...15 for QK_K = 256 => index of block of 32 is il/2
+ const float d = xb->d;
+ const int ib32 = il/2;
+ il = il%2;
+ // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16
+ device const uint8_t * qs = xb->qs + 4*ib32 + 2*il;
+ device const uint8_t * signs = qs + QK_K/8;
+ const uint8_t qh = xb->qh[ib32] >> 4*il;
+ const float dl = d * (0.5f + ((xb->scales[ib32] >> 4*il) & 0xf)) * 0.25f;
+ constant uint8_t * grid1 = (constant uint8_t *)(iq2s_grid + (qs[0] | ((qh << 8) & 0x300)));
+ constant uint8_t * grid2 = (constant uint8_t *)(iq2s_grid + (qs[1] | ((qh << 6) & 0x300)));
+ for (int i = 0; i < 8; ++i) {
+ reg[i/4+0][i%4] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i]);
+ reg[i/4+2][i%4] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i]);
+ }
+}
+
+template <typename type4x4>
+void dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & reg) {
+ // il is 0...15 for QK_K = 256 => index of block of 32 is il/2
+ const int ib32 = il/2;
+ il = il%2;
+ const float d = xb->d;
+ device const uint8_t * qs = xb->qs + 4*ib32 + 2*il;
+ device const uint16_t * qh = xb->qh;
+ const float dl = d * (2*((qh[ib32] >> 12) & 7) + 1);
+ const float ml = dl * (qh[ib32] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA);
+ const uint16_t h = qh[ib32] >> 6*il;
+ constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((h << 8) & 0x700)));
+ constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((h << 5) & 0x700)));
+ for (int i = 0; i < 4; ++i) {
+ reg[0][i] = dl * (grid1[i] & 0xf) + ml;
+ reg[1][i] = dl * (grid1[i] >> 4) + ml;
+ reg[2][i] = dl * (grid2[i] & 0xf) + ml;
+ reg[3][i] = dl * (grid2[i] >> 4) + ml;
+ }
+}
+
+template <typename type4x4>
+void dequantize_iq1_m(device const block_iq1_m * xb, short il, thread type4x4 & reg) {
+ // il is 0...15 for QK_K = 256 => index of block of 32 is il/2
+ const int ib32 = il/2;
+ il = il%2;
+ device const uint16_t * sc = (device const uint16_t *)xb->scales;
+
+ iq1m_scale_t scale;
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+ const float d = scale.f16;
+
+ device const uint8_t * qs = xb->qs + 4*ib32 + 2*il;
+ device const uint8_t * qh = xb->qh + 2*ib32 + il;
+
+ const float dl = d * (2*((sc[ib32/2] >> (6*(ib32%2)+3*il)) & 7) + 1);
+ const float ml1 = dl * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA);
+ const float ml2 = dl * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA);
+ constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700)));
+ constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 4) & 0x700)));
+ for (int i = 0; i < 4; ++i) {
+ reg[0][i] = dl * (grid1[i] & 0xf) + ml1;
+ reg[1][i] = dl * (grid1[i] >> 4) + ml1;
+ reg[2][i] = dl * (grid2[i] & 0xf) + ml2;
+ reg[3][i] = dl * (grid2[i] >> 4) + ml2;
+ }
+}
+
+template <typename type4x4>
+void dequantize_iq4_nl(device const block_iq4_nl * xb, short il, thread type4x4 & reg) {
+ device const uint16_t * q4 = (device const uint16_t *)xb->qs;
+ const float d = xb->d;
+ uint32_t aux32;
+ thread const uint8_t * q8 = (thread const uint8_t *)&aux32;
+ for (int i = 0; i < 4; ++i) {
+ aux32 = ((q4[2*i] | (q4[2*i+1] << 16)) >> 4*il) & 0x0f0f0f0f;
+ reg[i][0] = d * kvalues_iq4nl_f[q8[0]];
+ reg[i][1] = d * kvalues_iq4nl_f[q8[1]];
+ reg[i][2] = d * kvalues_iq4nl_f[q8[2]];
+ reg[i][3] = d * kvalues_iq4nl_f[q8[3]];
+ }
+}
+
+template <typename type4x4>
+void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4 & reg) {
+ // il is 0...15 for QK_K = 256 => index of block of 32 is il/2
+ const int ib32 = il/2;
+ il = il%2;
+ // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16
+ device const uint32_t * q4 = (device const uint32_t *)xb->qs + 4*ib32;
+ const int ls = ((xb->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((xb->scales_h >> 2*ib32) & 3) << 4);
+ const float d = (float)xb->d * (ls - 32);
+ uint32_t aux32;
+ thread const uint8_t * q8 = (thread const uint8_t *)&aux32;
+ for (int i = 0; i < 4; ++i) {
+ aux32 = (q4[i] >> 4*il) & 0x0f0f0f0f;
+ reg[i][0] = d * kvalues_iq4nl_f[q8[0]];
+ reg[i][1] = d * kvalues_iq4nl_f[q8[1]];
+ reg[i][2] = d * kvalues_iq4nl_f[q8[2]];
+ reg[i][3] = d * kvalues_iq4nl_f[q8[3]];
+ }
+}
+
+template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread float4x4 &)>
+kernel void kernel_get_rows_q(
+ device const void * src0,
+ device const void * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiitg[[thread_index_in_threadgroup]],
+ uint3 tptg [[threads_per_threadgroup]]) {
+ const int64_t i10 = tgpig.x;
+ const int64_t i11 = tgpig.y;
+
+ const int64_t r = ((const device int32_t *) ((const device char *) src1 + i11*nb11 + i10*nb10))[0];
+
+ const int64_t i02 = i11;
+
+ for (int64_t ind = tiitg; ind < ne00/16; ind += tptg.x) {
+ float4x4 temp;
+ dequantize_func(((device const block_q *) ((const device char *) src0 + r*nb01 + i02*nb02)) + ind/nl, ind%nl, temp);
+ *(((device float4x4 *) ((device char *) dst + i11*nb2 + i10*nb1)) + ind) = temp;
+ }
+}
+
+template<typename T>
+kernel void kernel_get_rows_f(
+ device const void * src0,
+ device const void * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiitg[[thread_index_in_threadgroup]],
+ uint3 tptg [[threads_per_threadgroup]]) {
+ const int64_t i10 = tgpig.x;
+ const int64_t i11 = tgpig.y;
+
+ const int64_t r = ((const device int32_t *) ((const device char *) src1 + i11*nb11 + i10*nb10))[0];
+
+ const int64_t i02 = i11;
+
+ for (int ind = tiitg; ind < ne00; ind += tptg.x) {
+ (( device float *) (( device char *) dst + i11*nb2 + i10*nb1))[ind] =
+ ((const device T *) ((const device char *) src0 + i02*nb02 + r*nb01))[ind];
+ }
+}
+
+kernel void kernel_get_rows_i32(
+ device const void * src0,
+ device const void * src1,
+ device int32_t * dst,
+ constant int64_t & ne00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb1,
+ constant uint64_t & nb2,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiitg[[thread_index_in_threadgroup]],
+ uint3 tptg [[threads_per_threadgroup]]) {
+ const int64_t i10 = tgpig.x;
+ const int64_t i11 = tgpig.y;
+
+ const int64_t r = ((const device int32_t *) ((const device char *) src1 + i11*nb11 + i10*nb10))[0];
+
+ const int64_t i02 = i11;
+
+ for (int ind = tiitg; ind < ne00; ind += tptg.x) {
+ (( device int32_t *) (( device char *) dst + i11*nb2 + i10*nb1))[ind] =
+ ((const device int32_t *) ((const device char *) src0 + i02*nb02 + r*nb01))[ind];
+ }
+}
+
+
+#define BLOCK_SIZE_M 64 // 8 simdgroup matrices from matrix A
+#define BLOCK_SIZE_N 32 // 4 simdgroup matrices from matrix B
+#define BLOCK_SIZE_K 32
+#define THREAD_MAT_M 4 // each thread take 4 simdgroup matrices from matrix A
+#define THREAD_MAT_N 2 // each thread take 2 simdgroup matrices from matrix B
+#define THREAD_PER_BLOCK 128
+#define THREAD_PER_ROW 2 // 2 thread for each row in matrix A to load numbers
+#define THREAD_PER_COL 4 // 4 thread for each row in matrix B to load numbers
+#define SG_MAT_SIZE 64 // simdgroup matrix is of shape 8x8
+#define SG_MAT_ROW 8
+
+// each block_q contains 16*nl weights
+template<typename T, typename T4x4, typename simdgroup_T8x8, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread T4x4 &)>
+kernel void kernel_mul_mm(device const uchar * src0,
+ device const uchar * src1,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne02,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint & r2,
+ constant uint & r3,
+ threadgroup uchar * shared_memory [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiitg[[thread_index_in_threadgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ threadgroup T * sa = (threadgroup T *)(shared_memory);
+ threadgroup float * sb = (threadgroup float *)(shared_memory + 4096);
+
+ const uint r0 = tgpig.y;
+ const uint r1 = tgpig.x;
+ const uint im = tgpig.z;
+
+ // if this block is of 64x32 shape or smaller
+ short n_rows = (ne0 - r0 * BLOCK_SIZE_M < BLOCK_SIZE_M) ? (ne0 - r0 * BLOCK_SIZE_M) : BLOCK_SIZE_M;
+ short n_cols = (ne1 - r1 * BLOCK_SIZE_N < BLOCK_SIZE_N) ? (ne1 - r1 * BLOCK_SIZE_N) : BLOCK_SIZE_N;
+
+ // a thread shouldn't load data outside of the matrix
+ short thread_row = ((short)tiitg/THREAD_PER_ROW) < n_rows ? ((short)tiitg/THREAD_PER_ROW) : n_rows - 1;
+ short thread_col = ((short)tiitg/THREAD_PER_COL) < n_cols ? ((short)tiitg/THREAD_PER_COL) : n_cols - 1;
+
+ simdgroup_T8x8 ma[4];
+ simdgroup_float8x8 mb[2];
+ simdgroup_float8x8 c_res[8];
+ for (int i = 0; i < 8; i++){
+ c_res[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
+ }
+
+ short il = (tiitg % THREAD_PER_ROW);
+
+ const uint i12 = im%ne12;
+ const uint i13 = im/ne12;
+
+ uint offset0 = (i12/r2)*nb02 + (i13/r3)*(nb02*ne02);
+ ushort offset1 = il/nl;
+
+ device const block_q * x = (device const block_q *)(src0 + (r0 * BLOCK_SIZE_M + thread_row) * nb01 + offset0) + offset1;
+ device const float * y = (device const float *)(src1
+ + nb12 * im
+ + nb11 * (r1 * BLOCK_SIZE_N + thread_col)
+ + nb10 * (BLOCK_SIZE_K / THREAD_PER_COL * (tiitg % THREAD_PER_COL)));
+
+ for (int loop_k = 0; loop_k < ne00; loop_k += BLOCK_SIZE_K) {
+ // load data and store to threadgroup memory
+ T4x4 temp_a;
+ dequantize_func(x, il, temp_a);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ #pragma unroll(16)
+ for (int i = 0; i < 16; i++) {
+ *(sa + SG_MAT_SIZE * ((tiitg / THREAD_PER_ROW / 8) \
+ + (tiitg % THREAD_PER_ROW) * 16 + (i / 8) * 8) \
+ + (tiitg / THREAD_PER_ROW) % 8 + (i & 7) * 8) = temp_a[i/4][i%4];
+ }
+
+ *(threadgroup float2x4 *)(sb + (tiitg % THREAD_PER_COL) * 8 * 32 + 8 * (tiitg / THREAD_PER_COL)) = *((device float2x4 *)y);
+
+ il = (il + 2 < nl) ? il + 2 : il % 2;
+ x = (il < 2) ? x + (2+nl-1)/nl : x;
+ y += BLOCK_SIZE_K;
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // load matrices from threadgroup memory and conduct outer products
+ threadgroup T * lsma = (sa + THREAD_MAT_M * SG_MAT_SIZE * (sgitg % 2));
+ threadgroup float * lsmb = (sb + THREAD_MAT_N * SG_MAT_SIZE * (sgitg / 2));
+
+ #pragma unroll(4)
+ for (int ik = 0; ik < BLOCK_SIZE_K / 8; ik++) {
+ #pragma unroll(4)
+ for (int i = 0; i < 4; i++) {
+ simdgroup_load(ma[i],lsma + SG_MAT_SIZE * i);
+ }
+ simdgroup_barrier(mem_flags::mem_none);
+ #pragma unroll(2)
+ for (int i = 0; i < 2; i++) {
+ simdgroup_load(mb[i],lsmb + SG_MAT_SIZE * i);
+ }
+
+ lsma += BLOCK_SIZE_M / SG_MAT_ROW * SG_MAT_SIZE;
+ lsmb += BLOCK_SIZE_N / SG_MAT_ROW * SG_MAT_SIZE;
+
+ #pragma unroll(8)
+ for (int i = 0; i < 8; i++){
+ simdgroup_multiply_accumulate(c_res[i], mb[i/4], ma[i%4], c_res[i]);
+ }
+ }
+ }
+
+ if ((r0 + 1) * BLOCK_SIZE_M <= ne0 && (r1 + 1) * BLOCK_SIZE_N <= ne1) {
+ device float * C = dst + (BLOCK_SIZE_M * r0 + 32 * (sgitg & 1)) \
+ + (BLOCK_SIZE_N * r1 + 16 * (sgitg >> 1)) * ne0 + im*ne1*ne0;
+ for (int i = 0; i < 8; i++) {
+ simdgroup_store(c_res[i], C + 8 * (i%4) + 8 * ne0 * (i/4), ne0);
+ }
+ } else {
+ // block is smaller than 64x32, we should avoid writing data outside of the matrix
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ threadgroup float * temp_str = ((threadgroup float *)shared_memory) \
+ + 32 * (sgitg&1) + (16 * (sgitg>>1)) * BLOCK_SIZE_M;
+ for (int i = 0; i < 8; i++) {
+ simdgroup_store(c_res[i], temp_str + 8 * (i%4) + 8 * BLOCK_SIZE_M * (i/4), BLOCK_SIZE_M);
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ device float * C = dst + (BLOCK_SIZE_M * r0) + (BLOCK_SIZE_N * r1) * ne0 + im*ne1*ne0;
+ if (sgitg == 0) {
+ for (int i = 0; i < n_rows; i++) {
+ for (int j = tiitg; j < n_cols; j += BLOCK_SIZE_N) {
+ *(C + i + j * ne0) = *(temp_str + i + j * BLOCK_SIZE_M);
+ }
+ }
+ }
+ }
+}
+
+// same as kernel_mul_mm_impl, but src1 and dst are accessed via indices stored in rowids
+template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread half4x4 &)>
+void kernel_mul_mm_id_impl(
+ device const uchar * src0,
+ device const uchar * src1,
+ threadgroup ushort2 * rowids,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne02,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ int64_t ne1,
+ int64_t ne0ne1,
+ threadgroup uchar * shared_memory,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiitg[[thread_index_in_threadgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ threadgroup half * sa = (threadgroup half *)(shared_memory);
+ threadgroup float * sb = (threadgroup float *)(shared_memory + 4096);
+
+ const uint r0 = tgpig.y;
+ const uint r1 = tgpig.x;
+
+ if (r1 * BLOCK_SIZE_N >= ne1) return;
+
+ // if this block is of 64x32 shape or smaller
+ short n_rows = (ne0 - r0 * BLOCK_SIZE_M < BLOCK_SIZE_M) ? (ne0 - r0 * BLOCK_SIZE_M) : BLOCK_SIZE_M;
+ short n_cols = (ne1 - r1 * BLOCK_SIZE_N < BLOCK_SIZE_N) ? (ne1 - r1 * BLOCK_SIZE_N) : BLOCK_SIZE_N;
+
+ // a thread shouldn't load data outside of the matrix
+ short thread_row = ((short)tiitg/THREAD_PER_ROW) < n_rows ? ((short)tiitg/THREAD_PER_ROW) : n_rows - 1;
+ short thread_col = ((short)tiitg/THREAD_PER_COL) < n_cols ? ((short)tiitg/THREAD_PER_COL) : n_cols - 1;
+
+ simdgroup_half8x8 ma[4];
+ simdgroup_float8x8 mb[2];
+ simdgroup_float8x8 c_res[8];
+ for (int i = 0; i < 8; i++){
+ c_res[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
+ }
+ short il = (tiitg % THREAD_PER_ROW);
+
+ ushort offset1 = il/nl;
+
+ threadgroup const auto & id = rowids[r1 * BLOCK_SIZE_N + thread_col];
+
+ device const block_q * x = (device const block_q *)(src0 + (r0 * BLOCK_SIZE_M + thread_row) * nb01) + offset1;
+ device const float * y = (device const float *)(src1
+ + nb12 * id[1]
+ + nb11 * (id[0] % ne11)
+ + nb10 * (BLOCK_SIZE_K / THREAD_PER_COL * (tiitg % THREAD_PER_COL)));
+
+ for (int loop_k = 0; loop_k < ne00; loop_k += BLOCK_SIZE_K) {
+ // load data and store to threadgroup memory
+ half4x4 temp_a;
+ dequantize_func(x, il, temp_a);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ for (int i = 0; i < 16; i++) {
+ *(sa + SG_MAT_SIZE * ((tiitg / THREAD_PER_ROW / 8) \
+ + (tiitg % THREAD_PER_ROW) * 16 + (i / 8) * 8) \
+ + (tiitg / THREAD_PER_ROW) % 8 + (i & 7) * 8) = temp_a[i/4][i%4];
+ }
+
+ *(threadgroup float2x4 *)(sb + (tiitg % THREAD_PER_COL) * 8 * 32 + 8 * (tiitg / THREAD_PER_COL)) = *((device float2x4 *)y);
+
+ il = (il + 2 < nl) ? il + 2 : il % 2;
+ x = (il < 2) ? x + (2+nl-1)/nl : x;
+ y += BLOCK_SIZE_K;
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // load matrices from threadgroup memory and conduct outer products
+ threadgroup half * lsma = (sa + THREAD_MAT_M * SG_MAT_SIZE * (sgitg % 2));
+ threadgroup float * lsmb = (sb + THREAD_MAT_N * SG_MAT_SIZE * (sgitg / 2));
+
+ for (int ik = 0; ik < BLOCK_SIZE_K / 8; ik++) {
+ for (int i = 0; i < 4; i++) {
+ simdgroup_load(ma[i], lsma + SG_MAT_SIZE * i);
+ }
+ simdgroup_barrier(mem_flags::mem_none);
+ for (int i = 0; i < 2; i++) {
+ simdgroup_load(mb[i], lsmb + SG_MAT_SIZE * i);
+ }
+
+ lsma += BLOCK_SIZE_M / SG_MAT_ROW * SG_MAT_SIZE;
+ lsmb += BLOCK_SIZE_N / SG_MAT_ROW * SG_MAT_SIZE;
+
+ for (int i = 0; i < 8; i++){
+ simdgroup_multiply_accumulate(c_res[i], mb[i/4], ma[i%4], c_res[i]);
+ }
+ }
+ }
+
+ {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ threadgroup float * temp_str = ((threadgroup float *)shared_memory) \
+ + 32 * (sgitg&1) + (16 * (sgitg>>1)) * BLOCK_SIZE_M;
+ for (int i = 0; i < 8; i++) {
+ simdgroup_store(c_res[i], temp_str + 8 * (i%4) + 8 * BLOCK_SIZE_M * (i/4), BLOCK_SIZE_M);
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ device float * C = dst + (BLOCK_SIZE_M * r0);
+ if (sgitg == 0) {
+ for (int j = tiitg; j < n_cols; j += BLOCK_SIZE_N) {
+ threadgroup const auto & jid = rowids[r1 * BLOCK_SIZE_N + j];
+ int joff = jid[0] * ne0 + jid[1] * ne0ne1;
+ for (int i = 0; i < n_rows; i++) {
+ *(C + i + joff) = *(temp_str + i + j * BLOCK_SIZE_M);
+ }
+ }
+ }
+ }
+}
+
+template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread half4x4 &)>
+kernel void kernel_mul_mm_id(
+ device const uchar * src0s,
+ device const uchar * src1,
+ device float * dst,
+ device const uchar * ids,
+ constant int64_t & nei0,
+ constant int64_t & nei1,
+ constant uint64_t & nbi1,
+ constant int64_t & ne00,
+ constant int64_t & ne02,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint64_t & nb1,
+ threadgroup uchar * shared_memory [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiitg[[thread_index_in_threadgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ const int32_t i02 = tgpig.z;
+ tgpig.z = 0;
+
+ device const uchar * src0 = src0s + i02*nb02;
+
+ // row indices
+ threadgroup ushort2 * rowids = (threadgroup ushort2 *)(shared_memory + 8192);
+
+ // TODO: parallelize this loop
+ int64_t _ne1 = 0;
+ for (ushort ii1 = 0; ii1 < nei1; ii1++) {
+ for (ushort ii0 = 0; ii0 < nei0; ii0++) {
+ int32_t id = ((device int32_t *) (ids + ii1*nbi1))[ii0];
+ if (id == i02) {
+ //if (tiitg == 0) {
+ rowids[_ne1] = ushort2(ii0, ii1);
+ //}
+ _ne1++;
+ }
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ kernel_mul_mm_id_impl<block_q, nl, dequantize_func>(
+ src0,
+ src1,
+ rowids,
+ dst,
+ ne00,
+ ne02,
+ nb01,
+ nb02,
+ ne11,
+ ne12,
+ nb10,
+ nb11,
+ nb12,
+ ne0,
+ _ne1,
+ ne0*ne1,
+ shared_memory,
+ tgpig,
+ tiitg,
+ sgitg);
+}
+
+#define QK_NL 16
+
+//
+// get rows
+//
+
+typedef decltype(kernel_get_rows_f<float>) get_rows_f_t;
+
+template [[host_name("kernel_get_rows_f32")]] kernel get_rows_f_t kernel_get_rows_f<float>;
+template [[host_name("kernel_get_rows_f16")]] kernel get_rows_f_t kernel_get_rows_f<half>;
+
+typedef decltype(kernel_get_rows_q<block_q4_0, 2, dequantize_q4_0>) get_rows_q_t;
+
+template [[host_name("kernel_get_rows_q4_0")]] kernel get_rows_q_t kernel_get_rows_q<block_q4_0, 2, dequantize_q4_0>;
+template [[host_name("kernel_get_rows_q4_1")]] kernel get_rows_q_t kernel_get_rows_q<block_q4_1, 2, dequantize_q4_1>;
+template [[host_name("kernel_get_rows_q5_0")]] kernel get_rows_q_t kernel_get_rows_q<block_q5_0, 2, dequantize_q5_0>;
+template [[host_name("kernel_get_rows_q5_1")]] kernel get_rows_q_t kernel_get_rows_q<block_q5_1, 2, dequantize_q5_1>;
+template [[host_name("kernel_get_rows_q8_0")]] kernel get_rows_q_t kernel_get_rows_q<block_q8_0, 2, dequantize_q8_0>;
+template [[host_name("kernel_get_rows_q2_K")]] kernel get_rows_q_t kernel_get_rows_q<block_q2_K, QK_NL, dequantize_q2_K>;
+template [[host_name("kernel_get_rows_q3_K")]] kernel get_rows_q_t kernel_get_rows_q<block_q3_K, QK_NL, dequantize_q3_K>;
+template [[host_name("kernel_get_rows_q4_K")]] kernel get_rows_q_t kernel_get_rows_q<block_q4_K, QK_NL, dequantize_q4_K>;
+template [[host_name("kernel_get_rows_q5_K")]] kernel get_rows_q_t kernel_get_rows_q<block_q5_K, QK_NL, dequantize_q5_K>;
+template [[host_name("kernel_get_rows_q6_K")]] kernel get_rows_q_t kernel_get_rows_q<block_q6_K, QK_NL, dequantize_q6_K>;
+template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_q_t kernel_get_rows_q<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;
+template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_q_t kernel_get_rows_q<block_iq2_xs, QK_NL, dequantize_iq2_xs>;
+template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_q_t kernel_get_rows_q<block_iq3_xxs, QK_NL, dequantize_iq3_xxs>;
+template [[host_name("kernel_get_rows_iq3_s")]] kernel get_rows_q_t kernel_get_rows_q<block_iq3_s, QK_NL, dequantize_iq3_s>;
+template [[host_name("kernel_get_rows_iq2_s")]] kernel get_rows_q_t kernel_get_rows_q<block_iq2_s, QK_NL, dequantize_iq2_s>;
+template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_q_t kernel_get_rows_q<block_iq1_s, QK_NL, dequantize_iq1_s>;
+template [[host_name("kernel_get_rows_iq1_m")]] kernel get_rows_q_t kernel_get_rows_q<block_iq1_m, QK_NL, dequantize_iq1_m>;
+template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_q_t kernel_get_rows_q<block_iq4_nl, 2, dequantize_iq4_nl>;
+template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_q_t kernel_get_rows_q<block_iq4_xs, QK_NL, dequantize_iq4_xs>;
+
+//
+// matrix-matrix multiplication
+//
+
+typedef decltype(kernel_mul_mm<half, half4x4, simdgroup_half8x8, float4x4, 1, dequantize_f32>) mat_mm_t;
+
+template [[host_name("kernel_mul_mm_f32_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, float4x4, 1, dequantize_f32>;
+template [[host_name("kernel_mul_mm_f16_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half4x4, 1, dequantize_f16>;
+template [[host_name("kernel_mul_mm_q4_0_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_q4_0, 2, dequantize_q4_0>;
+template [[host_name("kernel_mul_mm_q4_1_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_q4_1, 2, dequantize_q4_1>;
+template [[host_name("kernel_mul_mm_q5_0_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_q5_0, 2, dequantize_q5_0>;
+template [[host_name("kernel_mul_mm_q5_1_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_q5_1, 2, dequantize_q5_1>;
+template [[host_name("kernel_mul_mm_q8_0_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_q8_0, 2, dequantize_q8_0>;
+template [[host_name("kernel_mul_mm_q2_K_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K>;
+template [[host_name("kernel_mul_mm_q3_K_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_q3_K, QK_NL, dequantize_q3_K>;
+template [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_q4_K, QK_NL, dequantize_q4_K>;
+template [[host_name("kernel_mul_mm_q5_K_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_q5_K, QK_NL, dequantize_q5_K>;
+template [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_q6_K, QK_NL, dequantize_q6_K>;
+template [[host_name("kernel_mul_mm_iq2_xxs_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;
+template [[host_name("kernel_mul_mm_iq2_xs_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq2_xs, QK_NL, dequantize_iq2_xs>;
+template [[host_name("kernel_mul_mm_iq3_xxs_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq3_xxs, QK_NL, dequantize_iq3_xxs>;
+template [[host_name("kernel_mul_mm_iq3_s_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq3_s, QK_NL, dequantize_iq3_s>;
+template [[host_name("kernel_mul_mm_iq2_s_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq2_s, QK_NL, dequantize_iq2_s>;
+template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq1_s, QK_NL, dequantize_iq1_s>;
+template [[host_name("kernel_mul_mm_iq1_m_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq1_m, QK_NL, dequantize_iq1_m>;
+template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl>;
+template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mat_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs>;
+
+//
+// indirect matrix-matrix multiplication
+//
+
+typedef decltype(kernel_mul_mm_id<float4x4, 1, dequantize_f32>) mat_mm_id_t;
+
+template [[host_name("kernel_mul_mm_id_f32_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<float4x4, 1, dequantize_f32>;
+template [[host_name("kernel_mul_mm_id_f16_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<half4x4, 1, dequantize_f16>;
+template [[host_name("kernel_mul_mm_id_q4_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q4_0, 2, dequantize_q4_0>;
+template [[host_name("kernel_mul_mm_id_q4_1_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q4_1, 2, dequantize_q4_1>;
+template [[host_name("kernel_mul_mm_id_q5_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q5_0, 2, dequantize_q5_0>;
+template [[host_name("kernel_mul_mm_id_q5_1_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q5_1, 2, dequantize_q5_1>;
+template [[host_name("kernel_mul_mm_id_q8_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q8_0, 2, dequantize_q8_0>;
+template [[host_name("kernel_mul_mm_id_q2_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q2_K, QK_NL, dequantize_q2_K>;
+template [[host_name("kernel_mul_mm_id_q3_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q3_K, QK_NL, dequantize_q3_K>;
+template [[host_name("kernel_mul_mm_id_q4_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q4_K, QK_NL, dequantize_q4_K>;
+template [[host_name("kernel_mul_mm_id_q5_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q5_K, QK_NL, dequantize_q5_K>;
+template [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q6_K, QK_NL, dequantize_q6_K>;
+template [[host_name("kernel_mul_mm_id_iq2_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;
+template [[host_name("kernel_mul_mm_id_iq2_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq2_xs, QK_NL, dequantize_iq2_xs>;
+template [[host_name("kernel_mul_mm_id_iq3_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq3_xxs, QK_NL, dequantize_iq3_xxs>;
+template [[host_name("kernel_mul_mm_id_iq3_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq3_s, QK_NL, dequantize_iq3_s>;
+template [[host_name("kernel_mul_mm_id_iq2_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq2_s, QK_NL, dequantize_iq2_s>;
+template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq1_s, QK_NL, dequantize_iq1_s>;
+template [[host_name("kernel_mul_mm_id_iq1_m_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq1_m, QK_NL, dequantize_iq1_m>;
+template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq4_nl, 2, dequantize_iq4_nl>;
+template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq4_xs, QK_NL, dequantize_iq4_xs>;
+
+//
+// matrix-vector multiplication
+//
+
+typedef void (kernel_mul_mv_impl_t)(
+ device const char * src0,
+ device const char * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ uint64_t nb00,
+ uint64_t nb01,
+ uint64_t nb02,
+ int64_t ne10,
+ int64_t ne11,
+ int64_t ne12,
+ uint64_t nb10,
+ uint64_t nb11,
+ uint64_t nb12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ uint3 tgpig,
+ uint tiisg);
+
+typedef void (kernel_mul_mv2_impl_t)(
+ device const void * src0,
+ device const float * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ int64_t ne10,
+ int64_t ne12,
+ int64_t ne0,
+ int64_t ne1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiisg,
+ uint sgitg);
+
+template<kernel_mul_mv_impl_t impl_fn>
+void mmv_fn(
+ device const char * src0,
+ device const char * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ uint64_t nb00,
+ uint64_t nb01,
+ uint64_t nb02,
+ int64_t ne10,
+ int64_t ne11,
+ int64_t ne12,
+ int64_t ne13,
+ uint64_t nb10,
+ uint64_t nb11,
+ uint64_t nb12,
+ int64_t ne0,
+ int64_t ne1,
+ uint64_t nb1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiitg,
+ uint tiisg,
+ uint sgitg) {
+ impl_fn(src0,src1,dst,ne00,ne01,ne02,nb00,nb01,nb02,ne10,ne11,ne12,nb10,nb11,nb12,ne0,ne1,r2,r3,tgpig,tiisg);
+}
+
+template<kernel_mul_mv2_impl_t impl_fn>
+void mmv_fn(
+ device const char * src0,
+ device const char * src1,
+ device float * dst,
+ int64_t ne00,
+ int64_t ne01,
+ int64_t ne02,
+ uint64_t nb00,
+ uint64_t nb01,
+ uint64_t nb02,
+ int64_t ne10,
+ int64_t ne11,
+ int64_t ne12,
+ int64_t ne13,
+ uint64_t nb10,
+ uint64_t nb11,
+ uint64_t nb12,
+ int64_t ne0,
+ int64_t ne1,
+ uint64_t nb1,
+ uint r2,
+ uint r3,
+ threadgroup int8_t * shared_values,
+ uint3 tgpig,
+ uint tiitg,
+ uint tiisg,
+ uint sgitg) {
+ impl_fn(src0,(const device float *)src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,shared_values,tgpig,tiisg,sgitg);
+}
+
+typedef decltype(mmv_fn<kernel_mul_mv_impl<half, half4, half, half4>>) mul_mv_impl_fn_t;
+
+template<mul_mv_impl_fn_t impl_fn>
+kernel void kernel_mul_mv_id(
+ device const char * src0s,
+ device const char * src1,
+ device float * dst,
+ device const char * ids,
+ constant int64_t & nei0,
+ constant int64_t & nei1,
+ constant uint64_t & nbi1,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant uint64_t & nb1,
+ threadgroup int8_t * shared_values [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint tiitg[[thread_index_in_threadgroup]],
+ uint tiisg[[thread_index_in_simdgroup]],
+ uint sgitg[[simdgroup_index_in_threadgroup]]) {
+ const int iid1 = tgpig.z/nei0;
+ const int idx = tgpig.z%nei0;
+
+ tgpig.z = 0;
+
+ const int32_t i02 = ((device const int32_t *) (ids + iid1*nbi1))[idx];
+
+ const int64_t i11 = idx % ne11;
+ const int64_t i12 = iid1;
+
+ const int64_t i1 = idx;
+ const int64_t i2 = i12;
+
+ device const char * src0_cur = src0s + i02*nb02;
+ device const char * src1_cur = src1 + i11*nb11 + i12*nb12;
+ device float * dst_cur = dst + i1*ne0 + i2*ne1*ne0;
+
+ impl_fn(
+ /* src0 */ src0_cur,
+ /* src1 */ src1_cur,
+ /* dst */ dst_cur,
+ /* ne00 */ ne00,
+ /* ne01 */ ne01,
+ /* ne02 */ 1,//ne02,
+ /* nb00 */ nb00,
+ /* nb01 */ nb01,
+ /* nb02 */ nb02,
+ /* ne10 */ ne10,
+ /* ne11 */ 1,//ne11,
+ /* ne12 */ 1,//ne12,
+ /* ne13 */ 1,//ne13,
+ /* nb10 */ nb10,
+ /* nb11 */ nb11,
+ /* nb12 */ nb12,
+ /* ne0 */ ne0,
+ /* ne1 */ 1,//ne1,
+ /* nb1 */ nb1,
+ /* r2 */ 1,
+ /* r3 */ 1,
+ shared_values,
+ tgpig,
+ tiitg,
+ tiisg,
+ sgitg);
+}
+
+typedef decltype(kernel_mul_mv_id<mmv_fn<kernel_mul_mv_impl<float, float4, float, float4>>>) kernel_mul_mv_id_t;
+
+template [[host_name("kernel_mul_mv_id_f32_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_impl<float, float4, float, float4>>>;
+template [[host_name("kernel_mul_mv_id_f16_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_impl<half, half4, float, float4>>>;
+template [[host_name("kernel_mul_mv_id_q8_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q8_0_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_q4_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<mul_vec_q_n_f32_impl<block_q4_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>>>;
+template [[host_name("kernel_mul_mv_id_q4_1_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<mul_vec_q_n_f32_impl<block_q4_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>>>;
+template [[host_name("kernel_mul_mv_id_q5_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<mul_vec_q_n_f32_impl<block_q5_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>>>;
+template [[host_name("kernel_mul_mv_id_q5_1_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<mul_vec_q_n_f32_impl<block_q5_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>>>;
+template [[host_name("kernel_mul_mv_id_q2_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q2_K_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_q3_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q3_K_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_q4_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q4_K_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_q5_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q5_K_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_q6_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q6_K_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_s_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_m_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xxs_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xs_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_iq3_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq3_xxs_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq3_s_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_s_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_nl_f32_impl>>;
+template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_xs_f32_impl>>;
diff --git a/llama/ggml-metal_darwin_arm64.m b/llama/ggml-metal_darwin_arm64.m
new file mode 100644
index 00000000..952646bd
--- /dev/null
+++ b/llama/ggml-metal_darwin_arm64.m
@@ -0,0 +1,3512 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#import "ggml-metal.h"
+
+#import "ggml-backend-impl.h"
+#import "ggml.h"
+
+#import <Foundation/Foundation.h>
+
+#import <Metal/Metal.h>
+
+#undef MIN
+#undef MAX
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+#ifdef GGML_METAL_NDEBUG
+#define GGML_METAL_LOG_INFO(...)
+#define GGML_METAL_LOG_WARN(...)
+#define GGML_METAL_LOG_ERROR(...)
+#else
+#define GGML_METAL_LOG_INFO(...) ggml_metal_log(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
+#define GGML_METAL_LOG_WARN(...) ggml_metal_log(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
+#define GGML_METAL_LOG_ERROR(...) ggml_metal_log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
+#endif
+
+#define UNUSED(x) (void)(x)
+
+struct ggml_metal_kernel {
+ id<MTLComputePipelineState> pipeline;
+};
+
+enum ggml_metal_kernel_type {
+ GGML_METAL_KERNEL_TYPE_ADD,
+ GGML_METAL_KERNEL_TYPE_ADD_ROW,
+ GGML_METAL_KERNEL_TYPE_SUB,
+ GGML_METAL_KERNEL_TYPE_SUB_ROW,
+ GGML_METAL_KERNEL_TYPE_MUL,
+ GGML_METAL_KERNEL_TYPE_MUL_ROW,
+ GGML_METAL_KERNEL_TYPE_DIV,
+ GGML_METAL_KERNEL_TYPE_DIV_ROW,
+ GGML_METAL_KERNEL_TYPE_REPEAT_F32,
+ GGML_METAL_KERNEL_TYPE_REPEAT_F16,
+ GGML_METAL_KERNEL_TYPE_REPEAT_I32,
+ GGML_METAL_KERNEL_TYPE_REPEAT_I16,
+ GGML_METAL_KERNEL_TYPE_SCALE,
+ GGML_METAL_KERNEL_TYPE_SCALE_4,
+ GGML_METAL_KERNEL_TYPE_CLAMP,
+ GGML_METAL_KERNEL_TYPE_TANH,
+ GGML_METAL_KERNEL_TYPE_RELU,
+ GGML_METAL_KERNEL_TYPE_SIGMOID,
+ GGML_METAL_KERNEL_TYPE_GELU,
+ GGML_METAL_KERNEL_TYPE_GELU_4,
+ GGML_METAL_KERNEL_TYPE_GELU_QUICK,
+ GGML_METAL_KERNEL_TYPE_GELU_QUICK_4,
+ GGML_METAL_KERNEL_TYPE_SILU,
+ GGML_METAL_KERNEL_TYPE_SILU_4,
+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16,
+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4,
+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32,
+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4,
+ GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF,
+ GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_F32,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_F16,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_M,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS,
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_I32,
+ GGML_METAL_KERNEL_TYPE_RMS_NORM,
+ GGML_METAL_KERNEL_TYPE_GROUP_NORM,
+ GGML_METAL_KERNEL_TYPE_NORM,
+ GGML_METAL_KERNEL_TYPE_SSM_CONV_F32,
+ GGML_METAL_KERNEL_TYPE_SSM_SCAN_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_M_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32,
+ //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32,
+ //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW,
+ //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32,
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32,
+ GGML_METAL_KERNEL_TYPE_ROPE_NORM_F32,
+ GGML_METAL_KERNEL_TYPE_ROPE_NORM_F16,
+ GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F32,
+ GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F16,
+ GGML_METAL_KERNEL_TYPE_IM2COL_F16,
+ GGML_METAL_KERNEL_TYPE_IM2COL_F32,
+ GGML_METAL_KERNEL_TYPE_UPSCALE_F32,
+ GGML_METAL_KERNEL_TYPE_PAD_F32,
+ GGML_METAL_KERNEL_TYPE_ARANGE_F32,
+ GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32,
+ GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC,
+ GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC,
+ GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128,
+ //GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256, // https://github.com/ggerganov/llama.cpp/issues/7261
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128,
+ //GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256, // https://github.com/ggerganov/llama.cpp/issues/7261
+ GGML_METAL_KERNEL_TYPE_CPY_F32_F32,
+ GGML_METAL_KERNEL_TYPE_CPY_F32_F16,
+ GGML_METAL_KERNEL_TYPE_CPY_F16_F16,
+ GGML_METAL_KERNEL_TYPE_CPY_F16_F32,
+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0,
+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0,
+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1,
+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0,
+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1,
+ GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL,
+ GGML_METAL_KERNEL_TYPE_CONCAT,
+ GGML_METAL_KERNEL_TYPE_SQR,
+ GGML_METAL_KERNEL_TYPE_SQRT,
+ GGML_METAL_KERNEL_TYPE_SIN,
+ GGML_METAL_KERNEL_TYPE_COS,
+ GGML_METAL_KERNEL_TYPE_SUM_ROWS,
+
+ GGML_METAL_KERNEL_TYPE_COUNT
+};
+
+struct ggml_backend_metal_context {
+ int n_cb;
+
+ id<MTLDevice> device;
+ id<MTLCommandQueue> queue;
+
+ dispatch_queue_t d_queue;
+
+ struct ggml_metal_kernel kernels[GGML_METAL_KERNEL_TYPE_COUNT];
+
+ bool support_simdgroup_reduction;
+ bool support_simdgroup_mm;
+
+ bool should_capture_next_compute;
+
+ // abort ggml_metal_graph_compute if callback returns true
+ ggml_abort_callback abort_callback;
+ void * abort_callback_data;
+};
+
+// MSL code
+// TODO: move the contents here when ready
+// for now it is easier to work in a separate file
+// static NSString * const msl_library_source = @"see metal.metal";
+
+// Here to assist with NSBundle Path Hack
+@interface GGMLMetalClass : NSObject
+@end
+@implementation GGMLMetalClass
+@end
+
+static void ggml_metal_default_log_callback(enum ggml_log_level level, const char * msg, void * user_data) {
+ fprintf(stderr, "%s", msg);
+
+ UNUSED(level);
+ UNUSED(user_data);
+}
+
+ggml_log_callback ggml_metal_log_callback = ggml_metal_default_log_callback;
+void * ggml_metal_log_user_data = NULL;
+
+GGML_ATTRIBUTE_FORMAT(2, 3)
+static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){
+ if (ggml_metal_log_callback != NULL) {
+ va_list args;
+ va_start(args, format);
+ char buffer[128];
+ int len = vsnprintf(buffer, 128, format, args);
+ if (len < 128) {
+ ggml_metal_log_callback(level, buffer, ggml_metal_log_user_data);
+ } else {
+ char* buffer2 = malloc(len+1);
+ va_end(args);
+ va_start(args, format);
+ vsnprintf(buffer2, len+1, format, args);
+ buffer2[len] = 0;
+ ggml_metal_log_callback(level, buffer2, ggml_metal_log_user_data);
+ free(buffer2);
+ }
+ va_end(args);
+ }
+}
+
+static void * ggml_metal_host_malloc(size_t n) {
+ void * data = NULL;
+
+#if TARGET_OS_OSX
+ kern_return_t err = vm_allocate((vm_map_t) mach_task_self(), (void *) &data, n, VM_FLAGS_ANYWHERE);
+ if (err != KERN_SUCCESS) {
+ GGML_METAL_LOG_ERROR("%s: error: vm_allocate failed\n", __func__);
+ return NULL;
+ }
+#else
+ const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);
+ if (result != 0) {
+ GGML_METAL_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);
+ return NULL;
+ }
+#endif
+
+ return data;
+}
+
+static struct ggml_backend_metal_context * ggml_metal_init(int n_cb) {
+ GGML_METAL_LOG_INFO("%s: allocating\n", __func__);
+
+#if TARGET_OS_OSX && !GGML_METAL_NDEBUG
+ // Show all the Metal device instances in the system
+ NSArray * devices = MTLCopyAllDevices();
+ for (id<MTLDevice> device in devices) {
+ GGML_METAL_LOG_INFO("%s: found device: %s\n", __func__, [[device name] UTF8String]);
+ }
+ [devices release]; // since it was created by a *Copy* C method
+#endif
+
+ // Pick and show default Metal device
+ id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+ GGML_METAL_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]);
+
+ // Configure context
+ struct ggml_backend_metal_context * ctx = calloc(1, sizeof(struct ggml_backend_metal_context));
+ ctx->device = device;
+ ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);
+ ctx->queue = [ctx->device newCommandQueue];
+ ctx->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);
+
+ id<MTLLibrary> metal_library;
+
+ // load library
+ //
+ // - first check if the library is embedded
+ // - then check if the library is in the bundle
+ // - if not found, load the source and compile it
+ // - if that fails, return NULL
+ {
+ NSBundle * bundle = nil;
+#ifdef SWIFT_PACKAGE
+ bundle = SWIFTPM_MODULE_BUNDLE;
+#else
+ bundle = [NSBundle bundleForClass:[GGMLMetalClass class]];
+#endif
+
+ NSError * error = nil;
+
+#if GGML_METAL_EMBED_LIBRARY
+ const bool try_metallib = false;
+#else
+ const bool try_metallib = true;
+#endif
+
+ NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"];
+ if (try_metallib && path_lib != nil) {
+ // pre-compiled library found
+ NSURL * libURL = [NSURL fileURLWithPath:path_lib];
+ GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path_lib UTF8String]);
+
+ metal_library = [ctx->device newLibraryWithURL:libURL error:&error];
+ if (error) {
+ GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
+ return NULL;
+ }
+ } else {
+#if GGML_METAL_EMBED_LIBRARY
+ GGML_METAL_LOG_INFO("%s: using embedded metal library\n", __func__);
+
+ extern const char *ggml_metallib_start;
+ extern const char *ggml_metallib_end;
+
+ NSString * src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding];
+#else
+ GGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__);
+
+ NSString * path_source;
+ NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"];
+
+ GGML_METAL_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, path_resource ? [path_resource UTF8String] : "nil");
+
+ if (path_resource) {
+ path_source = [path_resource stringByAppendingPathComponent:@"ggml-metal.metal"];
+ } else {
+ path_source = [bundle pathForResource:@"ggml-metal" ofType:@"metal"];
+ }
+
+ if (path_source == nil) {
+ GGML_METAL_LOG_WARN("%s: error: could not use bundle path to find ggml-metal.metal, falling back to trying cwd\n", __func__);
+ path_source = @"ggml-metal.metal";
+ }
+
+ GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path_source UTF8String]);
+
+ NSString * src = [NSString stringWithContentsOfFile:path_source encoding:NSUTF8StringEncoding error:&error];
+ if (error) {
+ GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
+ return NULL;
+ }
+#endif // GGML_METAL_EMBED_LIBRARY
+
+ @autoreleasepool {
+ // dictionary of preprocessor macros
+ NSMutableDictionary * prep = [NSMutableDictionary dictionary];
+
+ MTLCompileOptions* options = [MTLCompileOptions new];
+ options.preprocessorMacros = prep;
+
+ //[options setFastMathEnabled:false];
+
+ metal_library = [ctx->device newLibraryWithSource:src options:options error:&error];
+ if (error) {
+ GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
+ return NULL;
+ }
+ }
+ }
+ }
+
+ // print MTL GPU family:
+ GGML_METAL_LOG_INFO("%s: GPU name: %s\n", __func__, [[ctx->device name] UTF8String]);
+
+ const NSInteger MTLGPUFamilyMetal3 = 5001;
+
+ // determine max supported GPU family
+ // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf
+ // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
+ {
+ for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
+ if ([ctx->device supportsFamily:i]) {
+ GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i);
+ break;
+ }
+ }
+
+ for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {
+ if ([ctx->device supportsFamily:i]) {
+ GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);
+ break;
+ }
+ }
+
+ for (int i = MTLGPUFamilyMetal3 + 5; i >= MTLGPUFamilyMetal3; --i) {
+ if ([ctx->device supportsFamily:i]) {
+ GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3 + 3, i);
+ break;
+ }
+ }
+ }
+
+ ctx->support_simdgroup_reduction = [ctx->device supportsFamily:MTLGPUFamilyApple7];
+ ctx->support_simdgroup_reduction |= [ctx->device supportsFamily:MTLGPUFamilyMetal3];
+
+ ctx->support_simdgroup_mm = [ctx->device supportsFamily:MTLGPUFamilyApple7];
+
+ GGML_METAL_LOG_INFO("%s: simdgroup reduction support = %s\n", __func__, ctx->support_simdgroup_reduction ? "true" : "false");
+ GGML_METAL_LOG_INFO("%s: simdgroup matrix mul. support = %s\n", __func__, ctx->support_simdgroup_mm ? "true" : "false");
+ GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false");
+
+ ctx->should_capture_next_compute = false;
+
+#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
+ if (@available(macOS 10.12, iOS 16.0, *)) {
+ GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6);
+ }
+#elif TARGET_OS_OSX
+ if (ctx->device.maxTransferRate != 0) {
+ GGML_METAL_LOG_INFO("%s: maxTransferRate = %8.2f MB/s\n", __func__, ctx->device.maxTransferRate / 1e6);
+ } else {
+ GGML_METAL_LOG_INFO("%s: maxTransferRate = built-in GPU\n", __func__);
+ }
+#endif
+
+ // load kernels
+ {
+ NSError * error = nil;
+
+ for (int i = 0; i < GGML_METAL_KERNEL_TYPE_COUNT; ++i) {
+ ctx->kernels[i].pipeline = nil;
+ }
+
+ /*
+ GGML_METAL_LOG_INFO("%s: loaded %-40s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) kernel->pipeline, \
+ (int) kernel->pipeline.maxTotalThreadsPerThreadgroup, \
+ (int) kernel->pipeline.threadExecutionWidth); \
+ */
+#define GGML_METAL_ADD_KERNEL(e, name, supported) \
+ if (supported) { \
+ struct ggml_metal_kernel * kernel = &ctx->kernels[e]; \
+ id<MTLFunction> metal_function = [metal_library newFunctionWithName:@"kernel_"#name]; \
+ kernel->pipeline = [ctx->device newComputePipelineStateWithFunction:metal_function error:&error]; \
+ [metal_function release]; \
+ if (error) { \
+ GGML_METAL_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \
+ [metal_library release]; \
+ return NULL; \
+ } \
+ } else { \
+ GGML_METAL_LOG_WARN("%s: skipping %-40s (not supported)\n", __func__, "kernel_"#name); \
+ }
+
+ // simd_sum and simd_max requires MTLGPUFamilyApple7
+
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD, add, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD_ROW, add_row, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUB, sub, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUB_ROW, sub_row, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL, mul, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_ROW, mul_row, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV, div, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV_ROW, div_row, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_REPEAT_F32, repeat_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_REPEAT_F16, repeat_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_REPEAT_I32, repeat_i32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_REPEAT_I16, repeat_i16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE, scale, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE_4, scale_4, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CLAMP, clamp, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TANH, tanh, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RELU, relu, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SIGMOID, sigmoid, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU, gelu, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_4, gelu_4, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK, gelu_quick, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK_4, gelu_quick_4, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU, silu, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU_4, silu_4, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16, soft_max_f16, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4, soft_max_f16_4, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32, soft_max_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4, soft_max_f32_4, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF, diag_mask_inf, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8, diag_mask_inf_8, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F32, get_rows_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F16, get_rows_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0, get_rows_q4_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1, get_rows_q4_1, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0, get_rows_q5_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1, get_rows_q5_1, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0, get_rows_q8_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K, get_rows_q2_K, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K, get_rows_q3_K, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K, get_rows_q4_K, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K, get_rows_q5_K, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K, get_rows_q6_K, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, get_rows_iq3_xxs, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S, get_rows_iq3_s, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S, get_rows_iq2_s, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_M, get_rows_iq1_m, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, get_rows_iq4_nl, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, get_rows_iq4_xs, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NORM, norm, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SSM_CONV_F32, ssm_conv_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SSM_SCAN_F32, ssm_scan_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, mul_mv_f32_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16, mul_mv_f16_f16, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, mul_mv_f16_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, mul_mv_f16_f32_1row, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4, mul_mv_f16_f32_l4, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32, mul_mv_q4_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32, mul_mv_q4_1_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, mul_mv_q5_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, mul_mv_q5_1_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, mul_mv_q8_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32, mul_mv_q2_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32, mul_mv_q3_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32, mul_mv_q4_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32, mul_mv_q5_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32, mul_mv_q6_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, mul_mv_iq3_xxs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32, mul_mv_iq3_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32, mul_mv_iq2_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_M_F32, mul_mv_iq1_m_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, mul_mv_iq4_nl_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32, mul_mv_iq4_xs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction);
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction);
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW, mul_mv_id_f16_f32_1row, ctx->support_simdgroup_reduction);
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4, mul_mv_id_f16_f32_l4, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32, mul_mv_id_q4_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32, mul_mv_id_q4_1_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32, mul_mv_id_q5_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32, mul_mv_id_q5_1_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32, mul_mv_id_q8_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32, mul_mv_id_q2_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32, mul_mv_id_q3_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32, mul_mv_id_q4_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32, mul_mv_id_q5_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32, mul_mv_id_q6_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, mul_mv_id_iq3_xxs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32, mul_mv_id_iq3_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32, mul_mv_id_iq2_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32, mul_mv_id_iq1_m_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32, mul_mv_id_iq4_xs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32, mul_mm_q4_1_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32, mul_mm_q5_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32, mul_mm_q5_1_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32, mul_mm_q8_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32, mul_mm_q2_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32, mul_mm_q3_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32, mul_mm_q4_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32, mul_mm_q5_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32, mul_mm_q6_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, mul_mm_iq3_xxs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32, mul_mm_iq3_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32, mul_mm_iq2_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32, mul_mm_iq1_m_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32, mul_mm_iq4_xs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32, mul_mm_id_q4_1_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32, mul_mm_id_q5_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32, mul_mm_id_q5_1_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32, mul_mm_id_q8_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32, mul_mm_id_q2_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32, mul_mm_id_q3_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32, mul_mm_id_q4_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32, mul_mm_id_q5_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32, mul_mm_id_q6_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, mul_mm_id_iq3_xxs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32, mul_mm_id_iq3_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32, mul_mm_id_iq2_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F32, mul_mm_id_iq1_m_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, mul_mm_id_iq4_nl_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32, mul_mm_id_iq4_xs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NORM_F32, rope_norm_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NORM_F16, rope_norm_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F32, rope_neox_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F16, rope_neox_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F32, im2col_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32, timestep_embedding_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARANGE_F32, arange_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64, flash_attn_ext_f16_h64, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80, flash_attn_ext_f16_h80, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96, flash_attn_ext_f16_h96, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112, flash_attn_ext_f16_h112, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128, flash_attn_ext_f16_h128, ctx->support_simdgroup_mm);
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256, flash_attn_ext_f16_h256, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128, flash_attn_ext_vec_f16_h128, ctx->support_simdgroup_reduction);
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256, flash_attn_ext_vec_f16_h256, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F16, cpy_f32_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F32, cpy_f32_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F16, cpy_f16_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F32, cpy_f16_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, cpy_f32_q4_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, cpy_f32_q4_1, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL, cpy_f32_iq4_nl, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQR, sqr, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQRT, sqrt, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SIN, sin, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_COS, cos, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS, sum_rows, true);
+ }
+
+ [metal_library release];
+ return ctx;
+}
+
+static void ggml_metal_free(struct ggml_backend_metal_context * ctx) {
+ GGML_METAL_LOG_INFO("%s: deallocating\n", __func__);
+
+ for (int i = 0; i < GGML_METAL_KERNEL_TYPE_COUNT; ++i) {
+ [ctx->kernels[i].pipeline release];
+ }
+
+ [ctx->queue release];
+ [ctx->device release];
+
+ dispatch_release(ctx->d_queue);
+
+ free(ctx);
+}
+
+// temporarily defined here for compatibility between ggml-backend and the old API
+
+struct ggml_backend_metal_buffer {
+ void * data;
+ size_t size;
+
+ id<MTLBuffer> metal;
+};
+
+struct ggml_backend_metal_buffer_context {
+ void * all_data;
+ size_t all_size;
+ bool owned;
+
+ // multiple buffers are used only to avoid the maximum buffer size limitation when using mmap
+ int n_buffers;
+ struct ggml_backend_metal_buffer buffers[GGML_METAL_MAX_BUFFERS];
+};
+
+// finds the Metal buffer that contains the tensor data on the GPU device
+// the assumption is that there is 1-to-1 mapping between the host and device memory buffers, so we can find the
+// Metal buffer based on the host memory pointer
+//
+static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_tensor * t, size_t * offs) {
+ //GGML_METAL_LOG_INFO("%s: data tensor '%16s', offs_data = %8ld, offs_eval = %8ld, offs_cach = %8ld\n", __func__, t->name, offs_data, offs_eval, offs_cach);
+
+ const int64_t tsize = ggml_nbytes(t);
+
+ ggml_backend_buffer_t buffer = t->view_src ? t->view_src->buffer : t->buffer;
+
+ struct ggml_backend_metal_buffer_context * buf_ctx = (struct ggml_backend_metal_buffer_context *) buffer->context;
+
+ // find the view that contains the tensor fully
+ for (int i = 0; i < buf_ctx->n_buffers; ++i) {
+ const int64_t ioffs = (int64_t) t->data - (int64_t) buf_ctx->buffers[i].data;
+
+ //GGML_METAL_LOG_INFO("ioffs = %10ld, tsize = %10ld, sum = %10ld, buf_ctx->buffers[%d].size = %10ld\n", ioffs, tsize, ioffs + tsize, i, buf_ctx->buffers[i].size);
+ if (ioffs >= 0 && ioffs + tsize <= (int64_t) buf_ctx->buffers[i].size) {
+ *offs = (size_t) ioffs;
+
+ //GGML_METAL_LOG_INFO("%s: tensor '%16s', offs = %8ld\n", __func__, t->name, *offs);
+
+ return buf_ctx->buffers[i].metal;
+ }
+ }
+
+ GGML_METAL_LOG_ERROR("%s: error: tensor '%s' buffer is nil\n", __func__, t->name);
+
+ return nil;
+}
+
+static bool ggml_metal_supports_op(const struct ggml_backend_metal_context * ctx, const struct ggml_tensor * op) {
+ for (size_t i = 0, n = 3; i < n; ++i) {
+ if (op->src[i] != NULL && op->src[i]->type == GGML_TYPE_BF16) {
+ return false;
+ }
+ }
+
+ switch (op->op) {
+ case GGML_OP_UNARY:
+ switch (ggml_get_unary_op(op)) {
+ case GGML_UNARY_OP_TANH:
+ case GGML_UNARY_OP_RELU:
+ case GGML_UNARY_OP_SIGMOID:
+ case GGML_UNARY_OP_GELU:
+ case GGML_UNARY_OP_GELU_QUICK:
+ case GGML_UNARY_OP_SILU:
+ return ggml_is_contiguous(op->src[0]);
+ default:
+ return false;
+ }
+ case GGML_OP_NONE:
+ case GGML_OP_RESHAPE:
+ case GGML_OP_VIEW:
+ case GGML_OP_TRANSPOSE:
+ case GGML_OP_PERMUTE:
+ case GGML_OP_CONCAT:
+ case GGML_OP_ADD:
+ case GGML_OP_SUB:
+ case GGML_OP_ACC:
+ case GGML_OP_MUL:
+ case GGML_OP_DIV:
+ case GGML_OP_REPEAT:
+ case GGML_OP_SCALE:
+ case GGML_OP_CLAMP:
+ return true;
+ case GGML_OP_SQR:
+ case GGML_OP_SQRT:
+ case GGML_OP_SIN:
+ case GGML_OP_COS:
+ return ggml_is_contiguous(op->src[0]);
+ case GGML_OP_SUM_ROWS:
+ case GGML_OP_SOFT_MAX:
+ case GGML_OP_RMS_NORM:
+ case GGML_OP_GROUP_NORM:
+ return ctx->support_simdgroup_reduction;
+ case GGML_OP_NORM:
+ case GGML_OP_ROPE:
+ case GGML_OP_IM2COL:
+ return true;
+ case GGML_OP_POOL_1D:
+ case GGML_OP_POOL_2D:
+ return false;
+ case GGML_OP_UPSCALE:
+ case GGML_OP_PAD:
+ case GGML_OP_ARANGE:
+ case GGML_OP_TIMESTEP_EMBEDDING:
+ case GGML_OP_ARGSORT:
+ case GGML_OP_LEAKY_RELU:
+ return true;
+ case GGML_OP_FLASH_ATTN_EXT:
+ if (op->src[1]->type != GGML_TYPE_F16) {
+ return false;
+ }
+ if (op->src[2]->type != GGML_TYPE_F16) {
+ return false;
+ }
+ if (op->src[0]->ne[0] == 256) {
+ return false;
+ }
+ return ctx->support_simdgroup_mm; // TODO: over-restricted for vec-kernels
+ case GGML_OP_SSM_CONV:
+ case GGML_OP_SSM_SCAN:
+ return true;
+ case GGML_OP_MUL_MAT:
+ case GGML_OP_MUL_MAT_ID:
+ return ctx->support_simdgroup_reduction &&
+ (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F32);
+ case GGML_OP_CPY:
+ case GGML_OP_DUP:
+ case GGML_OP_CONT:
+ {
+ switch (op->src[0]->type) {
+ case GGML_TYPE_F32:
+ switch (op->type) {
+ case GGML_TYPE_F32:
+ case GGML_TYPE_F16:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_IQ4_NL:
+ return true;
+ default:
+ return false;
+ }
+ case GGML_TYPE_F16:
+ switch (op->type) {
+ case GGML_TYPE_F32:
+ case GGML_TYPE_F16:
+ return true;
+ default:
+ return false;
+ }
+ default:
+ return false;
+ };
+ }
+ case GGML_OP_DIAG_MASK_INF:
+ case GGML_OP_GET_ROWS:
+ {
+ return op->ne[3] == 1;
+ }
+ default:
+ return false;
+ }
+}
+
+static enum ggml_status ggml_metal_graph_compute(
+ struct ggml_backend_metal_context * ctx,
+ struct ggml_cgraph * gf) {
+
+ @autoreleasepool {
+ MTLComputePassDescriptor * edesc = MTLComputePassDescriptor.computePassDescriptor;
+ edesc.dispatchType = MTLDispatchTypeSerial;
+
+ // create multiple command buffers and enqueue them
+ // then, we encode the graph into the command buffers in parallel
+
+ const int n_nodes = gf->n_nodes;
+ const int n_cb = ctx->n_cb;
+ const int n_nodes_per_cb = (n_nodes + n_cb - 1) / n_cb;
+
+ const bool should_capture = ctx->should_capture_next_compute;
+ if (should_capture) {
+ ctx->should_capture_next_compute = false;
+
+ MTLCaptureDescriptor * descriptor = [MTLCaptureDescriptor new];
+ descriptor.captureObject = ctx->queue;
+
+ NSError * error = nil;
+ if (![[MTLCaptureManager sharedCaptureManager] startCaptureWithDescriptor:descriptor error:&error]) {
+ GGML_METAL_LOG_ERROR("%s: error: unable to start capture '%s'\n", __func__, [[error localizedDescription] UTF8String]);
+ GGML_ABORT("capture failed");
+ }
+ }
+
+ id<MTLCommandBuffer> command_buffer_builder[n_cb];
+ for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) {
+ id<MTLCommandBuffer> command_buffer = [ctx->queue commandBufferWithUnretainedReferences];
+ command_buffer_builder[cb_idx] = command_buffer;
+
+ // always enqueue the first two command buffers
+ // enqueue all of the command buffers if we don't need to abort
+ if (cb_idx < 2 || ctx->abort_callback == NULL) {
+ [command_buffer enqueue];
+ }
+ }
+
+ const id<MTLCommandBuffer> *command_buffers = command_buffer_builder;
+
+ dispatch_apply(n_cb, ctx->d_queue, ^(size_t iter) {
+ const int cb_idx = iter;
+
+ size_t offs_src0 = 0;
+ size_t offs_src1 = 0;
+ size_t offs_src2 = 0;
+ size_t offs_dst = 0;
+
+ id<MTLCommandBuffer> command_buffer = command_buffers[cb_idx];
+ id<MTLComputeCommandEncoder> encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc];
+
+ const int node_start = (cb_idx + 0) * n_nodes_per_cb;
+ const int node_end = MIN((cb_idx == n_cb - 1) ? n_nodes : (cb_idx + 1) * n_nodes_per_cb, n_nodes);
+
+ for (int i = node_start; i < node_end; ++i) {
+ if (i == -1) {
+ [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers];
+ continue;
+ }
+
+ //GGML_METAL_LOG_INFO("%s: encoding node %3d, op = %8s\n", __func__, i, ggml_op_name(gf->nodes[i]->op));
+
+ struct ggml_tensor * src0 = gf->nodes[i]->src[0];
+ struct ggml_tensor * src1 = gf->nodes[i]->src[1];
+ struct ggml_tensor * src2 = gf->nodes[i]->src[2];
+ struct ggml_tensor * dst = gf->nodes[i];
+
+ if (ggml_is_empty(dst)) {
+ continue;
+ }
+
+ switch (dst->op) {
+ case GGML_OP_NONE:
+ case GGML_OP_RESHAPE:
+ case GGML_OP_VIEW:
+ case GGML_OP_TRANSPOSE:
+ case GGML_OP_PERMUTE:
+ {
+ // noop -> next node
+ } continue;
+ default:
+ {
+ } break;
+ }
+
+ if (!ggml_metal_supports_op(ctx, dst)) {
+ GGML_METAL_LOG_ERROR("%s: error: unsupported op '%s'\n", __func__, ggml_op_desc(dst));
+ GGML_ABORT("unsupported op");
+ }
+
+ if (should_capture) {
+ [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]];
+ }
+
+ const int64_t ne00 = src0 ? src0->ne[0] : 0;
+ const int64_t ne01 = src0 ? src0->ne[1] : 0;
+ const int64_t ne02 = src0 ? src0->ne[2] : 0;
+ const int64_t ne03 = src0 ? src0->ne[3] : 0;
+
+ const uint64_t nb00 = src0 ? src0->nb[0] : 0;
+ const uint64_t nb01 = src0 ? src0->nb[1] : 0;
+ const uint64_t nb02 = src0 ? src0->nb[2] : 0;
+ const uint64_t nb03 = src0 ? src0->nb[3] : 0;
+
+ const int64_t ne10 = src1 ? src1->ne[0] : 0;
+ const int64_t ne11 = src1 ? src1->ne[1] : 0;
+ const int64_t ne12 = src1 ? src1->ne[2] : 0;
+ const int64_t ne13 = src1 ? src1->ne[3] : 0;
+
+ const uint64_t nb10 = src1 ? src1->nb[0] : 0;
+ const uint64_t nb11 = src1 ? src1->nb[1] : 0;
+ const uint64_t nb12 = src1 ? src1->nb[2] : 0;
+ const uint64_t nb13 = src1 ? src1->nb[3] : 0;
+
+ const int64_t ne20 = src2 ? src2->ne[0] : 0;
+ const int64_t ne21 = src2 ? src2->ne[1] : 0;
+ const int64_t ne22 = src2 ? src2->ne[2] : 0; GGML_UNUSED(ne22);
+ const int64_t ne23 = src2 ? src2->ne[3] : 0; GGML_UNUSED(ne23);
+
+ const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20);
+ const uint64_t nb21 = src2 ? src2->nb[1] : 0;
+ const uint64_t nb22 = src2 ? src2->nb[2] : 0;
+ const uint64_t nb23 = src2 ? src2->nb[3] : 0;
+
+ const int64_t ne0 = dst ? dst->ne[0] : 0;
+ const int64_t ne1 = dst ? dst->ne[1] : 0;
+ const int64_t ne2 = dst ? dst->ne[2] : 0;
+ const int64_t ne3 = dst ? dst->ne[3] : 0;
+
+ const uint64_t nb0 = dst ? dst->nb[0] : 0;
+ const uint64_t nb1 = dst ? dst->nb[1] : 0;
+ const uint64_t nb2 = dst ? dst->nb[2] : 0;
+ const uint64_t nb3 = dst ? dst->nb[3] : 0;
+
+ const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT;
+ const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT;
+ const enum ggml_type dstt = dst ? dst->type : GGML_TYPE_COUNT;
+
+ id<MTLBuffer> id_src0 = src0 ? ggml_metal_get_buffer(src0, &offs_src0) : nil;
+ id<MTLBuffer> id_src1 = src1 ? ggml_metal_get_buffer(src1, &offs_src1) : nil;
+ id<MTLBuffer> id_src2 = src2 ? ggml_metal_get_buffer(src2, &offs_src2) : nil;
+ id<MTLBuffer> id_dst = dst ? ggml_metal_get_buffer(dst, &offs_dst) : nil;
+
+ //GGML_METAL_LOG_INFO("%s: op - %s\n", __func__, ggml_op_name(dst->op));
+ //if (src0) {
+ // GGML_METAL_LOG_INFO("%s: src0 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src0t), ne00, ne01, ne02,
+ // ggml_is_contiguous(src0), src0->name);
+ //}
+ //if (src1) {
+ // GGML_METAL_LOG_INFO("%s: src1 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src1t), ne10, ne11, ne12,
+ // ggml_is_contiguous(src1), src1->name);
+ //}
+ //if (dst) {
+ // GGML_METAL_LOG_INFO("%s: dst - %4s [%5lld, %5lld, %5lld], 1, %s\n", __func__, ggml_type_name(dstt), ne0, ne1, ne2,
+ // dst->name);
+ //}
+
+ switch (dst->op) {
+ case GGML_OP_CONCAT:
+ {
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CONCAT].pipeline;
+
+ const int32_t dim = ((int32_t *) dst->op_params)[0];
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];
+ [encoder setBytes:&dim length:sizeof(dim) atIndex:27];
+
+ const int nth = MIN(1024, ne0);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_ADD:
+ case GGML_OP_SUB:
+ case GGML_OP_MUL:
+ case GGML_OP_DIV:
+ {
+ GGML_ASSERT(src0t == GGML_TYPE_F32);
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
+
+ const size_t offs = 0;
+
+ bool bcast_row = false;
+
+ int64_t nb = ne00; // used by the "row" kernels
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) {
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ // src1 is a row
+ GGML_ASSERT(ne11 == 1);
+
+ nb = ne00 / 4;
+ switch (dst->op) {
+ case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD_ROW].pipeline; break;
+ case GGML_OP_SUB: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SUB_ROW].pipeline; break;
+ case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_ROW].pipeline; break;
+ case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV_ROW].pipeline; break;
+ default: GGML_ABORT("fatal error");
+ }
+
+ bcast_row = true;
+ } else {
+ switch (dst->op) {
+ case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline; break;
+ case GGML_OP_SUB: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SUB].pipeline; break;
+ case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL].pipeline; break;
+ case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV].pipeline; break;
+ default: GGML_ABORT("fatal error");
+ }
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];
+ [encoder setBytes:&offs length:sizeof(offs) atIndex:27];
+ [encoder setBytes:&nb length:sizeof(nb) atIndex:28];
+
+ if (bcast_row) {
+ const int64_t n = ggml_nelements(dst)/4;
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } else {
+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ }
+ } break;
+ case GGML_OP_REPEAT:
+ {
+ id<MTLComputePipelineState> pipeline;
+
+ switch (src0t) {
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_REPEAT_F32].pipeline; break;
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_REPEAT_F16].pipeline; break;
+ case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_REPEAT_I32].pipeline; break;
+ case GGML_TYPE_I16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_REPEAT_I16].pipeline; break;
+ default: GGML_ABORT("fatal error");
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];
+
+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_ACC:
+ {
+ GGML_ASSERT(src0t == GGML_TYPE_F32);
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
+ GGML_ASSERT(dstt == GGML_TYPE_F32);
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+ GGML_ASSERT(ggml_is_contiguous(src1));
+
+ const size_t pnb1 = ((int32_t *) dst->op_params)[0];
+ const size_t pnb2 = ((int32_t *) dst->op_params)[1];
+ const size_t pnb3 = ((int32_t *) dst->op_params)[2];
+ const size_t offs = ((int32_t *) dst->op_params)[3];
+
+ const bool inplace = (bool) ((int32_t *) dst->op_params)[4];
+
+ if (!inplace) {
+ // run a separete kernel to cpy src->dst
+ // not sure how to avoid this
+ // TODO: make a simpler cpy_bytes kernel
+
+ const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];
+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];
+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];
+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];
+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];
+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];
+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];
+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];
+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];
+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];
+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];
+
+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ }
+
+ const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
+ [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:8];
+ [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:9];
+ [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:10];
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
+ [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:24];
+ [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:25];
+ [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:26];
+ [encoder setBytes:&offs length:sizeof(offs) atIndex:27];
+
+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne11, ne12, ne13) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_SCALE:
+ {
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ float scale;
+ memcpy(&scale, dst->op_params, sizeof(scale));
+
+ int64_t n = ggml_nelements(dst);
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ if (n % 4 == 0) {
+ n /= 4;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE_4].pipeline;
+ } else {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE].pipeline;
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&scale length:sizeof(scale) atIndex:2];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_OP_CLAMP:
+ {
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CLAMP].pipeline;
+
+ float min;
+ float max;
+ memcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float));
+ memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float));
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&min length:sizeof(min) atIndex:2];
+ [encoder setBytes:&max length:sizeof(max) atIndex:3];
+
+ const int64_t n = ggml_nelements(dst);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_OP_UNARY:
+ switch (ggml_get_unary_op(gf->nodes[i])) {
+ // we are not taking into account the strides, so for now require contiguous tensors
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ case GGML_UNARY_OP_TANH:
+ {
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TANH].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+
+ const int64_t n = ggml_nelements(dst);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_UNARY_OP_RELU:
+ {
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RELU].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+
+ const int64_t n = ggml_nelements(dst);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_UNARY_OP_SIGMOID:
+ {
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SIGMOID].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+
+ const int64_t n = ggml_nelements(dst);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_UNARY_OP_GELU:
+ {
+ int64_t n = ggml_nelements(dst);
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ if (n % 4 == 0) {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU_4].pipeline;
+ n /= 4;
+ } else {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU].pipeline;
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_UNARY_OP_GELU_QUICK:
+ {
+ int64_t n = ggml_nelements(dst);
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ if (n % 4 == 0) {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU_QUICK_4].pipeline;
+ n /= 4;
+ } else {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU_QUICK].pipeline;
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_UNARY_OP_SILU:
+ {
+ int64_t n = ggml_nelements(dst);
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ if (n % 4 == 0) {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SILU_4].pipeline;
+ n /= 4;
+ } else {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SILU].pipeline;
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ default:
+ {
+ GGML_METAL_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
+ GGML_ABORT("fatal error");
+ }
+ } break;
+ case GGML_OP_SQR:
+ {
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SQR].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+
+ const int64_t n = ggml_nelements(dst);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_OP_SQRT:
+ {
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SQRT].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+
+ const int64_t n = ggml_nelements(dst);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_OP_SIN:
+ {
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SIN].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+
+ const int64_t n = ggml_nelements(dst);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_OP_COS:
+ {
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_COS].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+
+ const int64_t n = ggml_nelements(dst);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_OP_SUM_ROWS:
+ {
+ GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SUM_ROWS].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11];
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];
+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:18];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:19];
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:20];
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:21];
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:22];
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:23];
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:24];
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:25];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_OP_SOFT_MAX:
+ {
+ GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);
+
+ int nth = 32; // SIMD width
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);
+
+ if (ne00%4 == 0) {
+ while (nth < ne00/4 && nth*ne01*ne02*ne03 < 256) {
+ nth *= 2;
+ }
+ if (use_f16) {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4].pipeline;
+ } else {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4].pipeline;
+ }
+ } else {
+ while (nth < ne00 && nth*ne01*ne02*ne03 < 256) {
+ nth *= 2;
+ }
+ if (use_f16) {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16].pipeline;
+ } else {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32].pipeline;
+ }
+ }
+
+ float scale;
+ float max_bias;
+
+ memcpy(&scale, ((int32_t *) dst->op_params) + 0, sizeof(scale));
+ memcpy(&max_bias, ((int32_t *) dst->op_params) + 1, sizeof(max_bias));
+
+ const int64_t nrows_x = ggml_nrows(src0);
+ const int64_t nrows_y = src0->ne[1];
+
+ const uint32_t n_head = nrows_x/nrows_y;
+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));
+
+ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ if (id_src1) {
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ } else {
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1];
+ }
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
+ [encoder setBytes:&scale length:sizeof(scale) atIndex:6];
+ [encoder setBytes:&max_bias length:sizeof(max_bias) atIndex:7];
+ [encoder setBytes:&m0 length:sizeof(m0) atIndex:8];
+ [encoder setBytes:&m1 length:sizeof(m1) atIndex:9];
+ [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:10];
+ [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_DIAG_MASK_INF:
+ {
+ const int n_past = ((int32_t *)(dst->op_params))[0];
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ if (ne00%8 == 0) {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8].pipeline;
+ } else {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF].pipeline;
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
+ [encoder setBytes:&n_past length:sizeof(int) atIndex:4];
+
+ if (ne00%8 == 0) {
+ [encoder dispatchThreadgroups:MTLSizeMake(ne00*ne01*ne02/8, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ }
+ else {
+ [encoder dispatchThreadgroups:MTLSizeMake(ne00, ne01, ne02) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ }
+ } break;
+ case GGML_OP_SSM_CONV:
+ {
+ GGML_ASSERT(src0t == GGML_TYPE_F32);
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+ GGML_ASSERT(ggml_is_contiguous(src1));
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SSM_CONV_F32].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:9];
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:10];
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:11];
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:12];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:13];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:14];
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:15];
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:16];
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:17];
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:18];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne1, ne02) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_OP_SSM_SCAN:
+ {
+ struct ggml_tensor * src3 = gf->nodes[i]->src[3];
+ struct ggml_tensor * src4 = gf->nodes[i]->src[4];
+ struct ggml_tensor * src5 = gf->nodes[i]->src[5];
+
+ GGML_ASSERT(src3);
+ GGML_ASSERT(src4);
+ GGML_ASSERT(src5);
+
+ size_t offs_src3 = 0;
+ size_t offs_src4 = 0;
+ size_t offs_src5 = 0;
+
+ id<MTLBuffer> id_src3 = src3 ? ggml_metal_get_buffer(src3, &offs_src3) : nil;
+ id<MTLBuffer> id_src4 = src4 ? ggml_metal_get_buffer(src4, &offs_src4) : nil;
+ id<MTLBuffer> id_src5 = src5 ? ggml_metal_get_buffer(src5, &offs_src5) : nil;
+
+ const int64_t ne30 = src3->ne[0]; GGML_UNUSED(ne30);
+ const int64_t ne31 = src3->ne[1]; GGML_UNUSED(ne31);
+
+ const uint64_t nb30 = src3->nb[0];
+ const uint64_t nb31 = src3->nb[1];
+
+ const int64_t ne40 = src4->ne[0]; GGML_UNUSED(ne40);
+ const int64_t ne41 = src4->ne[1]; GGML_UNUSED(ne41);
+ const int64_t ne42 = src4->ne[2]; GGML_UNUSED(ne42);
+
+ const uint64_t nb40 = src4->nb[0];
+ const uint64_t nb41 = src4->nb[1];
+ const uint64_t nb42 = src4->nb[2];
+
+ const int64_t ne50 = src5->ne[0]; GGML_UNUSED(ne50);
+ const int64_t ne51 = src5->ne[1]; GGML_UNUSED(ne51);
+ const int64_t ne52 = src5->ne[2]; GGML_UNUSED(ne52);
+
+ const uint64_t nb50 = src5->nb[0];
+ const uint64_t nb51 = src5->nb[1];
+ const uint64_t nb52 = src5->nb[2];
+
+ const int64_t d_state = ne00;
+ const int64_t d_inner = ne01;
+ const int64_t n_seq_tokens = ne11;
+ const int64_t n_seqs = ne02;
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SSM_SCAN_F32].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];
+ [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3];
+ [encoder setBuffer:id_src4 offset:offs_src4 atIndex:4];
+ [encoder setBuffer:id_src5 offset:offs_src5 atIndex:5];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:6];
+
+ [encoder setBytes:&d_state length:sizeof(d_state) atIndex:7];
+ [encoder setBytes:&d_inner length:sizeof(d_inner) atIndex:8];
+ [encoder setBytes:&n_seq_tokens length:sizeof(n_seq_tokens) atIndex:9];
+ [encoder setBytes:&n_seqs length:sizeof(n_seqs) atIndex:10];
+
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:11];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:12];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:13];
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];
+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17];
+ [encoder setBytes:&nb20 length:sizeof(nb20) atIndex:18];
+ [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:19];
+ [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:20];
+ [encoder setBytes:&nb30 length:sizeof(nb30) atIndex:21];
+ [encoder setBytes:&nb31 length:sizeof(nb31) atIndex:22];
+ [encoder setBytes:&nb40 length:sizeof(nb40) atIndex:23];
+ [encoder setBytes:&nb41 length:sizeof(nb41) atIndex:24];
+ [encoder setBytes:&nb42 length:sizeof(nb42) atIndex:25];
+ [encoder setBytes:&nb50 length:sizeof(nb50) atIndex:26];
+ [encoder setBytes:&nb51 length:sizeof(nb51) atIndex:27];
+ [encoder setBytes:&nb52 length:sizeof(nb52) atIndex:28];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(d_inner, n_seqs, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_OP_MUL_MAT:
+ {
+ GGML_ASSERT(ne00 == ne10);
+
+ GGML_ASSERT(ne12 % ne02 == 0);
+ GGML_ASSERT(ne13 % ne03 == 0);
+
+ const uint r2 = ne12/ne02;
+ const uint r3 = ne13/ne03;
+
+ // find the break-even point where the matrix-matrix kernel becomes more efficient compared
+ // to the matrix-vector kernel
+ int ne11_mm_min = 1;
+
+ // the numbers below are measured on M2 Ultra for 7B and 13B models
+ // these numbers do not translate to other devices or model sizes
+ // TODO: need to find a better approach
+ switch (src0t) {
+ case GGML_TYPE_F16: ne11_mm_min = 2; break;
+ case GGML_TYPE_Q8_0: ne11_mm_min = 7; break;
+ case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;
+ case GGML_TYPE_Q3_K: ne11_mm_min = 7; break;
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;
+ case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;
+ case GGML_TYPE_Q5_0: // not tested yet
+ case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet
+ case GGML_TYPE_Q5_K: ne11_mm_min = 7; break;
+ case GGML_TYPE_Q6_K: ne11_mm_min = 7; break;
+ default: ne11_mm_min = 1; break;
+ }
+
+ // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
+ // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
+ if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&
+ !ggml_is_transposed(src0) &&
+ !ggml_is_transposed(src1) &&
+ src1t == GGML_TYPE_F32 &&
+ ne00 % 32 == 0 && ne00 >= 64 &&
+ (ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) {
+ //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
+
+ // some Metal matrix data types require aligned pointers
+ // ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5)
+ switch (src0->type) {
+ case GGML_TYPE_F32: GGML_ASSERT(nb01 % 16 == 0); break;
+ case GGML_TYPE_F16: GGML_ASSERT(nb01 % 8 == 0); break;
+ default: break;
+ }
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ switch (src0->type) {
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32 ].pipeline; break;
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32 ].pipeline; break;
+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32 ].pipeline; break;
+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32 ].pipeline; break;
+ case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32 ].pipeline; break;
+ case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32 ].pipeline; break;
+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32 ].pipeline; break;
+ case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32 ].pipeline; break;
+ case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32 ].pipeline; break;
+ case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32 ].pipeline; break;
+ case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32 ].pipeline; break;
+ case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32 ].pipeline; break;
+ case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32].pipeline; break;
+ case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break;
+ case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32].pipeline; break;
+ case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32 ].pipeline; break;
+ case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32 ].pipeline; break;
+ case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32 ].pipeline; break;
+ case GGML_TYPE_IQ1_M: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32 ].pipeline; break;
+ case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32 ].pipeline; break;
+ case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32 ].pipeline; break;
+ default: GGML_ABORT("MUL MAT-MAT not implemented");
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:5];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:6];
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:7];
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:8];
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:9];
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:10];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:11];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:12];
+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:13];
+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:14];
+ [encoder setThreadgroupMemoryLength:8192 atIndex:0];
+ [encoder dispatchThreadgroups:MTLSizeMake( (ne11 + 31)/32, (ne01 + 63)/64, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
+ } else {
+ int nth0 = 32;
+ int nth1 = 1;
+ int nrows = 1;
+ //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ // use custom matrix x vector kernel
+ switch (src0t) {
+ case GGML_TYPE_F32:
+ {
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32].pipeline;
+ nrows = 4;
+ } break;
+ case GGML_TYPE_F16:
+ {
+ nth0 = 32;
+ nth1 = 1;
+ if (src1t == GGML_TYPE_F32) {
+ if (ne11 * ne12 < 4) {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW].pipeline;
+ } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4].pipeline;
+ nrows = ne11;
+ } else {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32].pipeline;
+ nrows = 4;
+ }
+ } else {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16].pipeline;
+ nrows = 4;
+ }
+ } break;
+ case GGML_TYPE_Q4_0:
+ {
+ nth0 = 8;
+ nth1 = 8;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q4_1:
+ {
+ nth0 = 8;
+ nth1 = 8;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q5_0:
+ {
+ nth0 = 8;
+ nth1 = 8;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q5_1:
+ {
+ nth0 = 8;
+ nth1 = 8;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q8_0:
+ {
+ nth0 = 8;
+ nth1 = 8;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q2_K:
+ {
+ nth0 = 2;
+ nth1 = 32;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q3_K:
+ {
+ nth0 = 2;
+ nth1 = 32;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q4_K:
+ {
+ nth0 = 4; //1;
+ nth1 = 8; //32;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q5_K:
+ {
+ nth0 = 2;
+ nth1 = 32;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q6_K:
+ {
+ nth0 = 2;
+ nth1 = 32;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ2_XXS:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ2_XS:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ3_XXS:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ3_S:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ2_S:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ1_S:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ1_M:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_M_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ4_NL:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ4_XS:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32].pipeline;
+ } break;
+ default:
+ {
+ GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t);
+ GGML_ABORT("not implemented");
+ }
+ };
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:9];
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:10];
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:11];
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:12];
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:13];
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:14];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:15];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:16];
+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:17];
+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:18];
+
+ if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 || src0t == GGML_TYPE_Q5_0 ||
+ src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || src0t == GGML_TYPE_Q2_K ||
+ src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ2_S) {
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) {
+ const int mem_size = src0t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128;
+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_IQ3_XXS || src0t == GGML_TYPE_IQ3_S) {
+ const int mem_size = src0t == GGML_TYPE_IQ3_XXS ? 256*4+128 : 512*4;
+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_IQ4_NL || src0t == GGML_TYPE_IQ4_XS) {
+ const int mem_size = 32*sizeof(float);
+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_Q4_K) {
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_Q3_K) {
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_Q5_K) {
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_Q6_K) {
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ } else {
+ const int64_t ny = (ne11 + nrows - 1)/nrows;
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ }
+ } break;
+ case GGML_OP_MUL_MAT_ID:
+ {
+ const int n_as = src0->ne[2];
+
+ // src2 = ids
+ const enum ggml_type src2t = src2->type; GGML_UNUSED(src2t);
+
+ GGML_ASSERT(src2t == GGML_TYPE_I32);
+
+ GGML_ASSERT(!ggml_is_transposed(src0));
+ GGML_ASSERT(!ggml_is_transposed(src1));
+
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
+
+ // find the break-even point where the matrix-matrix kernel becomes more efficient compared
+ // to the matrix-vector kernel
+ // ne20 = n_used_experts
+ // ne21 = n_rows
+ const int dst_rows = ne20*ne21;
+ const int dst_rows_min = n_as;
+ const int dst_rows_max = (ctx->device.maxThreadgroupMemoryLength - 32 - 8192)/4;
+
+ // max size of the rowids array in the kernel shared buffer
+ GGML_ASSERT(dst_rows <= dst_rows_max);
+
+ // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
+ // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
+ // !!!
+ // TODO: for now, always use mat-vec kernels until we figure out how to improve the
+ // indirect matrix multiplication
+ // !!!
+ if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&
+ ne00 % 32 == 0 && ne00 >= 64 &&
+ dst_rows > dst_rows_min) {
+
+ // some Metal matrix data types require aligned pointers
+ // ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5)
+ switch (src0->type) {
+ case GGML_TYPE_F32: GGML_ASSERT(nb01 % 16 == 0); break;
+ case GGML_TYPE_F16: GGML_ASSERT(nb01 % 8 == 0); break;
+ default: break;
+ }
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ switch (src0->type) {
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32 ].pipeline; break;
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32 ].pipeline; break;
+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32 ].pipeline; break;
+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32 ].pipeline; break;
+ case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32 ].pipeline; break;
+ case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32 ].pipeline; break;
+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32 ].pipeline; break;
+ case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32 ].pipeline; break;
+ case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32 ].pipeline; break;
+ case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32 ].pipeline; break;
+ case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32 ].pipeline; break;
+ case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32 ].pipeline; break;
+ case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32].pipeline; break;
+ case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break;
+ case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32].pipeline; break;
+ case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32 ].pipeline; break;
+ case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32 ].pipeline; break;
+ case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32 ].pipeline; break;
+ case GGML_TYPE_IQ1_M: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F32 ].pipeline; break;
+ case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32 ].pipeline; break;
+ case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32 ].pipeline; break;
+ default: GGML_ABORT("MUL_MAT_ID not implemented");
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
+ [encoder setBuffer:id_src2 offset:offs_src2 atIndex:3];
+ [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];
+ [encoder setBytes:&ne21 length:sizeof(ne21) atIndex:5];
+ [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:6];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:7];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:8];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:9];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:10];
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11];
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:17];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:18];
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:19];
+
+ [encoder setThreadgroupMemoryLength:GGML_PAD(8192 + dst_rows*4/*sizeof(ushort2)*/, 16) atIndex:0];
+
+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 31)/32, (ne01 + 63)/64, n_as) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
+ } else {
+ int nth0 = 32;
+ int nth1 = 1;
+ int nrows = 1;
+ //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ // use custom matrix x vector kernel
+ switch (src0t) {
+ case GGML_TYPE_F32:
+ {
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32].pipeline;
+ } break;
+ case GGML_TYPE_F16:
+ {
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
+ nth0 = 32;
+ nth1 = 1;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q4_0:
+ {
+ nth0 = 8;
+ nth1 = 8;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q4_1:
+ {
+ nth0 = 8;
+ nth1 = 8;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q5_0:
+ {
+ nth0 = 8;
+ nth1 = 8;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q5_1:
+ {
+ nth0 = 8;
+ nth1 = 8;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q8_0:
+ {
+ nth0 = 8;
+ nth1 = 8;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q2_K:
+ {
+ nth0 = 2;
+ nth1 = 32;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q3_K:
+ {
+ nth0 = 2;
+ nth1 = 32;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q4_K:
+ {
+ nth0 = 4; //1;
+ nth1 = 8; //32;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q5_K:
+ {
+ nth0 = 2;
+ nth1 = 32;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32].pipeline;
+ } break;
+ case GGML_TYPE_Q6_K:
+ {
+ nth0 = 2;
+ nth1 = 32;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ2_XXS:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ2_XS:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ3_XXS:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ3_S:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ2_S:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ1_S:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ1_M:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ4_NL:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32].pipeline;
+ } break;
+ case GGML_TYPE_IQ4_XS:
+ {
+ nth0 = 4;
+ nth1 = 16;
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32].pipeline;
+ } break;
+ default:
+ {
+ GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t);
+ GGML_ABORT("not implemented");
+ }
+ };
+
+ if (ggml_is_quantized(src0t)) {
+ GGML_ASSERT(ne00 >= nth0*nth1);
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
+ [encoder setBuffer:id_src2 offset:offs_src2 atIndex:3];
+ [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];
+ [encoder setBytes:&ne21 length:sizeof(ne21) atIndex:5];
+ [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:6];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:7];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:8];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:9];
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:10];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:11];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:12];
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:13];
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:14];
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:15];
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:16];
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:17];
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:18];
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:19];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:20];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:21];
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:22];
+
+ const int64_t _ne1 = 1;
+ const int tgz = dst_rows;
+
+ if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 || src0t == GGML_TYPE_Q5_0 ||
+ src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || src0t == GGML_TYPE_Q2_K ||
+ src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ2_S) {
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) {
+ const int mem_size = src0t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128;
+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_IQ3_XXS || src0t == GGML_TYPE_IQ3_S) {
+ const int mem_size = src0t == GGML_TYPE_IQ3_XXS ? 256*4+128 : 512*4;
+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_IQ4_NL || src0t == GGML_TYPE_IQ4_XS) {
+ const int mem_size = 32*sizeof(float);
+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_Q4_K) {
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_Q3_K) {
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_Q5_K) {
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ else if (src0t == GGML_TYPE_Q6_K) {
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ } else {
+ const int64_t ny = (_ne1 + nrows - 1)/nrows; // = _ne1
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
+ }
+ }
+ } break;
+ case GGML_OP_GET_ROWS:
+ {
+ id<MTLComputePipelineState> pipeline = nil;
+
+ switch (src0->type) {
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F32 ].pipeline; break;
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F16 ].pipeline; break;
+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0 ].pipeline; break;
+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1 ].pipeline; break;
+ case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0 ].pipeline; break;
+ case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1 ].pipeline; break;
+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0 ].pipeline; break;
+ case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K ].pipeline; break;
+ case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K ].pipeline; break;
+ case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K ].pipeline; break;
+ case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K ].pipeline; break;
+ case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K ].pipeline; break;
+ case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS].pipeline; break;
+ case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break;
+ case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS].pipeline; break;
+ case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S ].pipeline; break;
+ case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S ].pipeline; break;
+ case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S ].pipeline; break;
+ case GGML_TYPE_IQ1_M: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_M ].pipeline; break;
+ case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL ].pipeline; break;
+ case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS ].pipeline; break;
+ case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break;
+ default: GGML_ABORT("not implemented");
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:4];
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:5];
+ [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:6];
+ [encoder setBytes:&nb10 length:sizeof( int64_t) atIndex:7];
+ [encoder setBytes:&nb11 length:sizeof( int64_t) atIndex:8];
+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:9];
+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:10];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne10, ne11, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
+ } break;
+ case GGML_OP_RMS_NORM:
+ {
+ GGML_ASSERT(ne00 % 4 == 0);
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
+
+ float eps;
+ memcpy(&eps, dst->op_params, sizeof(float));
+
+ int nth = 32; // SIMD width
+
+ while (nth < ne00/4 && nth < 1024) {
+ nth *= 2;
+ }
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RMS_NORM].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];
+ [encoder setBytes:&eps length:sizeof( float) atIndex:4];
+ [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
+
+ const int64_t nrows = ggml_nrows(src0);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_GROUP_NORM:
+ {
+ GGML_ASSERT(ne00 % 4 == 0);
+ GGML_ASSERT(ggml_is_contiguous(src0));
+
+ float eps;
+ memcpy(&eps, dst->op_params + 1, sizeof(float));
+
+ const int32_t n_groups = ((int32_t *) dst->op_params)[0];
+
+ int nth = 32; // SIMD width
+
+ //while (nth < ne00/4 && nth < 1024) {
+ // nth *= 2;
+ //}
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GROUP_NORM].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:5];
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:6];
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:7];
+ [encoder setBytes:&n_groups length:sizeof( int32_t) atIndex:8];
+ [encoder setBytes:&eps length:sizeof( float) atIndex:9];
+ [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_NORM:
+ {
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
+
+ float eps;
+ memcpy(&eps, dst->op_params, sizeof(float));
+
+ const int nth = MIN(256, ne00);
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_NORM].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];
+ [encoder setBytes:&eps length:sizeof( float) atIndex:4];
+ [encoder setThreadgroupMemoryLength:GGML_PAD(nth*sizeof(float), 16) atIndex:0];
+
+ const int64_t nrows = ggml_nrows(src0);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_ROPE:
+ {
+ GGML_ASSERT(ne10 == ne02);
+
+ const int nth = MIN(1024, ne00);
+
+ const int n_past = ((int32_t *) dst->op_params)[0];
+ const int n_dims = ((int32_t *) dst->op_params)[1];
+ const int mode = ((int32_t *) dst->op_params)[2];
+ // skip 3, n_ctx, used in GLM RoPE, unimplemented in metal
+ const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
+
+ float freq_base;
+ float freq_scale;
+ float ext_factor;
+ float attn_factor;
+ float beta_fast;
+ float beta_slow;
+
+ memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
+ memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
+ memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
+ memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
+ memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
+ memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
+
+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ if (!is_neox) {
+ switch (src0->type) {
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_NORM_F32].pipeline; break;
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_NORM_F16].pipeline; break;
+ default: GGML_ABORT("fatal error");
+ };
+ } else {
+ switch (src0->type) {
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F32].pipeline; break;
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F16].pipeline; break;
+ default: GGML_ABORT("fatal error");
+ };
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ if (id_src2 != nil) {
+ [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];
+ } else {
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:2];
+ }
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:3];
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:4];
+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:5];
+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:6];
+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:7];
+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:8];
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:9];
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:10];
+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:11];
+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:12];
+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:13];
+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:14];
+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:15];
+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:16];
+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:17];
+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:18];
+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:19];
+ [encoder setBytes:&n_past length:sizeof( int) atIndex:20];
+ [encoder setBytes:&n_dims length:sizeof( int) atIndex:21];
+ [encoder setBytes:&n_ctx_orig length:sizeof( int) atIndex:22];
+ [encoder setBytes:&freq_base length:sizeof( float) atIndex:23];
+ [encoder setBytes:&freq_scale length:sizeof( float) atIndex:24];
+ [encoder setBytes:&ext_factor length:sizeof( float) atIndex:25];
+ [encoder setBytes:&attn_factor length:sizeof( float) atIndex:26];
+ [encoder setBytes:&beta_fast length:sizeof( float) atIndex:27];
+ [encoder setBytes:&beta_slow length:sizeof( float) atIndex:28];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_IM2COL:
+ {
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);
+
+ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
+ const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
+ const int32_t p1 = ((const int32_t *)(dst->op_params))[3];
+ const int32_t d0 = ((const int32_t *)(dst->op_params))[4];
+ const int32_t d1 = ((const int32_t *)(dst->op_params))[5];
+
+ const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;
+
+ const int32_t N = src1->ne[is_2D ? 3 : 2];
+ const int32_t IC = src1->ne[is_2D ? 2 : 1];
+ const int32_t IH = is_2D ? src1->ne[1] : 1;
+ const int32_t IW = src1->ne[0];
+
+ const int32_t KH = is_2D ? src0->ne[1] : 1;
+ const int32_t KW = src0->ne[0];
+
+ const int32_t OH = is_2D ? dst->ne[2] : 1;
+ const int32_t OW = dst->ne[1];
+
+ const int32_t CHW = IC * KH * KW;
+
+ const int32_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4;
+ const int32_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4;
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ switch (dst->type) {
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F32].pipeline; break;
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F16].pipeline; break;
+ default: GGML_ABORT("fatal error");
+ };
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ofs0 length:sizeof( int32_t) atIndex:2];
+ [encoder setBytes:&ofs1 length:sizeof( int32_t) atIndex:3];
+ [encoder setBytes:&IW length:sizeof( int32_t) atIndex:4];
+ [encoder setBytes:&IH length:sizeof( int32_t) atIndex:5];
+ [encoder setBytes:&CHW length:sizeof( int32_t) atIndex:6];
+ [encoder setBytes:&s0 length:sizeof( int32_t) atIndex:7];
+ [encoder setBytes:&s1 length:sizeof( int32_t) atIndex:8];
+ [encoder setBytes:&p0 length:sizeof( int32_t) atIndex:9];
+ [encoder setBytes:&p1 length:sizeof( int32_t) atIndex:10];
+ [encoder setBytes:&d0 length:sizeof( int32_t) atIndex:11];
+ [encoder setBytes:&d1 length:sizeof( int32_t) atIndex:12];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(IC, OH, OW) threadsPerThreadgroup:MTLSizeMake(N, KH, KW)];
+ } break;
+ case GGML_OP_UPSCALE:
+ {
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+
+ const float sf0 = (float)ne0/src0->ne[0];
+ const float sf1 = (float)ne1/src0->ne[1];
+ const float sf2 = (float)ne2/src0->ne[2];
+ const float sf3 = (float)ne3/src0->ne[3];
+
+ const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_UPSCALE_F32].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];
+ [encoder setBytes:&sf0 length:sizeof(sf0) atIndex:18];
+ [encoder setBytes:&sf1 length:sizeof(sf1) atIndex:19];
+ [encoder setBytes:&sf2 length:sizeof(sf2) atIndex:20];
+ [encoder setBytes:&sf3 length:sizeof(sf3) atIndex:21];
+
+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_PAD:
+ {
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_PAD_F32].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];
+
+ const int nth = MIN(1024, ne0);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_ARANGE:
+ {
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
+
+ float start;
+ float step;
+
+ memcpy(&start, ((int32_t *) dst->op_params) + 0, sizeof(float));
+ memcpy(&step, ((int32_t *) dst->op_params) + 2, sizeof(float));
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARANGE_F32].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:0];
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:1];
+ [encoder setBytes:&start length:sizeof(start) atIndex:2];
+ [encoder setBytes:&step length:sizeof(step) atIndex:3];
+
+ const int nth = MIN(1024, ne0);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_TIMESTEP_EMBEDDING:
+ {
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+
+ const int dim = dst->op_params[0];
+ const int max_period = dst->op_params[1];
+
+ const int half = dim / 2;
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:2];
+ [encoder setBytes:&dim length:sizeof(dim) atIndex:3];
+ [encoder setBytes:&max_period length:sizeof(max_period) atIndex:4];
+
+ const int nth = MIN(1024, half);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne00, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ case GGML_OP_ARGSORT:
+ {
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_I32);
+
+ const int nrows = ggml_nrows(src0);
+
+ enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];
+
+ // bitonic sort requires the number of elements to be power of 2
+ int64_t ne00_padded = 1;
+ while (ne00_padded < ne00) {
+ ne00_padded *= 2;
+ }
+
+ // Metal kernels require the buffer size to be multiple of 16 bytes
+ // https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/1443142-setthreadgroupmemorylength
+ const int mem_size = GGML_PAD(ne00_padded*sizeof(int32_t), 16);
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ switch (order) {
+ case GGML_SORT_ORDER_ASC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC].pipeline; break;
+ case GGML_SORT_ORDER_DESC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC].pipeline; break;
+ default: GGML_ABORT("fatal error");
+ };
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
+ [encoder setBytes:&ne00_padded length:sizeof( int64_t) atIndex:3];
+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(1, nrows, 1) threadsPerThreadgroup:MTLSizeMake(ne00_padded, 1, 1)];
+ } break;
+ case GGML_OP_LEAKY_RELU:
+ {
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+
+ float slope;
+ memcpy(&slope, dst->op_params, sizeof(float));
+
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32].pipeline;
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&slope length:sizeof(slope) atIndex:2];
+
+ const int64_t n = ggml_nelements(dst);
+
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+ } break;
+ case GGML_OP_FLASH_ATTN_EXT:
+ {
+ GGML_ASSERT(ne00 % 4 == 0);
+ GGML_ASSERT(ne11 % 32 == 0);
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+
+ GGML_ASSERT(ggml_are_same_shape (src1, src2));
+
+ struct ggml_tensor * src3 = gf->nodes[i]->src[3];
+
+ size_t offs_src3 = 0;
+
+ id<MTLBuffer> id_src3 = src3 ? ggml_metal_get_buffer(src3, &offs_src3) : nil;
+
+ GGML_ASSERT(!src3 || src3->type == GGML_TYPE_F16);
+ GGML_ASSERT(!src3 || src3->ne[1] >= GGML_PAD(src0->ne[1], 8) &&
+ "the Flash-Attention Metal kernel requires the mask to be padded to 8 and at least n_queries big");
+
+ const int64_t ne30 = src3 ? src3->ne[0] : 0; GGML_UNUSED(ne30);
+ //const int64_t ne31 = src3 ? src3->ne[1] : 0;
+ const int64_t ne32 = src3 ? src3->ne[2] : 0; GGML_UNUSED(ne32);
+ const int64_t ne33 = src3 ? src3->ne[3] : 0; GGML_UNUSED(ne33);
+
+ const uint64_t nb30 = src3 ? src3->nb[0] : 0; GGML_UNUSED(nb30);
+ const uint64_t nb31 = src3 ? src3->nb[1] : 0;
+ const uint64_t nb32 = src3 ? src3->nb[2] : 0; GGML_UNUSED(nb32);
+ const uint64_t nb33 = src3 ? src3->nb[3] : 0; GGML_UNUSED(nb33);
+
+ const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t);
+
+ float scale;
+ float max_bias;
+ float logit_softcap;
+ memcpy(&scale, ((int32_t *) dst->op_params) + 0, sizeof(scale));
+ memcpy(&max_bias, ((int32_t *) dst->op_params) + 1, sizeof(max_bias));
+ memcpy(&logit_softcap, ((int32_t *) dst->op_params) + 2, sizeof(logit_softcap));
+
+ if (logit_softcap != 0.0f) {
+ scale /= logit_softcap;
+ }
+
+ const uint32_t n_head = src0->ne[2];
+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));
+
+ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ bool use_vec_kernel = false;
+
+ if (ne01 >= 4 || (ne00%128 != 0)) {
+ switch (ne00) {
+ case 64: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64 ].pipeline; break;
+ case 80: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80 ].pipeline; break;
+ case 96: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96 ].pipeline; break;
+ case 112: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112].pipeline; break;
+ case 128: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128].pipeline; break;
+ //case 256: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256].pipeline; break;
+ default:
+ {
+ GGML_METAL_LOG_ERROR("unsupported size: %lld\n", ne00);
+ GGML_METAL_LOG_ERROR("add template specialization for this size\n");
+ GGML_ABORT("add template specialization for this size");
+ }
+ }
+ } else {
+ use_vec_kernel = true;
+
+ switch (ne00) {
+ case 128: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128].pipeline; break;
+ //case 256: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256].pipeline; break;
+ default:
+ {
+ GGML_METAL_LOG_ERROR("unsupported size: %lld\n", ne00);
+ GGML_METAL_LOG_ERROR("add template specialization for this size\n");
+ GGML_ABORT("add template specialization for this size");
+ }
+ }
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];
+ if (id_src3) {
+ [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3];
+ } else {
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:3];
+ }
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:4];
+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:5];
+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:6];
+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:7];
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8];
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9];
+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10];
+ [encoder setBytes:&ne11 length:sizeof( int64_t) atIndex:11];
+ [encoder setBytes:&ne12 length:sizeof( int64_t) atIndex:12];
+ [encoder setBytes:&ne13 length:sizeof( int64_t) atIndex:13];
+ [encoder setBytes:&nb11 length:sizeof(uint64_t) atIndex:14];
+ [encoder setBytes:&nb12 length:sizeof(uint64_t) atIndex:15];
+ [encoder setBytes:&nb13 length:sizeof(uint64_t) atIndex:16];
+ [encoder setBytes:&nb21 length:sizeof(uint64_t) atIndex:17];
+ [encoder setBytes:&nb22 length:sizeof(uint64_t) atIndex:18];
+ [encoder setBytes:&nb23 length:sizeof(uint64_t) atIndex:19];
+ [encoder setBytes:&nb31 length:sizeof(uint64_t) atIndex:20];
+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:21];
+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:22];
+ [encoder setBytes:&scale length:sizeof( float) atIndex:23];
+ [encoder setBytes:&max_bias length:sizeof( float) atIndex:24];
+ [encoder setBytes:&m0 length:sizeof(m0) atIndex:25];
+ [encoder setBytes:&m1 length:sizeof(m1) atIndex:26];
+ [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:27];
+ [encoder setBytes:&logit_softcap length:sizeof(logit_softcap) atIndex:28];
+
+ if (!use_vec_kernel) {
+ // half8x8 kernel
+ const int64_t nqptg = 8; // queries per threadgroup !! sync with kernel template arguments !!
+ const int64_t ncpsg = 32; // cache values per simdgroup !! sync with kernel template arguments !!
+
+ GGML_ASSERT(nqptg <= 32);
+ GGML_ASSERT(nqptg % 8 == 0);
+ GGML_ASSERT(ncpsg % 32 == 0);
+
+ int64_t nsgmax = 2;
+
+ while (true) {
+ const size_t smem = nqptg*(ne00 + 2*nsgmax*(ncpsg + nqptg))*(sizeof(float)/2);
+ if (smem > ctx->device.maxThreadgroupMemoryLength) {
+ break;
+ }
+ nsgmax *= 2;
+ }
+ nsgmax /= 2;
+
+ // simdgroups per threadgroup (a.k.a. warps)
+ const int64_t nsg = ne01 <= nqptg ? MAX(4, MIN(nsgmax, MIN(ne11/ncpsg, (int64_t) pipeline.maxTotalThreadsPerThreadgroup/32))) : 4;
+
+ const size_t smem = nqptg*(ne00 + 2*nsg*(ncpsg + nqptg))*(sizeof(float)/2);
+
+ //printf("smem: %zu, max: %zu\n", smem, ctx->device.maxThreadgroupMemoryLength);
+ GGML_ASSERT(smem <= ctx->device.maxThreadgroupMemoryLength);
+
+ [encoder setThreadgroupMemoryLength:GGML_PAD(smem, 16) atIndex:0];
+
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + nqptg - 1)/nqptg, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)];
+ } else {
+ // half1x4 kernel
+ const int64_t nqptg = 1; // queries per threadgroup !! sync with kernel template arguments !!
+ const int64_t ncpsg = 32; // cache values per simdgroup !! sync with kernel template arguments !!
+
+ GGML_ASSERT(nqptg <= 32);
+ GGML_ASSERT(nqptg % 1 == 0);
+ GGML_ASSERT(ncpsg % 32 == 0);
+
+ // simdgroups per threadgroup (a.k.a. warps)
+ const int64_t nsgt = MAX(2, MIN(ne11/ncpsg, (int64_t) pipeline.maxTotalThreadsPerThreadgroup/32));
+
+ int64_t nsg = 1;
+ while (nsg <= nsgt) {
+ nsg *= 2;
+ }
+ nsg /= 2;
+
+ const size_t smem = (nqptg*(ne00 + 2*nsg*(ncpsg + nqptg)) + nsg*ne00)*(sizeof(float)/2);
+
+ //printf("smem: %zu, max: %zu\n", smem, ctx->device.maxThreadgroupMemoryLength);
+ GGML_ASSERT(smem <= ctx->device.maxThreadgroupMemoryLength);
+ [encoder setThreadgroupMemoryLength:GGML_PAD(smem, 16) atIndex:0];
+
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + nqptg - 1)/nqptg, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)];
+ }
+ } break;
+ case GGML_OP_DUP:
+ case GGML_OP_CPY:
+ case GGML_OP_CONT:
+ {
+ GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);
+
+ int nth = MIN(1024, ne00/ggml_blck_size(src0->type));
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ switch (src0t) {
+ case GGML_TYPE_F32:
+ {
+ GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0);
+
+ switch (dstt) {
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; break;
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F16].pipeline; break;
+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0].pipeline; break;
+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0].pipeline; break;
+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1].pipeline; break;
+ case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0].pipeline; break;
+ case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1].pipeline; break;
+ case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL].pipeline; break;
+ default: GGML_ABORT("not implemented");
+ };
+ } break;
+ case GGML_TYPE_F16:
+ {
+ switch (dstt) {
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F32].pipeline; break;
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F16].pipeline; break;
+ default: GGML_ABORT("not implemented");
+ };
+ } break;
+ default: GGML_ABORT("not implemented");
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];
+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];
+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];
+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];
+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];
+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];
+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];
+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];
+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];
+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];
+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];
+
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
+ } break;
+ default:
+ {
+ GGML_METAL_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
+ GGML_ABORT("fatal error");
+ }
+ }
+
+ if (should_capture) {
+ [encoder popDebugGroup];
+ }
+ }
+
+ [encoder endEncoding];
+
+ if (cb_idx < 2 || ctx->abort_callback == NULL) {
+ [command_buffer commit];
+ }
+ });
+
+ // Wait for completion and check status of each command buffer
+ // needed to detect if the device ran out-of-memory for example (#1881)
+
+ for (int i = 0; i < n_cb; ++i) {
+ id<MTLCommandBuffer> command_buffer = command_buffers[i];
+ [command_buffer waitUntilCompleted];
+
+ MTLCommandBufferStatus status = [command_buffer status];
+ if (status != MTLCommandBufferStatusCompleted) {
+ GGML_METAL_LOG_INFO("%s: command buffer %d failed with status %lu\n", __func__, i, status);
+ if (status == MTLCommandBufferStatusError) {
+ NSString * error_code = [command_buffer error].localizedDescription;
+ GGML_METAL_LOG_INFO("error: %s\n", [error_code UTF8String]);
+ }
+
+ return GGML_STATUS_FAILED;
+ }
+
+ id<MTLCommandBuffer> next_buffer = (i + 1 < n_cb ? command_buffers[i + 1] : nil);
+ if (!next_buffer) {
+ continue;
+ }
+
+ bool next_queued = ([next_buffer status] != MTLCommandBufferStatusNotEnqueued);
+ if (next_queued) {
+ continue;
+ }
+
+ if (ctx->abort_callback && ctx->abort_callback(ctx->abort_callback_data)) {
+ GGML_METAL_LOG_INFO("%s: command buffer %d aborted", __func__, i);
+ return GGML_STATUS_ABORTED;
+ }
+
+ [next_buffer commit];
+ }
+
+ if (should_capture) {
+ [[MTLCaptureManager sharedCaptureManager] stopCapture];
+ }
+
+ }
+ return GGML_STATUS_SUCCESS;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// backend interface
+
+// default buffer
+static id<MTLDevice> g_backend_device = nil;
+static int g_backend_device_ref_count = 0;
+
+static id<MTLDevice> ggml_backend_metal_get_device(void) {
+ if (g_backend_device == nil) {
+ g_backend_device = MTLCreateSystemDefaultDevice();
+ }
+
+ g_backend_device_ref_count++;
+
+ return g_backend_device;
+}
+
+static void ggml_backend_metal_free_device(void) {
+ assert(g_backend_device_ref_count > 0);
+
+ g_backend_device_ref_count--;
+
+ if (g_backend_device_ref_count == 0) {
+ [g_backend_device release];
+ g_backend_device = nil;
+ }
+}
+
+GGML_CALL static const char * ggml_backend_metal_buffer_get_name(ggml_backend_buffer_t buffer) {
+ return "Metal";
+
+ UNUSED(buffer);
+}
+
+GGML_CALL static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+ struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;
+
+ for (int i = 0; i < ctx->n_buffers; i++) {
+ [ctx->buffers[i].metal release];
+ }
+ ggml_backend_metal_free_device();
+
+ if (ctx->owned) {
+#if TARGET_OS_OSX
+ vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ctx->all_data, ctx->all_size);
+#else
+ free(ctx->all_data);
+#endif
+ }
+
+ free(ctx);
+}
+
+GGML_CALL static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) {
+ struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;
+
+ return ctx->all_data;
+}
+
+GGML_CALL static void ggml_backend_metal_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+ memcpy((char *)tensor->data + offset, data, size);
+
+ UNUSED(buffer);
+}
+
+GGML_CALL static void ggml_backend_metal_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+ memcpy(data, (const char *)tensor->data + offset, size);
+
+ UNUSED(buffer);
+}
+
+GGML_CALL static bool ggml_backend_metal_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {
+ if (ggml_backend_buffer_is_host(src->buffer)) {
+ memcpy(dst->data, src->data, ggml_nbytes(src));
+ return true;
+ }
+ return false;
+
+ UNUSED(buffer);
+}
+
+GGML_CALL static void ggml_backend_metal_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+ struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;
+
+ memset(ctx->all_data, value, ctx->all_size);
+}
+
+static struct ggml_backend_buffer_i ggml_backend_metal_buffer_i = {
+ /* .get_name = */ ggml_backend_metal_buffer_get_name,
+ /* .free_buffer = */ ggml_backend_metal_buffer_free_buffer,
+ /* .get_base = */ ggml_backend_metal_buffer_get_base,
+ /* .init_tensor = */ NULL,
+ /* .set_tensor = */ ggml_backend_metal_buffer_set_tensor,
+ /* .get_tensor = */ ggml_backend_metal_buffer_get_tensor,
+ /* .cpy_tensor = */ ggml_backend_metal_buffer_cpy_tensor,
+ /* .clear = */ ggml_backend_metal_buffer_clear,
+ /* .reset = */ NULL,
+};
+
+// default buffer type
+
+GGML_CALL static const char * ggml_backend_metal_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
+ return "Metal";
+
+ UNUSED(buft);
+}
+
+static void ggml_backend_metal_log_allocated_size(id<MTLDevice> device, size_t size_aligned) {
+#ifndef GGML_METAL_NDEBUG
+#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
+ if (@available(macOS 10.12, iOS 16.0, *)) {
+ GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, (%8.2f / %8.2f)",
+ __func__,
+ size_aligned / 1024.0 / 1024.0,
+ device.currentAllocatedSize / 1024.0 / 1024.0,
+ device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
+
+ if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {
+ GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);
+ } else {
+ GGML_METAL_LOG_INFO("\n");
+ }
+ } else {
+ GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, (%8.2f)\n",
+ __func__,
+ size_aligned / 1024.0 / 1024.0,
+ device.currentAllocatedSize / 1024.0 / 1024.0);
+ }
+#endif
+#endif
+ UNUSED(device);
+ UNUSED(size_aligned);
+}
+
+GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+ struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context));
+
+ const size_t size_page = sysconf(_SC_PAGESIZE);
+
+ size_t size_aligned = size;
+ if ((size_aligned % size_page) != 0) {
+ size_aligned += (size_page - (size_aligned % size_page));
+ }
+
+ id<MTLDevice> device = ggml_backend_metal_get_device();
+
+ ctx->all_data = ggml_metal_host_malloc(size_aligned);
+ ctx->all_size = size_aligned;
+ ctx->owned = true;
+ ctx->n_buffers = 1;
+
+ if (ctx->all_data != NULL) {
+ ctx->buffers[0].data = ctx->all_data;
+ ctx->buffers[0].size = size;
+ ctx->buffers[0].metal = [device newBufferWithBytesNoCopy:ctx->all_data
+ length:size_aligned
+ options:MTLResourceStorageModeShared
+ deallocator:nil];
+ }
+
+ if (ctx->all_data == NULL || ctx->buffers[0].metal == nil) {
+ GGML_METAL_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);
+ free(ctx);
+ ggml_backend_metal_free_device();
+ return NULL;
+ }
+
+ //ggml_backend_metal_log_allocated_size(device, size_aligned);
+
+ return ggml_backend_buffer_init(buft, ggml_backend_metal_buffer_i, ctx, size);
+}
+
+GGML_CALL static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
+ return 32;
+ UNUSED(buft);
+}
+
+GGML_CALL static size_t ggml_backend_metal_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
+ id<MTLDevice> device = ggml_backend_metal_get_device();
+ size_t max_size = device.maxBufferLength;
+ ggml_backend_metal_free_device();
+
+ return max_size;
+
+ UNUSED(buft);
+}
+
+GGML_CALL static bool ggml_backend_metal_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
+ return true;
+
+ UNUSED(buft);
+}
+
+GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {
+ static struct ggml_backend_buffer_type ggml_backend_buffer_type_metal = {
+ /* .iface = */ {
+ /* .get_name = */ ggml_backend_metal_buffer_type_get_name,
+ /* .alloc_buffer = */ ggml_backend_metal_buffer_type_alloc_buffer,
+ /* .get_alignment = */ ggml_backend_metal_buffer_type_get_alignment,
+ /* .get_max_size = */ ggml_backend_metal_buffer_type_get_max_size,
+ /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
+ /* .is_host = */ ggml_backend_metal_buffer_type_is_host,
+ },
+ /* .context = */ NULL,
+ };
+
+ return &ggml_backend_buffer_type_metal;
+}
+
+// buffer from ptr
+
+GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) {
+ struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context));
+
+ ctx->all_data = data;
+ ctx->all_size = size;
+ ctx->owned = false;
+ ctx->n_buffers = 0;
+
+ const size_t size_page = sysconf(_SC_PAGESIZE);
+
+ // page-align the data ptr
+ {
+ const uintptr_t offs = (uintptr_t) data % size_page;
+ data = (void *) ((char *) data - offs);
+ size += offs;
+ }
+
+ size_t size_aligned = size;
+ if ((size_aligned % size_page) != 0) {
+ size_aligned += (size_page - (size_aligned % size_page));
+ }
+
+ id<MTLDevice> device = ggml_backend_metal_get_device();
+
+ // the buffer fits into the max buffer size allowed by the device
+ if (size_aligned <= device.maxBufferLength) {
+ ctx->buffers[ctx->n_buffers].data = data;
+ ctx->buffers[ctx->n_buffers].size = size;
+
+ ctx->buffers[ctx->n_buffers].metal = [device newBufferWithBytesNoCopy:data length:size_aligned options:MTLResourceStorageModeShared deallocator:nil];
+
+ if (ctx->buffers[ctx->n_buffers].metal == nil) {
+ GGML_METAL_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);
+ return false;
+ }
+
+ ggml_backend_metal_log_allocated_size(device, size_aligned);
+
+ ++ctx->n_buffers;
+ } else {
+ // this overlap between the views will guarantee that the tensor with the maximum size will fully fit into
+ // one of the views
+ const size_t size_ovlp = ((max_size + size_page - 1) / size_page + 1) * size_page; // round-up 2 pages just in case
+ const size_t size_step = device.maxBufferLength - size_ovlp;
+ const size_t size_view = device.maxBufferLength;
+
+ for (size_t i = 0; i < size; i += size_step) {
+ const size_t size_step_aligned = (i + size_view <= size) ? size_view : (size_aligned - i);
+
+ ctx->buffers[ctx->n_buffers].data = (void *) ((uint8_t *) data + i);
+ ctx->buffers[ctx->n_buffers].size = size_step_aligned;
+
+ ctx->buffers[ctx->n_buffers].metal = [device newBufferWithBytesNoCopy:(void *) ((uint8_t *) data + i) length:size_step_aligned options:MTLResourceStorageModeShared deallocator:nil];
+
+ if (ctx->buffers[ctx->n_buffers].metal == nil) {
+ GGML_METAL_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_step_aligned / 1024.0 / 1024.0);
+ return false;
+ }
+
+ ggml_backend_metal_log_allocated_size(device, size_step_aligned);
+
+ if (i + size_step < size) {
+ GGML_METAL_LOG_INFO("\n");
+ }
+
+ ++ctx->n_buffers;
+ }
+ }
+
+ return ggml_backend_buffer_init(ggml_backend_metal_buffer_type(), ggml_backend_metal_buffer_i, ctx, size);
+}
+
+// backend
+
+GGML_CALL static const char * ggml_backend_metal_name(ggml_backend_t backend) {
+ return "Metal";
+
+ UNUSED(backend);
+}
+
+GGML_CALL static void ggml_backend_metal_free(ggml_backend_t backend) {
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
+ ggml_metal_free(ctx);
+ free(backend);
+}
+
+GGML_CALL static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffer_type(ggml_backend_t backend) {
+ return ggml_backend_metal_buffer_type();
+
+ UNUSED(backend);
+}
+
+GGML_CALL static enum ggml_status ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
+ struct ggml_backend_metal_context * metal_ctx = (struct ggml_backend_metal_context *)backend->context;
+
+ return ggml_metal_graph_compute(metal_ctx, cgraph);
+}
+
+GGML_CALL static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
+ struct ggml_backend_metal_context * metal_ctx = (struct ggml_backend_metal_context *)backend->context;
+
+ return ggml_metal_supports_op(metal_ctx, op);
+}
+
+GGML_CALL static bool ggml_backend_metal_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
+ return buft->iface.get_name == ggml_backend_metal_buffer_type_get_name;
+
+ UNUSED(backend);
+}
+
+static struct ggml_backend_i ggml_backend_metal_i = {
+ /* .get_name = */ ggml_backend_metal_name,
+ /* .free = */ ggml_backend_metal_free,
+ /* .get_default_buffer_type = */ ggml_backend_metal_get_default_buffer_type,
+ /* .set_tensor_async = */ NULL,
+ /* .get_tensor_async = */ NULL,
+ /* .cpy_tensor_async = */ NULL,
+ /* .synchronize = */ NULL,
+ /* .graph_plan_create = */ NULL,
+ /* .graph_plan_free = */ NULL,
+ /* .graph_plan_update = */ NULL,
+ /* .graph_plan_compute = */ NULL,
+ /* .graph_compute = */ ggml_backend_metal_graph_compute,
+ /* .supports_op = */ ggml_backend_metal_supports_op,
+ /* .supports_buft = */ ggml_backend_metal_supports_buft,
+ /* .offload_op = */ NULL,
+ /* .event_new = */ NULL,
+ /* .event_free = */ NULL,
+ /* .event_record = */ NULL,
+ /* .event_wait = */ NULL,
+ /* .event_synchronize = */ NULL,
+};
+
+void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {
+ ggml_metal_log_callback = log_callback;
+ ggml_metal_log_user_data = user_data;
+}
+
+static ggml_guid_t ggml_backend_metal_guid(void) {
+ static ggml_guid guid = { 0x81, 0xa1, 0x8b, 0x1e, 0x71, 0xec, 0x79, 0xed, 0x2b, 0x85, 0xdc, 0x8a, 0x61, 0x98, 0x30, 0xe6 };
+ return &guid;
+}
+
+ggml_backend_t ggml_backend_metal_init(void) {
+ struct ggml_backend_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);
+ if (ctx == NULL) {
+ GGML_METAL_LOG_ERROR("%s: error: failed to allocate context\n", __func__);
+ return NULL;
+ }
+
+ ggml_backend_t metal_backend = malloc(sizeof(struct ggml_backend));
+
+ *metal_backend = (struct ggml_backend) {
+ /* .guid = */ ggml_backend_metal_guid(),
+ /* .interface = */ ggml_backend_metal_i,
+ /* .context = */ ctx,
+ };
+
+ return metal_backend;
+}
+
+bool ggml_backend_is_metal(ggml_backend_t backend) {
+ return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_metal_guid());
+}
+
+void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {
+ GGML_ASSERT(ggml_backend_is_metal(backend));
+
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
+
+ ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);
+}
+
+void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data) {
+ GGML_ASSERT(ggml_backend_is_metal(backend));
+
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
+
+ ctx->abort_callback = abort_callback;
+ ctx->abort_callback_data = user_data;
+}
+
+bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {
+ GGML_ASSERT(ggml_backend_is_metal(backend));
+
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
+
+ return [ctx->device supportsFamily:(MTLGPUFamilyApple1 + family - 1)];
+}
+
+void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {
+ GGML_ASSERT(ggml_backend_is_metal(backend));
+
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
+ ctx->should_capture_next_compute = true;
+}
+
+GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data); // silence warning
+
+GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) {
+ return ggml_backend_metal_init();
+
+ GGML_UNUSED(params);
+ GGML_UNUSED(user_data);
+}
diff --git a/llama/ggml-quants.c b/llama/ggml-quants.c
new file mode 100644
index 00000000..c22ffac0
--- /dev/null
+++ b/llama/ggml-quants.c
@@ -0,0 +1,14898 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define GGML_COMMON_IMPL_C
+#include "ggml-common.h"
+
+#include "ggml-quants.h"
+#include "ggml-impl.h"
+
+
+#include <math.h>
+#include <string.h>
+#include <assert.h>
+#include <float.h>
+#include <stdlib.h> // for qsort
+#include <stdio.h> // for GGML_ASSERT
+
+#define GROUP_MAX_EPS 1e-15f
+#define GROUP_MAX_EPS_IQ3_XXS 1e-8f
+#define GROUP_MAX_EPS_IQ2_S 1e-8f
+#define GROUP_MAX_EPS_IQ1_M 1e-7f
+#define GROUP_MAX_EPS_IQ1_S 1e-12f
+
+#if defined(_MSC_VER)
+// disable "possible loss of data" to avoid warnings for hundreds of casts
+// we should just be careful :)
+#pragma warning(disable: 4244 4267)
+#endif
+
+#define UNUSED GGML_UNUSED
+
+// some compilers don't provide _mm256_set_m128i, e.g. gcc 7
+#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
+
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)
+// multiply int8_t, add results pairwise twice
+static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {
+ // Get absolute values of x vectors
+ const __m128i ax = _mm_sign_epi8(x, x);
+ // Sign the values of the y vectors
+ const __m128i sy = _mm_sign_epi8(y, x);
+ // Perform multiplication and create 16-bit values
+ const __m128i dot = _mm_maddubs_epi16(ax, sy);
+ const __m128i ones = _mm_set1_epi16(1);
+ return _mm_madd_epi16(ones, dot);
+}
+
+#if __AVX__ || __AVX2__ || __AVX512F__
+// horizontally add 8 floats
+static inline float hsum_float_8(const __m256 x) {
+ __m128 res = _mm256_extractf128_ps(x, 1);
+ res = _mm_add_ps(res, _mm256_castps256_ps128(x));
+ res = _mm_add_ps(res, _mm_movehl_ps(res, res));
+ res = _mm_add_ss(res, _mm_movehdup_ps(res));
+ return _mm_cvtss_f32(res);
+}
+
+// horizontally add 8 int32_t
+static inline int hsum_i32_8(const __m256i a) {
+ const __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));
+ const __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128);
+ const __m128i sum64 = _mm_add_epi32(hi64, sum128);
+ const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
+ return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));
+}
+
+// horizontally add 4 int32_t
+static inline int hsum_i32_4(const __m128i a) {
+ const __m128i hi64 = _mm_unpackhi_epi64(a, a);
+ const __m128i sum64 = _mm_add_epi32(hi64, a);
+ const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
+ return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));
+}
+
+#if defined(__AVX2__) || defined(__AVX512F__)
+// spread 32 bits to 32 bytes { 0x00, 0xFF }
+static inline __m256i bytes_from_bits_32(const uint8_t * x) {
+ uint32_t x32;
+ memcpy(&x32, x, sizeof(uint32_t));
+ const __m256i shuf_mask = _mm256_set_epi64x(
+ 0x0303030303030303, 0x0202020202020202,
+ 0x0101010101010101, 0x0000000000000000);
+ __m256i bytes = _mm256_shuffle_epi8(_mm256_set1_epi32(x32), shuf_mask);
+ const __m256i bit_mask = _mm256_set1_epi64x(0x7fbfdfeff7fbfdfe);
+ bytes = _mm256_or_si256(bytes, bit_mask);
+ return _mm256_cmpeq_epi8(bytes, _mm256_set1_epi64x(-1));
+}
+
+// Unpack 32 4-bit fields into 32 bytes
+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval
+static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)
+{
+ const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);
+ const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);
+ const __m256i lowMask = _mm256_set1_epi8( 0xF );
+ return _mm256_and_si256(lowMask, bytes);
+}
+
+// add int16_t pairwise and return as float vector
+static inline __m256 sum_i16_pairs_float(const __m256i x) {
+ const __m256i ones = _mm256_set1_epi16(1);
+ const __m256i summed_pairs = _mm256_madd_epi16(ones, x);
+ return _mm256_cvtepi32_ps(summed_pairs);
+}
+
+static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {
+#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))
+ const __m256i zero = _mm256_setzero_si256();
+ const __m256i summed_pairs = _mm256_dpbusd_epi32(zero, ax, sy);
+ return _mm256_cvtepi32_ps(summed_pairs);
+#else
+ // Perform multiplication and create 16-bit values
+ const __m256i dot = _mm256_maddubs_epi16(ax, sy);
+ return sum_i16_pairs_float(dot);
+#endif
+}
+
+// multiply int8_t, add results pairwise twice and return as float vector
+static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {
+#if __AVXVNNIINT8__
+ const __m256i zero = _mm256_setzero_si256();
+ const __m256i summed_pairs = _mm256_dpbssd_epi32(zero, x, y);
+ return _mm256_cvtepi32_ps(summed_pairs);
+#else
+ // Get absolute values of x vectors
+ const __m256i ax = _mm256_sign_epi8(x, x);
+ // Sign the values of the y vectors
+ const __m256i sy = _mm256_sign_epi8(y, x);
+ return mul_sum_us8_pairs_float(ax, sy);
+#endif
+}
+
+static inline __m128i packNibbles( __m256i bytes )
+{
+ // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh
+#if __AVX512F__
+ const __m256i bytes_srli_4 = _mm256_srli_epi16(bytes, 4); // 0000_0000_abcd_0000
+ bytes = _mm256_or_si256(bytes, bytes_srli_4); // 0000_abcd_abcd_efgh
+ return _mm256_cvtepi16_epi8(bytes); // abcd_efgh
+#else
+ const __m256i lowByte = _mm256_set1_epi16( 0xFF );
+ __m256i high = _mm256_andnot_si256( lowByte, bytes );
+ __m256i low = _mm256_and_si256( lowByte, bytes );
+ high = _mm256_srli_epi16( high, 4 );
+ bytes = _mm256_or_si256( low, high );
+
+ // Compress uint16_t lanes into bytes
+ __m128i r0 = _mm256_castsi256_si128( bytes );
+ __m128i r1 = _mm256_extracti128_si256( bytes, 1 );
+ return _mm_packus_epi16( r0, r1 );
+#endif
+}
+#elif defined(__AVX__)
+// spread 32 bits to 32 bytes { 0x00, 0xFF }
+static inline __m256i bytes_from_bits_32(const uint8_t * x) {
+ uint32_t x32;
+ memcpy(&x32, x, sizeof(uint32_t));
+ const __m128i shuf_maskl = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
+ const __m128i shuf_maskh = _mm_set_epi64x(0x0303030303030303, 0x0202020202020202);
+ __m128i bytesl = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskl);
+ __m128i bytesh = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskh);
+ const __m128i bit_mask = _mm_set1_epi64x(0x7fbfdfeff7fbfdfe);
+ bytesl = _mm_or_si128(bytesl, bit_mask);
+ bytesh = _mm_or_si128(bytesh, bit_mask);
+ bytesl = _mm_cmpeq_epi8(bytesl, _mm_set1_epi64x(-1));
+ bytesh = _mm_cmpeq_epi8(bytesh, _mm_set1_epi64x(-1));
+ return MM256_SET_M128I(bytesh, bytesl);
+}
+
+// Unpack 32 4-bit fields into 32 bytes
+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval
+static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)
+{
+ // Load 16 bytes from memory
+ __m128i tmpl = _mm_loadu_si128((const __m128i *)rsi);
+ __m128i tmph = _mm_srli_epi16(tmpl, 4);
+ const __m128i lowMask = _mm_set1_epi8(0xF);
+ tmpl = _mm_and_si128(lowMask, tmpl);
+ tmph = _mm_and_si128(lowMask, tmph);
+ return MM256_SET_M128I(tmph, tmpl);
+}
+
+// add int16_t pairwise and return as float vector
+static inline __m256 sum_i16_pairs_float(const __m128i xh, const __m128i xl) {
+ const __m128i ones = _mm_set1_epi16(1);
+ const __m128i summed_pairsl = _mm_madd_epi16(ones, xl);
+ const __m128i summed_pairsh = _mm_madd_epi16(ones, xh);
+ const __m256i summed_pairs = MM256_SET_M128I(summed_pairsh, summed_pairsl);
+ return _mm256_cvtepi32_ps(summed_pairs);
+}
+
+static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {
+ const __m128i axl = _mm256_castsi256_si128(ax);
+ const __m128i axh = _mm256_extractf128_si256(ax, 1);
+ const __m128i syl = _mm256_castsi256_si128(sy);
+ const __m128i syh = _mm256_extractf128_si256(sy, 1);
+ // Perform multiplication and create 16-bit values
+ const __m128i dotl = _mm_maddubs_epi16(axl, syl);
+ const __m128i doth = _mm_maddubs_epi16(axh, syh);
+ return sum_i16_pairs_float(doth, dotl);
+}
+
+// multiply int8_t, add results pairwise twice and return as float vector
+static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {
+ const __m128i xl = _mm256_castsi256_si128(x);
+ const __m128i xh = _mm256_extractf128_si256(x, 1);
+ const __m128i yl = _mm256_castsi256_si128(y);
+ const __m128i yh = _mm256_extractf128_si256(y, 1);
+ // Get absolute values of x vectors
+ const __m128i axl = _mm_sign_epi8(xl, xl);
+ const __m128i axh = _mm_sign_epi8(xh, xh);
+ // Sign the values of the y vectors
+ const __m128i syl = _mm_sign_epi8(yl, xl);
+ const __m128i syh = _mm_sign_epi8(yh, xh);
+ // Perform multiplication and create 16-bit values
+ const __m128i dotl = _mm_maddubs_epi16(axl, syl);
+ const __m128i doth = _mm_maddubs_epi16(axh, syh);
+ return sum_i16_pairs_float(doth, dotl);
+}
+
+static inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 )
+{
+ // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh
+ const __m128i lowByte = _mm_set1_epi16( 0xFF );
+ __m128i high = _mm_andnot_si128( lowByte, bytes1 );
+ __m128i low = _mm_and_si128( lowByte, bytes1 );
+ high = _mm_srli_epi16( high, 4 );
+ bytes1 = _mm_or_si128( low, high );
+ high = _mm_andnot_si128( lowByte, bytes2 );
+ low = _mm_and_si128( lowByte, bytes2 );
+ high = _mm_srli_epi16( high, 4 );
+ bytes2 = _mm_or_si128( low, high );
+
+ return _mm_packus_epi16( bytes1, bytes2);
+}
+#endif
+#elif defined(__SSSE3__)
+// horizontally add 4x4 floats
+static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 c, const __m128 d) {
+ __m128 res_0 =_mm_hadd_ps(a, b);
+ __m128 res_1 =_mm_hadd_ps(c, d);
+ __m128 res =_mm_hadd_ps(res_0, res_1);
+ res =_mm_hadd_ps(res, res);
+ res =_mm_hadd_ps(res, res);
+
+ return _mm_cvtss_f32(res);
+}
+#endif // __AVX__ || __AVX2__ || __AVX512F__
+#endif // defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)
+
+#if defined(__ARM_NEON) || defined(__wasm_simd128__) || defined(__POWER9_VECTOR__)
+#define B1(c,s,n) 0x ## n ## c , 0x ## n ## s
+#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)
+#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)
+#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s)
+#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s)
+#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s)
+#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s)
+#define B8(c,s ) B7(c,s, c), B7(c,s, s)
+
+// precomputed tables for expanding 8bits to 8 bytes:
+static const uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b) << 4
+static const uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4
+#endif
+
+#if defined(__loongarch_asx)
+
+#ifdef __clang__
+#define VREGS_PREFIX "$vr"
+#define XREGS_PREFIX "$xr"
+#else // GCC
+#define VREGS_PREFIX "$f"
+#define XREGS_PREFIX "$f"
+#endif
+#define __ALL_REGS "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31"
+// Convert __m128i to __m256i
+static inline __m256i ____m256i(__m128i in) {
+ __m256i out = __lasx_xvldi(0);
+ __asm__ volatile (
+ ".irp i," __ALL_REGS "\n\t"
+ " .ifc %[out], " XREGS_PREFIX"\\i \n\t"
+ " .irp j," __ALL_REGS "\n\t"
+ " .ifc %[in], " VREGS_PREFIX "\\j \n\t"
+ " xvpermi.q $xr\\i, $xr\\j, 0x20 \n\t"
+ " .endif \n\t"
+ " .endr \n\t"
+ " .endif \n\t"
+ ".endr \n\t"
+ : [out] "+f" (out) : [in] "f" (in)
+ );
+ return out;
+}
+// Convert two __m128i to __m256i
+static inline __m256i lasx_set_q(__m128i inhi, __m128i inlo) {
+ __m256i out;
+ __asm__ volatile (
+ ".irp i," __ALL_REGS "\n\t"
+ " .ifc %[hi], " VREGS_PREFIX "\\i \n\t"
+ " .irp j," __ALL_REGS "\n\t"
+ " .ifc %[lo], " VREGS_PREFIX "\\j \n\t"
+ " xvpermi.q $xr\\i, $xr\\j, 0x20 \n\t"
+ " .endif \n\t"
+ " .endr \n\t"
+ " .endif \n\t"
+ ".endr \n\t"
+ ".ifnc %[out], %[hi] \n\t"
+ ".irp i," __ALL_REGS "\n\t"
+ " .ifc %[out], " XREGS_PREFIX "\\i \n\t"
+ " .irp j," __ALL_REGS "\n\t"
+ " .ifc %[hi], " VREGS_PREFIX "\\j \n\t"
+ " xvori.b $xr\\i, $xr\\j, 0 \n\t"
+ " .endif \n\t"
+ " .endr \n\t"
+ " .endif \n\t"
+ ".endr \n\t"
+ ".endif \n\t"
+ : [out] "=f" (out), [hi] "+f" (inhi)
+ : [lo] "f" (inlo)
+ );
+ return out;
+}
+// Convert __m256i low part to __m128i
+static inline __m128i lasx_extracti128_lo(__m256i in) {
+ __m128i out;
+ __asm__ volatile (
+ ".ifnc %[out], %[in] \n\t"
+ ".irp i," __ALL_REGS "\n\t"
+ " .ifc %[out], " VREGS_PREFIX "\\i \n\t"
+ " .irp j," __ALL_REGS "\n\t"
+ " .ifc %[in], " XREGS_PREFIX "\\j \n\t"
+ " vori.b $vr\\i, $vr\\j, 0 \n\t"
+ " .endif \n\t"
+ " .endr \n\t"
+ " .endif \n\t"
+ ".endr \n\t"
+ ".endif \n\t"
+ : [out] "=f" (out) : [in] "f" (in)
+ );
+ return out;
+}
+// Convert __m256i high part to __m128i
+static inline __m128i lasx_extracti128_hi(__m256i in) {
+ __m128i out;
+ __asm__ volatile (
+ ".irp i," __ALL_REGS "\n\t"
+ " .ifc %[out], " VREGS_PREFIX "\\i \n\t"
+ " .irp j," __ALL_REGS "\n\t"
+ " .ifc %[in], " XREGS_PREFIX "\\j \n\t"
+ " xvpermi.q $xr\\i, $xr\\j, 0x11 \n\t"
+ " .endif \n\t"
+ " .endr \n\t"
+ " .endif \n\t"
+ ".endr \n\t"
+ : [out] "=f" (out) : [in] "f" (in)
+ );
+ return out;
+}
+
+static __m256i lasx_set_w(int e7, int e6, int e5, int e4, int e3, int e2, int e1, int e0) {
+ v8i32 __ret = {e0, e1, e2, e3, e4, e5, e6, e7};
+ return (__m256i)__ret;
+}
+
+static __m128i lsx_set_w(int32_t a, int32_t b, int32_t c, int32_t d) {
+ v4i32 __ret = {d, c, b, a};
+ return (__m128i)__ret;
+}
+
+static __m256i lasx_set_d(int64_t a, int64_t b, int64_t c, int64_t d) {
+ v4i64 __ret = {d, c, b, a};
+ return (__m256i)__ret;
+}
+
+static __m256i lasx_insertf128( __m128i x, __m128i y) {
+ return lasx_set_q(x, y);
+}
+
+static __m128i lsx_shuffle_b(__m128i a, __m128i b) {
+ __m128i mask_f, zero, tmp0, tmp2, mask;
+ int f = 0x8f;
+ mask_f = __lsx_vreplgr2vr_b(f);
+ zero = __lsx_vldi(0);
+ tmp0 = __lsx_vand_v(b, mask_f); // get mask with low 4 bit and sign bits
+ tmp0 = __lsx_vori_b(tmp0, 0x10); // make each mask or with 0x10 prepare for positive
+ mask = __lsx_vsle_b(zero, tmp0); // if mask >= 0, set mask
+ tmp2 = __lsx_vand_v(tmp0, mask); // maskout the in2 < ones
+ return __lsx_vshuf_b(a, zero, tmp2);
+}
+
+static __m256i lasx_shuffle_b(__m256i a, __m256i b) {
+ __m256i mask_f, zero, tmp0, tmp2, mask;
+ int f = 0x8f;
+ mask_f = __lasx_xvreplgr2vr_b(f);
+ zero = __lasx_xvldi(0);
+ tmp0 = __lasx_xvand_v(b, mask_f); // get mask with low 4 bit and sign bits
+ tmp0 = __lasx_xvori_b(tmp0, 0x10); // make each mask or with 0x10 prepare for positive
+ mask = __lasx_xvsle_b(zero, tmp0); // if mask >= 0, set mask
+ tmp2 = __lasx_xvand_v(tmp0, mask); // maskout the in2 < ones
+ return __lasx_xvshuf_b(a, zero, tmp2);
+}
+
+static __m256i lasx_extu8_16(__m128i a) {
+ __m128i zero = __lsx_vldi(0);
+ __m128i vlo = __lsx_vilvl_b(zero, a);
+ __m128i vhi = __lsx_vilvh_b(zero, a);
+ return lasx_set_q(vhi, vlo);
+}
+
+static __m256i lasx_ext8_16(__m128i a) {
+ __m128i sign = __lsx_vslti_b(a, 0);
+ __m128i vlo = __lsx_vilvl_b(sign, a);
+ __m128i vhi = __lsx_vilvh_b(sign, a);
+ return lasx_set_q(vhi, vlo);
+}
+
+static __m256i lasx_ext16_32(__m128i a) {
+ __m256i tmp1;
+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 0), 0);
+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 1), 1);
+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 2), 2);
+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 3), 3);
+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 4), 4);
+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 5), 5);
+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 6), 6);
+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 7), 7);
+ return tmp1;
+}
+
+static __m128i lasx_extracti128( __m256i a, int pos) {
+ __m128i ret;
+ if( pos == 0)
+ {
+ ret = lasx_extracti128_lo(a);
+ } else {
+ ret = lasx_extracti128_hi(a);
+ }
+ return ret;
+}
+
+static __m128 lasx_extractf128( __m256 a, int pos) {
+ __m128 ret;
+ if( pos == 0)
+ {
+ ret = (__m128)lasx_extracti128_lo((__m256i)a);
+ } else {
+ ret = (__m128)lasx_extracti128_hi((__m256i)a);
+ }
+ return ret;
+}
+
+static __m128i lsx_hadd_h(__m128i a, __m128i b) {
+ __m128i tmp1 = __lsx_vpickev_h(b, a);
+ __m128i tmp2 = __lsx_vpickod_h(b, a);
+ return __lsx_vadd_h(tmp1, tmp2);
+}
+
+static __m128i lsx_hadd_w(__m128i a, __m128i b) {
+ __m128i tmp1 = __lsx_vpickev_w(b, a);
+ __m128i tmp2 = __lsx_vpickod_w(b, a);
+ return __lsx_vadd_w(tmp1, tmp2);
+}
+
+static __m128 lsx_hadd_s(__m128 a, __m128 b) {
+ __m128 tmp1 = (__m128)__lsx_vpickev_w((__m128i)b, (__m128i)a);
+ __m128 tmp2 = (__m128)__lsx_vpickod_w((__m128i)b, (__m128i)a);
+
+ return __lsx_vfadd_s(tmp1, tmp2);
+}
+
+static __m256i lasx_maddubs_h(__m256i a, __m256i b) {
+ __m256i tmp1, tmp2;
+ tmp1 = __lasx_xvmulwev_h_b(a, b);
+ tmp2 = __lasx_xvmulwod_h_b(a, b);
+ return __lasx_xvsadd_h(tmp1, tmp2);
+}
+
+static __m256i lasx_madd_h(__m256i a, __m256i b) {
+ __m256i tmp1, tmp2;
+ tmp1 = __lasx_xvmulwev_w_h(a, b);
+ tmp2 = __lasx_xvmulwod_w_h(a, b);
+ return __lasx_xvadd_w(tmp1, tmp2);
+}
+
+static __m256i lasx_packs_w(__m256i a, __m256i b) {
+ __m256i tmp, tmp1;
+ tmp = __lasx_xvsat_w(a, 15);
+ tmp1 = __lasx_xvsat_w(b, 15);
+ return __lasx_xvpickev_h(tmp1, tmp);
+}
+
+static __m256i lasx_packs_h(__m256i a, __m256i b) {
+ __m256i tmp, tmp1;
+ tmp = __lasx_xvsat_h(a, 7);
+ tmp1 = __lasx_xvsat_h(b, 7);
+ return __lasx_xvpickev_b(tmp1, tmp);
+}
+
+static __m128i lsx_packs_w(__m128i a, __m128i b) {
+ __m128i tmp, tmp1;
+ tmp = __lsx_vsat_w(a, 15);
+ tmp1 = __lsx_vsat_w(b, 15);
+ return __lsx_vpickev_h(tmp1, tmp);
+}
+
+static __m128i lsx_packs_h(__m128i a, __m128i b) {
+ __m128i tmp, tmp1;
+ tmp = __lsx_vsat_h(a, 7);
+ tmp1 = __lsx_vsat_h(b, 7);
+ return __lsx_vpickev_b(tmp1, tmp);
+}
+
+static __m128i lsx_packus_h(__m128i a, __m128i b) {
+ __m128i tmp, tmp1;
+ tmp = __lsx_vsat_hu(a, 7);
+ tmp1 = __lsx_vsat_hu(b, 7);
+ return __lsx_vpickev_b(tmp1, tmp);
+}
+
+
+static __m128i lsx_maddubs_h(__m128i a, __m128i b) {
+ __m128i tmp1, tmp2;
+ tmp1 = __lsx_vmulwev_h_b(a, b);
+ tmp2 = __lsx_vmulwod_h_b(a, b);
+ return __lsx_vsadd_h(tmp1, tmp2);
+}
+
+static __m128i lsx_madd_h(__m128i a, __m128i b) {
+ __m128i tmp1, tmp2;
+ tmp1 = __lsx_vmulwev_w_h(a, b);
+ tmp2 = __lsx_vmulwod_w_h(a, b);
+ return __lsx_vadd_w(tmp1, tmp2);
+}
+
+// multiply int8_t, add results pairwise twice
+static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {
+ // Get absolute values of x vectors
+ const __m128i ax = __lsx_vsigncov_b(x, x);
+ // Sign the values of the y vectors
+ const __m128i sy = __lsx_vsigncov_b(x, y);
+ // Perform multiplication and create 16-bit values
+ const __m128i dot = lsx_maddubs_h(ax, sy);
+ const __m128i ones = __lsx_vreplgr2vr_h(1);
+ return lsx_madd_h(ones, dot);
+}
+
+// horizontally add 8 floats
+static inline float hsum_float_8(const __m256 x) {
+ __m128 res = lasx_extractf128(x, 1);
+ ft_union tmp;
+ res = __lsx_vfadd_s(res, lasx_extractf128(x, 0));
+ res = __lsx_vfadd_s(res, (__m128)__lsx_vpickod_d((__m128i)res, (__m128i)res));
+ res = __lsx_vfadd_s(res, (__m128)__lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w(res, 1), 0));
+ tmp.i = __lsx_vpickve2gr_w(res, 0);
+ return tmp.f;
+}
+
+// horizontally add 8 int32_t
+static inline int hsum_i32_8(const __m256i a) {
+
+ __m256i tmp1 = __lasx_xvpermi_q(a, a, 0x11);
+ __m256i tmp2 = __lasx_xvpermi_q(a, a, 0x00);
+
+ __m128i tmp1_128 = lasx_extracti128_lo(tmp1);
+ __m128i tmp2_128 = lasx_extracti128_lo(tmp2);
+
+ __m128i sum128 = __lsx_vadd_w(tmp1_128, tmp2_128);
+
+ __m128i ev = __lsx_vpickev_w(sum128, sum128);
+ __m128i od = __lsx_vpickod_w(sum128, sum128);
+ __m128i sum64 = __lsx_vadd_w(ev, od);
+
+ int sum64_1, sum64_2;
+ sum64_1 = __lsx_vpickve2gr_w(sum64, 0);
+ sum64_2 = __lsx_vpickve2gr_w(sum64, 1);
+
+ return sum64_1 + sum64_2;
+}
+
+// horizontally add 4 int32_t
+static inline int hsum_i32_4(const __m128i a) {
+ __m128i ev = __lsx_vpickev_w(a, a);
+ __m128i od = __lsx_vpickod_w(a, a);
+ __m128i sum64 = __lsx_vadd_w(ev, od);
+
+ int sum64_1, sum64_2;
+ sum64_1 = __lsx_vpickve2gr_w(sum64, 0);
+ sum64_2 = __lsx_vpickve2gr_w(sum64, 1);
+
+ return sum64_1 + sum64_2;
+}
+
+// spread 32 bits to 32 bytes { 0x00, 0xFF }
+static inline __m256i bytes_from_bits_32(const uint8_t * x) {
+
+ uint32_t x32;
+ memcpy(&x32, x, sizeof(uint32_t));
+ const __m256i shuf_mask = lasx_set_d(
+ 0x0303030303030303, 0x0202020202020202,
+ 0x0101010101010101, 0x0000000000000000);
+
+ __m256i bytes = lasx_shuffle_b(__lasx_xvreplgr2vr_w(x32), shuf_mask);
+ const __m256i bit_mask = __lasx_xvreplgr2vr_d(0x7fbfdfeff7fbfdfe);
+ bytes = __lasx_xvor_v(bytes, bit_mask);
+ return __lasx_xvseq_b(bytes, __lasx_xvreplgr2vr_d(-1));
+}
+
+// Unpack 32 4-bit fields into 32 bytes
+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval
+static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi) {
+ const __m128i lo = __lsx_vld((const __m128i *)rsi, 0);
+ __m128i hi = __lsx_vsrli_h(lo, 4);
+ return __lasx_xvandi_b(lasx_insertf128(hi, lo), 0xf);
+}
+
+// add int16_t pairwise and return as float vector
+static inline __m256 sum_i16_pairs_float(const __m256i x) {
+ __m256i v = __lasx_xvpackod_h(x, x);
+ __m256i summed_pairs = __lasx_xvaddwev_w_h(x, v);
+ return __lasx_xvffint_s_w(summed_pairs);
+}
+
+static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {
+ // Perform multiplication and create 16-bit values
+ const __m256i dot = lasx_maddubs_h(ax, sy);
+ return sum_i16_pairs_float(dot);
+}
+
+// multiply int8_t, add results pairwise twice and return as float vector
+static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {
+
+ // Get absolute values of x vectors
+ const __m256i ax = __lasx_xvsigncov_b(x, x);
+ // Sign the values of the y vectors
+ const __m256i sy = __lasx_xvsigncov_b(x, y);
+
+ return mul_sum_us8_pairs_float(ax, sy);
+}
+
+static inline __m128i packNibbles( __m256i bytes ) {
+ // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh
+ const __m256i lowByte = __lasx_xvreplgr2vr_h(0xFF);
+ __m256i high = __lasx_xvandn_v(lowByte, bytes);
+ __m256i low = __lasx_xvand_v(lowByte, bytes);
+ high = __lasx_xvsrli_h(high, 4);
+ bytes = __lasx_xvor_v(low, high);
+ // Compress uint16_t lanes into bytes
+ __m128i *r0 = (__m128i *)&bytes;
+ __m256i tmp_h128 = __lasx_xvpermi_q(bytes, bytes, 0x11);
+ __m128i *r1 = (__m128i *)&tmp_h128;
+
+ __m128i zero = __lsx_vldi(0);
+ __m128i tmp, tmp2, tmp3;
+
+ tmp = __lsx_vmax_h(zero, *r0);
+ tmp2 = __lsx_vsat_hu(tmp, 7);
+
+ tmp = __lsx_vmax_h(zero, *r1);
+ tmp3 = __lsx_vsat_hu(tmp, 7);
+ return __lsx_vpickev_b(tmp3, tmp2);
+}
+#endif //__loongarch_asx
+
+// reference implementation for deterministic creation of model files
+void quantize_row_q4_0_ref(const float * restrict x, block_q4_0 * restrict y, int64_t k) {
+ static const int qk = QK4_0;
+
+ assert(k % qk == 0);
+
+ const int nb = k / qk;
+
+ for (int i = 0; i < nb; i++) {
+ float amax = 0.0f; // absolute max
+ float max = 0.0f;
+
+ for (int j = 0; j < qk; j++) {
+ const float v = x[i*qk + j];
+ if (amax < fabsf(v)) {
+ amax = fabsf(v);
+ max = v;
+ }
+ }
+
+ const float d = max / -8;
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ for (int j = 0; j < qk/2; ++j) {
+ const float x0 = x[i*qk + 0 + j]*id;
+ const float x1 = x[i*qk + qk/2 + j]*id;
+
+ const uint8_t xi0 = MIN(15, (int8_t)(x0 + 8.5f));
+ const uint8_t xi1 = MIN(15, (int8_t)(x1 + 8.5f));
+
+ y[i].qs[j] = xi0;
+ y[i].qs[j] |= xi1 << 4;
+ }
+ }
+}
+
+void quantize_row_q4_0(const float * restrict x, void * restrict y, int64_t k) {
+ quantize_row_q4_0_ref(x, y, k);
+}
+
+
+void quantize_row_q4_1_ref(const float * restrict x, block_q4_1 * restrict y, int64_t k) {
+ const int qk = QK4_1;
+
+ assert(k % qk == 0);
+
+ const int nb = k / qk;
+
+ for (int i = 0; i < nb; i++) {
+ float min = FLT_MAX;
+ float max = -FLT_MAX;
+
+ for (int j = 0; j < qk; j++) {
+ const float v = x[i*qk + j];
+
+ if (v < min) min = v;
+ if (v > max) max = v;
+ }
+
+ const float d = (max - min) / ((1 << 4) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+ y[i].m = GGML_FP32_TO_FP16(min);
+
+ for (int j = 0; j < qk/2; ++j) {
+ const float x0 = (x[i*qk + 0 + j] - min)*id;
+ const float x1 = (x[i*qk + qk/2 + j] - min)*id;
+
+ const uint8_t xi0 = MIN(15, (int8_t)(x0 + 0.5f));
+ const uint8_t xi1 = MIN(15, (int8_t)(x1 + 0.5f));
+
+ y[i].qs[j] = xi0;
+ y[i].qs[j] |= xi1 << 4;
+ }
+ }
+}
+
+void quantize_row_q4_1(const float * restrict x, void * restrict y, int64_t k) {
+ quantize_row_q4_1_ref(x, y, k);
+}
+
+void quantize_row_q5_0_ref(const float * restrict x, block_q5_0 * restrict y, int64_t k) {
+ static const int qk = QK5_0;
+
+ assert(k % qk == 0);
+
+ const int nb = k / qk;
+
+ for (int i = 0; i < nb; i++) {
+ float amax = 0.0f; // absolute max
+ float max = 0.0f;
+
+ for (int j = 0; j < qk; j++) {
+ const float v = x[i*qk + j];
+ if (amax < fabsf(v)) {
+ amax = fabsf(v);
+ max = v;
+ }
+ }
+
+ const float d = max / -16;
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ uint32_t qh = 0;
+
+ for (int j = 0; j < qk/2; ++j) {
+ const float x0 = x[i*qk + 0 + j]*id;
+ const float x1 = x[i*qk + qk/2 + j]*id;
+
+ const uint8_t xi0 = MIN(31, (int8_t)(x0 + 16.5f));
+ const uint8_t xi1 = MIN(31, (int8_t)(x1 + 16.5f));
+
+ y[i].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);
+
+ // get the 5-th bit and store it in qh at the right position
+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);
+ qh |= ((xi1 & 0x10u) >> 4) << (j + qk/2);
+ }
+
+ memcpy(&y[i].qh, &qh, sizeof(qh));
+ }
+}
+
+void quantize_row_q5_0(const float * restrict x, void * restrict y, int64_t k) {
+ quantize_row_q5_0_ref(x, y, k);
+}
+
+void quantize_row_q5_1_ref(const float * restrict x, block_q5_1 * restrict y, int64_t k) {
+ const int qk = QK5_1;
+
+ assert(k % qk == 0);
+
+ const int nb = k / qk;
+
+ for (int i = 0; i < nb; i++) {
+ float min = FLT_MAX;
+ float max = -FLT_MAX;
+
+ for (int j = 0; j < qk; j++) {
+ const float v = x[i*qk + j];
+
+ if (v < min) min = v;
+ if (v > max) max = v;
+ }
+
+ const float d = (max - min) / ((1 << 5) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+ y[i].m = GGML_FP32_TO_FP16(min);
+
+ uint32_t qh = 0;
+
+ for (int j = 0; j < qk/2; ++j) {
+ const float x0 = (x[i*qk + 0 + j] - min)*id;
+ const float x1 = (x[i*qk + qk/2 + j] - min)*id;
+
+ const uint8_t xi0 = (uint8_t)(x0 + 0.5f);
+ const uint8_t xi1 = (uint8_t)(x1 + 0.5f);
+
+ y[i].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);
+
+ // get the 5-th bit and store it in qh at the right position
+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);
+ qh |= ((xi1 & 0x10u) >> 4) << (j + qk/2);
+ }
+
+ memcpy(&y[i].qh, &qh, sizeof(y[i].qh));
+ }
+}
+
+void quantize_row_q5_1(const float * restrict x, void * restrict y, int64_t k) {
+ quantize_row_q5_1_ref(x, y, k);
+}
+
+// reference implementation for deterministic creation of model files
+void quantize_row_q8_0_ref(const float * restrict x, block_q8_0 * restrict y, int64_t k) {
+ assert(k % QK8_0 == 0);
+ const int nb = k / QK8_0;
+
+ for (int i = 0; i < nb; i++) {
+ float amax = 0.0f; // absolute max
+
+ for (int j = 0; j < QK8_0; j++) {
+ const float v = x[i*QK8_0 + j];
+ amax = MAX(amax, fabsf(v));
+ }
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ for (int j = 0; j < QK8_0; ++j) {
+ const float x0 = x[i*QK8_0 + j]*id;
+
+ y[i].qs[j] = roundf(x0);
+ }
+ }
+}
+
+void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) {
+ assert(QK8_0 == 32);
+ assert(k % QK8_0 == 0);
+ const int nb = k / QK8_0;
+
+ block_q8_0 * restrict y = vy;
+
+#if defined(__ARM_NEON)
+ for (int i = 0; i < nb; i++) {
+ float32x4_t srcv [8];
+ float32x4_t asrcv[8];
+ float32x4_t amaxv[8];
+
+ for (int j = 0; j < 8; j++) srcv[j] = vld1q_f32(x + i*32 + 4*j);
+ for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]);
+
+ for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]);
+ for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]);
+ for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]);
+
+ const float amax = vmaxvq_f32(amaxv[0]);
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ for (int j = 0; j < 8; j++) {
+ const float32x4_t v = vmulq_n_f32(srcv[j], id);
+ const int32x4_t vi = vcvtnq_s32_f32(v);
+
+ y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);
+ y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);
+ y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);
+ y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);
+ }
+ }
+#elif defined(__wasm_simd128__)
+ for (int i = 0; i < nb; i++) {
+ v128_t srcv [8];
+ v128_t asrcv[8];
+ v128_t amaxv[8];
+
+ for (int j = 0; j < 8; j++) srcv[j] = wasm_v128_load(x + i*32 + 4*j);
+ for (int j = 0; j < 8; j++) asrcv[j] = wasm_f32x4_abs(srcv[j]);
+
+ for (int j = 0; j < 4; j++) amaxv[2*j] = wasm_f32x4_max(asrcv[2*j], asrcv[2*j+1]);
+ for (int j = 0; j < 2; j++) amaxv[4*j] = wasm_f32x4_max(amaxv[4*j], amaxv[4*j+2]);
+ for (int j = 0; j < 1; j++) amaxv[8*j] = wasm_f32x4_max(amaxv[8*j], amaxv[8*j+4]);
+
+ const float amax = MAX(MAX(wasm_f32x4_extract_lane(amaxv[0], 0),
+ wasm_f32x4_extract_lane(amaxv[0], 1)),
+ MAX(wasm_f32x4_extract_lane(amaxv[0], 2),
+ wasm_f32x4_extract_lane(amaxv[0], 3)));
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ for (int j = 0; j < 8; j++) {
+ const v128_t v = wasm_f32x4_mul(srcv[j], wasm_f32x4_splat(id));
+ const v128_t vi = wasm_i32x4_trunc_sat_f32x4(v);
+
+ y[i].qs[4*j + 0] = wasm_i32x4_extract_lane(vi, 0);
+ y[i].qs[4*j + 1] = wasm_i32x4_extract_lane(vi, 1);
+ y[i].qs[4*j + 2] = wasm_i32x4_extract_lane(vi, 2);
+ y[i].qs[4*j + 3] = wasm_i32x4_extract_lane(vi, 3);
+ }
+ }
+#elif defined(__AVX2__) || defined(__AVX__)
+ for (int i = 0; i < nb; i++) {
+ // Load elements into 4 AVX vectors
+ __m256 v0 = _mm256_loadu_ps( x );
+ __m256 v1 = _mm256_loadu_ps( x + 8 );
+ __m256 v2 = _mm256_loadu_ps( x + 16 );
+ __m256 v3 = _mm256_loadu_ps( x + 24 );
+ x += 32;
+
+ // Compute max(abs(e)) for the block
+ const __m256 signBit = _mm256_set1_ps( -0.0f );
+ __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );
+
+ __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );
+ max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );
+ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );
+ const float maxScalar = _mm_cvtss_f32( max4 );
+
+ // Quantize these floats
+ const float d = maxScalar / 127.f;
+ y[i].d = GGML_FP32_TO_FP16(d);
+ const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;
+ const __m256 mul = _mm256_set1_ps( id );
+
+ // Apply the multiplier
+ v0 = _mm256_mul_ps( v0, mul );
+ v1 = _mm256_mul_ps( v1, mul );
+ v2 = _mm256_mul_ps( v2, mul );
+ v3 = _mm256_mul_ps( v3, mul );
+
+ // Round to nearest integer
+ v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );
+ v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );
+ v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );
+ v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );
+
+ // Convert floats to integers
+ __m256i i0 = _mm256_cvtps_epi32( v0 );
+ __m256i i1 = _mm256_cvtps_epi32( v1 );
+ __m256i i2 = _mm256_cvtps_epi32( v2 );
+ __m256i i3 = _mm256_cvtps_epi32( v3 );
+
+#if defined(__AVX2__)
+ // Convert int32 to int16
+ i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15
+ i2 = _mm256_packs_epi32( i2, i3 ); // 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31
+ // Convert int16 to int8
+ i0 = _mm256_packs_epi16( i0, i2 ); // 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+
+ // We got our precious signed bytes, but the order is now wrong
+ // These AVX2 pack instructions process 16-byte pieces independently
+ // The following instruction is fixing the order
+ const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );
+ i0 = _mm256_permutevar8x32_epi32( i0, perm );
+
+ _mm256_storeu_si256((__m256i *)y[i].qs, i0);
+#else
+ // Since we don't have in AVX some necessary functions,
+ // we split the registers in half and call AVX2 analogs from SSE
+ __m128i ni0 = _mm256_castsi256_si128( i0 );
+ __m128i ni1 = _mm256_extractf128_si256( i0, 1);
+ __m128i ni2 = _mm256_castsi256_si128( i1 );
+ __m128i ni3 = _mm256_extractf128_si256( i1, 1);
+ __m128i ni4 = _mm256_castsi256_si128( i2 );
+ __m128i ni5 = _mm256_extractf128_si256( i2, 1);
+ __m128i ni6 = _mm256_castsi256_si128( i3 );
+ __m128i ni7 = _mm256_extractf128_si256( i3, 1);
+
+ // Convert int32 to int16
+ ni0 = _mm_packs_epi32( ni0, ni1 );
+ ni2 = _mm_packs_epi32( ni2, ni3 );
+ ni4 = _mm_packs_epi32( ni4, ni5 );
+ ni6 = _mm_packs_epi32( ni6, ni7 );
+ // Convert int16 to int8
+ ni0 = _mm_packs_epi16( ni0, ni2 );
+ ni4 = _mm_packs_epi16( ni4, ni6 );
+
+ _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);
+ _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);
+#endif
+ }
+#elif defined(__riscv_v_intrinsic)
+
+ size_t vl = __riscv_vsetvl_e32m4(QK8_0);
+
+ for (int i = 0; i < nb; i++) {
+ // load elements
+ vfloat32m4_t v_x = __riscv_vle32_v_f32m4(x+i*QK8_0, vl);
+
+ vfloat32m4_t vfabs = __riscv_vfabs_v_f32m4(v_x, vl);
+ vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl);
+ vfloat32m1_t vmax = __riscv_vfredmax_vs_f32m4_f32m1(vfabs, tmp, vl);
+ float amax = __riscv_vfmv_f_s_f32m1_f32(vmax);
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ vfloat32m4_t x0 = __riscv_vfmul_vf_f32m4(v_x, id, vl);
+
+ // convert to integer
+ vint16m2_t vi = __riscv_vfncvt_x_f_w_i16m2(x0, vl);
+ vint8m1_t vs = __riscv_vncvt_x_x_w_i8m1(vi, vl);
+
+ // store result
+ __riscv_vse8_v_i8m1(y[i].qs , vs, vl);
+ }
+
+#elif defined(__POWER9_VECTOR__)
+ for (int i = 0; i < nb; i++) {
+ vector float srcv [8];
+ vector float asrcv[8];
+ vector float amaxv[8];
+ vector signed int vi[8];
+
+ for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);
+ for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);
+
+ for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);
+ for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);
+ for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);
+
+ const float amax = MAX(MAX(vec_extract(amaxv[0], 0),
+ vec_extract(amaxv[0], 1)),
+ MAX(vec_extract(amaxv[0], 2),
+ vec_extract(amaxv[0], 3)));
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+ const vector float vid = vec_splats(id);
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ for (int j = 0; j < 8; j++) {
+ const vector float v = vec_round(vec_mul(srcv[j], vid));
+ vi[j] = vec_cts(v, 0);
+ }
+ vec_xst(vec_pack(vec_pack(vi[0], vi[1]), vec_pack(vi[2], vi[3])), 0, &y[i].qs[0]);
+ vec_xst(vec_pack(vec_pack(vi[4], vi[5]), vec_pack(vi[6], vi[7])), 16, &y[i].qs[0]);
+ }
+
+#elif defined(__loongarch_asx)
+ for (int i = 0; i < nb; i++) {
+ ft_union fi;
+ __m256 v0 = (__m256)__lasx_xvld( x , 0);
+ __m256 v1 = (__m256)__lasx_xvld( x , 32);
+ __m256 v2 = (__m256)__lasx_xvld( x , 64);
+ __m256 v3 = (__m256)__lasx_xvld( x , 96);
+ x += 32;
+
+ // Compute max(abs(e)) for the block
+ const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f );
+ __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 );
+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) );
+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) );
+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) );
+
+ __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs , 0) );
+ max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) );
+ __m128 tmp = max4;
+ max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vinsgr2vr_w(tmp, __lsx_vpickve2gr_w( max4, 1 ), 0 ));
+ fi.i = __lsx_vpickve2gr_w( (__m128i)max4, 0 );
+ const float max_scalar = fi.f;
+
+ // Quantize these floats
+ const float d = max_scalar / 127.f;
+ y[i].d = GGML_FP32_TO_FP16(d);
+ const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;
+ const __m256 mul = (__m256)__lasx_xvreplfr2vr_s( id );
+
+ // Apply the multiplier
+ v0 = __lasx_xvfmul_s( v0, mul );
+ v1 = __lasx_xvfmul_s( v1, mul );
+ v2 = __lasx_xvfmul_s( v2, mul );
+ v3 = __lasx_xvfmul_s( v3, mul );
+
+ // Round to nearest integer
+ __m256i i0 = __lasx_xvftintrne_w_s( v0 );
+ __m256i i1 = __lasx_xvftintrne_w_s( v1 );
+ __m256i i2 = __lasx_xvftintrne_w_s( v2 );
+ __m256i i3 = __lasx_xvftintrne_w_s( v3 );
+
+ __m128i ni0 = lasx_extracti128( i0, 0 );
+ __m128i ni1 = lasx_extracti128( i0, 1);
+ __m128i ni2 = lasx_extracti128( i1, 0);
+ __m128i ni3 = lasx_extracti128( i1, 1);
+ __m128i ni4 = lasx_extracti128( i2, 0);
+ __m128i ni5 = lasx_extracti128( i2, 1);
+ __m128i ni6 = lasx_extracti128( i3, 0);
+ __m128i ni7 = lasx_extracti128( i3, 1);
+
+ // Convert int32 to int16
+ ni0 = lsx_packs_w( ni0, ni1 );
+ ni2 = lsx_packs_w( ni2, ni3 );
+ ni4 = lsx_packs_w( ni4, ni5 );
+ ni6 = lsx_packs_w( ni6, ni7 );
+ // Convert int16 to int8
+ ni0 = lsx_packs_h( ni0, ni2 );
+ ni4 = lsx_packs_h( ni4, ni6 );
+
+ __lsx_vst(ni0, (__m128i *)(y[i].qs + 0), 0);
+ __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0);
+
+ }
+#else
+ GGML_UNUSED(nb);
+ // scalar
+ quantize_row_q8_0_ref(x, y, k);
+#endif
+}
+
+// reference implementation for deterministic creation of model files
+void quantize_row_q8_1_ref(const float * restrict x, block_q8_1 * restrict y, int64_t k) {
+ assert(QK8_1 == 32);
+ assert(k % QK8_1 == 0);
+ const int nb = k / QK8_1;
+
+ for (int i = 0; i < nb; i++) {
+ float amax = 0.0f; // absolute max
+
+ for (int j = 0; j < QK8_1; j++) {
+ const float v = x[i*QK8_1 + j];
+ amax = MAX(amax, fabsf(v));
+ }
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ int sum = 0;
+
+ for (int j = 0; j < QK8_1/2; ++j) {
+ const float v0 = x[i*QK8_1 + j]*id;
+ const float v1 = x[i*QK8_1 + QK8_1/2 + j]*id;
+
+ y[i].qs[ j] = roundf(v0);
+ y[i].qs[QK8_1/2 + j] = roundf(v1);
+
+ sum += y[i].qs[ j];
+ sum += y[i].qs[QK8_1/2 + j];
+ }
+
+ y[i].s = GGML_FP32_TO_FP16(sum*d);
+ }
+}
+
+void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k) {
+ assert(k % QK8_1 == 0);
+ const int nb = k / QK8_1;
+
+ block_q8_1 * restrict y = vy;
+
+#if defined(__ARM_NEON)
+ for (int i = 0; i < nb; i++) {
+ float32x4_t srcv [8];
+ float32x4_t asrcv[8];
+ float32x4_t amaxv[8];
+
+ for (int j = 0; j < 8; j++) srcv[j] = vld1q_f32(x + i*32 + 4*j);
+ for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]);
+
+ for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]);
+ for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]);
+ for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]);
+
+ const float amax = vmaxvq_f32(amaxv[0]);
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ int32x4_t accv = vdupq_n_s32(0);
+
+ for (int j = 0; j < 8; j++) {
+ const float32x4_t v = vmulq_n_f32(srcv[j], id);
+ const int32x4_t vi = vcvtnq_s32_f32(v);
+
+ y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);
+ y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);
+ y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);
+ y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);
+
+ accv = vaddq_s32(accv, vi);
+ }
+
+ y[i].s = GGML_FP32_TO_FP16(d * vaddvq_s32(accv));
+ }
+#elif defined(__wasm_simd128__)
+ for (int i = 0; i < nb; i++) {
+ v128_t srcv [8];
+ v128_t asrcv[8];
+ v128_t amaxv[8];
+
+ for (int j = 0; j < 8; j++) srcv[j] = wasm_v128_load(x + i*32 + 4*j);
+ for (int j = 0; j < 8; j++) asrcv[j] = wasm_f32x4_abs(srcv[j]);
+
+ for (int j = 0; j < 4; j++) amaxv[2*j] = wasm_f32x4_max(asrcv[2*j], asrcv[2*j+1]);
+ for (int j = 0; j < 2; j++) amaxv[4*j] = wasm_f32x4_max(amaxv[4*j], amaxv[4*j+2]);
+ for (int j = 0; j < 1; j++) amaxv[8*j] = wasm_f32x4_max(amaxv[8*j], amaxv[8*j+4]);
+
+ const float amax = MAX(MAX(wasm_f32x4_extract_lane(amaxv[0], 0),
+ wasm_f32x4_extract_lane(amaxv[0], 1)),
+ MAX(wasm_f32x4_extract_lane(amaxv[0], 2),
+ wasm_f32x4_extract_lane(amaxv[0], 3)));
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ v128_t accv = wasm_i32x4_splat(0);
+
+ for (int j = 0; j < 8; j++) {
+ const v128_t v = wasm_f32x4_mul(srcv[j], wasm_f32x4_splat(id));
+ const v128_t vi = wasm_i32x4_trunc_sat_f32x4(v);
+
+ y[i].qs[4*j + 0] = wasm_i32x4_extract_lane(vi, 0);
+ y[i].qs[4*j + 1] = wasm_i32x4_extract_lane(vi, 1);
+ y[i].qs[4*j + 2] = wasm_i32x4_extract_lane(vi, 2);
+ y[i].qs[4*j + 3] = wasm_i32x4_extract_lane(vi, 3);
+
+ accv = wasm_i32x4_add(accv, vi);
+ }
+
+ y[i].s = GGML_FP32_TO_FP16(
+ d * (wasm_i32x4_extract_lane(accv, 0) +
+ wasm_i32x4_extract_lane(accv, 1) +
+ wasm_i32x4_extract_lane(accv, 2) +
+ wasm_i32x4_extract_lane(accv, 3)));
+ }
+#elif defined(__AVX2__) || defined(__AVX__)
+ for (int i = 0; i < nb; i++) {
+ // Load elements into 4 AVX vectors
+ __m256 v0 = _mm256_loadu_ps( x );
+ __m256 v1 = _mm256_loadu_ps( x + 8 );
+ __m256 v2 = _mm256_loadu_ps( x + 16 );
+ __m256 v3 = _mm256_loadu_ps( x + 24 );
+ x += 32;
+
+ // Compute max(abs(e)) for the block
+ const __m256 signBit = _mm256_set1_ps( -0.0f );
+ __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );
+
+ __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );
+ max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );
+ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );
+ const float max_scalar = _mm_cvtss_f32( max4 );
+
+ // Quantize these floats
+ const float d = max_scalar / 127.f;
+ y[i].d = GGML_FP32_TO_FP16(d);
+ const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;
+ const __m256 mul = _mm256_set1_ps( id );
+
+ // Apply the multiplier
+ v0 = _mm256_mul_ps( v0, mul );
+ v1 = _mm256_mul_ps( v1, mul );
+ v2 = _mm256_mul_ps( v2, mul );
+ v3 = _mm256_mul_ps( v3, mul );
+
+ // Round to nearest integer
+ v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );
+ v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );
+ v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );
+ v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );
+
+ // Convert floats to integers
+ __m256i i0 = _mm256_cvtps_epi32( v0 );
+ __m256i i1 = _mm256_cvtps_epi32( v1 );
+ __m256i i2 = _mm256_cvtps_epi32( v2 );
+ __m256i i3 = _mm256_cvtps_epi32( v3 );
+
+#if defined(__AVX2__)
+ // Compute the sum of the quants and set y[i].s
+ y[i].s = GGML_FP32_TO_FP16(d * hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3))));
+
+ // Convert int32 to int16
+ i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15
+ i2 = _mm256_packs_epi32( i2, i3 ); // 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31
+ // Convert int16 to int8
+ i0 = _mm256_packs_epi16( i0, i2 ); // 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+
+ // We got our precious signed bytes, but the order is now wrong
+ // These AVX2 pack instructions process 16-byte pieces independently
+ // The following instruction is fixing the order
+ const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );
+ i0 = _mm256_permutevar8x32_epi32( i0, perm );
+
+ _mm256_storeu_si256((__m256i *)y[i].qs, i0);
+#else
+ // Since we don't have in AVX some necessary functions,
+ // we split the registers in half and call AVX2 analogs from SSE
+ __m128i ni0 = _mm256_castsi256_si128( i0 );
+ __m128i ni1 = _mm256_extractf128_si256( i0, 1);
+ __m128i ni2 = _mm256_castsi256_si128( i1 );
+ __m128i ni3 = _mm256_extractf128_si256( i1, 1);
+ __m128i ni4 = _mm256_castsi256_si128( i2 );
+ __m128i ni5 = _mm256_extractf128_si256( i2, 1);
+ __m128i ni6 = _mm256_castsi256_si128( i3 );
+ __m128i ni7 = _mm256_extractf128_si256( i3, 1);
+
+ // Compute the sum of the quants and set y[i].s
+ const __m128i s0 = _mm_add_epi32(_mm_add_epi32(ni0, ni1), _mm_add_epi32(ni2, ni3));
+ const __m128i s1 = _mm_add_epi32(_mm_add_epi32(ni4, ni5), _mm_add_epi32(ni6, ni7));
+ y[i].s = GGML_FP32_TO_FP16(d * hsum_i32_4(_mm_add_epi32(s0, s1)));
+
+ // Convert int32 to int16
+ ni0 = _mm_packs_epi32( ni0, ni1 );
+ ni2 = _mm_packs_epi32( ni2, ni3 );
+ ni4 = _mm_packs_epi32( ni4, ni5 );
+ ni6 = _mm_packs_epi32( ni6, ni7 );
+ // Convert int16 to int8
+ ni0 = _mm_packs_epi16( ni0, ni2 );
+ ni4 = _mm_packs_epi16( ni4, ni6 );
+
+ _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);
+ _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);
+#endif
+ }
+#elif defined(__riscv_v_intrinsic)
+
+ size_t vl = __riscv_vsetvl_e32m4(QK8_1);
+
+ for (int i = 0; i < nb; i++) {
+ // load elements
+ vfloat32m4_t v_x = __riscv_vle32_v_f32m4(x+i*QK8_1, vl);
+
+ vfloat32m4_t vfabs = __riscv_vfabs_v_f32m4(v_x, vl);
+ vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0, vl);
+ vfloat32m1_t vmax = __riscv_vfredmax_vs_f32m4_f32m1(vfabs, tmp, vl);
+ float amax = __riscv_vfmv_f_s_f32m1_f32(vmax);
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ vfloat32m4_t x0 = __riscv_vfmul_vf_f32m4(v_x, id, vl);
+
+ // convert to integer
+ vint16m2_t vi = __riscv_vfncvt_x_f_w_i16m2(x0, vl);
+ vint8m1_t vs = __riscv_vncvt_x_x_w_i8m1(vi, vl);
+
+ // store result
+ __riscv_vse8_v_i8m1(y[i].qs , vs, vl);
+
+ // compute sum for y[i].s
+ vint16m1_t tmp2 = __riscv_vmv_v_x_i16m1(0, vl);
+ vint16m1_t vwrs = __riscv_vwredsum_vs_i8m1_i16m1(vs, tmp2, vl);
+
+ // set y[i].s
+ int sum = __riscv_vmv_x_s_i16m1_i16(vwrs);
+ y[i].s = GGML_FP32_TO_FP16(sum*d);
+ }
+
+#elif defined(__POWER9_VECTOR__)
+ for (int i = 0; i < nb; i++) {
+ vector float srcv [8];
+ vector float asrcv[8];
+ vector float amaxv[8];
+ vector signed int vi[8];
+
+ for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);
+ for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);
+
+ for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);
+ for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);
+ for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);
+
+ const float amax = MAX(MAX(vec_extract(amaxv[0], 0),
+ vec_extract(amaxv[0], 1)),
+ MAX(vec_extract(amaxv[0], 2),
+ vec_extract(amaxv[0], 3)));
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+ const vector float vid = vec_splats(id);
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ vector int accv = vec_splats(0);
+
+ for (int j = 0; j < 8; j++) {
+ const vector float v = vec_round(vec_mul(srcv[j], vid));
+ vi[j] = vec_cts(v, 0);
+
+ accv = vec_add(accv, vi[j]);
+ }
+ vec_xst(vec_pack(vec_pack(vi[0], vi[1]), vec_pack(vi[2], vi[3])), 0, &y[i].qs[0]);
+ vec_xst(vec_pack(vec_pack(vi[4], vi[5]), vec_pack(vi[6], vi[7])), 16, &y[i].qs[0]);
+
+ accv = vec_add(accv, vec_sld(accv, accv, 4));
+ accv = vec_add(accv, vec_sld(accv, accv, 8));
+ y[i].s = GGML_FP32_TO_FP16(d * vec_extract(accv, 0));
+ }
+
+#elif defined(__loongarch_asx)
+ for (int i = 0; i < nb; i++) {
+ ft_union ft;
+ __m256 v0 = (__m256)__lasx_xvld( x , 0 );
+ __m256 v1 = (__m256)__lasx_xvld( x , 32 );
+ __m256 v2 = (__m256)__lasx_xvld( x , 64 );
+ __m256 v3 = (__m256)__lasx_xvld( x , 96 );
+ x += 32;
+
+ // Compute max(abs(e)) for the block
+ const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f );
+ __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 );
+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) );
+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) );
+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) );
+
+ __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs, 0) );
+ max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) );
+ __m128 tmp = max4;
+ max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vextrins_w((__m128i)tmp, (__m128i)max4, 0x10 ));
+ ft.i = __lsx_vpickve2gr_w( (__m128i)max4, 0 );
+ const float max_scalar = ft.f;
+
+ // Quantize these floats
+ const float d = max_scalar / 127.f;
+ y[i].d = GGML_FP32_TO_FP16(d);
+ const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;
+ const __m256 mul = __lasx_xvreplfr2vr_s( id );
+
+ // Apply the multiplier
+ v0 = __lasx_xvfmul_s( v0, mul );
+ v1 = __lasx_xvfmul_s( v1, mul );
+ v2 = __lasx_xvfmul_s( v2, mul );
+ v3 = __lasx_xvfmul_s( v3, mul );
+
+ // Round to nearest integer
+ __m256i i0 = __lasx_xvftintrne_w_s( v0 );
+ __m256i i1 = __lasx_xvftintrne_w_s( v1 );
+ __m256i i2 = __lasx_xvftintrne_w_s( v2 );
+ __m256i i3 = __lasx_xvftintrne_w_s( v3 );
+
+ __m128i ni0 = lasx_extracti128(i0, 0);
+ __m128i ni1 = lasx_extracti128( i0, 1);
+ __m128i ni2 = lasx_extracti128( i1, 0);
+ __m128i ni3 = lasx_extracti128( i1, 1);
+ __m128i ni4 = lasx_extracti128( i2, 0 );
+ __m128i ni5 = lasx_extracti128( i2, 1);
+ __m128i ni6 = lasx_extracti128( i3, 0);
+ __m128i ni7 = lasx_extracti128( i3, 1);
+
+ // Compute the sum of the quants and set y[i].s
+ const __m128i s0 = __lsx_vadd_w(__lsx_vadd_w(ni0, ni1), __lsx_vadd_w(ni2, ni3));
+ const __m128i s1 = __lsx_vadd_w(__lsx_vadd_w(ni4, ni5), __lsx_vadd_w(ni6, ni7));
+ y[i].s = GGML_FP32_TO_FP16(d * hsum_i32_4(__lsx_vadd_w(s0, s1)));
+
+ // Convert int32 to int16
+ ni0 = lsx_packs_w( ni0, ni1 );
+ ni2 = lsx_packs_w( ni2, ni3 );
+ ni4 = lsx_packs_w( ni4, ni5 );
+ ni6 = lsx_packs_w( ni6, ni7 );
+ // Convert int16 to int8
+ ni0 = lsx_packs_h( ni0, ni2 );
+ ni4 = lsx_packs_h( ni4, ni6 );
+
+ __lsx_vst(ni0, (__m128i *)(y[i].qs + 0), 0);
+ __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0);
+ }
+#else
+ GGML_UNUSED(nb);
+ // scalar
+ quantize_row_q8_1_ref(x, y, k);
+#endif
+}
+
+void dequantize_row_q4_0(const block_q4_0 * restrict x, float * restrict y, int64_t k) {
+ static const int qk = QK4_0;
+
+ assert(k % qk == 0);
+
+ const int nb = k / qk;
+
+ for (int i = 0; i < nb; i++) {
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+
+ for (int j = 0; j < qk/2; ++j) {
+ const int x0 = (x[i].qs[j] & 0x0F) - 8;
+ const int x1 = (x[i].qs[j] >> 4) - 8;
+
+ y[i*qk + j + 0 ] = x0*d;
+ y[i*qk + j + qk/2] = x1*d;
+ }
+ }
+}
+
+void dequantize_row_q4_1(const block_q4_1 * restrict x, float * restrict y, int64_t k) {
+ static const int qk = QK4_1;
+
+ assert(k % qk == 0);
+
+ const int nb = k / qk;
+
+ for (int i = 0; i < nb; i++) {
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ const float m = GGML_FP16_TO_FP32(x[i].m);
+
+ for (int j = 0; j < qk/2; ++j) {
+ const int x0 = (x[i].qs[j] & 0x0F);
+ const int x1 = (x[i].qs[j] >> 4);
+
+ y[i*qk + j + 0 ] = x0*d + m;
+ y[i*qk + j + qk/2] = x1*d + m;
+ }
+ }
+}
+
+void dequantize_row_q5_0(const block_q5_0 * restrict x, float * restrict y, int64_t k) {
+ static const int qk = QK5_0;
+
+ assert(k % qk == 0);
+
+ const int nb = k / qk;
+
+ for (int i = 0; i < nb; i++) {
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+
+ uint32_t qh;
+ memcpy(&qh, x[i].qh, sizeof(qh));
+
+ for (int j = 0; j < qk/2; ++j) {
+ const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;
+ const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;
+
+ const int32_t x0 = ((x[i].qs[j] & 0x0F) | xh_0) - 16;
+ const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16;
+
+ y[i*qk + j + 0 ] = x0*d;
+ y[i*qk + j + qk/2] = x1*d;
+ }
+ }
+}
+
+void dequantize_row_q5_1(const block_q5_1 * restrict x, float * restrict y, int64_t k) {
+ static const int qk = QK5_1;
+
+ assert(k % qk == 0);
+
+ const int nb = k / qk;
+
+ for (int i = 0; i < nb; i++) {
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ const float m = GGML_FP16_TO_FP32(x[i].m);
+
+ uint32_t qh;
+ memcpy(&qh, x[i].qh, sizeof(qh));
+
+ for (int j = 0; j < qk/2; ++j) {
+ const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;
+ const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;
+
+ const int x0 = (x[i].qs[j] & 0x0F) | xh_0;
+ const int x1 = (x[i].qs[j] >> 4) | xh_1;
+
+ y[i*qk + j + 0 ] = x0*d + m;
+ y[i*qk + j + qk/2] = x1*d + m;
+ }
+ }
+}
+
+void dequantize_row_q8_0(const block_q8_0 * restrict x, float * restrict y, int64_t k) {
+ static const int qk = QK8_0;
+
+ assert(k % qk == 0);
+
+ const int nb = k / qk;
+
+ for (int i = 0; i < nb; i++) {
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+
+ for (int j = 0; j < qk; ++j) {
+ y[i*qk + j] = x[i].qs[j]*d;
+ }
+ }
+}
+
+//
+// 2-6 bit quantization in super-blocks
+//
+
+//
+// ===================== Helper functions
+//
+static inline int nearest_int(float fval) {
+ assert(fval <= 4194303.f);
+ float val = fval + 12582912.f;
+ int i; memcpy(&i, &val, sizeof(int));
+ return (i & 0x007fffff) - 0x00400000;
+}
+
+static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t * restrict L, int rmse_type,
+ const float * restrict qw) {
+ float max = 0;
+ float amax = 0;
+ for (int i = 0; i < n; ++i) {
+ float ax = fabsf(x[i]);
+ if (ax > amax) { amax = ax; max = x[i]; }
+ }
+ if (amax < GROUP_MAX_EPS) { // all zero
+ for (int i = 0; i < n; ++i) {
+ L[i] = 0;
+ }
+ return 0.f;
+ }
+ float iscale = -nmax / max;
+ if (rmse_type == 0) {
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale * x[i]);
+ L[i] = nmax + MAX(-nmax, MIN(nmax-1, l));
+ }
+ return 1/iscale;
+ }
+ bool return_early = false;
+ if (rmse_type < 0) {
+ rmse_type = -rmse_type;
+ return_early = true;
+ }
+ float sumlx = 0;
+ float suml2 = 0;
+#ifdef HAVE_BUGGY_APPLE_LINKER
+ // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7
+ for (volatile int i = 0; i < n; ++i) {
+#else
+ for (int i = 0; i < n; ++i) {
+#endif
+ int l = nearest_int(iscale * x[i]);
+ l = MAX(-nmax, MIN(nmax-1, l));
+ L[i] = l + nmax;
+ float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i]));
+ sumlx += w*x[i]*l;
+ suml2 += w*l*l;
+ }
+ float scale = suml2 ? sumlx/suml2 : 0.0f;
+ if (return_early) return suml2 > 0 ? 0.5f*(scale + 1/iscale) : 1/iscale;
+ float best = scale * sumlx;
+ for (int is = -9; is <= 9; ++is) {
+ if (is == 0) {
+ continue;
+ }
+ iscale = -(nmax + 0.1f*is) / max;
+ sumlx = suml2 = 0;
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale * x[i]);
+ l = MAX(-nmax, MIN(nmax-1, l));
+ float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i]));
+ sumlx += w*x[i]*l;
+ suml2 += w*l*l;
+ }
+ if (suml2 > 0 && sumlx*sumlx > best*suml2) {
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale * x[i]);
+ L[i] = nmax + MAX(-nmax, MIN(nmax-1, l));
+ }
+ scale = sumlx/suml2; best = scale*sumlx;
+ }
+ }
+ return scale;
+}
+
+static float make_q3_quants(int n, int nmax, const float * restrict x, int8_t * restrict L, bool do_rmse) {
+ float max = 0;
+ float amax = 0;
+ for (int i = 0; i < n; ++i) {
+ float ax = fabsf(x[i]);
+ if (ax > amax) { amax = ax; max = x[i]; }
+ }
+ if (amax < GROUP_MAX_EPS) { // all zero
+ for (int i = 0; i < n; ++i) { L[i] = 0; }
+ return 0.f;
+ }
+ float iscale = -nmax / max;
+ if (do_rmse) {
+ float sumlx = 0;
+ float suml2 = 0;
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale * x[i]);
+ l = MAX(-nmax, MIN(nmax-1, l));
+ L[i] = l;
+ float w = x[i]*x[i];
+ sumlx += w*x[i]*l;
+ suml2 += w*l*l;
+ }
+ for (int itry = 0; itry < 5; ++itry) {
+ int n_changed = 0;
+ for (int i = 0; i < n; ++i) {
+ float w = x[i]*x[i];
+ float slx = sumlx - w*x[i]*L[i];
+ if (slx > 0) {
+ float sl2 = suml2 - w*L[i]*L[i];
+ int new_l = nearest_int(x[i] * sl2 / slx);
+ new_l = MAX(-nmax, MIN(nmax-1, new_l));
+ if (new_l != L[i]) {
+ slx += w*x[i]*new_l;
+ sl2 += w*new_l*new_l;
+ if (sl2 > 0 && slx*slx*suml2 > sumlx*sumlx*sl2) {
+ L[i] = new_l; sumlx = slx; suml2 = sl2;
+ ++n_changed;
+ }
+ }
+ }
+ }
+ if (!n_changed) {
+ break;
+ }
+ }
+ for (int i = 0; i < n; ++i) {
+ L[i] += nmax;
+ }
+ return sumlx / suml2;
+ }
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale * x[i]);
+ l = MAX(-nmax, MIN(nmax-1, l));
+ L[i] = l + nmax;
+ }
+ return 1/iscale;
+}
+
+static float make_qkx1_quants(int n, int nmax, const float * restrict x, uint8_t * restrict L, float * restrict the_min,
+ int ntry, float alpha) {
+ float min = x[0];
+ float max = x[0];
+ for (int i = 1; i < n; ++i) {
+ if (x[i] < min) min = x[i];
+ if (x[i] > max) max = x[i];
+ }
+ if (max == min) {
+ for (int i = 0; i < n; ++i) L[i] = 0;
+ *the_min = 0;
+ return 0.f;
+ }
+ if (min > 0) min = 0;
+ float iscale = nmax/(max - min);
+ float scale = 1/iscale;
+ for (int itry = 0; itry < ntry; ++itry) {
+ float sumlx = 0; int suml2 = 0;
+ bool did_change = false;
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale*(x[i] - min));
+ l = MAX(0, MIN(nmax, l));
+ if (l != L[i]) {
+ L[i] = l;
+ did_change = true;
+ }
+ sumlx += (x[i] - min)*l;
+ suml2 += l*l;
+ }
+ scale = sumlx/suml2;
+ float sum = 0;
+ for (int i = 0; i < n; ++i) {
+ sum += x[i] - scale*L[i];
+ }
+ min = alpha*min + (1 - alpha)*sum/n;
+ if (min > 0) min = 0;
+ iscale = 1/scale;
+ if (!did_change) break;
+ }
+ *the_min = -min;
+ return scale;
+}
+
+static float make_qkx2_quants(int n, int nmax, const float * restrict x, const float * restrict weights,
+ uint8_t * restrict L, float * restrict the_min, uint8_t * restrict Laux,
+ float rmin, float rdelta, int nstep, bool use_mad) {
+ float min = x[0];
+ float max = x[0];
+ float sum_w = weights[0];
+ float sum_x = sum_w * x[0];
+#ifdef HAVE_BUGGY_APPLE_LINKER
+ // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7
+ for (volatile int i = 1; i < n; ++i) {
+#else
+ for (int i = 1; i < n; ++i) {
+#endif
+ if (x[i] < min) min = x[i];
+ if (x[i] > max) max = x[i];
+ float w = weights[i];
+ sum_w += w;
+ sum_x += w * x[i];
+ }
+ if (min > 0) min = 0;
+ if (max == min) {
+ for (int i = 0; i < n; ++i) L[i] = 0;
+ *the_min = -min;
+ return 0.f;
+ }
+ float iscale = nmax/(max - min);
+ float scale = 1/iscale;
+ float best_mad = 0;
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale*(x[i] - min));
+ L[i] = MAX(0, MIN(nmax, l));
+ float diff = scale * L[i] + min - x[i];
+ diff = use_mad ? fabsf(diff) : diff * diff;
+ float w = weights[i];
+ best_mad += w * diff;
+ }
+ if (nstep < 1) {
+ *the_min = -min;
+ return scale;
+ }
+ for (int is = 0; is <= nstep; ++is) {
+ iscale = (rmin + rdelta*is + nmax)/(max - min);
+ float sum_l = 0, sum_l2 = 0, sum_xl = 0;
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale*(x[i] - min));
+ l = MAX(0, MIN(nmax, l));
+ Laux[i] = l;
+ float w = weights[i];
+ sum_l += w*l;
+ sum_l2 += w*l*l;
+ sum_xl += w*l*x[i];
+ }
+ float D = sum_w * sum_l2 - sum_l * sum_l;
+ if (D > 0) {
+ float this_scale = (sum_w * sum_xl - sum_x * sum_l)/D;
+ float this_min = (sum_l2 * sum_x - sum_l * sum_xl)/D;
+ if (this_min > 0) {
+ this_min = 0;
+ this_scale = sum_xl / sum_l2;
+ }
+ float mad = 0;
+ for (int i = 0; i < n; ++i) {
+ float diff = this_scale * Laux[i] + this_min - x[i];
+ diff = use_mad ? fabsf(diff) : diff * diff;
+ float w = weights[i];
+ mad += w * diff;
+ }
+ if (mad < best_mad) {
+ for (int i = 0; i < n; ++i) {
+ L[i] = Laux[i];
+ }
+ best_mad = mad;
+ scale = this_scale;
+ min = this_min;
+ }
+ }
+ }
+ *the_min = -min;
+ return scale;
+}
+
+static inline void get_scale_min_k4(int j, const uint8_t * restrict q, uint8_t * restrict d, uint8_t * restrict m) {
+ if (j < 4) {
+ *d = q[j] & 63; *m = q[j + 4] & 63;
+ } else {
+ *d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);
+ *m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);
+ }
+}
+
+//========================- 2-bit (de)-quantization
+
+void quantize_row_q2_K_ref(const float * restrict x, block_q2_K * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int nb = k / QK_K;
+
+ uint8_t L[QK_K];
+ uint8_t Laux[16];
+ float weights[16];
+ float mins[QK_K/16];
+ float scales[QK_K/16];
+
+ const float q4scale = 15.f;
+
+ for (int i = 0; i < nb; i++) {
+ float max_scale = 0; // as we are deducting the min, scales are always positive
+ float max_min = 0;
+ for (int j = 0; j < QK_K/16; ++j) {
+ for (int l = 0; l < 16; ++l) weights[l] = fabsf(x[16*j + l]);
+ scales[j] = make_qkx2_quants(16, 3, x + 16*j, weights, L + 16*j, &mins[j], Laux, -0.5f, 0.1f, 15, true);
+ float scale = scales[j];
+ if (scale > max_scale) {
+ max_scale = scale;
+ }
+ float min = mins[j];
+ if (min > max_min) {
+ max_min = min;
+ }
+ }
+
+ if (max_scale > 0) {
+ float iscale = q4scale/max_scale;
+ for (int j = 0; j < QK_K/16; ++j) {
+ int l = nearest_int(iscale*scales[j]);
+ y[i].scales[j] = l;
+ }
+ y[i].d = GGML_FP32_TO_FP16(max_scale/q4scale);
+ } else {
+ for (int j = 0; j < QK_K/16; ++j) y[i].scales[j] = 0;
+ y[i].d = GGML_FP32_TO_FP16(0.f);
+ }
+ if (max_min > 0) {
+ float iscale = q4scale/max_min;
+ for (int j = 0; j < QK_K/16; ++j) {
+ int l = nearest_int(iscale*mins[j]);
+ y[i].scales[j] |= (l << 4);
+ }
+ y[i].dmin = GGML_FP32_TO_FP16(max_min/q4scale);
+ } else {
+ y[i].dmin = GGML_FP32_TO_FP16(0.f);
+ }
+ for (int j = 0; j < QK_K/16; ++j) {
+ const float d = GGML_FP16_TO_FP32(y[i].d) * (y[i].scales[j] & 0xF);
+ if (!d) continue;
+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * (y[i].scales[j] >> 4);
+ for (int ii = 0; ii < 16; ++ii) {
+ int l = nearest_int((x[16*j + ii] + dm)/d);
+ l = MAX(0, MIN(3, l));
+ L[16*j + ii] = l;
+ }
+ }
+
+ for (int j = 0; j < QK_K; j += 128) {
+ for (int l = 0; l < 32; ++l) {
+ y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);
+ }
+ }
+
+ x += QK_K;
+ }
+}
+
+void dequantize_row_q2_K(const block_q2_K * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int nb = k / QK_K;
+
+ for (int i = 0; i < nb; i++) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ const float min = GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * q = x[i].qs;
+
+ int is = 0;
+ float dl, ml;
+ for (int n = 0; n < QK_K; n += 128) {
+ int shift = 0;
+ for (int j = 0; j < 4; ++j) {
+
+ uint8_t sc = x[i].scales[is++];
+ dl = d * (sc & 0xF); ml = min * (sc >> 4);
+ for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l] >> shift) & 3)) - ml;
+
+ sc = x[i].scales[is++];
+ dl = d * (sc & 0xF); ml = min * (sc >> 4);
+ for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3)) - ml;
+
+ shift += 2;
+ }
+ q += 32;
+ }
+ }
+}
+
+void quantize_row_q2_K(const float * restrict x, void * restrict vy, int64_t k) {
+ quantize_row_q2_K_ref(x, vy, k);
+}
+
+static float make_qkx3_quants(int n, int nmax, const float * restrict x, const float * restrict weights,
+ uint8_t * restrict L, float * restrict the_min, uint8_t * restrict Laux,
+ float rmin, float rdelta, int nstep, bool use_mad) {
+ float min = x[0];
+ float max = x[0];
+ float sum_w = weights ? weights[0] : x[0]*x[0];
+ float sum_x = sum_w * x[0];
+#ifdef HAVE_BUGGY_APPLE_LINKER
+ // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7
+ for (volatile int i = 1; i < n; ++i) {
+#else
+ for (int i = 1; i < n; ++i) {
+#endif
+ if (x[i] < min) min = x[i];
+ if (x[i] > max) max = x[i];
+ float w = weights ? weights[i] : x[i]*x[i];
+ sum_w += w;
+ sum_x += w * x[i];
+ }
+ if (min > 0) {
+ min = 0;
+ }
+ if (max <= min) {
+ memset(L, 0, n);
+ *the_min = -min;
+ return 0.f;
+ }
+ float iscale = nmax/(max - min);
+ float scale = 1/iscale;
+ float best_mad = 0;
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale*(x[i] - min));
+ L[i] = MAX(0, MIN(nmax, l));
+ float diff = scale * L[i] + min - x[i];
+ diff = use_mad ? fabsf(diff) : diff*diff;
+ float w = weights ? weights[i] : x[i]*x[i];
+ best_mad += w * diff;
+ }
+ if (nstep < 1) {
+ *the_min = -min;
+ return scale;
+ }
+ for (int is = 0; is <= nstep; ++is) {
+ iscale = (rmin + rdelta*is + nmax)/(max - min);
+ float sum_l = 0, sum_l2 = 0, sum_xl = 0;
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale*(x[i] - min));
+ l = MAX(0, MIN(nmax, l));
+ Laux[i] = l;
+ float w = weights ? weights[i] : x[i]*x[i];
+ sum_l += w*l;
+ sum_l2 += w*l*l;
+ sum_xl += w*l*x[i];
+ }
+ float D = sum_w * sum_l2 - sum_l * sum_l;
+ if (D > 0) {
+ float this_scale = (sum_w * sum_xl - sum_x * sum_l)/D;
+ float this_min = (sum_l2 * sum_x - sum_l * sum_xl)/D;
+ if (this_min > 0) {
+ this_min = 0;
+ this_scale = sum_xl / sum_l2;
+ }
+ float mad = 0;
+ for (int i = 0; i < n; ++i) {
+ float diff = this_scale * Laux[i] + this_min - x[i];
+ diff = use_mad ? fabsf(diff) : diff*diff;
+ float w = weights ? weights[i] : x[i]*x[i];
+ mad += w * diff;
+ }
+ if (mad < best_mad) {
+ for (int i = 0; i < n; ++i) {
+ L[i] = Laux[i];
+ }
+ best_mad = mad;
+ scale = this_scale;
+ min = this_min;
+ }
+ }
+ }
+ *the_min = -min;
+ return scale;
+}
+
+static float make_qp_quants(int n, int nmax, const float * restrict x, uint8_t * restrict L, const float * quant_weights) {
+ float max = 0;
+ for (int i = 0; i < n; ++i) {
+ max = MAX(max, x[i]);
+ }
+ if (!max) { // all zero
+ for (int i = 0; i < n; ++i) { L[i] = 0; }
+ return 0.f;
+ }
+ float iscale = nmax / max;
+ for (int i = 0; i < n; ++i) {
+ L[i] = nearest_int(iscale * x[i]);
+ }
+ float scale = 1/iscale;
+ float best_mse = 0;
+ for (int i = 0; i < n; ++i) {
+ float diff = x[i] - scale*L[i];
+ float w = quant_weights[i];
+ best_mse += w*diff*diff;
+ }
+ for (int is = -4; is <= 4; ++is) {
+ if (is == 0) continue;
+ float iscale_is = (0.1f*is + nmax)/max;
+ float scale_is = 1/iscale_is;
+ float mse = 0;
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale_is*x[i]);
+ l = MIN(nmax, l);
+ float diff = x[i] - scale_is*l;
+ float w = quant_weights[i];
+ mse += w*diff*diff;
+ }
+ if (mse < best_mse) {
+ best_mse = mse;
+ iscale = iscale_is;
+ }
+ }
+ float sumlx = 0;
+ float suml2 = 0;
+ for (int i = 0; i < n; ++i) {
+ int l = nearest_int(iscale * x[i]);
+ l = MIN(nmax, l);
+ L[i] = l;
+ float w = quant_weights[i];
+ sumlx += w*x[i]*l;
+ suml2 += w*l*l;
+ }
+ for (int itry = 0; itry < 5; ++itry) {
+ int n_changed = 0;
+ for (int i = 0; i < n; ++i) {
+ float w = quant_weights[i];
+ float slx = sumlx - w*x[i]*L[i];
+ float sl2 = suml2 - w*L[i]*L[i];
+ if (slx > 0 && sl2 > 0) {
+ int new_l = nearest_int(x[i] * sl2 / slx);
+ new_l = MIN(nmax, new_l);
+ if (new_l != L[i]) {
+ slx += w*x[i]*new_l;
+ sl2 += w*new_l*new_l;
+ if (slx*slx*suml2 > sumlx*sumlx*sl2) {
+ L[i] = new_l; sumlx = slx; suml2 = sl2;
+ ++n_changed;
+ }
+ }
+ }
+ }
+ if (!n_changed) {
+ break;
+ }
+ }
+ return sumlx/suml2;
+}
+
+static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restrict y, int k, const float * restrict quant_weights) {
+ GGML_ASSERT(quant_weights);
+ assert(k % QK_K == 0);
+ const int nb = k / QK_K;
+ const bool requantize = true;
+
+ uint8_t L[QK_K];
+ uint8_t Laux[16];
+ float mins[QK_K/16];
+ float scales[QK_K/16];
+ float sw[QK_K/16];
+ float weight[16];
+ uint8_t Ls[QK_K/16], Lm[QK_K/16];
+
+ for (int i = 0; i < nb; i++) {
+ memset(sw, 0, QK_K/16*sizeof(float));
+ float sumx2 = 0;
+ for (int j = 0; j < QK_K; ++j) sumx2 += x[j]*x[j];
+ float sigma2 = sumx2/QK_K;
+ for (int j = 0; j < QK_K/16; ++j) {
+ const float * restrict qw = quant_weights + QK_K * i + 16*j;
+ for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j + l]*x[16*j + l]);
+ for (int l = 0; l < QK_K/16; ++l) sw[j] += weight[l];
+ scales[j] = make_qkx3_quants(16, 3, x + 16*j, weight, L + 16*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);
+ }
+
+ float dm, mm;
+ dm = make_qp_quants(QK_K/16, 15, scales, Ls, sw);
+ mm = make_qp_quants(QK_K/16, 15, mins, Lm, sw);
+
+ y[i].d = GGML_FP32_TO_FP16(dm);
+ y[i].dmin = GGML_FP32_TO_FP16(mm);
+ dm = GGML_FP16_TO_FP32(y[i].d);
+ mm = GGML_FP16_TO_FP32(y[i].dmin);
+
+ for (int j = 0; j < QK_K/16; ++j) {
+ y[i].scales[j] = Ls[j] | (Lm[j] << 4);
+ }
+
+ if (requantize) {
+ for (int j = 0; j < QK_K/16; ++j) {
+ const float d = dm * (y[i].scales[j] & 0xF);
+ if (!d) continue;
+ const float m = mm * (y[i].scales[j] >> 4);
+ for (int ii = 0; ii < 16; ++ii) {
+ int l = nearest_int((x[16*j + ii] + m)/d);
+ l = MAX(0, MIN(3, l));
+ L[16*j + ii] = l;
+ }
+ }
+ }
+
+ for (int j = 0; j < QK_K; j += 128) {
+ for (int l = 0; l < 32; ++l) {
+ y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);
+ }
+ }
+
+ x += QK_K;
+ }
+}
+
+size_t quantize_q2_K(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ size_t row_size = ggml_row_size(GGML_TYPE_Q2_K, n_per_row);
+ if (!quant_weights) {
+ quantize_row_q2_K_ref(src, dst, (int64_t)nrow*n_per_row);
+ }
+ else {
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_q2_K_impl(src, (block_q2_K*)qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += row_size;
+ }
+ }
+ return nrow * row_size;
+}
+
+//========================= 3-bit (de)-quantization
+
+void quantize_row_q3_K_ref(const float * restrict x, block_q3_K * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int nb = k / QK_K;
+
+ int8_t L[QK_K];
+ float scales[QK_K / 16];
+
+ for (int i = 0; i < nb; i++) {
+
+ float max_scale = 0;
+ float amax = 0;
+ for (int j = 0; j < QK_K/16; ++j) {
+ scales[j] = make_q3_quants(16, 4, x + 16*j, L + 16*j, true);
+ float scale = fabsf(scales[j]);
+ if (scale > amax) {
+ amax = scale; max_scale = scales[j];
+ }
+ }
+
+ memset(y[i].scales, 0, 12);
+ if (max_scale) {
+ float iscale = -32.f/max_scale;
+ for (int j = 0; j < QK_K/16; ++j) {
+ int8_t l = nearest_int(iscale*scales[j]);
+ l = MAX(-32, MIN(31, l)) + 32;
+ if (j < 8) {
+ y[i].scales[j] = l & 0xF;
+ } else {
+ y[i].scales[j-8] |= ((l & 0xF) << 4);
+ }
+ l >>= 4;
+ y[i].scales[j%4 + 8] |= (l << (2*(j/4)));
+ }
+ y[i].d = GGML_FP32_TO_FP16(1/iscale);
+ } else {
+ y[i].d = GGML_FP32_TO_FP16(0.f);
+ }
+
+ int8_t sc;
+ for (int j = 0; j < QK_K/16; ++j) {
+ sc = j < 8 ? y[i].scales[j] & 0xF : y[i].scales[j-8] >> 4;
+ sc = (sc | (((y[i].scales[8 + j%4] >> (2*(j/4))) & 3) << 4)) - 32;
+ float d = GGML_FP16_TO_FP32(y[i].d) * sc;
+ if (!d) {
+ continue;
+ }
+ for (int ii = 0; ii < 16; ++ii) {
+ int l = nearest_int(x[16*j + ii]/d);
+ l = MAX(-4, MIN(3, l));
+ L[16*j + ii] = l + 4;
+ }
+ }
+
+ memset(y[i].hmask, 0, QK_K/8);
+ // We put the high-bit for the 1st 8 quants into bit 0, the next 8 into bit 1, etc.
+ int m = 0;
+ uint8_t hm = 1;
+ for (int j = 0; j < QK_K; ++j) {
+ if (L[j] > 3) {
+ y[i].hmask[m] |= hm;
+ L[j] -= 4;
+ }
+ if (++m == QK_K/8) {
+ m = 0; hm <<= 1;
+ }
+ }
+ for (int j = 0; j < QK_K; j += 128) {
+ for (int l = 0; l < 32; ++l) {
+ y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);
+ }
+ }
+
+ x += QK_K;
+ }
+}
+
+void dequantize_row_q3_K(const block_q3_K * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int nb = k / QK_K;
+
+ const uint32_t kmask1 = 0x03030303;
+ const uint32_t kmask2 = 0x0f0f0f0f;
+
+ uint32_t aux[4];
+ const int8_t * scales = (const int8_t*)aux;
+
+ for (int i = 0; i < nb; i++) {
+
+ const float d_all = GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q = x[i].qs;
+ const uint8_t * restrict hm = x[i].hmask;
+ uint8_t m = 1;
+
+ memcpy(aux, x[i].scales, 12);
+ uint32_t tmp = aux[2];
+ aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
+ aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
+ aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
+ aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
+
+ int is = 0;
+ float dl;
+ for (int n = 0; n < QK_K; n += 128) {
+ int shift = 0;
+ for (int j = 0; j < 4; ++j) {
+
+ dl = d_all * (scales[is++] - 32);
+ for (int l = 0; l < 16; ++l) {
+ *y++ = dl * ((int8_t)((q[l+ 0] >> shift) & 3) - ((hm[l+ 0] & m) ? 0 : 4));
+ }
+
+ dl = d_all * (scales[is++] - 32);
+ for (int l = 0; l < 16; ++l) {
+ *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3) - ((hm[l+16] & m) ? 0 : 4));
+ }
+
+ shift += 2;
+ m <<= 1;
+ }
+ q += 32;
+ }
+
+ }
+}
+
+void quantize_row_q3_K(const float * restrict x, void * restrict vy, int64_t k) {
+ quantize_row_q3_K_ref(x, vy, k);
+}
+
+static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restrict y, int64_t n_per_row, const float * restrict quant_weights) {
+ assert(n_per_row % QK_K == 0);
+ const int nb = n_per_row / QK_K;
+
+ int8_t L[QK_K];
+ float scales[QK_K / 16];
+ float weight[16];
+ float sw[QK_K / 16];
+ int8_t Ls[QK_K / 16];
+
+ for (int i = 0; i < nb; i++) {
+
+ float sumx2 = 0;
+ for (int j = 0; j < QK_K; ++j) sumx2 += x[j]*x[j];
+ float sigma2 = 2*sumx2/QK_K;
+
+ for (int j = 0; j < QK_K/16; ++j) {
+ if (quant_weights) {
+ const float * qw = quant_weights + QK_K * i + 16*j;
+ for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j+l]*x[16*j+l]);
+ } else {
+ for (int l = 0; l < 16; ++l) weight[l] = x[16*j+l]*x[16*j+l];
+ }
+ float sumw = 0;
+ for (int l = 0; l < 16; ++l) sumw += weight[l];
+ sw[j] = sumw;
+
+ scales[j] = make_qx_quants(16, 4, x + 16*j, L + 16*j, 1, weight);
+
+ }
+
+ memset(y[i].scales, 0, 12);
+
+ float d_block = make_qx_quants(QK_K/16, 32, scales, Ls, 1, sw);
+ for (int j = 0; j < QK_K/16; ++j) {
+ int l = Ls[j];
+ if (j < 8) {
+ y[i].scales[j] = l & 0xF;
+ } else {
+ y[i].scales[j-8] |= ((l & 0xF) << 4);
+ }
+ l >>= 4;
+ y[i].scales[j%4 + 8] |= (l << (2*(j/4)));
+ }
+ y[i].d = GGML_FP32_TO_FP16(d_block);
+
+ int8_t sc;
+ for (int j = 0; j < QK_K/16; ++j) {
+ sc = j < 8 ? y[i].scales[j] & 0xF : y[i].scales[j-8] >> 4;
+ sc = (sc | (((y[i].scales[8 + j%4] >> (2*(j/4))) & 3) << 4)) - 32;
+ float d = GGML_FP16_TO_FP32(y[i].d) * sc;
+ if (!d) {
+ continue;
+ }
+ for (int ii = 0; ii < 16; ++ii) {
+ int l = nearest_int(x[16*j + ii]/d);
+ l = MAX(-4, MIN(3, l));
+ L[16*j + ii] = l + 4;
+ }
+ }
+
+ memset(y[i].hmask, 0, QK_K/8);
+ // We put the high-bit for the 1st 8 quants into bit 0, the next 8 into bit 1, etc.
+ int m = 0;
+ uint8_t hm = 1;
+ for (int j = 0; j < QK_K; ++j) {
+ if (L[j] > 3) {
+ y[i].hmask[m] |= hm;
+ L[j] -= 4;
+ }
+ if (++m == QK_K/8) {
+ m = 0; hm <<= 1;
+ }
+ }
+ for (int j = 0; j < QK_K; j += 128) {
+ for (int l = 0; l < 32; ++l) {
+ y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);
+ }
+ }
+
+ x += QK_K;
+ }
+}
+
+size_t quantize_q3_K(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ size_t row_size = ggml_row_size(GGML_TYPE_Q3_K, n_per_row);
+ if (!quant_weights) {
+ quantize_row_q3_K_ref(src, dst, (int64_t)nrow*n_per_row);
+ }
+ else {
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_q3_K_impl(src, (block_q3_K*)qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += row_size;
+ }
+ }
+ return nrow * row_size;
+}
+
+// ====================== 4-bit (de)-quantization
+
+void quantize_row_q4_K_ref(const float * restrict x, block_q4_K * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int nb = k / QK_K;
+
+ uint8_t L[QK_K];
+ uint8_t Laux[32];
+ float weights[32];
+ float mins[QK_K/32];
+ float scales[QK_K/32];
+
+ for (int i = 0; i < nb; i++) {
+ float max_scale = 0; // as we are deducting the min, scales are always positive
+ float max_min = 0;
+ for (int j = 0; j < QK_K/32; ++j) {
+ //scales[j] = make_qkx1_quants(32, 15, x + 32*j, L + 32*j, &mins[j], 9, 0.5f);
+ float sum_x2 = 0;
+ for (int l = 0; l < 32; ++l) sum_x2 += x[32*j + l] * x[32*j + l];
+ float av_x = sqrtf(sum_x2/32);
+ for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);
+ scales[j] = make_qkx2_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -1.f, 0.1f, 20, false);
+ float scale = scales[j];
+ if (scale > max_scale) {
+ max_scale = scale;
+ }
+ float min = mins[j];
+ if (min > max_min) {
+ max_min = min;
+ }
+ }
+
+ float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f;
+ float inv_min = max_min > 0 ? 63.f/max_min : 0.f;
+ for (int j = 0; j < QK_K/32; ++j) {
+ uint8_t ls = nearest_int(inv_scale*scales[j]);
+ uint8_t lm = nearest_int(inv_min*mins[j]);
+ ls = MIN(63, ls);
+ lm = MIN(63, lm);
+ if (j < 4) {
+ y[i].scales[j] = ls;
+ y[i].scales[j+4] = lm;
+ } else {
+ y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);
+ y[i].scales[j-4] |= ((ls >> 4) << 6);
+ y[i].scales[j-0] |= ((lm >> 4) << 6);
+ }
+ }
+ y[i].d = GGML_FP32_TO_FP16(max_scale/63.f);
+ y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f);
+
+ uint8_t sc, m;
+ for (int j = 0; j < QK_K/32; ++j) {
+ get_scale_min_k4(j, y[i].scales, &sc, &m);
+ const float d = GGML_FP16_TO_FP32(y[i].d) * sc;
+ if (!d) continue;
+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;
+ for (int ii = 0; ii < 32; ++ii) {
+ int l = nearest_int((x[32*j + ii] + dm)/d);
+ l = MAX(0, MIN(15, l));
+ L[32*j + ii] = l;
+ }
+ }
+
+ uint8_t * q = y[i].qs;
+ for (int j = 0; j < QK_K; j += 64) {
+ for (int l = 0; l < 32; ++l) q[l] = L[j + l] | (L[j + l + 32] << 4);
+ q += 32;
+ }
+
+ x += QK_K;
+ }
+}
+
+void dequantize_row_q4_K(const block_q4_K * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int nb = k / QK_K;
+
+ for (int i = 0; i < nb; i++) {
+ const uint8_t * q = x[i].qs;
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ const float min = GGML_FP16_TO_FP32(x[i].dmin);
+
+ int is = 0;
+ uint8_t sc, m;
+ for (int j = 0; j < QK_K; j += 64) {
+ get_scale_min_k4(is + 0, x[i].scales, &sc, &m);
+ const float d1 = d * sc; const float m1 = min * m;
+ get_scale_min_k4(is + 1, x[i].scales, &sc, &m);
+ const float d2 = d * sc; const float m2 = min * m;
+ for (int l = 0; l < 32; ++l) *y++ = d1 * (q[l] & 0xF) - m1;
+ for (int l = 0; l < 32; ++l) *y++ = d2 * (q[l] >> 4) - m2;
+ q += 32; is += 2;
+ }
+ }
+}
+
+void quantize_row_q4_K(const float * restrict x, void * restrict vy, int64_t k) {
+ assert(k % QK_K == 0);
+ block_q4_K * restrict y = vy;
+ quantize_row_q4_K_ref(x, y, k);
+}
+
+static void quantize_row_q4_K_impl(const float * restrict x, block_q4_K * restrict y, int64_t n_per_row, const float * quant_weights) {
+ assert(n_per_row % QK_K == 0);
+ const int64_t nb = n_per_row / QK_K;
+
+ uint8_t L[QK_K];
+ uint8_t Laux[32];
+ uint8_t Ls[QK_K/32];
+ uint8_t Lm[QK_K/32];
+ float weights[32];
+ float sw[QK_K/32];
+ float mins[QK_K/32];
+ float scales[QK_K/32];
+
+ for (int i = 0; i < nb; i++) {
+
+ float sum_x2 = 0;
+ for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l];
+ float sigma2 = 2*sum_x2/QK_K;
+ float av_x = sqrtf(sigma2);
+
+ for (int j = 0; j < QK_K/32; ++j) {
+ if (quant_weights) {
+ const float * qw = quant_weights + QK_K*i + 32*j;
+ for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]);
+ } else {
+ for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);
+ }
+ float sumw = 0;
+ for (int l = 0; l < 32; ++l) sumw += weights[l];
+ sw[j] = sumw;
+ scales[j] = make_qkx3_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);
+ }
+
+ float d_block = make_qp_quants(QK_K/32, 63, scales, Ls, sw);
+ float m_block = make_qp_quants(QK_K/32, 63, mins, Lm, sw);
+ for (int j = 0; j < QK_K/32; ++j) {
+ uint8_t ls = Ls[j];
+ uint8_t lm = Lm[j];
+ if (j < 4) {
+ y[i].scales[j] = ls;
+ y[i].scales[j+4] = lm;
+ } else {
+ y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);
+ y[i].scales[j-4] |= ((ls >> 4) << 6);
+ y[i].scales[j-0] |= ((lm >> 4) << 6);
+ }
+ }
+ y[i].d = GGML_FP32_TO_FP16(d_block);
+ y[i].dmin = GGML_FP32_TO_FP16(m_block);
+
+ uint8_t sc, m;
+ for (int j = 0; j < QK_K/32; ++j) {
+ get_scale_min_k4(j, y[i].scales, &sc, &m);
+ const float d = GGML_FP16_TO_FP32(y[i].d) * sc;
+ if (!d) continue;
+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;
+ for (int ii = 0; ii < 32; ++ii) {
+ int l = nearest_int((x[32*j + ii] + dm)/d);
+ l = MAX(0, MIN(15, l));
+ L[32*j + ii] = l;
+ }
+ }
+ uint8_t * q = y[i].qs;
+ for (int j = 0; j < QK_K; j += 64) {
+ for (int l = 0; l < 32; ++l) q[l] = L[j + l] | (L[j + l + 32] << 4);
+ q += 32;
+ }
+
+ x += QK_K;
+
+ }
+}
+
+size_t quantize_q4_K(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ size_t row_size = ggml_row_size(GGML_TYPE_Q4_K, n_per_row);
+ if (!quant_weights) {
+ quantize_row_q4_K_ref(src, dst, (int64_t)nrow*n_per_row);
+ }
+ else {
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_q4_K_impl(src, (block_q4_K*)qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += row_size;
+ }
+ }
+ return nrow * row_size;
+}
+
+// ====================== 5-bit (de)-quantization
+
+void quantize_row_q5_K_ref(const float * restrict x, block_q5_K * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ uint8_t L[QK_K];
+ float mins[QK_K/32];
+ float scales[QK_K/32];
+ float weights[32];
+ uint8_t Laux[32];
+
+ for (int i = 0; i < nb; i++) {
+ float max_scale = 0; // as we are deducting the min, scales are always positive
+ float max_min = 0;
+ for (int j = 0; j < QK_K/32; ++j) {
+ //scales[j] = make_qkx1_quants(32, 31, x + 32*j, L + 32*j, &mins[j], 9, 0.5f);
+ float sum_x2 = 0;
+ for (int l = 0; l < 32; ++l) sum_x2 += x[32*j + l] * x[32*j + l];
+ float av_x = sqrtf(sum_x2/32);
+ for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);
+ scales[j] = make_qkx2_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.5f, 0.1f, 15, false);
+ float scale = scales[j];
+ if (scale > max_scale) {
+ max_scale = scale;
+ }
+ float min = mins[j];
+ if (min > max_min) {
+ max_min = min;
+ }
+ }
+
+ float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f;
+ float inv_min = max_min > 0 ? 63.f/max_min : 0.f;
+ for (int j = 0; j < QK_K/32; ++j) {
+ uint8_t ls = nearest_int(inv_scale*scales[j]);
+ uint8_t lm = nearest_int(inv_min*mins[j]);
+ ls = MIN(63, ls);
+ lm = MIN(63, lm);
+ if (j < 4) {
+ y[i].scales[j] = ls;
+ y[i].scales[j+4] = lm;
+ } else {
+ y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);
+ y[i].scales[j-4] |= ((ls >> 4) << 6);
+ y[i].scales[j-0] |= ((lm >> 4) << 6);
+ }
+ }
+ y[i].d = GGML_FP32_TO_FP16(max_scale/63.f);
+ y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f);
+
+ uint8_t sc, m;
+ for (int j = 0; j < QK_K/32; ++j) {
+ get_scale_min_k4(j, y[i].scales, &sc, &m);
+ const float d = GGML_FP16_TO_FP32(y[i].d) * sc;
+ if (!d) continue;
+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;
+ for (int ii = 0; ii < 32; ++ii) {
+ int l = nearest_int((x[32*j + ii] + dm)/d);
+ l = MAX(0, MIN(31, l));
+ L[32*j + ii] = l;
+ }
+ }
+
+ uint8_t * restrict qh = y[i].qh;
+ uint8_t * restrict ql = y[i].qs;
+ memset(qh, 0, QK_K/8);
+
+ uint8_t m1 = 1, m2 = 2;
+ for (int n = 0; n < QK_K; n += 64) {
+ for (int j = 0; j < 32; ++j) {
+ int l1 = L[n + j];
+ if (l1 > 15) {
+ l1 -= 16; qh[j] |= m1;
+ }
+ int l2 = L[n + j + 32];
+ if (l2 > 15) {
+ l2 -= 16; qh[j] |= m2;
+ }
+ ql[j] = l1 | (l2 << 4);
+ }
+ m1 <<= 2; m2 <<= 2;
+ ql += 32;
+ }
+
+ x += QK_K;
+ }
+}
+
+void dequantize_row_q5_K(const block_q5_K * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ for (int i = 0; i < nb; i++) {
+ const uint8_t * ql = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ const float min = GGML_FP16_TO_FP32(x[i].dmin);
+
+ int is = 0;
+ uint8_t sc, m;
+ uint8_t u1 = 1, u2 = 2;
+ for (int j = 0; j < QK_K; j += 64) {
+ get_scale_min_k4(is + 0, x[i].scales, &sc, &m);
+ const float d1 = d * sc; const float m1 = min * m;
+ get_scale_min_k4(is + 1, x[i].scales, &sc, &m);
+ const float d2 = d * sc; const float m2 = min * m;
+ for (int l = 0; l < 32; ++l) *y++ = d1 * ((ql[l] & 0xF) + (qh[l] & u1 ? 16 : 0)) - m1;
+ for (int l = 0; l < 32; ++l) *y++ = d2 * ((ql[l] >> 4) + (qh[l] & u2 ? 16 : 0)) - m2;
+ ql += 32; is += 2;
+ u1 <<= 2; u2 <<= 2;
+ }
+ }
+}
+
+void quantize_row_q5_K(const float * restrict x, void * restrict vy, int64_t k) {
+ assert(k % QK_K == 0);
+ block_q5_K * restrict y = vy;
+ quantize_row_q5_K_ref(x, y, k);
+}
+
+static void quantize_row_q5_K_impl(const float * restrict x, block_q5_K * restrict y, int64_t n_per_row, const float * quant_weights) {
+ assert(n_per_row % QK_K == 0);
+ const int64_t nb = n_per_row / QK_K;
+
+ uint8_t L[QK_K];
+ uint8_t Laux[32];
+ uint8_t Ls[QK_K/32];
+ uint8_t Lm[QK_K/32];
+ float mins[QK_K/32];
+ float scales[QK_K/32];
+ float sw[QK_K/32];
+ float weights[32];
+
+ for (int i = 0; i < nb; i++) {
+
+ float sum_x2 = 0;
+ for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l];
+ float sigma2 = 2*sum_x2/QK_K;
+ float av_x = sqrtf(sigma2);
+
+ for (int j = 0; j < QK_K/32; ++j) {
+ if (quant_weights) {
+ const float * qw = quant_weights + QK_K*i + 32*j;
+ for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]);
+ } else {
+ for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);
+ }
+ float sumw = 0;
+ for (int l = 0; l < 32; ++l) sumw += weights[l];
+ sw[j] = sumw;
+
+ scales[j] = make_qkx3_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);
+ }
+
+ float d_block = make_qp_quants(QK_K/32, 63, scales, Ls, sw);
+ float m_block = make_qp_quants(QK_K/32, 63, mins, Lm, sw);
+
+ for (int j = 0; j < QK_K/32; ++j) {
+ uint8_t ls = Ls[j];
+ uint8_t lm = Lm[j];
+ ls = MIN(63, ls);
+ lm = MIN(63, lm);
+ if (j < 4) {
+ y[i].scales[j] = ls;
+ y[i].scales[j+4] = lm;
+ } else {
+ y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);
+ y[i].scales[j-4] |= ((ls >> 4) << 6);
+ y[i].scales[j-0] |= ((lm >> 4) << 6);
+ }
+ }
+ y[i].d = GGML_FP32_TO_FP16(d_block);
+ y[i].dmin = GGML_FP32_TO_FP16(m_block);
+
+ uint8_t sc, m;
+ for (int j = 0; j < QK_K/32; ++j) {
+ get_scale_min_k4(j, y[i].scales, &sc, &m);
+ const float d = GGML_FP16_TO_FP32(y[i].d) * sc;
+ if (!d) continue;
+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;
+ for (int ii = 0; ii < 32; ++ii) {
+ int l = nearest_int((x[32*j + ii] + dm)/d);
+ l = MAX(0, MIN(31, l));
+ L[32*j + ii] = l;
+ }
+ }
+
+ uint8_t * restrict qh = y[i].qh;
+ uint8_t * restrict ql = y[i].qs;
+ memset(qh, 0, QK_K/8);
+
+ uint8_t m1 = 1, m2 = 2;
+ for (int n = 0; n < QK_K; n += 64) {
+ for (int j = 0; j < 32; ++j) {
+ int l1 = L[n + j];
+ if (l1 > 15) {
+ l1 -= 16; qh[j] |= m1;
+ }
+ int l2 = L[n + j + 32];
+ if (l2 > 15) {
+ l2 -= 16; qh[j] |= m2;
+ }
+ ql[j] = l1 | (l2 << 4);
+ }
+ m1 <<= 2; m2 <<= 2;
+ ql += 32;
+ }
+
+ x += QK_K;
+
+ }
+}
+
+size_t quantize_q5_K(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ size_t row_size = ggml_row_size(GGML_TYPE_Q5_K, n_per_row);
+ if (!quant_weights) {
+ quantize_row_q5_K_ref(src, dst, (int64_t)nrow*n_per_row);
+ }
+ else {
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_q5_K_impl(src, (block_q5_K*)qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += row_size;
+ }
+ }
+ return nrow * row_size;
+}
+
+// ====================== 6-bit (de)-quantization
+
+void quantize_row_q6_K_ref(const float * restrict x, block_q6_K * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ int8_t L[QK_K];
+ float scales[QK_K/16];
+
+ for (int i = 0; i < nb; i++) {
+
+ float max_scale = 0;
+ float max_abs_scale = 0;
+
+ for (int ib = 0; ib < QK_K/16; ++ib) {
+
+ const float scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL);
+ scales[ib] = scale;
+
+ const float abs_scale = fabsf(scale);
+ if (abs_scale > max_abs_scale) {
+ max_abs_scale = abs_scale;
+ max_scale = scale;
+ }
+
+ }
+
+ if (max_abs_scale < GROUP_MAX_EPS) {
+ memset(&y[i], 0, sizeof(block_q6_K));
+ y[i].d = GGML_FP32_TO_FP16(0.f);
+ x += QK_K;
+ continue;
+ }
+
+ float iscale = -128.f/max_scale;
+ y[i].d = GGML_FP32_TO_FP16(1/iscale);
+ for (int ib = 0; ib < QK_K/16; ++ib) {
+ y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib]));
+ }
+
+ for (int j = 0; j < QK_K/16; ++j) {
+ float d = GGML_FP16_TO_FP32(y[i].d) * y[i].scales[j];
+ if (!d) {
+ continue;
+ }
+ for (int ii = 0; ii < 16; ++ii) {
+ int l = nearest_int(x[16*j + ii]/d);
+ l = MAX(-32, MIN(31, l));
+ L[16*j + ii] = l + 32;
+ }
+ }
+
+ uint8_t * restrict ql = y[i].ql;
+ uint8_t * restrict qh = y[i].qh;
+ for (int j = 0; j < QK_K; j += 128) {
+ for (int l = 0; l < 32; ++l) {
+ const uint8_t q1 = L[j + l + 0] & 0xF;
+ const uint8_t q2 = L[j + l + 32] & 0xF;
+ const uint8_t q3 = L[j + l + 64] & 0xF;
+ const uint8_t q4 = L[j + l + 96] & 0xF;
+ ql[l+ 0] = q1 | (q3 << 4);
+ ql[l+32] = q2 | (q4 << 4);
+ qh[l] = (L[j + l] >> 4) | ((L[j + l + 32] >> 4) << 2) | ((L[j + l + 64] >> 4) << 4) | ((L[j + l + 96] >> 4) << 6);
+ }
+ ql += 64;
+ qh += 32;
+ }
+
+ x += QK_K;
+ }
+}
+
+void dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ for (int i = 0; i < nb; i++) {
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict ql = x[i].ql;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict sc = x[i].scales;
+
+ for (int n = 0; n < QK_K; n += 128) {
+ for (int l = 0; l < 32; ++l) {
+ int is = l/16;
+ const int8_t q1 = (int8_t)((ql[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;
+ const int8_t q2 = (int8_t)((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;
+ const int8_t q3 = (int8_t)((ql[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;
+ const int8_t q4 = (int8_t)((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;
+ y[l + 0] = d * sc[is + 0] * q1;
+ y[l + 32] = d * sc[is + 2] * q2;
+ y[l + 64] = d * sc[is + 4] * q3;
+ y[l + 96] = d * sc[is + 6] * q4;
+ }
+ y += 128;
+ ql += 64;
+ qh += 32;
+ sc += 8;
+ }
+ }
+}
+
+void quantize_row_q6_K(const float * restrict x, void * restrict vy, int64_t k) {
+ assert(k % QK_K == 0);
+ block_q6_K * restrict y = vy;
+ quantize_row_q6_K_ref(x, y, k);
+}
+
+static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restrict y, int64_t n_per_row, const float * quant_weights) {
+ assert(n_per_row % QK_K == 0);
+ const int64_t nb = n_per_row / QK_K;
+
+ int8_t L[QK_K];
+ float scales[QK_K/16];
+ //float weights[16];
+
+ for (int i = 0; i < nb; i++) {
+
+ //float sum_x2 = 0;
+ //for (int j = 0; j < QK_K; ++j) sum_x2 += x[j]*x[j];
+ //float sigma2 = sum_x2/QK_K;
+
+ float max_scale = 0;
+ float max_abs_scale = 0;
+
+ for (int ib = 0; ib < QK_K/16; ++ib) {
+
+ float scale;
+ if (quant_weights) {
+ const float * qw = quant_weights + QK_K*i + 16*ib;
+ //for (int j = 0; j < 16; ++j) weights[j] = qw[j] * sqrtf(sigma2 + x[16*ib + j]*x[16*ib + j]);
+ //scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, weights);
+ scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, qw);
+ } else {
+ scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL);
+ }
+ scales[ib] = scale;
+
+ const float abs_scale = fabsf(scale);
+ if (abs_scale > max_abs_scale) {
+ max_abs_scale = abs_scale;
+ max_scale = scale;
+ }
+
+ }
+
+ if (max_abs_scale < GROUP_MAX_EPS) {
+ memset(&y[i], 0, sizeof(block_q6_K));
+ y[i].d = GGML_FP32_TO_FP16(0.f);
+ x += QK_K;
+ continue;
+ }
+
+ float iscale = -128.f/max_scale;
+ y[i].d = GGML_FP32_TO_FP16(1/iscale);
+ for (int ib = 0; ib < QK_K/16; ++ib) {
+ y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib]));
+ }
+
+ for (int j = 0; j < QK_K/16; ++j) {
+ float d = GGML_FP16_TO_FP32(y[i].d) * y[i].scales[j];
+ if (!d) {
+ continue;
+ }
+ for (int ii = 0; ii < 16; ++ii) {
+ int l = nearest_int(x[16*j + ii]/d);
+ l = MAX(-32, MIN(31, l));
+ L[16*j + ii] = l + 32;
+ }
+ }
+
+ uint8_t * restrict ql = y[i].ql;
+ uint8_t * restrict qh = y[i].qh;
+ for (int j = 0; j < QK_K; j += 128) {
+ for (int l = 0; l < 32; ++l) {
+ const uint8_t q1 = L[j + l + 0] & 0xF;
+ const uint8_t q2 = L[j + l + 32] & 0xF;
+ const uint8_t q3 = L[j + l + 64] & 0xF;
+ const uint8_t q4 = L[j + l + 96] & 0xF;
+ ql[l+ 0] = q1 | (q3 << 4);
+ ql[l+32] = q2 | (q4 << 4);
+ qh[l] = (L[j + l] >> 4) | ((L[j + l + 32] >> 4) << 2) | ((L[j + l + 64] >> 4) << 4) | ((L[j + l + 96] >> 4) << 6);
+ }
+ ql += 64;
+ qh += 32;
+ }
+
+ x += QK_K;
+
+ }
+}
+
+size_t quantize_q6_K(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ size_t row_size = ggml_row_size(GGML_TYPE_Q6_K, n_per_row);
+ if (!quant_weights) {
+ quantize_row_q6_K_ref(src, dst, (int64_t)nrow*n_per_row);
+ }
+ else {
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_q6_K_impl(src, (block_q6_K*)qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += row_size;
+ }
+ }
+ return nrow * row_size;
+}
+
+static void quantize_row_q4_0_impl(const float * restrict x, block_q4_0 * restrict y, int64_t n_per_row, const float * quant_weights) {
+ static_assert(QK4_0 == 32, "QK4_0 must be 32");
+
+ if (!quant_weights) {
+ quantize_row_q4_0_ref(x, y, n_per_row);
+ return;
+ }
+
+ float weight[QK4_0];
+ int8_t L[QK4_0];
+
+ float sum_x2 = 0;
+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];
+ float sigma2 = sum_x2/n_per_row;
+
+ const int64_t nb = n_per_row/QK4_0;
+ for (int ib = 0; ib < nb; ++ib) {
+ const float * xb = x + QK4_0 * ib;
+ const float * qw = quant_weights + QK4_0 * ib;
+ for (int j = 0; j < QK4_0; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
+ float d = make_qx_quants(QK4_0, 8, xb, L, 1, weight);
+ y[ib].d = GGML_FP32_TO_FP16(d);
+ for (int j = 0; j < 16; ++j) {
+ y[ib].qs[j] = L[j] | (L[j+16] << 4);
+ }
+ }
+}
+
+size_t quantize_q4_0(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ if (!quant_weights) {
+ quantize_row_q4_0_ref(src, dst, (int64_t)nrow*n_per_row);
+ return nrow * ggml_row_size(GGML_TYPE_Q4_0, n_per_row);
+ }
+ size_t row_size = ggml_row_size(GGML_TYPE_Q4_0, n_per_row);
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_q4_0_impl(src, (block_q4_0*)qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += row_size;
+ }
+ return nrow * row_size;
+}
+
+static void quantize_row_q4_1_impl(const float * restrict x, block_q4_1 * restrict y, int64_t n_per_row, const float * quant_weights) {
+ static_assert(QK4_1 == 32, "QK4_1 must be 32");
+
+ if (!quant_weights) {
+ quantize_row_q4_1_ref(x, y, n_per_row);
+ return;
+ }
+
+ float weight[QK4_1];
+ uint8_t L[QK4_1], Laux[QK4_1];
+
+ float sum_x2 = 0;
+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];
+ float sigma2 = sum_x2/n_per_row;
+
+ const int64_t nb = n_per_row/QK4_1;
+ for (int ib = 0; ib < nb; ++ib) {
+ const float * xb = x + QK4_1 * ib;
+ const float * qw = quant_weights + QK4_1 * ib;
+ for (int j = 0; j < QK4_1; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
+ float min;
+ float d = make_qkx3_quants(QK4_1, 15, xb, weight, L, &min, Laux, -0.9f, 0.05f, 36, false);
+ y[ib].d = GGML_FP32_TO_FP16(d);
+ y[ib].m = GGML_FP32_TO_FP16(-min);
+ for (int j = 0; j < 16; ++j) {
+ y[ib].qs[j] = L[j] | (L[j+16] << 4);
+ }
+ }
+}
+
+size_t quantize_q4_1(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ if (!quant_weights) {
+ quantize_row_q4_1_ref(src, dst, (int64_t)nrow*n_per_row);
+ return nrow * ggml_row_size(GGML_TYPE_Q4_1, n_per_row);
+ }
+ size_t row_size = ggml_row_size(GGML_TYPE_Q4_1, n_per_row);
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_q4_1_impl(src, (block_q4_1*)qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += row_size;
+ }
+ return nrow * row_size;
+}
+
+static void quantize_row_q5_0_impl(const float * restrict x, block_q5_0 * restrict y, int64_t n_per_row, const float * quant_weights) {
+ static_assert(QK5_0 == 32, "QK5_0 must be 32");
+
+ if (!quant_weights) {
+ quantize_row_q5_0_ref(x, y, n_per_row);
+ return;
+ }
+
+ float weight[QK5_0];
+ int8_t L[QK5_0];
+
+ float sum_x2 = 0;
+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];
+ float sigma2 = sum_x2/n_per_row;
+
+ const int64_t nb = n_per_row/QK5_0;
+ for (int ib = 0; ib < nb; ++ib) {
+ const float * xb = x + QK5_0 * ib;
+ const float * qw = quant_weights + QK5_0 * ib;
+ for (int j = 0; j < QK5_0; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
+ float d = make_qx_quants(QK5_0, 16, xb, L, 1, weight);
+ y[ib].d = GGML_FP32_TO_FP16(d);
+
+ uint32_t qh = 0;
+
+ for (int j = 0; j < 16; ++j) {
+ const uint8_t xi0 = L[j];
+ const uint8_t xi1 = L[j+16];
+ y[ib].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);
+
+ // get the 5-th bit and store it in qh at the right position
+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);
+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);
+ }
+
+ memcpy(&y[ib].qh, &qh, sizeof(qh));
+ }
+}
+
+size_t quantize_q5_0(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ if (!quant_weights) {
+ quantize_row_q5_0_ref(src, dst, (int64_t)nrow*n_per_row);
+ return nrow * ggml_row_size(GGML_TYPE_Q5_0, n_per_row);
+ }
+ size_t row_size = ggml_row_size(GGML_TYPE_Q5_0, n_per_row);
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_q5_0_impl(src, (block_q5_0*)qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += row_size;
+ }
+ return nrow * row_size;
+}
+
+static void quantize_row_q5_1_impl(const float * restrict x, block_q5_1 * restrict y, int64_t n_per_row, const float * quant_weights) {
+ static_assert(QK5_1 == 32, "QK5_1 must be 32");
+
+ if (!quant_weights) {
+ quantize_row_q5_1_ref(x, y, n_per_row);
+ return;
+ }
+
+ float weight[QK5_1];
+ uint8_t L[QK5_1], Laux[QK5_1];
+
+ float sum_x2 = 0;
+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];
+ float sigma2 = sum_x2/n_per_row;
+
+ const int64_t nb = n_per_row/QK5_1;
+ for (int ib = 0; ib < nb; ++ib) {
+ const float * xb = x + QK5_1 * ib;
+ const float * qw = quant_weights + QK5_1 * ib;
+ for (int j = 0; j < QK5_1; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
+ float min;
+ float d = make_qkx3_quants(QK5_1, 31, xb, weight, L, &min, Laux, -0.9f, 0.05f, 36, false);
+ y[ib].d = GGML_FP32_TO_FP16(d);
+ y[ib].m = GGML_FP32_TO_FP16(-min);
+
+ uint32_t qh = 0;
+ for (int j = 0; j < 16; ++j) {
+ const uint8_t xi0 = L[j];
+ const uint8_t xi1 = L[j+16];
+ y[ib].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);
+ // get the 5-th bit and store it in qh at the right position
+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);
+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);
+ }
+ memcpy(&y[ib].qh, &qh, sizeof(qh));
+ }
+}
+
+size_t quantize_q5_1(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ if (!quant_weights) {
+ quantize_row_q5_1_ref(src, dst, (int64_t)nrow*n_per_row);
+ return nrow * ggml_row_size(GGML_TYPE_Q5_1, n_per_row);
+ }
+ size_t row_size = ggml_row_size(GGML_TYPE_Q5_1, n_per_row);
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_q5_1_impl(src, (block_q5_1*)qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += row_size;
+ }
+ return nrow * row_size;
+}
+
+size_t quantize_q8_0(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ (void)quant_weights; // not used
+ const size_t row_size = ggml_row_size(GGML_TYPE_Q8_0, n_per_row);
+ quantize_row_q8_0_ref(src, dst, (int64_t)nrow*n_per_row);
+ return nrow * row_size;
+}
+
+// ====================== "True" 2-bit (de)-quantization
+
+void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ uint32_t aux32[2];
+ const uint8_t * aux8 = (const uint8_t *)aux32;
+
+ for (int i = 0; i < nb; i++) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+
+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
+ memcpy(aux32, x[i].qs + 4*ib32, 2*sizeof(uint32_t));
+ const float db = d * (0.5f + (aux32[1] >> 28)) * 0.25f;
+ for (int l = 0; l < 4; ++l) {
+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);
+ const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];
+ for (int j = 0; j < 8; ++j) {
+ y[j] = db * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
+ }
+ y += 8;
+ }
+ }
+ }
+}
+
+// ====================== 2.3125 bpw (de)-quantization
+
+void dequantize_row_iq2_xs(const block_iq2_xs * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ float db[2];
+
+ for (int i = 0; i < nb; i++) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+
+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
+ db[0] = d * (0.5f + (x[i].scales[ib32] & 0xf)) * 0.25f;
+ db[1] = d * (0.5f + (x[i].scales[ib32] >> 4)) * 0.25f;
+ for (int l = 0; l < 4; ++l) {
+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (x[i].qs[4*ib32 + l] & 511));
+ const uint8_t signs = ksigns_iq2xs[x[i].qs[4*ib32 + l] >> 9];
+ for (int j = 0; j < 8; ++j) {
+ y[j] = db[l/2] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
+ }
+ y += 8;
+ }
+ }
+ }
+}
+
+// ====================== 2.5625 bpw (de)-quantization
+
+void dequantize_row_iq2_s(const block_iq2_s * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ float db[2];
+
+ for (int i = 0; i < nb; i++) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+ const uint8_t * signs = qs + QK_K/8;
+
+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
+ db[0] = d * (0.5f + (x[i].scales[ib32] & 0xf)) * 0.25f;
+ db[1] = d * (0.5f + (x[i].scales[ib32] >> 4)) * 0.25f;
+ for (int l = 0; l < 4; ++l) {
+ const float dl = db[l/2];
+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));
+ for (int j = 0; j < 8; ++j) {
+ y[j] = dl * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1.f : 1.f);
+ }
+ y += 8;
+ }
+ qs += 4;
+ signs += 4;
+ }
+ }
+}
+
+// ====================== 3.0625 bpw (de)-quantization
+
+void dequantize_row_iq3_xxs(const block_iq3_xxs * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ uint32_t aux32;
+
+ for (int i = 0; i < nb; i++) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * scales_and_signs = qs + QK_K/4;
+
+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
+ memcpy(&aux32, scales_and_signs + 4*ib32, sizeof(uint32_t));
+ const float db = d * (0.5f + (aux32 >> 28)) * 0.5f;
+ for (int l = 0; l < 4; ++l) {
+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127];
+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + qs[2*l+0]);
+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + qs[2*l+1]);
+ for (int j = 0; j < 4; ++j) {
+ y[j+0] = db * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
+ y[j+4] = db * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
+ }
+ y += 8;
+ }
+ qs += 8;
+ }
+ }
+}
+
+// ====================== 3.3125 bpw (de)-quantization
+
+void dequantize_row_iq3_s(const block_iq3_s * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ for (int i = 0; i < nb; i++) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+ const uint8_t * signs = x[i].signs;
+
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const float db1 = d * (1 + 2*(x[i].scales[ib32/2] & 0xf));
+ const float db2 = d * (1 + 2*(x[i].scales[ib32/2] >> 4));
+ for (int l = 0; l < 4; ++l) {
+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256)));
+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256)));
+ for (int j = 0; j < 4; ++j) {
+ y[j+0] = db1 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f);
+ y[j+4] = db1 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f);
+ }
+ y += 8;
+ }
+ qs += 8;
+ signs += 4;
+ for (int l = 0; l < 4; ++l) {
+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[1] << (8-2*l)) & 256)));
+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[1] << (7-2*l)) & 256)));
+ for (int j = 0; j < 4; ++j) {
+ y[j+0] = db2 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f);
+ y[j+4] = db2 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f);
+ }
+ y += 8;
+ }
+ qh += 2;
+ qs += 8;
+ signs += 4;
+ }
+ }
+}
+
+// ====================== 1.5625 bpw (de)-quantization
+
+void dequantize_row_iq1_s(const block_iq1_s * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ for (int i = 0; i < nb; i++) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ const uint8_t * qs = x[i].qs;
+ const uint16_t * qh = x[i].qh;
+
+ for (int ib = 0; ib < QK_K/32; ++ib) {
+ const float dl = d * (2*((qh[ib] >> 12) & 7) + 1);
+ const float delta = qh[ib] & 0x8000 ? -IQ1S_DELTA : IQ1S_DELTA;
+ for (int l = 0; l < 4; ++l) {
+ const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8)));
+ for (int j = 0; j < 8; ++j) {
+ y[j] = dl * (grid[j] + delta);
+ }
+ y += 8;
+ }
+ qs += 4;
+ }
+ }
+}
+
+void dequantize_row_iq1_m(const block_iq1_m * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ float delta[4];
+ uint16_t idx[4];
+
+ iq1m_scale_t scale;
+
+ for (int i = 0; i < nb; i++) {
+
+ const uint16_t * sc = (const uint16_t *)x[i].scales;
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+ const float d = GGML_FP16_TO_FP32(scale.f16);
+
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+
+ for (int ib = 0; ib < QK_K/32; ++ib) {
+ const float dl1 = d * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1);
+ const float dl2 = d * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1);
+
+ idx[0] = qs[0] | ((qh[0] << 8) & 0x700);
+ idx[1] = qs[1] | ((qh[0] << 4) & 0x700);
+ idx[2] = qs[2] | ((qh[1] << 8) & 0x700);
+ idx[3] = qs[3] | ((qh[1] << 4) & 0x700);
+ delta[0] = qh[0] & 0x08 ? -IQ1S_DELTA : IQ1S_DELTA;
+ delta[1] = qh[0] & 0x80 ? -IQ1S_DELTA : IQ1S_DELTA;
+ delta[2] = qh[1] & 0x08 ? -IQ1S_DELTA : IQ1S_DELTA;
+ delta[3] = qh[1] & 0x80 ? -IQ1S_DELTA : IQ1S_DELTA;
+ for (int l = 0; l < 2; ++l) {
+ const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]);
+ for (int j = 0; j < 8; ++j) {
+ y[j] = dl1 * (grid[j] + delta[l]);
+ }
+ y += 8;
+ }
+ for (int l = 2; l < 4; ++l) {
+ const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]);
+ for (int j = 0; j < 8; ++j) {
+ y[j] = dl2 * (grid[j] + delta[l]);
+ }
+ y += 8;
+ }
+ qs += 4;
+ qh += 2;
+ }
+ }
+}
+
+static const int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
+
+void dequantize_row_iq4_nl(const block_iq4_nl * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK4_NL == 0);
+ const int64_t nb = k / QK4_NL;
+
+ for (int i = 0; i < nb; i++) {
+
+ const uint8_t * qs = x[i].qs;
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ for (int j = 0; j < QK4_NL/2; ++j) {
+ y[j+ 0] = d * kvalues_iq4nl[qs[j] & 0xf];
+ y[j+QK4_NL/2] = d * kvalues_iq4nl[qs[j] >> 4];
+ }
+ y += QK4_NL;
+ qs += QK4_NL/2;
+ }
+}
+
+void dequantize_row_iq4_xs(const block_iq4_xs * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ for (int i = 0; i < nb; i++) {
+
+ const uint8_t * qs = x[i].qs;
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+
+ for (int ib = 0; ib < QK_K/32; ++ib) {
+ const int ls = ((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4);
+ const float dl = d * (ls - 32);
+ for (int j = 0; j < 16; ++j) {
+ y[j+ 0] = dl * kvalues_iq4nl[qs[j] & 0xf];
+ y[j+16] = dl * kvalues_iq4nl[qs[j] >> 4];
+ }
+ y += 32;
+ qs += 16;
+ }
+ }
+}
+
+//===================================== Q8_K ==============================================
+
+void quantize_row_q8_K_ref(const float * restrict x, block_q8_K * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ for (int i = 0; i < nb; i++) {
+
+ float max = 0;
+ float amax = 0;
+ for (int j = 0; j < QK_K; ++j) {
+ float ax = fabsf(x[j]);
+ if (ax > amax) {
+ amax = ax; max = x[j];
+ }
+ }
+ if (!amax) {
+ y[i].d = 0;
+ memset(y[i].qs, 0, QK_K);
+ x += QK_K;
+ continue;
+ }
+ //const float iscale = -128.f/max;
+ // We need this change for IQ2_XXS, else the AVX implementation becomes very awkward
+ const float iscale = -127.f/max;
+ for (int j = 0; j < QK_K; ++j) {
+ int v = nearest_int(iscale*x[j]);
+ y[i].qs[j] = MIN(127, v);
+ }
+ for (int j = 0; j < QK_K/16; ++j) {
+ int sum = 0;
+ for (int ii = 0; ii < 16; ++ii) {
+ sum += y[i].qs[j*16 + ii];
+ }
+ y[i].bsums[j] = sum;
+ }
+ y[i].d = 1/iscale;
+ x += QK_K;
+ }
+}
+
+void dequantize_row_q8_K(const block_q8_K * restrict x, float * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int64_t nb = k / QK_K;
+
+ for (int i = 0; i < nb; i++) {
+ for (int j = 0; j < QK_K; ++j) {
+ *y++ = x[i].d * x[i].qs[j];
+ }
+ }
+}
+
+void quantize_row_q8_K(const float * restrict x, void * restrict y, int64_t k) {
+ quantize_row_q8_K_ref(x, y, k);
+}
+
+//===================================== Dot products =================================
+
+//
+// Helper functions
+//
+#if __AVX__ || __AVX2__ || __AVX512F__
+
+// shuffles to pick the required scales in dot products
+static inline __m256i get_scale_shuffle_q3k(int i) {
+ static const uint8_t k_shuffle[128] = {
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
+ 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,
+ 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,
+ 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13, 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,
+ };
+ return _mm256_loadu_si256((const __m256i*)k_shuffle + i);
+}
+static inline __m256i get_scale_shuffle_k4(int i) {
+ static const uint8_t k_shuffle[256] = {
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
+ 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,
+ 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,
+ 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,
+ 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,
+ 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,
+ 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15
+ };
+ return _mm256_loadu_si256((const __m256i*)k_shuffle + i);
+}
+static inline __m128i get_scale_shuffle(int i) {
+ static const uint8_t k_shuffle[128] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,
+ 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,
+ 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,
+ 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
+ 10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11,
+ 12,12,12,12,12,12,12,12, 13,13,13,13,13,13,13,13,
+ 14,14,14,14,14,14,14,14, 15,15,15,15,15,15,15,15
+ };
+ return _mm_loadu_si128((const __m128i*)k_shuffle + i);
+}
+#elif defined(__loongarch_asx)
+// shuffles to pick the required scales in dot products
+static inline __m256i get_scale_shuffle_q3k(int i) {
+ static const uint8_t k_shuffle[128] = {
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
+ 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,
+ 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,
+ 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13, 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,
+ };
+ return __lasx_xvld((const __m256i*)k_shuffle + i, 0);
+}
+static inline __m256i get_scale_shuffle_k4(int i) {
+ static const uint8_t k_shuffle[256] = {
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
+ 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,
+ 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,
+ 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,
+ 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,
+ 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,
+ 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15
+ };
+ return __lasx_xvld((const __m256i*)k_shuffle + i, 0);
+}
+static inline __m128i get_scale_shuffle(int i) {
+ static const uint8_t k_shuffle[128] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,
+ 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,
+ 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,
+ 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
+ 10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11,
+ 12,12,12,12,12,12,12,12, 13,13,13,13,13,13,13,13,
+ 14,14,14,14,14,14,14,14, 15,15,15,15,15,15,15,15
+ };
+ return __lsx_vld((const __m128i*)k_shuffle + i, 0);
+}
+#endif
+
+void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ const int qk = QK8_0;
+ const int nb = n / qk;
+
+ assert(n % qk == 0);
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ assert((nrc == 2) || (nrc == 1));
+#else
+ assert(nrc == 1);
+#endif
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q4_0 * restrict x = vx;
+ const block_q8_0 * restrict y = vy;
+
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ if (nrc == 2) {
+ const block_q4_0 * restrict vx0 = vx;
+ const block_q4_0 * restrict vx1 = (const block_q4_0 *) ((const uint8_t*)vx + bx);
+ const block_q8_0 * restrict vy0 = vy;
+ const block_q8_0 * restrict vy1 = (const block_q8_0 *) ((const uint8_t*)vy + by);
+
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+
+ for (int i = 0; i < nb; i++) {
+ const block_q4_0 * restrict b_x0 = &vx0[i];
+ const block_q4_0 * restrict b_x1 = &vx1[i];
+ const block_q8_0 * restrict b_y0 = &vy0[i];
+ const block_q8_0 * restrict b_y1 = &vy1[i];
+
+ const uint8x16_t m4b = vdupq_n_u8(0x0F);
+ const int8x16_t s8b = vdupq_n_s8(0x8);
+
+ const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);
+ const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);
+
+ // 4-bit -> 8-bit
+ const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
+ const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
+ const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
+ const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+
+ // sub 8
+ const int8x16_t x0_l = vsubq_s8(v0_0l, s8b);
+ const int8x16_t x0_h = vsubq_s8(v0_0h, s8b);
+ const int8x16_t x1_l = vsubq_s8(v0_1l, s8b);
+ const int8x16_t x1_h = vsubq_s8(v0_1h, s8b);
+
+ // load y
+ const int8x16_t y0_l = vld1q_s8(b_y0->qs);
+ const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);
+ const int8x16_t y1_l = vld1q_s8(b_y1->qs);
+ const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
+
+ float32_t _scale[4] = { GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),
+ GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),
+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),
+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};
+
+ float32x4_t scale = vld1q_f32(_scale);
+
+ int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
+ int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
+
+ int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
+ int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
+
+ int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+ int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+
+ int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+ int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+
+ sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
+ l1, r1)), l2, r2)), l3, r3))), scale);
+ }
+ float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
+ float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
+
+ vst1_f32(s, vget_low_f32(sumv2));
+ vst1_f32(s + bs, vget_high_f32(sumv2));
+ return;
+ }
+#endif
+
+ int ib = 0;
+ float sumf = 0;
+
+#if defined(__ARM_FEATURE_SVE)
+ if (ggml_sve_cnt_b == QK8_0) {
+ const svbool_t ptrueh = svptrue_pat_b8(SV_VL16);
+ const svbool_t ptruel = svnot_b_z(svptrue_b8(), ptrueh);
+
+ svfloat32_t sumv0 = svdup_n_f32(0.0f);
+ svfloat32_t sumv1 = svdup_n_f32(0.0f);
+
+ for (; ib + 1 < nb; ib += 2) {
+ const block_q4_0 * restrict x0 = &x[ib + 0];
+ const block_q4_0 * restrict x1 = &x[ib + 1];
+ const block_q8_0 * restrict y0 = &y[ib + 0];
+ const block_q8_0 * restrict y1 = &y[ib + 1];
+
+ // load x
+ const svuint8_t qx0r = svld1rq_u8(svptrue_b8(), x0->qs);
+ const svuint8_t qx1r = svld1rq_u8(svptrue_b8(), x1->qs);
+
+ // 4-bit -> 8-bit
+ const svint8_t qx0 = svreinterpret_s8_u8(svlsr_n_u8_m(ptruel, svand_n_u8_m(ptrueh, qx0r, 0x0F), 0x04));
+ const svint8_t qx1 = svreinterpret_s8_u8(svlsr_n_u8_m(ptruel, svand_n_u8_m(ptrueh, qx1r, 0x0F), 0x04));
+
+ // sub 8
+ const svint8_t qx0s = svsub_n_s8_x(svptrue_b8(), qx0, 8);
+ const svint8_t qx1s = svsub_n_s8_x(svptrue_b8(), qx1, 8);
+
+ // load y
+ const svint8_t qy0 = svld1_s8(svptrue_b8(), y0->qs);
+ const svint8_t qy1 = svld1_s8(svptrue_b8(), y1->qs);
+
+ // dot product
+ sumv0 = svmla_n_f32_x(svptrue_b32(), sumv0, svcvt_f32_s32_x(svptrue_b32(), svdot_s32(svdup_n_s32(0), qx0s, qy0)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+ sumv1 = svmla_n_f32_x(svptrue_b32(), sumv1, svcvt_f32_s32_x(svptrue_b32(), svdot_s32(svdup_n_s32(0), qx1s, qy1)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
+ }
+
+ sumf = svaddv_f32(svptrue_b32(), svadd_f32_x(svptrue_b32(), sumv0, sumv1));
+ }
+#elif defined(__ARM_NEON)
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t sumv1 = vdupq_n_f32(0.0f);
+
+ for (; ib + 1 < nb; ib += 2) {
+ const block_q4_0 * restrict x0 = &x[ib + 0];
+ const block_q4_0 * restrict x1 = &x[ib + 1];
+ const block_q8_0 * restrict y0 = &y[ib + 0];
+ const block_q8_0 * restrict y1 = &y[ib + 1];
+
+ const uint8x16_t m4b = vdupq_n_u8(0x0F);
+ const int8x16_t s8b = vdupq_n_s8(0x8);
+
+ const uint8x16_t v0_0 = vld1q_u8(x0->qs);
+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);
+
+ // 4-bit -> 8-bit
+ const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
+ const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
+ const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
+ const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+
+ // sub 8
+ const int8x16_t v0_0ls = vsubq_s8(v0_0l, s8b);
+ const int8x16_t v0_0hs = vsubq_s8(v0_0h, s8b);
+ const int8x16_t v0_1ls = vsubq_s8(v0_1l, s8b);
+ const int8x16_t v0_1hs = vsubq_s8(v0_1h, s8b);
+
+ // load y
+ const int8x16_t v1_0l = vld1q_s8(y0->qs);
+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);
+ const int8x16_t v1_1l = vld1q_s8(y1->qs);
+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);
+
+ // dot product into int32x4_t
+ const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0l), v0_0hs, v1_0h);
+ const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1l), v0_1hs, v1_1h);
+
+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
+ }
+
+ sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);
+#elif defined(__AVX2__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ /* Compute combined scale for the block */
+ const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d) );
+
+ __m256i qx = bytes_from_nibbles_32(x[ib].qs);
+
+ // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.
+ const __m256i off = _mm256_set1_epi8( 8 );
+ qx = _mm256_sub_epi8( qx, off );
+
+ __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);
+
+ const __m256 q = mul_sum_i8_pairs_float(qx, qy);
+
+ /* Multiply q with scale and accumulate */
+ acc = _mm256_fmadd_ps( d, q, acc );
+ }
+
+ sumf = hsum_float_8(acc);
+#elif defined(__AVX__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ // Compute combined scale for the block
+ const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d) );
+
+ const __m128i lowMask = _mm_set1_epi8(0xF);
+ const __m128i off = _mm_set1_epi8(8);
+
+ const __m128i tmp = _mm_loadu_si128((const __m128i *)x[ib].qs);
+
+ __m128i bx_0 = _mm_and_si128(lowMask, tmp);
+ __m128i by_0 = _mm_loadu_si128((const __m128i *)y[ib].qs);
+ bx_0 = _mm_sub_epi8(bx_0, off);
+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);
+
+ bx_0 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp, 4));
+ by_0 = _mm_loadu_si128((const __m128i *)(y[ib].qs + 16));
+ bx_0 = _mm_sub_epi8(bx_0, off);
+ const __m128i i32_1 = mul_sum_i8_pairs(bx_0, by_0);
+
+ // Convert int32_t to float
+ __m256 p = _mm256_cvtepi32_ps(MM256_SET_M128I(i32_0, i32_1));
+
+ // Apply the scale, and accumulate
+ acc = _mm256_add_ps(_mm256_mul_ps( d, p ), acc);
+ }
+
+ sumf = hsum_float_8(acc);
+#elif defined(__SSSE3__)
+ // set constants
+ const __m128i lowMask = _mm_set1_epi8(0xF);
+ const __m128i off = _mm_set1_epi8(8);
+
+ // Initialize accumulator with zeros
+ __m128 acc_0 = _mm_setzero_ps();
+ __m128 acc_1 = _mm_setzero_ps();
+ __m128 acc_2 = _mm_setzero_ps();
+ __m128 acc_3 = _mm_setzero_ps();
+
+ for (; ib + 1 < nb; ib += 2) {
+ _mm_prefetch(&x[ib] + sizeof(block_q4_0), _MM_HINT_T0);
+ _mm_prefetch(&y[ib] + sizeof(block_q8_0), _MM_HINT_T0);
+
+ // Compute combined scale for the block 0 and 1
+ const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d) );
+
+ const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[ib].qs);
+
+ __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);
+ __m128i by_0 = _mm_loadu_si128((const __m128i *)y[ib].qs);
+ bx_0 = _mm_sub_epi8(bx_0, off);
+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);
+
+ __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));
+ __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[ib].qs + 16));
+ bx_1 = _mm_sub_epi8(bx_1, off);
+ const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);
+
+ _mm_prefetch(&x[ib] + 2 * sizeof(block_q4_0), _MM_HINT_T0);
+ _mm_prefetch(&y[ib] + 2 * sizeof(block_q8_0), _MM_HINT_T0);
+
+ // Compute combined scale for the block 2 and 3
+ const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[ib + 1].d) * GGML_FP16_TO_FP32(y[ib + 1].d) );
+
+ const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);
+
+ __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);
+ __m128i by_2 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);
+ bx_2 = _mm_sub_epi8(bx_2, off);
+ const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);
+
+ __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));
+ __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[ib + 1].qs + 16));
+ bx_3 = _mm_sub_epi8(bx_3, off);
+ const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);
+
+ // Convert int32_t to float
+ __m128 p0 = _mm_cvtepi32_ps(i32_0);
+ __m128 p1 = _mm_cvtepi32_ps(i32_1);
+ __m128 p2 = _mm_cvtepi32_ps(i32_2);
+ __m128 p3 = _mm_cvtepi32_ps(i32_3);
+
+ // Apply the scale
+ __m128 p0_d = _mm_mul_ps( d_0_1, p0 );
+ __m128 p1_d = _mm_mul_ps( d_0_1, p1 );
+ __m128 p2_d = _mm_mul_ps( d_2_3, p2 );
+ __m128 p3_d = _mm_mul_ps( d_2_3, p3 );
+
+ // Acummulate
+ acc_0 = _mm_add_ps(p0_d, acc_0);
+ acc_1 = _mm_add_ps(p1_d, acc_1);
+ acc_2 = _mm_add_ps(p2_d, acc_2);
+ acc_3 = _mm_add_ps(p3_d, acc_3);
+ }
+
+ sumf = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);
+#elif defined(__riscv_v_intrinsic)
+ size_t vl = __riscv_vsetvl_e8m1(qk/2);
+
+ for (; ib < nb; ++ib) {
+ // load elements
+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[ib].qs, vl);
+
+ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[ib].qs, vl);
+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[ib].qs+16, vl);
+
+ // mask and store lower part of x, and then upper part
+ vuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);
+ vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);
+
+ vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
+ vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
+
+ // subtract offset
+ vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 8, vl);
+ vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 8, vl);
+
+ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);
+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);
+
+ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);
+
+ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);
+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);
+
+ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);
+
+ sumf += sumi*GGML_FP16_TO_FP32(x[ib].d)*GGML_FP16_TO_FP32(y[ib].d);
+ }
+
+#elif defined(__POWER9_VECTOR__)
+ const vector signed char lowMask = vec_splats((signed char)0xF);
+ const vector signed int v0 = vec_splats((int32_t)0);
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
+ const vector signed char v8 = vec_splats((signed char)0x8);
+
+ vector float vsumf0 = vec_splats(0.0f);
+
+#pragma GCC unroll 8
+ for (; ib < nb; ++ib) {
+ __builtin_prefetch(x[ib].qs, 0, 1);
+ __builtin_prefetch(y[ib].qs, 0, 1);
+
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ib].d));
+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[ib].d));
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);
+ vector signed char q8y0 = vec_xl( 0, y[ib].qs);
+ vector signed char q8y1 = vec_xl(16, y[ib].qs);
+
+ vector signed char q4x0 = vec_and(qxs, lowMask);
+ vector signed char q4x1 = vec_sr(qxs, v4);
+
+ q4x0 = vec_sub(q4x0, v8);
+ q4x1 = vec_sub(q4x1, v8);
+
+ vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));
+ vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));
+
+ vector signed int vsumi0 = v0;
+
+ vsumi0 = vec_sum4s(qv0, vsumi0);
+ vsumi0 = vec_sum4s(qv1, vsumi0);
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ }
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ sumf = vec_extract(vsumf0, 0);
+
+#elif defined(__loongarch_asx)
+ // Initialize accumulator with zeros
+ __m256 acc = (__m256)__lasx_xvldi(0);
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ /* Compute combined scale for the block */
+ const __m256 d = __lasx_xvreplfr2vr_s( GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d) );
+
+ __m256i qx = bytes_from_nibbles_32(x[ib].qs);
+
+ // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.
+ const __m256i off = __lasx_xvreplgr2vr_b( 8 );
+ qx = __lasx_xvsub_b( qx, off );
+
+ __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);
+
+ const __m256 q = mul_sum_i8_pairs_float(qx, qy);
+
+ /* Multiply q with scale and accumulate */
+ acc = __lasx_xvfmadd_s( d, q, acc );
+ }
+
+ sumf = hsum_float_8(acc);
+#elif defined(__loongarch_sx)
+ // set constants
+ const __m128i low_mask = __lsx_vreplgr2vr_b(0xF);
+ const __m128i off = __lsx_vreplgr2vr_b(8);
+
+ // Initialize accumulator with zeros
+ __m128 acc_0 = __lsx_vldi(0);
+ __m128 acc_1 = __lsx_vldi(0);
+ __m128 acc_2 = __lsx_vldi(0);
+ __m128 acc_3 = __lsx_vldi(0);
+
+ for (; ib + 1 < nb; ib += 2) {
+
+ // Compute combined scale for the block 0 and 1
+ const __m128 d_0_1 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d) );
+
+ const __m128i tmp_0_1 = __lsx_vld((const __m128i *)x[ib].qs, 0);
+
+ __m128i bx_0 = __lsx_vand_v(low_mask, tmp_0_1);
+ __m128i by_0 = __lsx_vld((const __m128i *)y[ib].qs, 0);
+ bx_0 = __lsx_vsub_b(bx_0, off);
+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);
+
+ __m128i bx_1 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_0_1, 4));
+ __m128i by_1 = __lsx_vld((const __m128i *)(y[ib].qs + 16), 0);
+ bx_1 = __lsx_vsub_b(bx_1, off);
+ const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);
+
+ //_mm_prefetch(&x[ib] + 2 * sizeof(block_q4_0), _MM_HINT_T0);
+ //_mm_prefetch(&y[ib] + 2 * sizeof(block_q8_0), _MM_HINT_T0);
+
+ // Compute combined scale for the block 2 and 3
+ const __m128 d_2_3 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[ib + 1].d) * GGML_FP16_TO_FP32(y[ib + 1].d) );
+
+ const __m128i tmp_2_3 = __lsx_vld((const __m128i *)x[ib + 1].qs, 0);
+
+ __m128i bx_2 = __lsx_vand_v(low_mask, tmp_2_3);
+ __m128i by_2 = __lsx_vld((const __m128i *)y[ib + 1].qs, 0);
+ bx_2 = __lsx_vsub_b(bx_2, off);
+ const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);
+
+ __m128i bx_3 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_2_3, 4));
+ __m128i by_3 = __lsx_vld((const __m128i *)(y[ib + 1].qs + 16), 0);
+ bx_3 = __lsx_vsub_b(bx_3, off);
+ const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);
+
+ // Convert int32_t to float
+ __m128 p0 = __lsx_vffint_s_w(i32_0);
+ __m128 p1 = __lsx_vffint_s_w(i32_1);
+ __m128 p2 = __lsx_vffint_s_w(i32_2);
+ __m128 p3 = __lsx_vffint_s_w(i32_3);
+
+ // Apply the scale
+ __m128 p0_d = __lsx_vfmul_s( d_0_1, p0 );
+ __m128 p1_d = __lsx_vfmul_s( d_0_1, p1 );
+ __m128 p2_d = __lsx_vfmul_s( d_2_3, p2 );
+ __m128 p3_d = __lsx_vfmul_s( d_2_3, p3 );
+
+ // Acummulate
+ acc_0 = __lsx_vfadd_s(p0_d, acc_0);
+ acc_1 = __lsx_vfadd_s(p1_d, acc_1);
+ acc_2 = __lsx_vfadd_s(p2_d, acc_2);
+ acc_3 = __lsx_vfadd_s(p3_d, acc_3);
+ }
+
+ sumf = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);
+#endif
+ for (; ib < nb; ++ib) {
+ int sumi0 = 0;
+ int sumi1 = 0;
+
+ for (int j = 0; j < qk/2; ++j) {
+ const int v0 = (x[ib].qs[j] & 0x0F) - 8;
+ const int v1 = (x[ib].qs[j] >> 4) - 8;
+
+ sumi0 += (v0 * y[ib].qs[j]);
+ sumi1 += (v1 * y[ib].qs[j + qk/2]);
+ }
+
+ int sumi = sumi0 + sumi1;
+ sumf += sumi*GGML_FP16_TO_FP32(x[ib].d)*GGML_FP16_TO_FP32(y[ib].d);
+ }
+
+ *s = sumf;
+}
+
+void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ const int qk = QK8_1;
+ const int nb = n / qk;
+
+ assert(n % qk == 0);
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ assert((nrc == 2) || (nrc == 1));
+#else
+ assert(nrc == 1);
+#endif
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q4_1 * restrict x = vx;
+ const block_q8_1 * restrict y = vy;
+
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ if (nrc == 2) {
+ const block_q4_1 * restrict vx0 = vx;
+ const block_q4_1 * restrict vx1 = (const block_q4_1 *) ((const uint8_t*)vx + bx);
+ const block_q8_1 * restrict vy0 = vy;
+ const block_q8_1 * restrict vy1 = (const block_q8_1 *) ((const uint8_t*)vy + by);
+
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t summs0 = vdupq_n_f32(0.0f);
+
+ for (int i = 0; i < nb; i++) {
+ const block_q4_1 * restrict b_x0 = &vx0[i];
+ const block_q4_1 * restrict b_x1 = &vx1[i];
+ const block_q8_1 * restrict b_y0 = &vy0[i];
+ const block_q8_1 * restrict b_y1 = &vy1[i];
+
+ float32_t summs_t[4] = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s),
+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s),
+ GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s),
+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)};
+ summs0 = vaddq_f32(summs0, vld1q_f32(summs_t));
+
+ const uint8x16_t m4b = vdupq_n_u8(0x0F);
+
+ const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);
+ const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);
+
+ // 4-bit -> 8-bit
+ const int8x16_t x0_l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
+ const int8x16_t x0_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
+ const int8x16_t x1_l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
+ const int8x16_t x1_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+
+ // load y
+ const int8x16_t y0_l = vld1q_s8(b_y0->qs);
+ const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);
+ const int8x16_t y1_l = vld1q_s8(b_y1->qs);
+ const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
+
+ // mmla into int32x4_t
+ float32_t _scale[4] = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d,
+ GGML_FP16_TO_FP32(b_x0->d)*b_y1->d,
+ GGML_FP16_TO_FP32(b_x1->d)*b_y0->d,
+ GGML_FP16_TO_FP32(b_x1->d)*b_y1->d};
+ float32x4_t scale = vld1q_f32(_scale);
+
+ int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
+ int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
+
+ int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
+ int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
+
+ int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+ int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+
+ int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+ int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+ sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
+ l1, r1)), l2, r2)), l3, r3))), scale);
+ }
+
+ float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
+ float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
+ sumv2 = vaddq_f32(sumv2, summs0);
+
+ vst1_f32(s, vget_low_f32 (sumv2));
+ vst1_f32(s + bs, vget_high_f32(sumv2));
+ return;
+ }
+#endif
+
+ int ib = 0;
+ float sumf = 0;
+
+ // TODO: add WASM SIMD
+#if defined(__ARM_NEON)
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t sumv1 = vdupq_n_f32(0.0f);
+
+ float summs = 0;
+
+ for (; ib + 1 < nb; ib += 2) {
+ const block_q4_1 * restrict x0 = &x[ib + 0];
+ const block_q4_1 * restrict x1 = &x[ib + 1];
+ const block_q8_1 * restrict y0 = &y[ib + 0];
+ const block_q8_1 * restrict y1 = &y[ib + 1];
+
+ summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s) + GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);
+
+ const uint8x16_t m4b = vdupq_n_u8(0x0F);
+
+ const uint8x16_t v0_0 = vld1q_u8(x0->qs);
+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);
+
+ // 4-bit -> 8-bit
+ const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
+ const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
+ const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
+ const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+
+ // load y
+ const int8x16_t v1_0l = vld1q_s8(y0->qs);
+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);
+ const int8x16_t v1_1l = vld1q_s8(y1->qs);
+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);
+
+ // dot product into int32x4_t
+ const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);
+ const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);
+
+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
+ }
+
+ sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;
+#elif defined(__AVX2__) || defined(__AVX__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+
+ float summs = 0;
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ const float d0 = GGML_FP16_TO_FP32(x[ib].d);
+ const float d1 = GGML_FP16_TO_FP32(y[ib].d);
+
+ summs += GGML_FP16_TO_FP32(x[ib].m) * GGML_FP16_TO_FP32(y[ib].s);
+
+ const __m256 d0v = _mm256_set1_ps( d0 );
+ const __m256 d1v = _mm256_set1_ps( d1 );
+
+ // Compute combined scales
+ const __m256 d0d1 = _mm256_mul_ps( d0v, d1v );
+
+ // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes
+ const __m256i qx = bytes_from_nibbles_32(x[ib].qs);
+ const __m256i qy = _mm256_loadu_si256( (const __m256i *)y[ib].qs );
+
+ const __m256 xy = mul_sum_us8_pairs_float(qx, qy);
+
+ // Accumulate d0*d1*x*y
+#if defined(__AVX2__)
+ acc = _mm256_fmadd_ps( d0d1, xy, acc );
+#else
+ acc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );
+#endif
+ }
+
+ sumf = hsum_float_8(acc) + summs;
+#elif defined(__riscv_v_intrinsic)
+ size_t vl = __riscv_vsetvl_e8m1(qk/2);
+
+ for (; ib < nb; ++ib) {
+ // load elements
+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[ib].qs, vl);
+
+ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[ib].qs, vl);
+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[ib].qs+16, vl);
+
+ // mask and store lower part of x, and then upper part
+ vuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);
+ vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);
+
+ vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
+ vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
+
+ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);
+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);
+
+ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);
+
+ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);
+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);
+
+ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);
+
+ sumf += (GGML_FP16_TO_FP32(x[ib].d)*GGML_FP16_TO_FP32(y[ib].d))*sumi + GGML_FP16_TO_FP32(x[ib].m)*GGML_FP16_TO_FP32(y[ib].s);
+ }
+
+#elif defined(__POWER9_VECTOR__)
+ const vector signed char lowMask = vec_splats((signed char)0xF);
+ const vector signed int v0 = vec_splats((int32_t)0);
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
+
+ vector float vsumf0 = vec_splats(0.0f);
+
+#pragma GCC unroll 4
+ for (; ib < nb; ++ib) {
+ __builtin_prefetch(x[ib].qs, 0, 1);
+ __builtin_prefetch(y[ib].qs, 0, 1);
+
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ib].d));
+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[ib].d));
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[ib].m));
+ vector float vys = {GGML_FP16_TO_FP32(y[ib].s), 0.0f, 0.0f, 0.0f};
+ vsumf0 = vec_madd(vxmin, vys, vsumf0);
+
+ vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);
+ vector signed char q8y0 = vec_xl( 0, y[ib].qs);
+ vector signed char q8y1 = vec_xl(16, y[ib].qs);
+
+ vector unsigned char q4x0 = (vector unsigned char)vec_and(qxs, lowMask);
+ vector unsigned char q4x1 = (vector unsigned char)vec_sr(qxs, v4);
+
+ vector signed int vsumi0 = v0;
+
+ vsumi0 = vec_msum(q8y0, q4x0, vsumi0);
+ vsumi0 = vec_msum(q8y1, q4x1, vsumi0);
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ }
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ sumf = vec_extract(vsumf0, 0);
+
+#elif defined(__loongarch_asx)
+ // Initialize accumulator with zeros
+ __m256 acc = (__m256)__lasx_xvldi(0);
+
+ float summs = 0;
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ const float d0 = GGML_FP16_TO_FP32(x[ib].d);
+ const float d1 = GGML_FP16_TO_FP32(y[ib].d);
+
+ summs += GGML_FP16_TO_FP32(x[ib].m) * GGML_FP16_TO_FP32(y[ib].s);
+
+ const __m256 d0v = __lasx_xvreplfr2vr_s( d0 );
+ const __m256 d1v = __lasx_xvreplfr2vr_s( d1 );
+
+ // Compute combined scales
+ const __m256 d0d1 = __lasx_xvfmul_s( d0v, d1v );
+
+ // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes
+ const __m256i qx = bytes_from_nibbles_32(x[ib].qs);
+ const __m256i qy = __lasx_xvld( (const __m256i *)y[ib].qs, 0);
+
+ const __m256 xy = mul_sum_us8_pairs_float(qx, qy);
+
+ // Accumulate d0*d1*x*y
+ acc = __lasx_xvfmadd_s( d0d1, xy, acc );
+ }
+
+ sumf = hsum_float_8(acc) + summs;
+#endif
+ for (; ib < nb; ++ib) {
+ int sumi0 = 0;
+ int sumi1 = 0;
+
+ for (int j = 0; j < qk/2; ++j) {
+ const int v0 = (x[ib].qs[j] & 0x0F);
+ const int v1 = (x[ib].qs[j] >> 4);
+
+ sumi0 += (v0 * y[ib].qs[j]);
+ sumi1 += (v1 * y[ib].qs[j + qk/2]);
+ }
+
+ int sumi = sumi0 + sumi1;
+ sumf += (GGML_FP16_TO_FP32(x[ib].d)*GGML_FP16_TO_FP32(y[ib].d))*sumi + GGML_FP16_TO_FP32(x[ib].m)*GGML_FP16_TO_FP32(y[ib].s);
+ }
+
+ *s = sumf;
+}
+
+void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ const int qk = QK8_0;
+ const int nb = n / qk;
+
+ int ib = 0;
+ float sumf = 0;
+
+ assert(n % qk == 0);
+ assert(qk == QK5_0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q5_0 * restrict x = vx;
+ const block_q8_0 * restrict y = vy;
+
+#if defined(__ARM_NEON)
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t sumv1 = vdupq_n_f32(0.0f);
+
+ uint32_t qh0;
+ uint32_t qh1;
+
+ uint64_t tmp0[4];
+ uint64_t tmp1[4];
+
+ for (; ib + 1 < nb; ib += 2) {
+ const block_q5_0 * restrict x0 = &x[ib];
+ const block_q5_0 * restrict x1 = &x[ib + 1];
+ const block_q8_0 * restrict y0 = &y[ib];
+ const block_q8_0 * restrict y1 = &y[ib + 1];
+
+ const uint8x16_t m4b = vdupq_n_u8(0x0F);
+
+ // extract the 5th bit via lookup table ((!b) << 4)
+ memcpy(&qh0, x0->qh, sizeof(qh0));
+ memcpy(&qh1, x1->qh, sizeof(qh1));
+
+ tmp0[0] = table_b2b_1[(qh0 >> 0) & 0xFF];
+ tmp0[1] = table_b2b_1[(qh0 >> 8) & 0xFF];
+ tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];
+ tmp0[3] = table_b2b_1[(qh0 >> 24) ];
+
+ tmp1[0] = table_b2b_1[(qh1 >> 0) & 0xFF];
+ tmp1[1] = table_b2b_1[(qh1 >> 8) & 0xFF];
+ tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];
+ tmp1[3] = table_b2b_1[(qh1 >> 24) ];
+
+ const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));
+ const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));
+ const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));
+ const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));
+
+ const uint8x16_t v0_0 = vld1q_u8(x0->qs);
+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);
+
+ // 4-bit -> 8-bit
+ int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
+ int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
+ int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
+ int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+
+ // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
+ const int8x16_t v0_0lf = vsubq_s8(v0_0l, qhl0);
+ const int8x16_t v0_0hf = vsubq_s8(v0_0h, qhh0);
+ const int8x16_t v0_1lf = vsubq_s8(v0_1l, qhl1);
+ const int8x16_t v0_1hf = vsubq_s8(v0_1h, qhh1);
+
+ // load y
+ const int8x16_t v1_0l = vld1q_s8(y0->qs);
+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);
+ const int8x16_t v1_1l = vld1q_s8(y1->qs);
+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);
+
+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
+ }
+
+ sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);
+#elif defined(__wasm_simd128__)
+ v128_t sumv = wasm_f32x4_splat(0.0f);
+
+ uint32_t qh;
+ uint64_t tmp[4];
+
+ // TODO: check if unrolling this is better
+ for (; ib < nb; ++ib) {
+ const block_q5_0 * restrict x0 = &x[ib];
+ const block_q8_0 * restrict y0 = &y[ib];
+
+ const v128_t m4b = wasm_i8x16_splat(0x0F);
+
+ // extract the 5th bit
+ memcpy(&qh, x0->qh, sizeof(qh));
+
+ tmp[0] = table_b2b_1[(qh >> 0) & 0xFF];
+ tmp[1] = table_b2b_1[(qh >> 8) & 0xFF];
+ tmp[2] = table_b2b_1[(qh >> 16) & 0xFF];
+ tmp[3] = table_b2b_1[(qh >> 24) ];
+
+ const v128_t qhl = wasm_v128_load(tmp + 0);
+ const v128_t qhh = wasm_v128_load(tmp + 2);
+
+ const v128_t v0 = wasm_v128_load(x0->qs);
+
+ // 4-bit -> 8-bit
+ const v128_t v0l = wasm_v128_and (v0, m4b);
+ const v128_t v0h = wasm_u8x16_shr(v0, 4);
+
+ // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
+ const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);
+ const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);
+
+ // load y
+ const v128_t v1l = wasm_v128_load(y0->qs);
+ const v128_t v1h = wasm_v128_load(y0->qs + 16);
+
+ // int8x16 -> int16x8
+ const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
+ const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
+ const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);
+ const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);
+
+ const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);
+ const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);
+ const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);
+ const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);
+
+ // dot product
+ sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(
+ wasm_i32x4_add(
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),
+ wasm_i32x4_dot_i16x8(v0lfh, v1lh)),
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
+ wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
+ wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d))));
+ }
+
+ sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
+ wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);
+#elif defined(__AVX2__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ /* Compute combined scale for the block */
+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d));
+
+ __m256i qx = bytes_from_nibbles_32(x[ib].qs);
+ __m256i bxhi = bytes_from_bits_32(x[ib].qh);
+ bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));
+ qx = _mm256_or_si256(qx, bxhi);
+
+ __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);
+
+ const __m256 q = mul_sum_i8_pairs_float(qx, qy);
+
+ /* Multiply q with scale and accumulate */
+ acc = _mm256_fmadd_ps(d, q, acc);
+ }
+
+ sumf = hsum_float_8(acc);
+#elif defined(__AVX__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+ __m128i mask = _mm_set1_epi8((char)0xF0);
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ /* Compute combined scale for the block */
+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d));
+
+ __m256i bx_0 = bytes_from_nibbles_32(x[ib].qs);
+ const __m256i bxhi = bytes_from_bits_32(x[ib].qh);
+ __m128i bxhil = _mm256_castsi256_si128(bxhi);
+ __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);
+ bxhil = _mm_andnot_si128(bxhil, mask);
+ bxhih = _mm_andnot_si128(bxhih, mask);
+ __m128i bxl = _mm256_castsi256_si128(bx_0);
+ __m128i bxh = _mm256_extractf128_si256(bx_0, 1);
+ bxl = _mm_or_si128(bxl, bxhil);
+ bxh = _mm_or_si128(bxh, bxhih);
+ bx_0 = MM256_SET_M128I(bxh, bxl);
+
+ const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[ib].qs);
+
+ const __m256 q = mul_sum_i8_pairs_float(bx_0, by_0);
+
+ /* Multiply q with scale and accumulate */
+ acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);
+ }
+
+ sumf = hsum_float_8(acc);
+#elif defined(__riscv_v_intrinsic)
+ uint32_t qh;
+
+ size_t vl = __riscv_vsetvl_e8m1(qk/2);
+
+ // These temporary registers are for masking and shift operations
+ vuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);
+ vuint32m2_t vt_2 = __riscv_vsll_vv_u32m2(__riscv_vmv_v_x_u32m2(1, vl), vt_1, vl);
+
+ vuint32m2_t vt_3 = __riscv_vsll_vx_u32m2(vt_2, 16, vl);
+ vuint32m2_t vt_4 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);
+
+ for (; ib < nb; ++ib) {
+ memcpy(&qh, x[ib].qh, sizeof(uint32_t));
+
+ // ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;
+ vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(vt_2, qh, vl);
+ vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(xha_0, vt_1, vl);
+ vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);
+
+ // ((qh & (1u << (j + 16))) >> (j + 12));
+ vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(vt_3, qh, vl);
+ vuint32m2_t xhl_1 = __riscv_vsrl_vv_u32m2(xha_1, vt_4, vl);
+
+ // narrowing
+ vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xhl_0, vl);
+ vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);
+
+ vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xhl_1, vl);
+ vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);
+
+ // load
+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[ib].qs, vl);
+
+ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[ib].qs, vl);
+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[ib].qs+16, vl);
+
+ vuint8mf2_t x_at = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);
+ vuint8mf2_t x_lt = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);
+
+ vuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);
+ vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);
+
+ vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
+ vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
+
+ vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 16, vl);
+ vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 16, vl);
+
+ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);
+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);
+
+ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);
+
+ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);
+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);
+
+ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);
+
+ sumf += (GGML_FP16_TO_FP32(x[ib].d)*GGML_FP16_TO_FP32(y[ib].d)) * sumi;
+ }
+
+#elif defined(__POWER9_VECTOR__)
+ const vector signed char lowMask = vec_splats((signed char)0xF);
+ const vector unsigned char v4 = vec_splats((unsigned char)4);
+
+ vector float vsumf0 = vec_splats(0.0f);
+
+#pragma GCC unroll 4
+ for (; ib < nb; ++ib) {
+ __builtin_prefetch(x[ib].qs, 0, 1);
+ __builtin_prefetch(y[ib].qs, 0, 1);
+
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ib].d));
+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[ib].d));
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector signed long long aux64x2_0 = {(uint64_t)(table_b2b_1[x[ib].qh[0]]), (uint64_t)(table_b2b_1[x[ib].qh[1]])};
+ vector signed long long aux64x2_1 = {(uint64_t)(table_b2b_1[x[ib].qh[2]]), (uint64_t)(table_b2b_1[x[ib].qh[3]])};
+
+ vector signed char qh0 = (vector signed char)aux64x2_0;
+ vector signed char qh1 = (vector signed char)aux64x2_1;
+
+ vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);
+
+ vector signed char q5x0 = vec_sub(vec_and (qxs, lowMask), qh0);
+ vector signed char q5x1 = vec_sub(vec_sr(qxs, v4), qh1);
+
+ vector signed char q8y0 = vec_xl( 0, y[ib].qs);
+ vector signed char q8y1 = vec_xl( 16, y[ib].qs);
+
+ vector signed short qv0 = vec_add(vec_mule(q5x0, q8y0), vec_mulo(q5x0, q8y0));
+ vector signed short qv1 = vec_add(vec_mule(q5x1, q8y1), vec_mulo(q5x1, q8y1));
+
+ qv0 = vec_add(qv0, qv1);
+
+ vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0));
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ }
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ sumf = vec_extract(vsumf0, 0);
+
+#elif defined(__loongarch_asx)
+ // Initialize accumulator with zeros
+ __m256 acc = (__m256)__lasx_xvldi(0);
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ /* Compute combined scale for the block */
+ const __m256 d = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); //FIXME
+
+ __m256i qx = bytes_from_nibbles_32(x[ib].qs);
+ __m256i bxhi = bytes_from_bits_32(x[ib].qh);
+ bxhi = __lasx_xvandn_v(bxhi, __lasx_xvreplgr2vr_b((char)0xF0));
+ qx = __lasx_xvor_v(qx, bxhi);
+
+ __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);
+
+ const __m256 q = mul_sum_i8_pairs_float(qx, qy);
+
+ /* Multiply q with scale and accumulate */
+ acc = __lasx_xvfmadd_s(d, q, acc);
+ }
+
+ sumf = hsum_float_8(acc);
+#endif
+ for (; ib < nb; ++ib) {
+ uint32_t qh;
+ memcpy(&qh, x[ib].qh, sizeof(qh));
+
+ int sumi0 = 0;
+ int sumi1 = 0;
+
+ for (int j = 0; j < qk/2; ++j) {
+ const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;
+ const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));
+
+ const int32_t x0 = (int8_t)(((x[ib].qs[j] & 0x0F) | xh_0) - 16);
+ const int32_t x1 = (int8_t)(((x[ib].qs[j] >> 4) | xh_1) - 16);
+
+ sumi0 += (x0 * y[ib].qs[j]);
+ sumi1 += (x1 * y[ib].qs[j + qk/2]);
+ }
+
+ int sumi = sumi0 + sumi1;
+ sumf += (GGML_FP16_TO_FP32(x[ib].d)*GGML_FP16_TO_FP32(y[ib].d)) * sumi;
+ }
+
+ *s = sumf;
+}
+
+void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ const int qk = QK8_1;
+ const int nb = n / qk;
+
+ int ib = 0;
+ float sumf = 0;
+
+ assert(n % qk == 0);
+ assert(qk == QK5_1);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q5_1 * restrict x = vx;
+ const block_q8_1 * restrict y = vy;
+
+#if defined(__ARM_NEON)
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t sumv1 = vdupq_n_f32(0.0f);
+
+ float summs0 = 0.0f;
+ float summs1 = 0.0f;
+
+ uint32_t qh0;
+ uint32_t qh1;
+
+ uint64_t tmp0[4];
+ uint64_t tmp1[4];
+
+ for (; ib + 1 < nb; ib += 2) {
+ const block_q5_1 * restrict x0 = &x[ib];
+ const block_q5_1 * restrict x1 = &x[ib + 1];
+ const block_q8_1 * restrict y0 = &y[ib];
+ const block_q8_1 * restrict y1 = &y[ib + 1];
+
+ const uint8x16_t m4b = vdupq_n_u8(0x0F);
+
+ summs0 += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s);
+ summs1 += GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);
+
+ // extract the 5th bit via lookup table ((b) << 4)
+ memcpy(&qh0, x0->qh, sizeof(qh0));
+ memcpy(&qh1, x1->qh, sizeof(qh1));
+
+ tmp0[0] = table_b2b_0[(qh0 >> 0) & 0xFF];
+ tmp0[1] = table_b2b_0[(qh0 >> 8) & 0xFF];
+ tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];
+ tmp0[3] = table_b2b_0[(qh0 >> 24) ];
+
+ tmp1[0] = table_b2b_0[(qh1 >> 0) & 0xFF];
+ tmp1[1] = table_b2b_0[(qh1 >> 8) & 0xFF];
+ tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];
+ tmp1[3] = table_b2b_0[(qh1 >> 24) ];
+
+ const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));
+ const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));
+ const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));
+ const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));
+
+ const uint8x16_t v0_0 = vld1q_u8(x0->qs);
+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);
+
+ // 4-bit -> 8-bit
+ const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
+ const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
+ const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
+ const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+
+ // add high bit
+ const int8x16_t v0_0lf = vorrq_s8(v0_0l, qhl0);
+ const int8x16_t v0_0hf = vorrq_s8(v0_0h, qhh0);
+ const int8x16_t v0_1lf = vorrq_s8(v0_1l, qhl1);
+ const int8x16_t v0_1hf = vorrq_s8(v0_1h, qhh1);
+
+ // load y
+ const int8x16_t v1_0l = vld1q_s8(y0->qs);
+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);
+ const int8x16_t v1_1l = vld1q_s8(y1->qs);
+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);
+
+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
+ }
+
+ sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs0 + summs1;
+#elif defined(__wasm_simd128__)
+ v128_t sumv = wasm_f32x4_splat(0.0f);
+
+ float summs = 0.0f;
+
+ uint32_t qh;
+ uint64_t tmp[4];
+
+ // TODO: check if unrolling this is better
+ for (; ib < nb; ++ib) {
+ const block_q5_1 * restrict x0 = &x[ib];
+ const block_q8_1 * restrict y0 = &y[ib];
+
+ summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s);
+
+ const v128_t m4b = wasm_i8x16_splat(0x0F);
+
+ // extract the 5th bit
+ memcpy(&qh, x0->qh, sizeof(qh));
+
+ tmp[0] = table_b2b_0[(qh >> 0) & 0xFF];
+ tmp[1] = table_b2b_0[(qh >> 8) & 0xFF];
+ tmp[2] = table_b2b_0[(qh >> 16) & 0xFF];
+ tmp[3] = table_b2b_0[(qh >> 24) ];
+
+ const v128_t qhl = wasm_v128_load(tmp + 0);
+ const v128_t qhh = wasm_v128_load(tmp + 2);
+
+ const v128_t v0 = wasm_v128_load(x0->qs);
+
+ // 4-bit -> 8-bit
+ const v128_t v0l = wasm_v128_and (v0, m4b);
+ const v128_t v0h = wasm_u8x16_shr(v0, 4);
+
+ // add high bit
+ const v128_t v0lf = wasm_v128_or(v0l, qhl);
+ const v128_t v0hf = wasm_v128_or(v0h, qhh);
+
+ // load y
+ const v128_t v1l = wasm_v128_load(y0->qs);
+ const v128_t v1h = wasm_v128_load(y0->qs + 16);
+
+ // int8x16 -> int16x8
+ const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
+ const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
+ const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);
+ const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);
+
+ const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);
+ const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);
+ const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);
+ const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);
+
+ // dot product
+ sumv = wasm_f32x4_add(sumv,
+ wasm_f32x4_mul(wasm_f32x4_convert_i32x4(wasm_i32x4_add(
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),
+ wasm_i32x4_dot_i16x8(v0lfh, v1lh)),
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
+ wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
+ wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d))));
+ }
+
+ sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
+ wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3) + summs;
+#elif defined(__AVX2__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+
+ float summs = 0.0f;
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[ib].d));
+
+ summs += GGML_FP16_TO_FP32(x[ib].m) * GGML_FP16_TO_FP32(y[ib].s);
+
+ __m256i qx = bytes_from_nibbles_32(x[ib].qs);
+ __m256i bxhi = bytes_from_bits_32(x[ib].qh);
+ bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10));
+ qx = _mm256_or_si256(qx, bxhi);
+
+ const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[ib].d));
+ const __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);
+
+ const __m256 q = mul_sum_us8_pairs_float(qx, qy);
+
+ acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc);
+ }
+
+ sumf = hsum_float_8(acc) + summs;
+#elif defined(__AVX__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+ __m128i mask = _mm_set1_epi8(0x10);
+
+ float summs = 0.0f;
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[ib].d));
+
+ summs += GGML_FP16_TO_FP32(x[ib].m) * GGML_FP16_TO_FP32(y[ib].s);
+
+ __m256i bx_0 = bytes_from_nibbles_32(x[ib].qs);
+ const __m256i bxhi = bytes_from_bits_32(x[ib].qh);
+ __m128i bxhil = _mm256_castsi256_si128(bxhi);
+ __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);
+ bxhil = _mm_and_si128(bxhil, mask);
+ bxhih = _mm_and_si128(bxhih, mask);
+ __m128i bxl = _mm256_castsi256_si128(bx_0);
+ __m128i bxh = _mm256_extractf128_si256(bx_0, 1);
+ bxl = _mm_or_si128(bxl, bxhil);
+ bxh = _mm_or_si128(bxh, bxhih);
+ bx_0 = MM256_SET_M128I(bxh, bxl);
+
+ const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[ib].d));
+ const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[ib].qs);
+
+ const __m256 q = mul_sum_us8_pairs_float(bx_0, by_0);
+
+ acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc);
+ }
+
+ sumf = hsum_float_8(acc) + summs;
+#elif defined(__riscv_v_intrinsic)
+ uint32_t qh;
+
+ size_t vl = __riscv_vsetvl_e8m1(qk/2);
+
+ // temporary registers for shift operations
+ vuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);
+ vuint32m2_t vt_2 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);
+
+ for (; ib < nb; ++ib) {
+ memcpy(&qh, x[ib].qh, sizeof(uint32_t));
+
+ // load qh
+ vuint32m2_t vqh = __riscv_vmv_v_x_u32m2(qh, vl);
+
+ // ((qh >> (j + 0)) << 4) & 0x10;
+ vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(vqh, vt_1, vl);
+ vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);
+ vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(xhl_0, 0x10, vl);
+
+ // ((qh >> (j + 12)) ) & 0x10;
+ vuint32m2_t xhr_1 = __riscv_vsrl_vv_u32m2(vqh, vt_2, vl);
+ vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(xhr_1, 0x10, vl);
+
+ // narrowing
+ vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xha_0, vl);
+ vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);
+
+ vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xha_1, vl);
+ vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);
+
+ // load
+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[ib].qs, vl);
+
+ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[ib].qs, vl);
+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[ib].qs+16, vl);
+
+ vuint8mf2_t x_at = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);
+ vuint8mf2_t x_lt = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);
+
+ vuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);
+ vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);
+
+ vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
+ vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
+
+ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);
+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);
+
+ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);
+
+ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);
+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);
+
+ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);
+
+ sumf += (GGML_FP16_TO_FP32(x[ib].d)*GGML_FP16_TO_FP32(y[ib].d))*sumi + GGML_FP16_TO_FP32(x[ib].m)*GGML_FP16_TO_FP32(y[ib].s);
+ }
+
+#elif defined(__POWER9_VECTOR__)
+ const vector signed char lowMask = vec_splats((signed char)0xF);
+ const vector signed int v0 = vec_splats((int32_t)0);
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
+
+ vector float vsumf0 = vec_splats(0.0f);
+
+#pragma GCC unroll 4
+ for (; ib < nb; ++ib) {
+ __builtin_prefetch(x[ib].qs, 0, 1);
+ __builtin_prefetch(y[ib].qs, 0, 1);
+
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ib].d));
+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[ib].d));
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[ib].m));
+ vector float vys = {GGML_FP16_TO_FP32(y[ib].s), 0.f, 0.f, 0.f};
+ vsumf0 = vec_madd(vxmin, vys, vsumf0);
+
+ vector unsigned long long aux64x2_0 = {(uint64_t)(table_b2b_0[x[ib].qh[0]]), (uint64_t)(table_b2b_0[x[ib].qh[1]])};
+ vector unsigned long long aux64x2_1 = {(uint64_t)(table_b2b_0[x[ib].qh[2]]), (uint64_t)(table_b2b_0[x[ib].qh[3]])};
+
+ vector signed char qh0 = (vector signed char)aux64x2_0;
+ vector signed char qh1 = (vector signed char)aux64x2_1;
+
+ vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);
+
+ vector unsigned char q5x0 = (vector unsigned char)vec_or(vec_and(qxs, lowMask), qh0);
+ vector unsigned char q5x1 = (vector unsigned char)vec_or(vec_sr(qxs, v4), qh1);
+
+ vector signed char q8y0 = vec_xl( 0, y[ib].qs);
+ vector signed char q8y1 = vec_xl( 16, y[ib].qs);
+
+ vector signed int vsumi0 = v0;
+
+ vsumi0 = vec_msum(q8y0, q5x0, vsumi0);
+ vsumi0 = vec_msum(q8y1, q5x1, vsumi0);
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ }
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ sumf = vec_extract(vsumf0, 0);
+
+#elif defined(__loongarch_asx)
+ // Initialize accumulator with zeros
+ __m256 acc = (__m256)__lasx_xvldi(0);
+
+ float summs = 0.0f;
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ const __m256 dx = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[ib].d));
+
+ summs += GGML_FP16_TO_FP32(x[ib].m) * GGML_FP16_TO_FP32(y[ib].s);
+
+ __m256i qx = bytes_from_nibbles_32(x[ib].qs);
+ __m256i bxhi = bytes_from_bits_32(x[ib].qh);
+ bxhi = __lasx_xvand_v(bxhi, __lasx_xvreplgr2vr_b(0x10));
+ qx = __lasx_xvor_v(qx, bxhi);
+
+ const __m256 dy = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(y[ib].d));
+ const __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);
+
+ const __m256 q = mul_sum_us8_pairs_float(qx, qy);
+
+ acc = __lasx_xvfmadd_s(q, __lasx_xvfmul_s(dx, dy), acc);
+ }
+
+ sumf = hsum_float_8(acc) + summs;
+#endif
+ for (; ib < nb; ++ib) {
+ uint32_t qh;
+ memcpy(&qh, x[ib].qh, sizeof(qh));
+
+ int sumi0 = 0;
+ int sumi1 = 0;
+
+ for (int j = 0; j < qk/2; ++j) {
+ const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;
+ const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;
+
+ const int32_t x0 = (x[ib].qs[j] & 0xF) | xh_0;
+ const int32_t x1 = (x[ib].qs[j] >> 4) | xh_1;
+
+ sumi0 += (x0 * y[ib].qs[j]);
+ sumi1 += (x1 * y[ib].qs[j + qk/2]);
+ }
+
+ int sumi = sumi0 + sumi1;
+ sumf += (GGML_FP16_TO_FP32(x[ib].d)*GGML_FP16_TO_FP32(y[ib].d))*sumi + GGML_FP16_TO_FP32(x[ib].m)*GGML_FP16_TO_FP32(y[ib].s);
+ }
+
+ *s = sumf;
+}
+
+void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ const int qk = QK8_0;
+ const int nb = n / qk;
+
+ assert(n % qk == 0);
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ assert((nrc == 2) || (nrc == 1));
+#else
+ assert(nrc == 1);
+#endif
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q8_0 * restrict x = vx;
+ const block_q8_0 * restrict y = vy;
+
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ if (nrc == 2) {
+ const block_q8_0 * restrict vx0 = vx;
+ const block_q8_0 * restrict vx1 = (const block_q8_0 *) ((const uint8_t*)vx + bx);
+ const block_q8_0 * restrict vy0 = vy;
+ const block_q8_0 * restrict vy1 = (const block_q8_0 *) ((const uint8_t*)vy + by);
+
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+
+ for (int i = 0; i < nb; i++) {
+ const block_q8_0 * restrict b_x0 = &vx0[i];
+ const block_q8_0 * restrict b_y0 = &vy0[i];
+
+ const block_q8_0 * restrict b_x1 = &vx1[i];
+ const block_q8_0 * restrict b_y1 = &vy1[i];
+
+ const int8x16_t x0_l = vld1q_s8(b_x0->qs);
+ const int8x16_t x0_h = vld1q_s8(b_x0->qs + 16);
+ const int8x16_t x1_l = vld1q_s8(b_x1->qs);
+ const int8x16_t x1_h = vld1q_s8(b_x1->qs + 16);
+
+ // load y
+ const int8x16_t y0_l = vld1q_s8(b_y0->qs);
+ const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);
+ const int8x16_t y1_l = vld1q_s8(b_y1->qs);
+ const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
+
+ float32_t _scale[4] = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),
+ GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),
+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),
+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};
+ float32x4_t scale = vld1q_f32(_scale);
+
+ int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
+ int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
+
+ int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
+ int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
+
+ int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+ int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+
+ int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+ int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+
+ sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
+ l1, r1)), l2, r2)), l3, r3))), scale);
+ }
+ float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
+ float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
+
+ vst1_f32(s, vget_low_f32(sumv2));
+ vst1_f32(s + bs, vget_high_f32(sumv2));
+ return;
+ }
+#endif
+
+ int ib = 0;
+ float sumf = 0;
+
+#if defined(__ARM_FEATURE_SVE)
+ if (ggml_sve_cnt_b == QK8_0) {
+ svfloat32_t sumv0 = svdup_n_f32(0.0f);
+ svfloat32_t sumv1 = svdup_n_f32(0.0f);
+
+ for (; ib + 1 < nb; ib += 2) {
+ const block_q8_0 * restrict x0 = &x[ib + 0];
+ const block_q8_0 * restrict x1 = &x[ib + 1];
+ const block_q8_0 * restrict y0 = &y[ib + 0];
+ const block_q8_0 * restrict y1 = &y[ib + 1];
+
+ // load x
+ const svint8_t qx0 = svld1_s8(svptrue_b8(), x0->qs);
+ const svint8_t qx1 = svld1_s8(svptrue_b8(), x1->qs);
+
+ // load y
+ const svint8_t qy0 = svld1_s8(svptrue_b8(), y0->qs);
+ const svint8_t qy1 = svld1_s8(svptrue_b8(), y1->qs);
+
+ sumv0 = svmla_n_f32_x(svptrue_b32(), sumv0, svcvt_f32_s32_x(svptrue_b32(), svdot_s32(svdup_n_s32(0), qx0, qy0)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+ sumv1 = svmla_n_f32_x(svptrue_b32(), sumv1, svcvt_f32_s32_x(svptrue_b32(), svdot_s32(svdup_n_s32(0), qx1, qy1)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
+ }
+
+ sumf = svaddv_f32(svptrue_b32(), svadd_f32_x(svptrue_b32(), sumv0, sumv1));
+ }
+#elif defined(__ARM_NEON)
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t sumv1 = vdupq_n_f32(0.0f);
+
+ for (; ib + 1 < nb; ib += 2) {
+ const block_q8_0 * restrict x0 = &x[ib + 0];
+ const block_q8_0 * restrict x1 = &x[ib + 1];
+ const block_q8_0 * restrict y0 = &y[ib + 0];
+ const block_q8_0 * restrict y1 = &y[ib + 1];
+
+ const int8x16_t x0_0 = vld1q_s8(x0->qs);
+ const int8x16_t x0_1 = vld1q_s8(x0->qs + 16);
+ const int8x16_t x1_0 = vld1q_s8(x1->qs);
+ const int8x16_t x1_1 = vld1q_s8(x1->qs + 16);
+
+ // load y
+ const int8x16_t y0_0 = vld1q_s8(y0->qs);
+ const int8x16_t y0_1 = vld1q_s8(y0->qs + 16);
+ const int8x16_t y1_0 = vld1q_s8(y1->qs);
+ const int8x16_t y1_1 = vld1q_s8(y1->qs + 16);
+
+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), x0_0, y0_0),
+ ggml_vdotq_s32(vdupq_n_s32(0), x0_1, y0_1))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+
+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), x1_0, y1_0),
+ ggml_vdotq_s32(vdupq_n_s32(0), x1_1, y1_1))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
+ }
+
+ sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);
+#elif defined(__AVX2__) || defined(__AVX__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ // Compute combined scale for the block
+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d));
+ __m256i qx = _mm256_loadu_si256((const __m256i *)x[ib].qs);
+ __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);
+
+ const __m256 q = mul_sum_i8_pairs_float(qx, qy);
+
+ // Multiply q with scale and accumulate
+#if defined(__AVX2__)
+ acc = _mm256_fmadd_ps( d, q, acc );
+#else
+ acc = _mm256_add_ps( _mm256_mul_ps( d, q ), acc );
+#endif
+ }
+
+ sumf = hsum_float_8(acc);
+#elif defined(__riscv_v_intrinsic)
+ size_t vl = __riscv_vsetvl_e8m1(qk);
+
+ for (; ib < nb; ++ib) {
+ // load elements
+ vint8m1_t bx_0 = __riscv_vle8_v_i8m1(x[ib].qs, vl);
+ vint8m1_t by_0 = __riscv_vle8_v_i8m1(y[ib].qs, vl);
+
+ vint16m2_t vw_mul = __riscv_vwmul_vv_i16m2(bx_0, by_0, vl);
+
+ vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, vl);
+ vint32m1_t v_sum = __riscv_vwredsum_vs_i16m2_i32m1(vw_mul, v_zero, vl);
+
+ int sumi = __riscv_vmv_x_s_i32m1_i32(v_sum);
+
+ sumf += sumi*(GGML_FP16_TO_FP32(x[ib].d)*GGML_FP16_TO_FP32(y[ib].d));
+ }
+#elif defined(__POWER9_VECTOR__)
+ const vector signed int v0 = vec_splats((int32_t)0);
+ vector float vsumf0 = vec_splats(0.0f);
+
+#pragma GCC unroll 8
+ for (; ib < nb; ++ib) {
+ __builtin_prefetch(x[ib].qs, 0, 1);
+ __builtin_prefetch(y[ib].qs, 0, 1);
+
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ib].d));
+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[ib].d));
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector signed char q8x0 = vec_xl( 0, x[ib].qs);
+ vector signed char q8x1 = vec_xl(16, x[ib].qs);
+ vector signed char q8y0 = vec_xl( 0, y[ib].qs);
+ vector signed char q8y1 = vec_xl(16, y[ib].qs);
+
+ vector signed short qv0 = vec_mule(q8x0, q8y0);
+ vector signed short qv1 = vec_mulo(q8x0, q8y0);
+ vector signed short qv2 = vec_mule(q8x1, q8y1);
+ vector signed short qv3 = vec_mulo(q8x1, q8y1);
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+
+ vsumi0 = vec_sum4s(qv0, vsumi0);
+ vsumi1 = vec_sum4s(qv1, vsumi1);
+ vsumi0 = vec_sum4s(qv2, vsumi0);
+ vsumi1 = vec_sum4s(qv3, vsumi1);
+
+ vsumi0 = vec_add(vsumi0, vsumi1);
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ }
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ sumf = vec_extract(vsumf0, 0);
+
+#elif defined(__loongarch_asx)
+ // Initialize accumulator with zeros
+ __m256 acc = (__m256)__lasx_xvldi(0);
+
+ // Main loop
+ for (; ib < nb; ++ib) {
+ // Compute combined scale for the block
+ const __m256 d = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d));
+ __m256i qx = __lasx_xvld((const __m256i *)x[ib].qs, 0);
+ __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);
+
+ const __m256 q = mul_sum_i8_pairs_float(qx, qy);
+
+ // Multiply q with scale and accumulate
+ acc = __lasx_xvfmadd_s( d, q, acc );
+ }
+
+ sumf = hsum_float_8(acc);
+#endif
+ for (; ib < nb; ++ib) {
+ int sumi = 0;
+
+ for (int j = 0; j < qk; j++) {
+ sumi += x[ib].qs[j]*y[ib].qs[j];
+ }
+
+ sumf += sumi*(GGML_FP16_TO_FP32(x[ib].d)*GGML_FP16_TO_FP32(y[ib].d));
+ }
+
+ *s = sumf;
+}
+
+void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q2_K * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#ifdef __ARM_NEON
+ const uint8x16_t m3 = vdupq_n_u8(0x3);
+ const uint8x16_t m4 = vdupq_n_u8(0xF);
+
+ const int32x4_t vzero = vdupq_n_s32(0);
+
+ ggml_int8x16x2_t q2bytes;
+ uint8_t aux[16];
+
+ float sum = 0;
+
+ for (int i = 0; i < nb; ++i) {
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ const uint8_t * restrict sc = x[i].scales;
+
+ const uint8x16_t mins_and_scales = vld1q_u8(sc);
+ const uint8x16_t scales = vandq_u8(mins_and_scales, m4);
+ vst1q_u8(aux, scales);
+
+ const uint8x16_t mins = vshrq_n_u8(mins_and_scales, 4);
+ const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);
+ const ggml_int16x8x2_t mins16 = {{vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins)))}};
+ const int32x4_t s0 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[0]), vget_low_s16 (q8sums.val[0])),
+ vmull_s16(vget_high_s16(mins16.val[0]), vget_high_s16(q8sums.val[0])));
+ const int32x4_t s1 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[1]), vget_low_s16 (q8sums.val[1])),
+ vmull_s16(vget_high_s16(mins16.val[1]), vget_high_s16(q8sums.val[1])));
+ sum += dmin * vaddvq_s32(vaddq_s32(s0, s1));
+
+ int isum = 0;
+ int is = 0;
+
+// We use this macro instead of a function call because for some reason
+// the code runs 2-3% slower, even if the function is declared inline
+#define MULTIPLY_ACCUM_WITH_SCALE(index)\
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * aux[is+(index)];\
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * aux[is+1+(index)];
+
+#define SHIFT_MULTIPLY_ACCUM_WITH_SCALE(shift, index)\
+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\
+ q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[0], (shift)), m3));\
+ q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[1], (shift)), m3));\
+ MULTIPLY_ACCUM_WITH_SCALE((index));
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ const ggml_uint8x16x2_t q2bits = ggml_vld1q_u8_x2(q2); q2 += 32;
+
+ ggml_int8x16x2_t q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;
+ q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[0], m3));
+ q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[1], m3));
+
+ MULTIPLY_ACCUM_WITH_SCALE(0);
+
+ SHIFT_MULTIPLY_ACCUM_WITH_SCALE(2, 2);
+ SHIFT_MULTIPLY_ACCUM_WITH_SCALE(4, 4);
+ SHIFT_MULTIPLY_ACCUM_WITH_SCALE(6, 6);
+
+ is += 8;
+ }
+
+ sum += d * isum;
+ }
+
+ *s = sum;
+
+#elif defined __AVX2__
+
+ const __m256i m3 = _mm256_set1_epi8(3);
+ const __m128i m4 = _mm_set1_epi8(0xF);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);
+ const __m128i scales8 = _mm_and_si128(mins_and_scales, m4);
+ const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);
+ const __m256i mins = _mm256_cvtepi8_epi16(mins8);
+ const __m256i prod = _mm256_madd_epi16(mins, _mm256_loadu_si256((const __m256i*)y[i].bsums));
+
+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc);
+
+ const __m256i all_scales = _mm256_cvtepi8_epi16(scales8);
+ const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);
+ const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);
+ const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};
+
+ __m256i sumi = _mm256_setzero_si256();
+
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ const __m256i q2bits = _mm256_loadu_si256((const __m256i*)q2); q2 += 32;
+
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+
+ const __m256i q2_0 = _mm256_and_si256(q2bits, m3);
+ const __m256i q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3);
+ const __m256i q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3);
+ const __m256i q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3);
+
+ __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);
+ __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);
+ __m256i p2 = _mm256_maddubs_epi16(q2_2, q8_2);
+ __m256i p3 = _mm256_maddubs_epi16(q2_3, q8_3);
+
+ p0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(0)), p0);
+ p1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(1)), p1);
+ p2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(2)), p2);
+ p3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(3)), p3);
+
+ p0 = _mm256_add_epi32(p0, p1);
+ p2 = _mm256_add_epi32(p2, p3);
+
+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2));
+ }
+
+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);
+
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __AVX__
+
+ const __m128i m3 = _mm_set1_epi8(0x3);
+ const __m128i m4 = _mm_set1_epi8(0xF);
+ const __m128i m2 = _mm_set1_epi8(0x2);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ // load mins and scales from block_q2_K.scales[QK_K/16]
+ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);
+ const __m128i scales16 = _mm_and_si128(mins_and_scales, m4);
+ const __m128i mins16 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);
+ const __m128i mins_0 = _mm_cvtepi8_epi16(mins16);
+ const __m128i mins_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(mins16, mins16));
+
+ // summs = y[i].bsums * (x[i].scales >> 4) in 16bits*8*2 to 32bits*4*2
+ const __m128i summs_0 = _mm_madd_epi16(mins_0, _mm_loadu_si128((const __m128i*)&y[i].bsums[0]));
+ const __m128i summs_1 = _mm_madd_epi16(mins_1, _mm_loadu_si128((const __m128i*)&y[i].bsums[8]));
+
+ // sumf += -dmin * summs in 32bits*8
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(MM256_SET_M128I(summs_1, summs_0))), acc);
+
+ const __m128i scales_0 = _mm_cvtepi8_epi16(scales16);
+ const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales16, scales16));
+ const __m128i scales[2] = { scales_0, scales_1 };
+
+ __m128i sumi_0 = _mm_setzero_si128();
+ __m128i sumi_1 = _mm_setzero_si128();
+
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ // load Q8 quants int8*16*8 from block_q8_K.qs[QK_K]
+ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+
+ // load 2bits*16*8 from block_q2_K.qs[QK_K/4]
+ __m128i q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;
+ const __m128i q2_0 = _mm_and_si128(q2bits, m3);
+ const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);
+ const __m128i q2_4 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);
+ const __m128i q2_6 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);
+ q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;
+ const __m128i q2_1 = _mm_and_si128(q2bits, m3);
+ const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);
+ const __m128i q2_5 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);
+ const __m128i q2_7 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);
+
+ // isuml = q8[l] * ((q2[l] >> shift) & 3) in 8bits*16*8 to 16bits*8*8
+ __m128i p0 = _mm_maddubs_epi16(q2_0, q8_0);
+ __m128i p1 = _mm_maddubs_epi16(q2_1, q8_1);
+ __m128i p2 = _mm_maddubs_epi16(q2_2, q8_2);
+ __m128i p3 = _mm_maddubs_epi16(q2_3, q8_3);
+ __m128i p4 = _mm_maddubs_epi16(q2_4, q8_4);
+ __m128i p5 = _mm_maddubs_epi16(q2_5, q8_5);
+ __m128i p6 = _mm_maddubs_epi16(q2_6, q8_6);
+ __m128i p7 = _mm_maddubs_epi16(q2_7, q8_7);
+
+ // isum += (x[i].scales[is++] & 0xF) * isuml in 16bits*8*8 to 32bits*4*8
+ __m128i shuffle = _mm_set1_epi16(0x0100);
+ p0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p0);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p1);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p2);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p3);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p4);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p5);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p6);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p7);
+
+ p0 = _mm_add_epi32(p0, p1);
+ p2 = _mm_add_epi32(p2, p3);
+ p4 = _mm_add_epi32(p4, p5);
+ p6 = _mm_add_epi32(p6, p7);
+
+ // isum in 32bits*4*2
+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p0, p2));
+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p4, p6));
+ }
+
+ // sumf += dall * isum - dmin * summs in 32bits
+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dall), _mm256_cvtepi32_ps(sumi)), acc);
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __riscv_v_intrinsic
+
+ float sumf = 0;
+ uint8_t temp_01[32] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+
+ for (int i = 0; i < nb; ++i) {
+
+ const uint8_t * q2 = x[i].qs;
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * sc = x[i].scales;
+
+ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ size_t vl = 16;
+
+ vuint8m1_t scales = __riscv_vle8_v_u8m1(sc, vl);
+ vuint8m1_t aux = __riscv_vand_vx_u8m1(scales, 0x0F, vl);
+
+ vint16m1_t q8sums = __riscv_vle16_v_i16m1(y[i].bsums, vl);
+
+ vuint8mf2_t scales_2 = __riscv_vle8_v_u8mf2(sc, vl);
+ vuint8mf2_t mins8 = __riscv_vsrl_vx_u8mf2(scales_2, 0x4, vl);
+ vint16m1_t mins = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(mins8, vl));
+ vint32m2_t prod = __riscv_vwmul_vv_i32m2(q8sums, mins, vl);
+ vint32m1_t vsums = __riscv_vredsum_vs_i32m2_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);
+
+ sumf += dmin * __riscv_vmv_x_s_i32m1_i32(vsums);
+
+ vl = 32;
+
+ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
+ vuint8m1_t v_b = __riscv_vle8_v_u8m1(temp_01, vl);
+
+ uint8_t is=0;
+ int isum=0;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ // load Q2
+ vuint8m1_t q2_x = __riscv_vle8_v_u8m1(q2, vl);
+
+ vuint8m1_t q2_0 = __riscv_vand_vx_u8m1(q2_x, 0x03, vl);
+ vuint8m1_t q2_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x2, vl), 0x03 , vl);
+ vuint8m1_t q2_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x4, vl), 0x03 , vl);
+ vuint8m1_t q2_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x6, vl), 0x03 , vl);
+
+ // duplicate scale elements for product
+ vuint8m1_t sc0 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 0+is, vl), vl);
+ vuint8m1_t sc1 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 2+is, vl), vl);
+ vuint8m1_t sc2 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 4+is, vl), vl);
+ vuint8m1_t sc3 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 6+is, vl), vl);
+
+ vint16m2_t p0 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_0, sc0, vl));
+ vint16m2_t p1 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_1, sc1, vl));
+ vint16m2_t p2 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_2, sc2, vl));
+ vint16m2_t p3 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_3, sc3, vl));
+
+ // load Q8
+ vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);
+ vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);
+ vint8m1_t q8_2 = __riscv_vle8_v_i8m1(q8+64, vl);
+ vint8m1_t q8_3 = __riscv_vle8_v_i8m1(q8+96, vl);
+
+ vint32m4_t s0 = __riscv_vwmul_vv_i32m4(p0, __riscv_vwcvt_x_x_v_i16m2(q8_0, vl), vl);
+ vint32m4_t s1 = __riscv_vwmul_vv_i32m4(p1, __riscv_vwcvt_x_x_v_i16m2(q8_1, vl), vl);
+ vint32m4_t s2 = __riscv_vwmul_vv_i32m4(p2, __riscv_vwcvt_x_x_v_i16m2(q8_2, vl), vl);
+ vint32m4_t s3 = __riscv_vwmul_vv_i32m4(p3, __riscv_vwcvt_x_x_v_i16m2(q8_3, vl), vl);
+
+ vint32m1_t isum0 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s0, s1, vl), vzero, vl);
+ vint32m1_t isum1 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s2, s3, vl), isum0, vl);
+
+ isum += __riscv_vmv_x_s_i32m1_i32(isum1);
+
+ q2+=32; q8+=128; is=8;
+
+ }
+
+ sumf += dall * isum;
+
+ }
+
+ *s = sumf;
+
+#elif defined(__POWER9_VECTOR__)
+ const vector signed char lowMask = vec_splats((signed char)0x3);
+ const vector signed char lowScaleMask = vec_splats((signed char)0xF);
+ const vector int v0 = vec_splats((int32_t)0);
+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);
+ const vector unsigned char v6 = vec_splats((unsigned char)0x6);
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
+
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ for (int i = 0; i < nb; ++i) {
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
+ vector float vyd = vec_splats(y[i].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin));
+ vector float vdmin = vec_mul(vxmin, vyd);
+
+ vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);
+ vector signed short q8ysums1 = vec_xl(16, y[i].bsums);
+
+ vector signed char q2xmins = (vector signed char)vec_xl( 0, x[i].scales);
+ vector signed char vscales = vec_and(q2xmins, lowScaleMask);
+
+ q2xmins = vec_sr(q2xmins, v4);
+ vector signed short q2xmins0 = vec_unpackh(q2xmins);
+ vector signed short q2xmins1 = vec_unpackl(q2xmins);
+
+ vector signed int prod0 = vec_mule(q2xmins0, q8ysums0);
+ vector signed int prod1 = vec_mulo(q2xmins0, q8ysums0);
+ vector signed int prod2 = vec_mule(q2xmins1, q8ysums1);
+ vector signed int prod3 = vec_mulo(q2xmins1, q8ysums1);
+
+ vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);
+ vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);
+ vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);
+ vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+ vector signed int vsumi2 = v0;
+ vector signed int vsumi3 = v0;
+ vector signed int vsumi4 = v0;
+ vector signed int vsumi5 = v0;
+ vector signed int vsumi6 = v0;
+ vector signed int vsumi7 = v0;
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ __builtin_prefetch(q2, 0, 1);
+ __builtin_prefetch(q8, 0, 1);
+
+ vector signed char qxs0 = (vector signed char)vec_xl( 0, q2);
+ vector signed char qxs1 = (vector signed char)vec_xl(16, q2);
+ q2 += 32;
+
+ vector unsigned char q2x00 = (vector unsigned char)vec_and(qxs0, lowMask);
+ vector unsigned char q2x01 = (vector unsigned char)vec_and(vec_sr(qxs0, v2), lowMask);
+ vector unsigned char q2x02 = (vector unsigned char)vec_and(vec_sr(qxs0, v4), lowMask);
+ vector unsigned char q2x03 = (vector unsigned char)vec_and(vec_sr(qxs0, v6), lowMask);
+ vector unsigned char q2x10 = (vector unsigned char)vec_and(qxs1, lowMask);
+ vector unsigned char q2x11 = (vector unsigned char)vec_and(vec_sr(qxs1, v2), lowMask);
+ vector unsigned char q2x12 = (vector unsigned char)vec_and(vec_sr(qxs1, v4), lowMask);
+ vector unsigned char q2x13 = (vector unsigned char)vec_and(vec_sr(qxs1, v6), lowMask);
+
+ vector signed char q8y00 = vec_xl( 0, q8);
+ vector signed char q8y10 = vec_xl( 16, q8);
+ vector signed char q8y01 = vec_xl( 32, q8);
+ vector signed char q8y11 = vec_xl( 48, q8);
+ vector signed char q8y02 = vec_xl( 64, q8);
+ vector signed char q8y12 = vec_xl( 80, q8);
+ vector signed char q8y03 = vec_xl( 96, q8);
+ vector signed char q8y13 = vec_xl(112, q8);
+ q8 += 128;
+
+ vector signed int qv0 = vec_msum(q8y00, q2x00, v0);
+ vector signed int qv1 = vec_msum(q8y01, q2x01, v0);
+ vector signed int qv2 = vec_msum(q8y02, q2x02, v0);
+ vector signed int qv3 = vec_msum(q8y03, q2x03, v0);
+ vector signed int qv4 = vec_msum(q8y10, q2x10, v0);
+ vector signed int qv5 = vec_msum(q8y11, q2x11, v0);
+ vector signed int qv6 = vec_msum(q8y12, q2x12, v0);
+ vector signed int qv7 = vec_msum(q8y13, q2x13, v0);
+
+ vector signed short vscales_07 = vec_unpackh(vscales);
+ vector signed int vscales_03 = vec_unpackh(vscales_07);
+ vector signed int vscales_47 = vec_unpackl(vscales_07);
+ vector signed int vs0 = vec_splat(vscales_03, 0);
+ vector signed int vs1 = vec_splat(vscales_03, 1);
+ vector signed int vs2 = vec_splat(vscales_03, 2);
+ vector signed int vs3 = vec_splat(vscales_03, 3);
+ vector signed int vs4 = vec_splat(vscales_47, 0);
+ vector signed int vs5 = vec_splat(vscales_47, 1);
+ vector signed int vs6 = vec_splat(vscales_47, 2);
+ vector signed int vs7 = vec_splat(vscales_47, 3);
+ vscales = vec_sld(vscales, vscales, 8);
+
+ vsumi0 = vec_add(vec_mul(qv0, vs0), vsumi0);
+ vsumi1 = vec_add(vec_mul(qv1, vs2), vsumi1);
+ vsumi2 = vec_add(vec_mul(qv2, vs4), vsumi2);
+ vsumi3 = vec_add(vec_mul(qv3, vs6), vsumi3);
+ vsumi4 = vec_add(vec_mul(qv4, vs1), vsumi4);
+ vsumi5 = vec_add(vec_mul(qv5, vs3), vsumi5);
+ vsumi6 = vec_add(vec_mul(qv6, vs5), vsumi6);
+ vsumi7 = vec_add(vec_mul(qv7, vs7), vsumi7);
+ }
+
+ vsumi0 = vec_add(vsumi0, vsumi4);
+ vsumi1 = vec_add(vsumi1, vsumi5);
+ vsumi2 = vec_add(vsumi2, vsumi6);
+ vsumi3 = vec_add(vsumi3, vsumi7);
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = vec_extract(vsumf0, 0);
+
+#elif defined __loongarch_asx
+
+ const __m256i m3 = __lasx_xvreplgr2vr_b(3);
+ const __m128i m4 = __lsx_vreplgr2vr_b(0xF);
+
+ __m256 acc = (__m256)__lasx_xvldi(0);
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const __m128i mins_and_scales = __lsx_vld((const __m128i*)x[i].scales, 0);
+ const __m128i scales8 = __lsx_vand_v(mins_and_scales, m4);
+ const __m128i mins8 = __lsx_vand_v(__lsx_vsrli_h(mins_and_scales, 4), m4);
+ const __m256i mins = lasx_ext8_16(mins8);
+ const __m256i prod = lasx_madd_h(mins, __lasx_xvld((const __m256i*)y[i].bsums, 0));
+
+ acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(dmin), __lasx_xvffint_s_w(prod), acc);
+
+ const __m256i all_scales = lasx_ext8_16(scales8);
+ const __m128i l_scales = lasx_extracti128(all_scales, 0);
+ const __m128i h_scales = lasx_extracti128(all_scales, 1);
+ const __m256i scales[2] = {lasx_insertf128(l_scales, l_scales), lasx_insertf128(h_scales, h_scales)};
+
+ __m256i sumi = __lasx_xvldi(0);
+
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ const __m256i q2bits = __lasx_xvld((const __m256i*)q2, 0); q2 += 32;
+
+ const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+
+ const __m256i q2_0 = __lasx_xvand_v(q2bits, m3);
+ const __m256i q2_1 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 2), m3);
+ const __m256i q2_2 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 4), m3);
+ const __m256i q2_3 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 6), m3);
+
+ __m256i p0 = lasx_maddubs_h(q2_0, q8_0);
+ __m256i p1 = lasx_maddubs_h(q2_1, q8_1);
+ __m256i p2 = lasx_maddubs_h(q2_2, q8_2);
+ __m256i p3 = lasx_maddubs_h(q2_3, q8_3);
+
+ p0 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(0)), p0);
+ p1 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(1)), p1);
+ p2 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(2)), p2);
+ p3 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(3)), p3);
+
+ p0 = __lasx_xvadd_w(p0, p1);
+ p2 = __lasx_xvadd_w(p2, p3);
+
+ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p0, p2));
+ }
+
+ acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);
+
+ }
+
+ *s = hsum_float_8(acc);
+
+#else
+
+ float sumf = 0;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const uint8_t * q2 = x[i].qs;
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * sc = x[i].scales;
+
+ int summs = 0;
+ for (int j = 0; j < 16; ++j) {
+ summs += y[i].bsums[j] * (sc[j] >> 4);
+ }
+
+ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ int isum = 0;
+ int is = 0;
+ int d;
+ for (int k = 0; k < QK_K/128; ++k) {
+ int shift = 0;
+ for (int j = 0; j < 4; ++j) {
+ d = sc[is++] & 0xF;
+ int isuml = 0;
+ for (int l = 0; l < 16; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);
+ isum += d * isuml;
+ d = sc[is++] & 0xF;
+ isuml = 0;
+ for (int l = 16; l < 32; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);
+ isum += d * isuml;
+ shift += 2;
+ q8 += 32;
+ }
+ q2 += 32;
+ }
+ sumf += dall * isum - dmin * summs;
+ }
+ *s = sumf;
+#endif
+}
+
+void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const uint32_t kmask1 = 0x03030303;
+ const uint32_t kmask2 = 0x0f0f0f0f;
+
+ const block_q3_K * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#ifdef __ARM_NEON
+
+ uint32_t aux[3];
+ uint32_t utmp[4];
+
+ const uint8x16_t m3b = vdupq_n_u8(0x3);
+ const int32x4_t vzero = vdupq_n_s32(0);
+
+ const uint8x16_t m0 = vdupq_n_u8(1);
+ const uint8x16_t m1 = vshlq_n_u8(m0, 1);
+ const uint8x16_t m2 = vshlq_n_u8(m0, 2);
+ const uint8x16_t m3 = vshlq_n_u8(m0, 3);
+ const int8_t m32 = 32;
+
+ ggml_int8x16x4_t q3bytes;
+
+ float sum = 0;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict qh = x[i].hmask;
+ const int8_t * restrict q8 = y[i].qs;
+
+ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);
+
+ ggml_uint8x16x4_t q3h;
+
+ int32_t isum = 0;
+
+ // Set up scales
+ memcpy(aux, x[i].scales, 12);
+ utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);
+ utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);
+ utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);
+ utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);
+
+ int8_t * scale = (int8_t *)utmp;
+ for (int j = 0; j < 16; ++j) scale[j] -= m32;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ const ggml_uint8x16x2_t q3bits = ggml_vld1q_u8_x2(q3); q3 += 32;
+ const ggml_int8x16x4_t q8bytes_1 = ggml_vld1q_s8_x4(q8); q8 += 64;
+ const ggml_int8x16x4_t q8bytes_2 = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ q3h.val[0] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[0]), 2);
+ q3h.val[1] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[1]), 2);
+ q3h.val[2] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[0]), 1);
+ q3h.val[3] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[1]), 1);
+
+ q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[0], m3b)), vreinterpretq_s8_u8(q3h.val[0]));
+ q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[1], m3b)), vreinterpretq_s8_u8(q3h.val[1]));
+ q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 2), m3b)), vreinterpretq_s8_u8(q3h.val[2]));
+ q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 2), m3b)), vreinterpretq_s8_u8(q3h.val[3]));
+
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_1.val[0])) * scale[0];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_1.val[1])) * scale[1];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_1.val[2])) * scale[2];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_1.val[3])) * scale[3];
+
+ scale += 4;
+
+ q3h.val[0] = vbicq_u8(m2, qhbits.val[0]);
+ q3h.val[1] = vbicq_u8(m2, qhbits.val[1]);
+ q3h.val[2] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[0]), 1);
+ q3h.val[3] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[1]), 1);
+
+ q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 4), m3b)), vreinterpretq_s8_u8(q3h.val[0]));
+ q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 4), m3b)), vreinterpretq_s8_u8(q3h.val[1]));
+ q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 6), m3b)), vreinterpretq_s8_u8(q3h.val[2]));
+ q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 6), m3b)), vreinterpretq_s8_u8(q3h.val[3]));
+
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_2.val[0])) * scale[0];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_2.val[1])) * scale[1];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_2.val[2])) * scale[2];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_2.val[3])) * scale[3];
+
+ scale += 4;
+
+ if (j == 0) {
+ qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 4);
+ qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 4);
+ }
+
+ }
+ sum += d * isum;
+
+ }
+
+ *s = sum;
+
+#elif defined __AVX2__
+
+ const __m256i m3 = _mm256_set1_epi8(3);
+ const __m256i mone = _mm256_set1_epi8(1);
+ const __m128i m32 = _mm_set1_epi8(32);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ uint32_t aux[3];
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ // Set up scales
+ memcpy(aux, x[i].scales, 12);
+ __m128i scales128 = _mm_set_epi32(
+ ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),
+ ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),
+ (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),
+ (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));
+ scales128 = _mm_sub_epi8(scales128, m32);
+ const __m256i all_scales = _mm256_cvtepi8_epi16(scales128);
+ const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);
+ const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);
+ const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};
+
+ // high bit
+ const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].hmask);
+
+ // integer accumulator
+ __m256i sumi = _mm256_setzero_si256();
+
+ int bit = 0;
+ int is = 0;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ // load low 2 bits
+ const __m256i q3bits = _mm256_loadu_si256((const __m256i*)q3); q3 += 32;
+
+ // prepare low and high bits
+ const __m256i q3l_0 = _mm256_and_si256(q3bits, m3);
+ const __m256i q3h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
+ ++bit;
+
+ const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3);
+ const __m256i q3h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
+ ++bit;
+
+ const __m256i q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3);
+ const __m256i q3h_2 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
+ ++bit;
+
+ const __m256i q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3);
+ const __m256i q3h_3 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
+ ++bit;
+
+ // load Q8 quants
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+
+ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,
+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,
+ // and 2 if the high bit was set)
+ __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);
+ __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);
+ __m256i q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2);
+ __m256i q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3);
+
+ __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);
+ __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);
+ __m256i p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2);
+ __m256i p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3);
+
+ p16_0 = _mm256_sub_epi16(p16_0, q8s_0);
+ p16_1 = _mm256_sub_epi16(p16_1, q8s_1);
+ p16_2 = _mm256_sub_epi16(p16_2, q8s_2);
+ p16_3 = _mm256_sub_epi16(p16_3, q8s_3);
+
+ // multiply with scales
+ p16_0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0);
+ p16_1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1);
+ p16_2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2);
+ p16_3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3);
+
+ // accumulate
+ p16_0 = _mm256_add_epi32(p16_0, p16_1);
+ p16_2 = _mm256_add_epi32(p16_2, p16_3);
+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2));
+
+ }
+
+ // multiply with block scale and accumulate
+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);
+
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __AVX__
+
+ const __m128i m3 = _mm_set1_epi8(3);
+ const __m128i mone = _mm_set1_epi8(1);
+ const __m128i m32 = _mm_set1_epi8(32);
+ const __m128i m2 = _mm_set1_epi8(2);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ const uint32_t *aux;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ // Set up scales
+ aux = (const uint32_t *)x[i].scales;
+ __m128i scales128 = _mm_set_epi32(
+ ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),
+ ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),
+ (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),
+ (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));
+ scales128 = _mm_sub_epi8(scales128, m32);
+ const __m128i scales_0 = _mm_cvtepi8_epi16(scales128);
+ const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales128, scales128));
+ const __m128i scales[2] = { scales_0, scales_1 };
+
+ // high bit *128*2 from block_q3_K.hmask[QK_K/8]
+ const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].hmask[0]);
+ const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].hmask[16]);
+
+ // integer accumulator
+ __m128i sumi_0 = _mm_setzero_si128();
+ __m128i sumi_1 = _mm_setzero_si128();
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ // load low 2 bits *64*2 from block_q3_K.qs[QK_K/4]
+ const __m128i q3bits_0 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;
+ const __m128i q3bits_1 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;
+
+ // prepare low and high bits
+ const int bit = j << 2;
+
+ const __m128i q3l_0 = _mm_and_si128(q3bits_0, m3);
+ const __m128i q3l_1 = _mm_and_si128(q3bits_1, m3);
+ const __m128i q3h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit)), bit), 2);
+ const __m128i q3h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit)), bit), 2);
+
+ const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 2), m3);
+ const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 2), m3);
+ const __m128i q3h_2 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+1)), bit+1), 2);
+ const __m128i q3h_3 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+1)), bit+1), 2);
+
+ const __m128i q3l_4 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 4), m3);
+ const __m128i q3l_5 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 4), m3);
+ const __m128i q3h_4 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+2)), bit+2), 2);
+ const __m128i q3h_5 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+2)), bit+2), 2);
+
+ const __m128i q3l_6 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 6), m3);
+ const __m128i q3l_7 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 6), m3);
+ const __m128i q3h_6 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+3)), bit+3), 2);
+ const __m128i q3h_7 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+3)), bit+3), 2);
+
+ // load Q8 quants from block_q8_K.qs[QK_K]
+ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+
+ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,
+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,
+ // and 2 if the high bit was set)
+ __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, q8_0);
+ __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, q8_1);
+ __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, q8_2);
+ __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, q8_3);
+ __m128i q8s_4 = _mm_maddubs_epi16(q3h_4, q8_4);
+ __m128i q8s_5 = _mm_maddubs_epi16(q3h_5, q8_5);
+ __m128i q8s_6 = _mm_maddubs_epi16(q3h_6, q8_6);
+ __m128i q8s_7 = _mm_maddubs_epi16(q3h_7, q8_7);
+
+ __m128i p16_0 = _mm_maddubs_epi16(q3l_0, q8_0);
+ __m128i p16_1 = _mm_maddubs_epi16(q3l_1, q8_1);
+ __m128i p16_2 = _mm_maddubs_epi16(q3l_2, q8_2);
+ __m128i p16_3 = _mm_maddubs_epi16(q3l_3, q8_3);
+ __m128i p16_4 = _mm_maddubs_epi16(q3l_4, q8_4);
+ __m128i p16_5 = _mm_maddubs_epi16(q3l_5, q8_5);
+ __m128i p16_6 = _mm_maddubs_epi16(q3l_6, q8_6);
+ __m128i p16_7 = _mm_maddubs_epi16(q3l_7, q8_7);
+
+ p16_0 = _mm_sub_epi16(p16_0, q8s_0);
+ p16_1 = _mm_sub_epi16(p16_1, q8s_1);
+ p16_2 = _mm_sub_epi16(p16_2, q8s_2);
+ p16_3 = _mm_sub_epi16(p16_3, q8s_3);
+ p16_4 = _mm_sub_epi16(p16_4, q8s_4);
+ p16_5 = _mm_sub_epi16(p16_5, q8s_5);
+ p16_6 = _mm_sub_epi16(p16_6, q8s_6);
+ p16_7 = _mm_sub_epi16(p16_7, q8s_7);
+
+ // multiply with scales
+ __m128i shuffle = _mm_set1_epi16(0x0100);
+ p16_0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_0);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_1);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_2);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_3);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_4);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_5);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_6);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_7);
+
+ // accumulate
+ p16_0 = _mm_add_epi32(p16_0, p16_1);
+ p16_2 = _mm_add_epi32(p16_2, p16_3);
+ p16_4 = _mm_add_epi32(p16_4, p16_5);
+ p16_6 = _mm_add_epi32(p16_6, p16_7);
+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));
+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_4, p16_6));
+
+ }
+
+ // multiply with block scale and accumulate
+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);
+
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __riscv_v_intrinsic
+
+ uint32_t aux[3];
+ uint32_t utmp[4];
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict qh = x[i].hmask;
+ const int8_t * restrict q8 = y[i].qs;
+
+ memcpy(aux, x[i].scales, 12);
+ utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);
+ utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);
+ utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);
+ utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);
+
+ int8_t * scale = (int8_t *)utmp;
+ for (int j = 0; j < 16; ++j) scale[j] -= 32;
+
+
+ size_t vl = 32;
+ uint8_t m = 1;
+
+ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
+ vuint8m1_t vqh = __riscv_vle8_v_u8m1(qh, vl);
+
+ int sum_t = 0;
+
+ for (int j = 0; j < QK_K; j += 128) {
+
+ vl = 32;
+
+ // load Q3
+ vuint8m1_t q3_x = __riscv_vle8_v_u8m1(q3, vl);
+
+ vint8m1_t q3_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q3_x, 0x03, vl));
+ vint8m1_t q3_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x2, vl), 0x03 , vl));
+ vint8m1_t q3_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x4, vl), 0x03 , vl));
+ vint8m1_t q3_3 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x6, vl), 0x03 , vl));
+
+ // compute mask for subtraction
+ vuint8m1_t qh_m0 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_0 = __riscv_vmseq_vx_u8m1_b8(qh_m0, 0, vl);
+ vint8m1_t q3_m0 = __riscv_vsub_vx_i8m1_mu(vmask_0, q3_0, q3_0, 0x4, vl);
+ m <<= 1;
+
+ vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_1 = __riscv_vmseq_vx_u8m1_b8(qh_m1, 0, vl);
+ vint8m1_t q3_m1 = __riscv_vsub_vx_i8m1_mu(vmask_1, q3_1, q3_1, 0x4, vl);
+ m <<= 1;
+
+ vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_2 = __riscv_vmseq_vx_u8m1_b8(qh_m2, 0, vl);
+ vint8m1_t q3_m2 = __riscv_vsub_vx_i8m1_mu(vmask_2, q3_2, q3_2, 0x4, vl);
+ m <<= 1;
+
+ vuint8m1_t qh_m3 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_3 = __riscv_vmseq_vx_u8m1_b8(qh_m3, 0, vl);
+ vint8m1_t q3_m3 = __riscv_vsub_vx_i8m1_mu(vmask_3, q3_3, q3_3, 0x4, vl);
+ m <<= 1;
+
+ // load Q8 and take product with Q3
+ vint16m2_t a0 = __riscv_vwmul_vv_i16m2(q3_m0, __riscv_vle8_v_i8m1(q8, vl), vl);
+ vint16m2_t a1 = __riscv_vwmul_vv_i16m2(q3_m1, __riscv_vle8_v_i8m1(q8+32, vl), vl);
+ vint16m2_t a2 = __riscv_vwmul_vv_i16m2(q3_m2, __riscv_vle8_v_i8m1(q8+64, vl), vl);
+ vint16m2_t a3 = __riscv_vwmul_vv_i16m2(q3_m3, __riscv_vle8_v_i8m1(q8+96, vl), vl);
+
+ vl = 16;
+
+ // retrieve lane to multiply with scale
+ vint32m2_t aux0_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 0), (scale[0]), vl);
+ vint32m2_t aux0_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 1), (scale[1]), vl);
+ vint32m2_t aux1_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 0), (scale[2]), vl);
+ vint32m2_t aux1_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 1), (scale[3]), vl);
+ vint32m2_t aux2_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 0), (scale[4]), vl);
+ vint32m2_t aux2_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 1), (scale[5]), vl);
+ vint32m2_t aux3_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 0), (scale[6]), vl);
+ vint32m2_t aux3_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 1), (scale[7]), vl);
+
+ vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux0_0, aux0_1, vl), vzero, vl);
+ vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux1_0, aux1_1, vl), isum0, vl);
+ vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux2_0, aux2_1, vl), isum1, vl);
+ vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux3_0, aux3_1, vl), isum2, vl);
+
+ sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);
+
+ q3 += 32; q8 += 128; scale += 8;
+
+ }
+
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+
+ sumf += d*sum_t;
+
+ }
+
+ *s = sumf;
+
+#elif defined(__POWER9_VECTOR__)
+ const vector signed char lowMask = vec_splats((signed char)0x3);
+ const vector signed char lowMask1 = vec_splats((int8_t)0xf);
+ const vector signed char lowMask2 = vec_splats((int8_t)0x30);
+ const vector int v0 = vec_splats((int32_t)0);
+ const vector signed char v1 = vec_splats((signed char)0x1);
+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);
+ const vector unsigned char v3 = vec_splats((unsigned char)0x3);
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
+ const vector unsigned char v6 = vec_splats((unsigned char)0x6);
+ const vector signed char off = vec_splats((signed char)0x20);
+
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ for (int i = 0; i < nb; ++i) {
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
+ vector float vyd = vec_splats(y[i].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ UNUSED(kmask1);
+ UNUSED(kmask2);
+
+ vector signed char u0 = (vector signed char)vec_xl_len(x[i].scales, 8);
+ vector signed char u1 = vec_and(u0, lowMask1);
+ vector signed char u2 = (vector signed char)vec_xl_len(x[i].scales + 8, 4);
+ vector signed char u3 = (vector signed char)vec_mergeh((vector signed int)u2, (vector signed int)vec_sr(u2, v2));
+ vector signed char u30 = vec_sl(vec_and(u3, lowMask), v4);
+ vector signed char u31 = vec_and(u3, lowMask2);
+
+ u1 = vec_or(u1, u30);
+ u2 = vec_or(vec_sr(u0, v4), u31);
+
+ vector signed char vscales = (vector signed char)vec_mergeh((vector signed long long)u1, (vector signed long long)u2);
+ vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].hmask);
+ vector signed char qxhs1 = (vector signed char)vec_xl(16, x[i].hmask);
+
+ vscales = vec_sub(vscales, off);
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+ vector signed int vsumi2 = v0;
+ vector signed int vsumi3 = v0;
+ vector signed int vsumi4 = v0;
+ vector signed int vsumi5 = v0;
+ vector signed int vsumi6 = v0;
+ vector signed int vsumi7 = v0;
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ __builtin_prefetch(q3, 0, 1);
+ __builtin_prefetch(q8, 0, 1);
+
+ vector signed char qxs0 = (vector signed char)vec_xl( 0, q3);
+ vector signed char qxs1 = (vector signed char)vec_xl(16, q3);
+ q3 += 32;
+
+ //the low 2 bits
+ vector signed char qxs00 = vec_and(qxs0, lowMask);
+ vector signed char qxs01 = vec_and(vec_sr(qxs0, v2), lowMask);
+ vector signed char qxs02 = vec_and(vec_sr(qxs0, v4), lowMask);
+ vector signed char qxs03 = vec_and(vec_sr(qxs0, v6), lowMask);
+ vector signed char qxs10 = vec_and(qxs1, lowMask);
+ vector signed char qxs11 = vec_and(vec_sr(qxs1, v2), lowMask);
+ vector signed char qxs12 = vec_and(vec_sr(qxs1, v4), lowMask);
+ vector signed char qxs13 = vec_and(vec_sr(qxs1, v6), lowMask);
+
+ //the 3rd bit
+ vector signed char qxh00 = vec_sl(vec_andc(v1, qxhs0), v2);
+ vector signed char qxh01 = vec_sl(vec_andc(v1, vec_sr(qxhs0, (vector unsigned char)v1)), v2);
+ vector signed char qxh02 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v2)), v2);
+ vector signed char qxh03 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v3)), v2);
+ vector signed char qxh10 = vec_sl(vec_andc(v1, qxhs1), v2);
+ vector signed char qxh11 = vec_sl(vec_andc(v1, vec_sr(qxhs1, (vector unsigned char)v1)), v2);
+ vector signed char qxh12 = vec_sl(vec_andc(v1, vec_sr(qxhs1, v2)), v2);
+ vector signed char qxh13 = vec_sl(vec_andc(v1, vec_sr(qxhs1, v3)), v2);
+ qxhs0 = vec_sr(qxhs0, v4);
+ qxhs1 = vec_sr(qxhs1, v4);
+
+ vector signed char q3x00 = vec_sub(qxs00, qxh00);
+ vector signed char q3x01 = vec_sub(qxs01, qxh01);
+ vector signed char q3x02 = vec_sub(qxs02, qxh02);
+ vector signed char q3x03 = vec_sub(qxs03, qxh03);
+ vector signed char q3x10 = vec_sub(qxs10, qxh10);
+ vector signed char q3x11 = vec_sub(qxs11, qxh11);
+ vector signed char q3x12 = vec_sub(qxs12, qxh12);
+ vector signed char q3x13 = vec_sub(qxs13, qxh13);
+
+ vector signed char q8y00 = vec_xl( 0, q8);
+ vector signed char q8y10 = vec_xl( 16, q8);
+ vector signed char q8y01 = vec_xl( 32, q8);
+ vector signed char q8y11 = vec_xl( 48, q8);
+ vector signed char q8y02 = vec_xl( 64, q8);
+ vector signed char q8y12 = vec_xl( 80, q8);
+ vector signed char q8y03 = vec_xl( 96, q8);
+ vector signed char q8y13 = vec_xl(112, q8);
+ q8 += 128;
+
+ vector signed short vscales_h = vec_unpackh(vscales);
+ vector signed short vs0 = vec_splat(vscales_h, 0);
+ vector signed short vs1 = vec_splat(vscales_h, 1);
+ vector signed short vs2 = vec_splat(vscales_h, 2);
+ vector signed short vs3 = vec_splat(vscales_h, 3);
+ vector signed short vs4 = vec_splat(vscales_h, 4);
+ vector signed short vs5 = vec_splat(vscales_h, 5);
+ vector signed short vs6 = vec_splat(vscales_h, 6);
+ vector signed short vs7 = vec_splat(vscales_h, 7);
+ vscales = vec_sld(vscales, vscales, 8);
+
+ vector signed short qv00 = vec_add(vec_mule(q3x00, q8y00), vec_mulo(q3x00, q8y00));
+ vector signed short qv01 = vec_add(vec_mule(q3x01, q8y01), vec_mulo(q3x01, q8y01));
+ vector signed short qv02 = vec_add(vec_mule(q3x02, q8y02), vec_mulo(q3x02, q8y02));
+ vector signed short qv03 = vec_add(vec_mule(q3x03, q8y03), vec_mulo(q3x03, q8y03));
+ vector signed short qv10 = vec_add(vec_mule(q3x10, q8y10), vec_mulo(q3x10, q8y10));
+ vector signed short qv11 = vec_add(vec_mule(q3x11, q8y11), vec_mulo(q3x11, q8y11));
+ vector signed short qv12 = vec_add(vec_mule(q3x12, q8y12), vec_mulo(q3x12, q8y12));
+ vector signed short qv13 = vec_add(vec_mule(q3x13, q8y13), vec_mulo(q3x13, q8y13));
+
+ vsumi0 = vec_msum(qv00, vs0, vsumi0);
+ vsumi1 = vec_msum(qv01, vs2, vsumi1);
+ vsumi2 = vec_msum(qv02, vs4, vsumi2);
+ vsumi3 = vec_msum(qv03, vs6, vsumi3);
+ vsumi4 = vec_msum(qv10, vs1, vsumi4);
+ vsumi5 = vec_msum(qv11, vs3, vsumi5);
+ vsumi6 = vec_msum(qv12, vs5, vsumi6);
+ vsumi7 = vec_msum(qv13, vs7, vsumi7);
+ }
+
+ vsumi0 = vec_add(vsumi0, vsumi4);
+ vsumi1 = vec_add(vsumi1, vsumi5);
+ vsumi2 = vec_add(vsumi2, vsumi6);
+ vsumi3 = vec_add(vsumi3, vsumi7);
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = vec_extract(vsumf0, 0);
+
+#elif defined __loongarch_asx
+
+ const __m256i m3 = __lasx_xvreplgr2vr_b(3);
+ const __m256i mone = __lasx_xvreplgr2vr_b(1);
+ const __m128i m32 = __lsx_vreplgr2vr_b(32);
+
+ __m256 acc = (__m256)__lasx_xvldi(0);
+
+ uint32_t aux[3];
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const uint8_t * restrict q3 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ // Set up scales
+ memcpy(aux, x[i].scales, 12);
+ __m128i scales128 = lsx_set_w(
+ ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),
+ ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),
+ (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),
+ (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));
+ scales128 = __lsx_vsub_b(scales128, m32);
+ const __m256i all_scales = lasx_ext8_16(scales128);
+ const __m128i l_scales = lasx_extracti128(all_scales, 0);
+ const __m128i h_scales = lasx_extracti128(all_scales, 1);
+ const __m256i scales[2] = {lasx_insertf128(l_scales, l_scales), lasx_insertf128(h_scales, h_scales)};
+
+ // high bit
+ const __m256i hbits = __lasx_xvld((const __m256i*)x[i].hmask, 0);
+
+ // integer accumulator
+ __m256i sumi = __lasx_xvldi(0);
+
+ int bit = 0;
+ int is = 0;
+ __m256i xvbit;
+
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ // load low 2 bits
+ const __m256i q3bits = __lasx_xvld((const __m256i*)q3, 0); q3 += 32;
+
+ xvbit = __lasx_xvreplgr2vr_h(bit);
+ // prepare low and high bits
+ const __m256i q3l_0 = __lasx_xvand_v(q3bits, m3);
+ const __m256i q3h_0 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvandn_v(hbits, __lasx_xvsll_h(mone, xvbit)), xvbit), 2);
+ ++bit;
+
+ xvbit = __lasx_xvreplgr2vr_h(bit);
+ const __m256i q3l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 2), m3);
+ const __m256i q3h_1 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvandn_v(hbits, __lasx_xvsll_h(mone, xvbit)), xvbit), 2);
+ ++bit;
+
+ xvbit = __lasx_xvreplgr2vr_h(bit);
+ const __m256i q3l_2 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 4), m3);
+ const __m256i q3h_2 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvandn_v(hbits, __lasx_xvsll_h(mone, xvbit)), xvbit), 2);
+ ++bit;
+
+ xvbit = __lasx_xvreplgr2vr_h(bit);
+ const __m256i q3l_3 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 6), m3);
+ const __m256i q3h_3 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvandn_v(hbits, __lasx_xvsll_h(mone, xvbit)), xvbit), 2);
+ ++bit;
+
+ // load Q8 quants
+ const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+
+ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use lasx_maddubs_h,
+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,
+ // and 2 if the high bit was set)
+ __m256i q8s_0 = lasx_maddubs_h(q3h_0, q8_0);
+ __m256i q8s_1 = lasx_maddubs_h(q3h_1, q8_1);
+ __m256i q8s_2 = lasx_maddubs_h(q3h_2, q8_2);
+ __m256i q8s_3 = lasx_maddubs_h(q3h_3, q8_3);
+
+ __m256i p16_0 = lasx_maddubs_h(q3l_0, q8_0);
+ __m256i p16_1 = lasx_maddubs_h(q3l_1, q8_1);
+ __m256i p16_2 = lasx_maddubs_h(q3l_2, q8_2);
+ __m256i p16_3 = lasx_maddubs_h(q3l_3, q8_3);
+
+ p16_0 = __lasx_xvsub_h(p16_0, q8s_0);
+ p16_1 = __lasx_xvsub_h(p16_1, q8s_1);
+ p16_2 = __lasx_xvsub_h(p16_2, q8s_2);
+ p16_3 = __lasx_xvsub_h(p16_3, q8s_3);
+
+ // multiply with scales
+ p16_0 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0);
+ p16_1 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1);
+ p16_2 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2);
+ p16_3 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3);
+
+ // accumulate
+ p16_0 = __lasx_xvadd_w(p16_0, p16_1);
+ p16_2 = __lasx_xvadd_w(p16_2, p16_3);
+ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_2));
+ }
+ // multiply with block scale and accumulate
+ acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);//FIXME
+ }
+
+ *s = hsum_float_8(acc);
+
+#else
+ // scalar version
+ // This function is written like this so the compiler can manage to vectorize most of it
+ // Using -Ofast, GCC and clang manage to produce code that is within a factor of 2 or so from the
+ // manually vectorized version above. Every other version I tried would run at least 4 times slower.
+ // The ideal situation would be if we could just write the code once, and the compiler would
+ // automatically produce the best possible set of machine instructions, instead of us having to manually
+ // write vectorized versions for AVX, ARM_NEON, etc.
+
+ int8_t aux8[QK_K];
+ int16_t aux16[8];
+ float sums [8];
+ int32_t aux32[8];
+ memset(sums, 0, 8*sizeof(float));
+
+ uint32_t auxs[4];
+ const int8_t * scales = (const int8_t*)auxs;
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict hm = x[i].hmask;
+ const int8_t * restrict q8 = y[i].qs;
+ memset(aux32, 0, 8*sizeof(int32_t));
+ int8_t * restrict a = aux8;
+ uint8_t m = 1;
+ for (int j = 0; j < QK_K; j += 128) {
+ for (int l = 0; l < 32; ++l) a[l] = q3[l] & 3;
+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
+ a += 32; m <<= 1;
+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 2) & 3;
+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
+ a += 32; m <<= 1;
+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 4) & 3;
+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
+ a += 32; m <<= 1;
+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 6) & 3;
+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
+ a += 32; m <<= 1;
+ q3 += 32;
+ }
+ a = aux8;
+
+ memcpy(auxs, x[i].scales, 12);
+ uint32_t tmp = auxs[2];
+ auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
+ auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
+ auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
+ auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
+ for (int j = 0; j < QK_K/16; ++j) {
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];
+ q8 += 8; a += 8;
+ }
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
+ }
+ for (int l = 0; l < 8; ++l) sumf += sums[l];
+ *s = sumf;
+
+#endif
+
+}
+
+void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q4_K * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+ static const uint32_t kmask1 = 0x3f3f3f3f;
+ static const uint32_t kmask2 = 0x0f0f0f0f;
+ static const uint32_t kmask3 = 0x03030303;
+
+ uint32_t utmp[4];
+
+#ifdef __ARM_NEON
+ const uint8x16_t m4b = vdupq_n_u8(0xf);
+ const int32x4_t mzero = vdupq_n_s32(0);
+
+ ggml_int8x16x2_t q4bytes;
+ ggml_int8x16x2_t q8bytes;
+
+ float sumf = 0;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));
+
+ memcpy(utmp, x[i].scales, 12);
+
+ uint32x2_t mins8 = { 0 };
+ mins8 = vset_lane_u32(utmp[1] & kmask1, mins8, 0);
+ mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), mins8, 1);
+
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[0] &= kmask1;
+
+ const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8)));
+ const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),
+ vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));
+ sumf -= dmin * vaddvq_s32(prod);
+
+ const uint8_t * scales = (const uint8_t *)utmp;
+
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ int32_t sumi1 = 0;
+ int32_t sumi2 = 0;
+
+ for (int j = 0; j < QK_K/64; ++j) {
+ const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;
+
+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;
+ q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));
+ q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));
+
+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);
+ sumi1 += vaddvq_s32(p1) * scales[2*j+0];
+
+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;
+ q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));
+ q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));
+
+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);
+
+ sumi2 += vaddvq_s32(p2) * scales[2*j+1];
+ }
+
+ sumf += d * (sumi1 + sumi2);
+
+ }
+
+ *s = sumf;
+
+#elif defined __AVX2__
+
+ const __m256i m4 = _mm256_set1_epi8(0xF);
+
+ __m256 acc = _mm256_setzero_ps();
+ __m128 acc_m = _mm_setzero_ps();
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));
+
+ const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);
+ const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));
+ const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);
+ acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);
+
+ const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);
+ const __m256i scales = MM256_SET_M128I(sc128, sc128);
+
+ __m256i sumi = _mm256_setzero_si256();
+
+ for (int j = 0; j < QK_K/64; ++j) {
+
+ const __m256i scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));
+ const __m256i scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));
+
+ const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;
+ const __m256i q4l = _mm256_and_si256(q4bits, m4);
+ const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);
+
+ const __m256i q8l = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);
+ p16l = _mm256_madd_epi16(scale_l, p16l);
+
+ const __m256i q8h = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);
+ p16h = _mm256_madd_epi16(scale_h, p16h);
+ const __m256i sumj = _mm256_add_epi32(p16l, p16h);
+
+ sumi = _mm256_add_epi32(sumi, sumj);
+ }
+
+ __m256 vd = _mm256_set1_ps(d);
+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);
+
+ }
+
+ acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));
+ acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));
+
+ *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);
+
+#elif defined __AVX__
+
+ const __m128i m4 = _mm_set1_epi8(0xF);
+ const __m128i m2 = _mm_set1_epi8(0x2);
+
+ __m256 acc = _mm256_setzero_ps();
+ __m128 acc_m = _mm_setzero_ps();
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);
+ const __m128i scales = _mm_cvtepu8_epi16(utmps);
+ const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));
+
+ const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);
+ const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);
+ const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);
+ const __m128i prod = _mm_madd_epi16(mins, q8s);
+ acc_m = _mm_add_ps(_mm_mul_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod)), acc_m);
+
+ __m128i sumi_0 = _mm_setzero_si128();
+ __m128i sumi_1 = _mm_setzero_si128();
+
+ __m128i shuffle = _mm_set1_epi16(0x0100);
+ for (int j = 0; j < QK_K/64; ++j) {
+
+ const __m128i scale_l = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ const __m128i scale_h = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi16(shuffle, m2);
+
+ __m128i q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+ const __m128i q4l_0 = _mm_and_si128(q4bits, m4);
+ const __m128i q4h_0 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);
+ q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+ const __m128i q4l_1 = _mm_and_si128(q4bits, m4);
+ const __m128i q4h_1 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);
+
+ const __m128i q8l_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ __m128i p16l = _mm_maddubs_epi16(q4l_0, q8l_0);
+ p16l = _mm_madd_epi16(scale_l, p16l);
+ sumi_0 = _mm_add_epi32(sumi_0, p16l);
+ const __m128i q8l_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ p16l = _mm_maddubs_epi16(q4l_1, q8l_1);
+ p16l = _mm_madd_epi16(scale_l, p16l);
+ sumi_1 = _mm_add_epi32(sumi_1, p16l);
+
+ const __m128i q8h_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ __m128i p16h = _mm_maddubs_epi16(q4h_0, q8h_0);
+ p16h = _mm_madd_epi16(scale_h, p16h);
+ sumi_0 = _mm_add_epi32(sumi_0, p16h);
+ const __m128i q8h_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ p16h = _mm_maddubs_epi16(q4h_1, q8h_1);
+ p16h = _mm_madd_epi16(scale_h, p16h);
+ sumi_1 = _mm_add_epi32(sumi_1, p16h);
+
+ }
+
+ __m256 vd = _mm256_set1_ps(d);
+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);
+
+ }
+
+ acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));
+ acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));
+
+ *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);
+
+#elif defined __riscv_v_intrinsic
+
+ const uint8_t * scales = (const uint8_t*)&utmp[0];
+ const uint8_t * mins = (const uint8_t*)&utmp[2];
+
+ float sumf = 0;
+
+ for (int i = 0; i < nb; ++i) {
+
+ size_t vl = 8;
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);
+ vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);
+ vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);
+
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);
+ vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));
+ vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);
+
+ vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);
+ sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);
+
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ vl = 32;
+
+ int32_t sum_1 = 0;
+ int32_t sum_2 = 0;
+
+ vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);
+
+ for (int j = 0; j < QK_K/64; ++j) {
+ // load Q4
+ vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);
+
+ // load Q8 and multiply it with lower Q4 nibble
+ vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);
+ vint8m1_t q4_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));
+ vint16m2_t qv_0 = __riscv_vwmul_vv_i16m2(q4_0, q8_0, vl);
+ vint16m1_t vs_0 = __riscv_vredsum_vs_i16m2_i16m1(qv_0, vzero, vl);
+
+ sum_1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[2*j+0];
+
+ // load Q8 and multiply it with upper Q4 nibble
+ vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);
+ vint8m1_t q4_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));
+ vint16m2_t qv_1 = __riscv_vwmul_vv_i16m2(q4_1, q8_1, vl);
+ vint16m1_t vs_1 = __riscv_vredsum_vs_i16m2_i16m1(qv_1, vzero, vl);
+
+ sum_2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[2*j+1];
+
+ q4 += 32; q8 += 64;
+
+ }
+
+ sumf += d*(sum_1 + sum_2);
+
+ }
+
+ *s = sumf;
+
+#elif defined(__POWER9_VECTOR__)
+ const vector signed char lowMask = vec_splats((signed char)0xF);
+ const vector signed char lowMask1 = vec_splats((int8_t)0x3f);
+ const vector signed char lowMask2 = vec_splats((int8_t)0x30);
+ const vector int v0 = vec_splats((int32_t)0);
+ const vector unsigned char v2 = vec_splats((uint8_t)2);
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
+
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ for (int i = 0; i < nb; ++i) {
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
+ vector float vyd = vec_splats(y[i].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin));
+ vector float vdmin = vec_mul(vxmin, vyd);
+
+ vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);
+ vector signed short q8ysums1 = vec_xl(16, y[i].bsums);
+
+ UNUSED(kmask1);
+ UNUSED(kmask2);
+ UNUSED(kmask3);
+ UNUSED(utmp);
+
+ vector signed char u0 = (vector signed char)vec_xl_len(x[i].scales, 8);
+ vector signed char u1 = vec_and(vec_sr(u0, v2), lowMask2);
+ vector signed char u2 = (vector signed char)vec_xl_len(x[i].scales + 8, 4);
+ vector signed char u3 = vec_sr(u2, v4);
+
+ vector signed char u30 = u1;
+ vector signed char u31 = (vector signed char)vec_mergeh((vector signed int)vec_and(u2, lowMask), (vector signed int)u3);
+
+ u1 = vec_and(u0, lowMask1);
+ u2 = vec_or(u30, u31);
+
+ vector signed char utmps = (vector signed char)vec_mergeh((vector signed int)u1, (vector signed int)u2);
+
+ vector signed short vscales = vec_unpackh(utmps);
+ vector signed short q4xmins = vec_unpackl(utmps);
+ vector signed short q4xmins0 = vec_mergeh(q4xmins, q4xmins);
+ vector signed short q4xmins1 = vec_mergel(q4xmins, q4xmins);
+
+ vector signed int prod0 = vec_mule(q4xmins0, q8ysums0);
+ vector signed int prod1 = vec_mule(q4xmins1, q8ysums1);
+ vector signed int prod2 = vec_mulo(q4xmins0, q8ysums0);
+ vector signed int prod3 = vec_mulo(q4xmins1, q8ysums1);
+
+ vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);
+ vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);
+ vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);
+ vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+ vector signed int vsumi2 = v0;
+ vector signed int vsumi3 = v0;
+
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ for (int j = 0; j < QK_K/64; j+=2) {
+ __builtin_prefetch(q4, 0, 1);
+ __builtin_prefetch(q8, 0, 1);
+
+ vector signed char qxs0 = (vector signed char)vec_xl( 0, q4);
+ vector signed char qxs1 = (vector signed char)vec_xl(16, q4);
+ vector signed char qxs2 = (vector signed char)vec_xl(32, q4);
+ vector signed char qxs3 = (vector signed char)vec_xl(48, q4);
+ q4 += 64;
+
+ vector unsigned char q4x00 = (vector unsigned char)vec_and(qxs0, lowMask);
+ vector unsigned char q4x01 = (vector unsigned char)vec_sr(qxs0, v4);
+ vector unsigned char q4x10 = (vector unsigned char)vec_and(qxs1, lowMask);
+ vector unsigned char q4x11 = (vector unsigned char)vec_sr(qxs1, v4);
+ vector unsigned char q4x20 = (vector unsigned char)vec_and(qxs2, lowMask);
+ vector unsigned char q4x21 = (vector unsigned char)vec_sr(qxs2, v4);
+ vector unsigned char q4x30 = (vector unsigned char)vec_and(qxs3, lowMask);
+ vector unsigned char q4x31 = (vector unsigned char)vec_sr(qxs3, v4);
+
+ vector signed char q8y00 = vec_xl( 0, q8);
+ vector signed char q8y10 = vec_xl( 16, q8);
+ vector signed char q8y01 = vec_xl( 32, q8);
+ vector signed char q8y11 = vec_xl( 48, q8);
+ vector signed char q8y20 = vec_xl( 64, q8);
+ vector signed char q8y30 = vec_xl( 80, q8);
+ vector signed char q8y21 = vec_xl( 96, q8);
+ vector signed char q8y31 = vec_xl(112, q8);
+ q8 += 128;
+
+ vector signed int qv00 = vec_msum(q8y00, q4x00, v0);
+ vector signed int qv01 = vec_msum(q8y01, q4x01, v0);
+ vector signed int qv10 = vec_msum(q8y10, q4x10, v0);
+ vector signed int qv11 = vec_msum(q8y11, q4x11, v0);
+ vector signed int qv20 = vec_msum(q8y20, q4x20, v0);
+ vector signed int qv21 = vec_msum(q8y21, q4x21, v0);
+ vector signed int qv30 = vec_msum(q8y30, q4x30, v0);
+ vector signed int qv31 = vec_msum(q8y31, q4x31, v0);
+
+ vector signed int vscales_h = vec_unpackh(vscales);
+ vector signed int vs0 = vec_splat(vscales_h, 0);
+ vector signed int vs1 = vec_splat(vscales_h, 1);
+ vector signed int vs2 = vec_splat(vscales_h, 2);
+ vector signed int vs3 = vec_splat(vscales_h, 3);
+ vscales = vec_sld(vscales, vscales, 8);
+
+ vsumi0 = vec_add(vec_mul(qv00, vs0), vsumi0);
+ vsumi1 = vec_add(vec_mul(qv01, vs1), vsumi1);
+ vsumi2 = vec_add(vec_mul(qv20, vs2), vsumi2);
+ vsumi3 = vec_add(vec_mul(qv21, vs3), vsumi3);
+
+ vsumi0 = vec_add(vec_mul(qv10, vs0), vsumi0);
+ vsumi1 = vec_add(vec_mul(qv11, vs1), vsumi1);
+ vsumi2 = vec_add(vec_mul(qv30, vs2), vsumi2);
+ vsumi3 = vec_add(vec_mul(qv31, vs3), vsumi3);
+ }
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = vec_extract(vsumf0, 0);
+
+#elif defined __loongarch_asx
+ GGML_UNUSED(kmask1);
+ GGML_UNUSED(kmask2);
+ GGML_UNUSED(kmask3);
+
+ const __m256i m4 = __lasx_xvreplgr2vr_b(0xF);
+
+ __m256 acc = (__m256)__lasx_xvldi(0);
+ __m128 acc_m = (__m128)__lsx_vldi(0);
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const __m256i mins_and_scales = lasx_extu8_16(lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]));
+
+ const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0);
+ const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1));
+ const __m128i prod = lsx_madd_h(lasx_extracti128(mins_and_scales, 1), q8s);
+ acc_m = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(dmin), __lsx_vffint_s_w(prod), acc_m);
+
+ const __m128i sc128 = lasx_extracti128(mins_and_scales, 0);
+ const __m256i scales = lasx_insertf128(sc128, sc128);
+
+ __m256i sumi = __lasx_xvldi(0);
+
+ for (int j = 0; j < QK_K/64; ++j) {
+
+ const __m256i scale_l = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+0));
+ const __m256i scale_h = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+1));
+
+ const __m256i q4bits = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;
+ const __m256i q4l = __lasx_xvand_v(q4bits, m4);
+ const __m256i q4h = __lasx_xvand_v(__lasx_xvsrli_h(q4bits, 4), m4);
+
+ const __m256i q8l = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ __m256i p16l = lasx_maddubs_h(q4l, q8l);
+ p16l = lasx_madd_h(scale_l, p16l);
+
+ const __m256i q8h = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ __m256i p16h = lasx_maddubs_h(q4h, q8h);
+ p16h = lasx_madd_h(scale_h, p16h);
+ const __m256i sumj = __lasx_xvadd_w(p16l, p16h);
+
+ sumi = __lasx_xvadd_w(sumi, sumj);
+ }
+
+ __m256 vd = __lasx_xvreplfr2vr_s(d);
+ acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc);
+
+ }
+
+ acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vpermi_w((__m128i)acc_m, (__m128i)acc_m, 0xee));
+ __m128i tmp1 = __lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w((__m128i)acc_m, 1), 0);
+ acc_m = __lsx_vfadd_s(acc_m, (__m128)tmp1);
+
+
+ ft_union fi;
+ fi.i = __lsx_vpickve2gr_w(acc_m, 0);
+ *s = hsum_float_8(acc) + fi.f ;
+#else
+
+ const uint8_t * scales = (const uint8_t*)&utmp[0];
+ const uint8_t * mins = (const uint8_t*)&utmp[2];
+
+ int8_t aux8[QK_K];
+ int16_t aux16[8];
+ float sums [8];
+ int32_t aux32[8];
+ memset(sums, 0, 8*sizeof(float));
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ memset(aux32, 0, 8*sizeof(int32_t));
+ int8_t * restrict a = aux8;
+ for (int j = 0; j < QK_K/64; ++j) {
+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);
+ a += 32;
+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);
+ a += 32; q4 += 32;
+ }
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ int sumi = 0;
+ for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];
+ a = aux8;
+ int is = 0;
+ for (int j = 0; j < QK_K/32; ++j) {
+ int32_t scale = scales[is++];
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ }
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
+ const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;
+ sumf -= dmin * sumi;
+ }
+ for (int l = 0; l < 8; ++l) sumf += sums[l];
+ *s = sumf;
+#endif
+}
+
+void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q5_K * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+ static const uint32_t kmask1 = 0x3f3f3f3f;
+ static const uint32_t kmask2 = 0x0f0f0f0f;
+ static const uint32_t kmask3 = 0x03030303;
+
+ uint32_t utmp[4];
+
+#ifdef __ARM_NEON
+ const uint8x16_t m4b = vdupq_n_u8(0xf);
+ const uint8x16_t mone = vdupq_n_u8(1);
+ const uint8x16_t mtwo = vdupq_n_u8(2);
+ const int32x4_t mzero = vdupq_n_s32(0);
+
+ ggml_int8x16x4_t q5bytes;
+
+ float sumf = 0;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));
+
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ const uint8x8_t mins8 = vld1_u8((const uint8_t*)utmp + 8);
+ const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(mins8));
+ const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),
+ vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));
+ int32_t sumi_mins = vaddvq_s32(prod);
+
+ const uint8_t * scales = (const uint8_t *)utmp;
+
+ const uint8_t * restrict q5 = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
+
+ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);
+
+ ggml_uint8x16x4_t q5h;
+
+ int32_t sumi = 0;
+
+ for (int j = 0; j < QK_K/64; ++j) {
+
+ const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5); q5 += 32;
+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ q5h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);
+ q5h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);
+ q5h.val[2] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[0]), 3);
+ q5h.val[3] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[1]), 3);
+ qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 2);
+ qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 2);
+
+ q5bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[0], m4b), q5h.val[0]));
+ q5bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[1], m4b), q5h.val[1]));
+ q5bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[0], 4), q5h.val[2]));
+ q5bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[1], 4), q5h.val[3]));
+
+ sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]), q5bytes.val[1], q8bytes.val[1])) * *scales++;
+ sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]), q5bytes.val[3], q8bytes.val[3])) * *scales++;
+ }
+
+ sumf += d * sumi - dmin * sumi_mins;
+ }
+
+ *s = sumf;
+
+#elif defined __AVX2__
+
+ const __m256i m4 = _mm256_set1_epi8(0xF);
+ const __m128i mzero = _mm_setzero_si128();
+ const __m256i mone = _mm256_set1_epi8(1);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ float summs = 0.f;
+
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t * restrict q5 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));
+
+ const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);
+ const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));
+ const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);
+ const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);
+ summs += dmin * _mm_extract_epi32(hsum, 0);
+
+ const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);
+ const __m256i scales = MM256_SET_M128I(sc128, sc128);
+
+ const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].qh);
+ __m256i hmask = mone;
+
+ __m256i sumi = _mm256_setzero_si256();
+
+ int bit = 0;
+
+ for (int j = 0; j < QK_K/64; ++j) {
+
+ const __m256i scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));
+ const __m256i scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));
+
+ const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5); q5 += 32;
+
+ const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);
+ const __m256i q5h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);
+ const __m256i q5_0 = _mm256_add_epi8(q5l_0, q5h_0);
+ hmask = _mm256_slli_epi16(hmask, 1);
+
+ const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);
+ const __m256i q5h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);
+ const __m256i q5_1 = _mm256_add_epi8(q5l_1, q5h_1);
+ hmask = _mm256_slli_epi16(hmask, 1);
+
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+
+ __m256i p16_0 = _mm256_maddubs_epi16(q5_0, q8_0);
+ __m256i p16_1 = _mm256_maddubs_epi16(q5_1, q8_1);
+
+ p16_0 = _mm256_madd_epi16(scale_0, p16_0);
+ p16_1 = _mm256_madd_epi16(scale_1, p16_1);
+
+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));
+
+ }
+
+ __m256 vd = _mm256_set1_ps(d);
+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);
+
+ }
+
+ *s = hsum_float_8(acc) + summs;
+
+#elif defined __AVX__
+
+ const __m128i m4 = _mm_set1_epi8(0xF);
+ const __m128i mzero = _mm_setzero_si128();
+ const __m128i mone = _mm_set1_epi8(1);
+ const __m128i m2 = _mm_set1_epi8(2);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ float summs = 0.f;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q5 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);
+ const __m128i scales = _mm_cvtepu8_epi16(utmps);
+ const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));
+
+ const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);
+ const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);
+ const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);
+ const __m128i prod = _mm_madd_epi16(mins, q8s);
+ const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);
+ summs += dmin * _mm_extract_epi32(hsum, 0);
+
+ const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].qh[0]);
+ const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].qh[16]);
+ __m128i hmask = mone;
+
+ __m128i sumi_0 = _mm_setzero_si128();
+ __m128i sumi_1 = _mm_setzero_si128();
+
+ int bit = 0;
+
+ __m128i shuffle = _mm_set1_epi16(0x0100);
+ for (int j = 0; j < QK_K/64; ++j) {
+
+ const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi16(shuffle, m2);
+
+ const __m128i q5bits_0 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;
+ const __m128i q5bits_1 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;
+
+ __m128i q5l_0 = _mm_and_si128(q5bits_0, m4);
+ __m128i q5l_1 = _mm_and_si128(q5bits_1, m4);
+ __m128i q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);
+ __m128i q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);
+ __m128i q5_0 = _mm_add_epi8(q5l_0, q5h_0);
+ __m128i q5_1 = _mm_add_epi8(q5l_1, q5h_1);
+ hmask = _mm_slli_epi16(hmask, 1);
+
+ __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ __m128i p16_0 = _mm_maddubs_epi16(q5_0, q8_0);
+ __m128i p16_1 = _mm_maddubs_epi16(q5_1, q8_1);
+ p16_0 = _mm_madd_epi16(scale_0, p16_0);
+ p16_1 = _mm_madd_epi16(scale_0, p16_1);
+
+ q5l_0 = _mm_and_si128(_mm_srli_epi16(q5bits_0, 4), m4);
+ q5l_1 = _mm_and_si128(_mm_srli_epi16(q5bits_1, 4), m4);
+ q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);
+ q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);
+ q5_0 = _mm_add_epi8(q5l_0, q5h_0);
+ q5_1 = _mm_add_epi8(q5l_1, q5h_1);
+ hmask = _mm_slli_epi16(hmask, 1);
+
+ q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ __m128i p16_2 = _mm_maddubs_epi16(q5_0, q8_0);
+ __m128i p16_3 = _mm_maddubs_epi16(q5_1, q8_1);
+ p16_2 = _mm_madd_epi16(scale_1, p16_2);
+ p16_3 = _mm_madd_epi16(scale_1, p16_3);
+
+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));
+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));
+
+ }
+
+ __m256 vd = _mm256_set1_ps(d);
+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);
+
+ }
+
+ *s = hsum_float_8(acc) + summs;
+
+#elif defined __riscv_v_intrinsic
+
+ const uint8_t * scales = (const uint8_t*)&utmp[0];
+ const uint8_t * mins = (const uint8_t*)&utmp[2];
+
+ float sumf = 0;
+ float sums = 0.0;
+
+ size_t vl;
+
+ for (int i = 0; i < nb; ++i) {
+
+ vl = 8;
+
+ const uint8_t * restrict q5 = x[i].qs;
+ const uint8_t * restrict hm = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;
+
+ vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);
+ vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);
+ vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);
+
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);
+ vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));
+ vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);
+
+ vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);
+ sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);
+
+ vl = 32;
+ int32_t aux32 = 0;
+ int is = 0;
+
+ uint8_t m = 1;
+ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
+ vuint8m1_t vqh = __riscv_vle8_v_u8m1(hm, vl);
+
+ for (int j = 0; j < QK_K/64; ++j) {
+ // load Q5 and Q8
+ vuint8m1_t q5_x = __riscv_vle8_v_u8m1(q5, vl);
+ vint8m1_t q8_y1 = __riscv_vle8_v_i8m1(q8, vl);
+ vint8m1_t q8_y2 = __riscv_vle8_v_i8m1(q8+32, vl);
+
+ // compute mask for addition
+ vint8m1_t q5_a = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q5_x, 0x0F, vl));
+ vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_1 = __riscv_vmsne_vx_u8m1_b8(qh_m1, 0, vl);
+ vint8m1_t q5_m1 = __riscv_vadd_vx_i8m1_mu(vmask_1, q5_a, q5_a, 16, vl);
+ m <<= 1;
+
+ vint8m1_t q5_l = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q5_x, 0x04, vl));
+ vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_2 = __riscv_vmsne_vx_u8m1_b8(qh_m2, 0, vl);
+ vint8m1_t q5_m2 = __riscv_vadd_vx_i8m1_mu(vmask_2, q5_l, q5_l, 16, vl);
+ m <<= 1;
+
+ vint16m2_t v0 = __riscv_vwmul_vv_i16m2(q5_m1, q8_y1, vl);
+ vint16m2_t v1 = __riscv_vwmul_vv_i16m2(q5_m2, q8_y2, vl);
+
+ vint32m4_t vs1 = __riscv_vwmul_vx_i32m4(v0, scales[is++], vl);
+ vint32m4_t vs2 = __riscv_vwmul_vx_i32m4(v1, scales[is++], vl);
+
+ vint32m1_t vacc1 = __riscv_vredsum_vs_i32m4_i32m1(vs1, vzero, vl);
+ vint32m1_t vacc2 = __riscv_vredsum_vs_i32m4_i32m1(vs2, vzero, vl);
+
+ aux32 += __riscv_vmv_x_s_i32m1_i32(vacc1) + __riscv_vmv_x_s_i32m1_i32(vacc2);
+ q5 += 32; q8 += 64;
+
+ }
+
+ vfloat32m1_t vaux = __riscv_vfmul_vf_f32m1(__riscv_vfmv_v_f_f32m1(aux32, 1), d, 1);
+ sums += __riscv_vfmv_f_s_f32m1_f32(vaux);
+
+ }
+
+ *s = sumf+sums;
+
+#elif defined(__POWER9_VECTOR__)
+ const vector signed char lowMask = vec_splats((signed char)0xF);
+ const vector signed char lowMask1 = vec_splats((int8_t)0x3f);
+ const vector signed char lowMask2 = vec_splats((int8_t)0x30);
+ const vector int v0 = vec_splats((int32_t)0);
+ const vector unsigned char v1 = vec_splats((unsigned char)0x1);
+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);
+ const vector unsigned char v3 = vec_splats((unsigned char)0x3);
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
+
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ for (int i = 0; i < nb; ++i) {
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
+ vector float vyd = vec_splats(y[i].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin));
+ vector float vdmin = vec_mul(vxmin, vyd);
+
+ UNUSED(kmask1);
+ UNUSED(kmask2);
+ UNUSED(kmask3);
+ UNUSED(utmp);
+
+ vector signed char u0 = (vector signed char)vec_xl_len(x[i].scales, 8);
+ vector signed char u1 = vec_and(vec_sr(u0, v2), lowMask2);
+ vector signed char u2 = (vector signed char)vec_xl_len(x[i].scales + 8, 4);
+ vector signed char u3 = vec_sr(u2, v4);
+
+ vector signed char u30 = u1;
+ vector signed char u31 = (vector signed char)vec_mergeh((vector signed int)vec_and(u2, lowMask), (vector signed int)u3);
+
+ u1 = vec_and(u0, lowMask1);
+ u2 = vec_or(u30, u31);
+
+ vector signed char utmps = (vector signed char)vec_mergeh((vector signed int)u1, (vector signed int)u2);
+
+ vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);
+ vector signed short q8ysums1 = vec_xl(16, y[i].bsums);
+
+ vector signed short vscales = vec_unpackh(utmps);
+
+ vector signed short q5xmins = vec_unpackl(utmps);
+ vector signed short q5xmins0 = vec_mergeh(q5xmins, q5xmins);
+ vector signed short q5xmins1 = vec_mergel(q5xmins, q5xmins);
+
+ vector signed int prod0 = vec_mule(q5xmins0, q8ysums0);
+ vector signed int prod1 = vec_mule(q5xmins1, q8ysums1);
+ vector signed int prod2 = vec_mulo(q5xmins0, q8ysums0);
+ vector signed int prod3 = vec_mulo(q5xmins1, q8ysums1);
+
+ vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);
+ vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);
+ vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);
+ vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);
+
+ vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].qh);
+ vector signed char qxhs1 = (vector signed char)vec_xl(16, x[i].qh);
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+ vector signed int vsumi2 = v0;
+ vector signed int vsumi3 = v0;
+
+ const uint8_t * restrict q5 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ for (int j = 0; j < QK_K/64; ++j) {
+ __builtin_prefetch(q5, 0, 1);
+ __builtin_prefetch(q8, 0, 1);
+
+ vector signed char qxs0 = (vector signed char)vec_xl( 0, q5);
+ vector signed char qxs1 = (vector signed char)vec_xl(16, q5);
+ q5 += 32;
+
+ vector signed char qxs00 = vec_and(qxs0, lowMask);
+ vector signed char qxs01 = vec_sr(qxs0, v4);
+ vector signed char qxs10 = vec_and(qxs1, lowMask);
+ vector signed char qxs11 = vec_sr(qxs1, v4);
+
+ vector signed char q5h00 = vec_sl(vec_and((vector signed char)v1, qxhs0), v4);
+ vector signed char q5h01 = vec_sl(vec_and((vector signed char)v2, qxhs0), v3);
+ vector signed char q5h10 = vec_sl(vec_and((vector signed char)v1, qxhs1), v4);
+ vector signed char q5h11 = vec_sl(vec_and((vector signed char)v2, qxhs1), v3);
+ qxhs0 = vec_sr(qxhs0, v2);
+ qxhs1 = vec_sr(qxhs1, v2);
+
+ vector unsigned char q5x00 = (vector unsigned char)vec_or(q5h00, qxs00);
+ vector unsigned char q5x01 = (vector unsigned char)vec_or(q5h01, qxs01);
+ vector unsigned char q5x10 = (vector unsigned char)vec_or(q5h10, qxs10);
+ vector unsigned char q5x11 = (vector unsigned char)vec_or(q5h11, qxs11);
+
+ vector signed char q8y00 = vec_xl( 0, q8);
+ vector signed char q8y10 = vec_xl(16, q8);
+ vector signed char q8y01 = vec_xl(32, q8);
+ vector signed char q8y11 = vec_xl(48, q8);
+ q8 += 64;
+
+ vector signed int qv00 = vec_msum(q8y00, q5x00, v0);
+ vector signed int qv01 = vec_msum(q8y01, q5x01, v0);
+ vector signed int qv10 = vec_msum(q8y10, q5x10, v0);
+ vector signed int qv11 = vec_msum(q8y11, q5x11, v0);
+
+ vector signed int vscales_h = vec_unpackh(vscales);
+ vector signed int vs0 = vec_splat(vscales_h, 0);
+ vector signed int vs1 = vec_splat(vscales_h, 1);
+ vscales = vec_sld(vscales, vscales, 12);
+
+ vsumi0 = vec_add(vec_mul(qv00, vs0), vsumi0);
+ vsumi1 = vec_add(vec_mul(qv10, vs0), vsumi1);
+ vsumi2 = vec_add(vec_mul(qv01, vs1), vsumi2);
+ vsumi3 = vec_add(vec_mul(qv11, vs1), vsumi3);
+ }
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = vec_extract(vsumf0, 0);
+
+#elif defined __loongarch_asx
+ GGML_UNUSED(kmask1);
+ GGML_UNUSED(kmask2);
+ GGML_UNUSED(kmask3);
+
+ const __m256i m4 = __lasx_xvreplgr2vr_b(0xF);
+ const __m128i mzero = __lsx_vldi(0);
+ const __m256i mone = __lasx_xvreplgr2vr_b(1);
+
+ __m256 acc = (__m256)__lasx_xvldi(0);
+
+ float summs = 0.f;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const uint8_t * restrict q5 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ const __m256i mins_and_scales = lasx_extu8_16(lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]));
+
+ const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0);
+ const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1));
+ const __m128i prod = lsx_madd_h(lasx_extracti128(mins_and_scales, 1), q8s);
+ const __m128i hsum = lsx_hadd_w(lsx_hadd_w(prod, mzero), mzero);
+ summs += dmin * __lsx_vpickve2gr_w(hsum, 0); //TODO check
+
+ const __m128i sc128 = lasx_extracti128(mins_and_scales, 0);
+ const __m256i scales = lasx_insertf128(sc128, sc128);
+
+ const __m256i hbits = __lasx_xvld((const __m256i*)x[i].qh, 0);
+ __m256i hmask = mone;
+
+ __m256i sumi = __lasx_xvldi(0);
+
+ int bit = 0;
+ __m256i xvbit;
+
+ for (int j = 0; j < QK_K/64; ++j) {
+
+ const __m256i scale_0 = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+0));
+ const __m256i scale_1 = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+1));
+
+ const __m256i q5bits = __lasx_xvld((const __m256i*)q5, 0); q5 += 32;
+
+ xvbit = __lasx_xvreplgr2vr_h(bit++);
+ const __m256i q5l_0 = __lasx_xvand_v(q5bits, m4);
+ const __m256i q5h_0 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvand_v(hbits, hmask), xvbit), 4);
+ const __m256i q5_0 = __lasx_xvadd_b(q5l_0, q5h_0);
+ hmask = __lasx_xvslli_h(hmask, 1);
+
+ xvbit = __lasx_xvreplgr2vr_h(bit++);
+ const __m256i q5l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q5bits, 4), m4);
+ const __m256i q5h_1 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvand_v(hbits, hmask), xvbit), 4);
+ const __m256i q5_1 = __lasx_xvadd_b(q5l_1, q5h_1);
+ hmask = __lasx_xvslli_h(hmask, 1);
+
+ const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+
+ __m256i p16_0 = lasx_maddubs_h(q5_0, q8_0);
+ __m256i p16_1 = lasx_maddubs_h(q5_1, q8_1);
+
+ p16_0 = lasx_madd_h(scale_0, p16_0);
+ p16_1 = lasx_madd_h(scale_1, p16_1);
+
+ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1));
+
+ }
+
+ __m256 vd = __lasx_xvreplfr2vr_s(d);
+ acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc);
+
+ }
+
+ *s = hsum_float_8(acc) + summs;
+
+#else
+
+ const uint8_t * scales = (const uint8_t*)&utmp[0];
+ const uint8_t * mins = (const uint8_t*)&utmp[2];
+
+ int8_t aux8[QK_K];
+ int16_t aux16[8];
+ float sums [8];
+ int32_t aux32[8];
+ memset(sums, 0, 8*sizeof(float));
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t * restrict q4 = x[i].qs;
+ const uint8_t * restrict hm = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
+ memset(aux32, 0, 8*sizeof(int32_t));
+ int8_t * restrict a = aux8;
+ uint8_t m = 1;
+ for (int j = 0; j < QK_K/64; ++j) {
+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);
+ for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);
+ a += 32; m <<= 1;
+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);
+ for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);
+ a += 32; m <<= 1;
+ q4 += 32;
+ }
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ int sumi = 0;
+ for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];
+ a = aux8;
+ int is = 0;
+ for (int j = 0; j < QK_K/32; ++j) {
+ int32_t scale = scales[is++];
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ }
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
+ const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;
+ sumf -= dmin * sumi;
+ }
+ for (int l = 0; l < 8; ++l) sumf += sums[l];
+ *s = sumf;
+#endif
+}
+
+void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q6_K * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#ifdef __ARM_NEON
+ float sum = 0;
+
+ const uint8x16_t m4b = vdupq_n_u8(0xF);
+ const int32x4_t vzero = vdupq_n_s32(0);
+ //const int8x16_t m32s = vdupq_n_s8(32);
+
+ const uint8x16_t mone = vdupq_n_u8(3);
+
+ ggml_int8x16x4_t q6bytes;
+ ggml_uint8x16x4_t q6h;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d_all = GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q6 = x[i].ql;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const int8_t * restrict scale = x[i].scales;
+
+ const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);
+ const int8x16_t scales = vld1q_s8(scale);
+ const ggml_int16x8x2_t q6scales = {{vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))}};
+
+ const int32x4_t prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[0]), vget_low_s16 (q6scales.val[0])),
+ vmull_s16(vget_high_s16(q8sums.val[0]), vget_high_s16(q6scales.val[0]))),
+ vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[1]), vget_low_s16 (q6scales.val[1])),
+ vmull_s16(vget_high_s16(q8sums.val[1]), vget_high_s16(q6scales.val[1]))));
+ int32_t isum_mins = vaddvq_s32(prod);
+
+ int32_t isum = 0;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh); qh += 32;
+ ggml_uint8x16x4_t q6bits = ggml_vld1q_u8_x4(q6); q6 += 64;
+ ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);
+ q6h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);
+ uint8x16_t shifted = vshrq_n_u8(qhbits.val[0], 2);
+ q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+ shifted = vshrq_n_u8(qhbits.val[1], 2);
+ q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+
+ //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0])), m32s);
+ //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1])), m32s);
+ //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2])), m32s);
+ //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3])), m32s);
+ q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0]));
+ q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1]));
+ q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2]));
+ q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3]));
+
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];
+
+ scale += 4;
+
+ q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ shifted = vshrq_n_u8(qhbits.val[0], 4);
+ q6h.val[0] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+ shifted = vshrq_n_u8(qhbits.val[1], 4);
+ q6h.val[1] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+ shifted = vshrq_n_u8(qhbits.val[0], 6);
+ q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+ shifted = vshrq_n_u8(qhbits.val[1], 6);
+ q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+
+ //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0])), m32s);
+ //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1])), m32s);
+ //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2])), m32s);
+ //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3])), m32s);
+ q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0]));
+ q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1]));
+ q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2]));
+ q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3]));
+
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];
+ scale += 4;
+ }
+ //sum += isum * d_all * y[i].d;
+ sum += d_all * y[i].d * (isum - 32 * isum_mins);
+
+ }
+ *s = sum;
+
+#elif defined __AVX2__
+
+ const __m256i m4 = _mm256_set1_epi8(0xF);
+ const __m256i m2 = _mm256_set1_epi8(3);
+ const __m256i m32s = _mm256_set1_epi8(32);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q4 = x[i].ql;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);
+
+ __m256i sumi = _mm256_setzero_si256();
+
+ int is = 0;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));
+ const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));
+ const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));
+ const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));
+ is += 4;
+
+ const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;
+ const __m256i q4bits2 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;
+ const __m256i q4bitsH = _mm256_loadu_si256((const __m256i*)qh); qh += 32;
+
+ const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m2), 4);
+ const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 2), m2), 4);
+ const __m256i q4h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 4), m2), 4);
+ const __m256i q4h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 6), m2), 4);
+
+ const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0);
+ const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1);
+ const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2);
+ const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3);
+
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+
+ __m256i q8s_0 = _mm256_maddubs_epi16(m32s, q8_0);
+ __m256i q8s_1 = _mm256_maddubs_epi16(m32s, q8_1);
+ __m256i q8s_2 = _mm256_maddubs_epi16(m32s, q8_2);
+ __m256i q8s_3 = _mm256_maddubs_epi16(m32s, q8_3);
+
+ __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0);
+ __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1);
+ __m256i p16_2 = _mm256_maddubs_epi16(q4_2, q8_2);
+ __m256i p16_3 = _mm256_maddubs_epi16(q4_3, q8_3);
+
+ p16_0 = _mm256_sub_epi16(p16_0, q8s_0);
+ p16_1 = _mm256_sub_epi16(p16_1, q8s_1);
+ p16_2 = _mm256_sub_epi16(p16_2, q8s_2);
+ p16_3 = _mm256_sub_epi16(p16_3, q8s_3);
+
+ p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0);
+ p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1);
+ p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2);
+ p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3);
+
+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));
+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3));
+
+ }
+
+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __AVX__
+
+ const __m128i m4 = _mm_set1_epi8(0xF);
+ const __m128i m3 = _mm_set1_epi8(3);
+ const __m128i m32s = _mm_set1_epi8(32);
+ const __m128i m2 = _mm_set1_epi8(2);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q4 = x[i].ql;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);
+
+ __m128i sumi_0 = _mm_setzero_si128();
+ __m128i sumi_1 = _mm_setzero_si128();
+
+ __m128i shuffle = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ const __m128i q4bitsH_0 = _mm_loadu_si128((const __m128i*)qh); qh += 16;
+ const __m128i q4bitsH_1 = _mm_loadu_si128((const __m128i*)qh); qh += 16;
+
+ const __m128i q4h_0 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, m3), 4);
+ const __m128i q4h_1 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, m3), 4);
+ const __m128i q4h_2 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 2), m3), 4);
+ const __m128i q4h_3 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 2), m3), 4);
+ const __m128i q4h_4 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 4), m3), 4);
+ const __m128i q4h_5 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 4), m3), 4);
+ const __m128i q4h_6 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 6), m3), 4);
+ const __m128i q4h_7 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 6), m3), 4);
+
+ const __m128i q4bits1_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+ const __m128i q4bits1_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+ const __m128i q4bits2_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+ const __m128i q4bits2_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+
+ const __m128i q4_0 = _mm_or_si128(_mm_and_si128(q4bits1_0, m4), q4h_0);
+ const __m128i q4_1 = _mm_or_si128(_mm_and_si128(q4bits1_1, m4), q4h_1);
+ const __m128i q4_2 = _mm_or_si128(_mm_and_si128(q4bits2_0, m4), q4h_2);
+ const __m128i q4_3 = _mm_or_si128(_mm_and_si128(q4bits2_1, m4), q4h_3);
+ const __m128i q4_4 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_0, 4), m4), q4h_4);
+ const __m128i q4_5 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_1, 4), m4), q4h_5);
+ const __m128i q4_6 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_0, 4), m4), q4h_6);
+ const __m128i q4_7 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_1, 4), m4), q4h_7);
+
+ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+
+ __m128i q8s_0 = _mm_maddubs_epi16(m32s, q8_0);
+ __m128i q8s_1 = _mm_maddubs_epi16(m32s, q8_1);
+ __m128i q8s_2 = _mm_maddubs_epi16(m32s, q8_2);
+ __m128i q8s_3 = _mm_maddubs_epi16(m32s, q8_3);
+ __m128i q8s_4 = _mm_maddubs_epi16(m32s, q8_4);
+ __m128i q8s_5 = _mm_maddubs_epi16(m32s, q8_5);
+ __m128i q8s_6 = _mm_maddubs_epi16(m32s, q8_6);
+ __m128i q8s_7 = _mm_maddubs_epi16(m32s, q8_7);
+
+ __m128i p16_0 = _mm_maddubs_epi16(q4_0, q8_0);
+ __m128i p16_1 = _mm_maddubs_epi16(q4_1, q8_1);
+ __m128i p16_2 = _mm_maddubs_epi16(q4_2, q8_2);
+ __m128i p16_3 = _mm_maddubs_epi16(q4_3, q8_3);
+ __m128i p16_4 = _mm_maddubs_epi16(q4_4, q8_4);
+ __m128i p16_5 = _mm_maddubs_epi16(q4_5, q8_5);
+ __m128i p16_6 = _mm_maddubs_epi16(q4_6, q8_6);
+ __m128i p16_7 = _mm_maddubs_epi16(q4_7, q8_7);
+
+ p16_0 = _mm_sub_epi16(p16_0, q8s_0);
+ p16_1 = _mm_sub_epi16(p16_1, q8s_1);
+ p16_2 = _mm_sub_epi16(p16_2, q8s_2);
+ p16_3 = _mm_sub_epi16(p16_3, q8s_3);
+ p16_4 = _mm_sub_epi16(p16_4, q8s_4);
+ p16_5 = _mm_sub_epi16(p16_5, q8s_5);
+ p16_6 = _mm_sub_epi16(p16_6, q8s_6);
+ p16_7 = _mm_sub_epi16(p16_7, q8s_7);
+
+ const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi8(shuffle, m2);
+ const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi8(shuffle, m2);
+ const __m128i scale_2 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi8(shuffle, m2);
+ const __m128i scale_3 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi8(shuffle, m2);
+
+ p16_0 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_0), p16_0);
+ p16_1 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_0, scale_0)), p16_1);
+ p16_2 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_1), p16_2);
+ p16_3 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_1, scale_1)), p16_3);
+ p16_4 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_2), p16_4);
+ p16_5 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_2, scale_2)), p16_5);
+ p16_6 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_3), p16_6);
+ p16_7 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_3, scale_3)), p16_7);
+
+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));
+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));
+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_4, p16_6));
+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_5, p16_7));
+
+ }
+
+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __riscv_v_intrinsic
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+
+ const uint8_t * restrict q6 = x[i].ql;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const int8_t * restrict scale = x[i].scales;
+
+ size_t vl;
+
+ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
+
+ int sum_t = 0;
+ int is = 0;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ vl = 32;
+
+ // load qh
+ vuint8m1_t qh_x = __riscv_vle8_v_u8m1(qh, vl);
+
+ // load Q6
+ vuint8m1_t q6_0 = __riscv_vle8_v_u8m1(q6, vl);
+ vuint8m1_t q6_1 = __riscv_vle8_v_u8m1(q6+32, vl);
+
+ vuint8m1_t q6a_0 = __riscv_vand_vx_u8m1(q6_0, 0x0F, vl);
+ vuint8m1_t q6a_1 = __riscv_vand_vx_u8m1(q6_1, 0x0F, vl);
+ vuint8m1_t q6s_0 = __riscv_vsrl_vx_u8m1(q6_0, 0x04, vl);
+ vuint8m1_t q6s_1 = __riscv_vsrl_vx_u8m1(q6_1, 0x04, vl);
+
+ vuint8m1_t qh_0 = __riscv_vand_vx_u8m1(qh_x, 0x03, vl);
+ vuint8m1_t qh_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x2, vl), 0x03 , vl);
+ vuint8m1_t qh_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x4, vl), 0x03 , vl);
+ vuint8m1_t qh_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x6, vl), 0x03 , vl);
+
+ vuint8m1_t qhi_0 = __riscv_vor_vv_u8m1(q6a_0, __riscv_vsll_vx_u8m1(qh_0, 0x04, vl), vl);
+ vuint8m1_t qhi_1 = __riscv_vor_vv_u8m1(q6a_1, __riscv_vsll_vx_u8m1(qh_1, 0x04, vl), vl);
+ vuint8m1_t qhi_2 = __riscv_vor_vv_u8m1(q6s_0, __riscv_vsll_vx_u8m1(qh_2, 0x04, vl), vl);
+ vuint8m1_t qhi_3 = __riscv_vor_vv_u8m1(q6s_1, __riscv_vsll_vx_u8m1(qh_3, 0x04, vl), vl);
+
+ vint8m1_t a_0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_0), 32, vl);
+ vint8m1_t a_1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_1), 32, vl);
+ vint8m1_t a_2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_2), 32, vl);
+ vint8m1_t a_3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_3), 32, vl);
+
+ // load Q8 and take product
+ vint16m2_t va_q_0 = __riscv_vwmul_vv_i16m2(a_0, __riscv_vle8_v_i8m1(q8, vl), vl);
+ vint16m2_t va_q_1 = __riscv_vwmul_vv_i16m2(a_1, __riscv_vle8_v_i8m1(q8+32, vl), vl);
+ vint16m2_t va_q_2 = __riscv_vwmul_vv_i16m2(a_2, __riscv_vle8_v_i8m1(q8+64, vl), vl);
+ vint16m2_t va_q_3 = __riscv_vwmul_vv_i16m2(a_3, __riscv_vle8_v_i8m1(q8+96, vl), vl);
+
+ vl = 16;
+
+ vint32m2_t vaux_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 0), scale[is+0], vl);
+ vint32m2_t vaux_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 1), scale[is+1], vl);
+ vint32m2_t vaux_2 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 0), scale[is+2], vl);
+ vint32m2_t vaux_3 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 1), scale[is+3], vl);
+ vint32m2_t vaux_4 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 0), scale[is+4], vl);
+ vint32m2_t vaux_5 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 1), scale[is+5], vl);
+ vint32m2_t vaux_6 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 0), scale[is+6], vl);
+ vint32m2_t vaux_7 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 1), scale[is+7], vl);
+
+ vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl);
+ vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl);
+ vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_4, vaux_5, vl), isum1, vl);
+ vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_6, vaux_7, vl), isum2, vl);
+
+ sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);
+
+ q6 += 64; qh += 32; q8 += 128; is=8;
+
+ }
+
+ sumf += d * sum_t;
+
+ }
+
+ *s = sumf;
+
+#elif defined(__POWER9_VECTOR__)
+ const vector signed char lowMask = vec_splats((signed char)0xF);
+ const vector int v0 = vec_splats((int32_t)0);
+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);
+ const vector unsigned char v3 = vec_splats((unsigned char)0x3);
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
+ const vector unsigned char v6 = vec_splats((unsigned char)0x6);
+ const vector signed char off = vec_splats((signed char)0x20);
+
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ for (int i = 0; i < nb; ++i) {
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
+ vector float vyd = vec_splats(y[i].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+ vector signed int vsumi2 = v0;
+ vector signed int vsumi3 = v0;
+ vector signed int vsumi4 = v0;
+ vector signed int vsumi5 = v0;
+ vector signed int vsumi6 = v0;
+ vector signed int vsumi7 = v0;
+
+ const uint8_t * restrict q6 = x[i].ql;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict qs = x[i].scales;
+ const int8_t * restrict q8 = y[i].qs;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ __builtin_prefetch(q6, 0, 0);
+ __builtin_prefetch(qh, 0, 0);
+ __builtin_prefetch(q8, 0, 0);
+
+ vector signed char qxs0 = (vector signed char)vec_xl( 0, q6);
+ vector signed char qxs1 = (vector signed char)vec_xl(16, q6);
+ vector signed char qxs2 = (vector signed char)vec_xl(32, q6);
+ vector signed char qxs3 = (vector signed char)vec_xl(48, q6);
+ q6 += 64;
+
+ vector signed char qxs00 = vec_and(qxs0, lowMask);
+ vector signed char qxs01 = vec_sr(qxs0, v4);
+ vector signed char qxs10 = vec_and(qxs1, lowMask);
+ vector signed char qxs11 = vec_sr(qxs1, v4);
+ vector signed char qxs20 = vec_and(qxs2, lowMask);
+ vector signed char qxs21 = vec_sr(qxs2, v4);
+ vector signed char qxs30 = vec_and(qxs3, lowMask);
+ vector signed char qxs31 = vec_sr(qxs3, v4);
+
+ vector signed char qxhs0 = (vector signed char)vec_xl( 0, qh);
+ vector signed char qxhs1 = (vector signed char)vec_xl(16, qh);
+ qh += 32;
+
+ vector signed char qxh00 = vec_sl(vec_and((vector signed char)v3, qxhs0), v4);
+ vector signed char qxh01 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v4)), v4);
+ vector signed char qxh10 = vec_sl(vec_and((vector signed char)v3, qxhs1), v4);
+ vector signed char qxh11 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v4)), v4);
+ vector signed char qxh20 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v2)), v4);
+ vector signed char qxh21 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v6)), v4);
+ vector signed char qxh30 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v2)), v4);
+ vector signed char qxh31 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v6)), v4);
+
+ vector signed char q6x00 = vec_sub(vec_or(qxh00, qxs00), off);
+ vector signed char q6x01 = vec_sub(vec_or(qxh01, qxs01), off);
+ vector signed char q6x10 = vec_sub(vec_or(qxh10, qxs10), off);
+ vector signed char q6x11 = vec_sub(vec_or(qxh11, qxs11), off);
+ vector signed char q6x20 = vec_sub(vec_or(qxh20, qxs20), off);
+ vector signed char q6x21 = vec_sub(vec_or(qxh21, qxs21), off);
+ vector signed char q6x30 = vec_sub(vec_or(qxh30, qxs30), off);
+ vector signed char q6x31 = vec_sub(vec_or(qxh31, qxs31), off);
+
+ vector signed char q8y00 = vec_xl( 0, q8);
+ vector signed char q8y10 = vec_xl( 16, q8);
+ vector signed char q8y20 = vec_xl( 32, q8);
+ vector signed char q8y30 = vec_xl( 48, q8);
+ vector signed char q8y01 = vec_xl( 64, q8);
+ vector signed char q8y11 = vec_xl( 80, q8);
+ vector signed char q8y21 = vec_xl( 96, q8);
+ vector signed char q8y31 = vec_xl(112, q8);
+ q8 += 128;
+
+ vector signed short qv00 = vec_add(vec_mule(q6x00, q8y00), vec_mulo(q6x00, q8y00));
+ vector signed short qv10 = vec_add(vec_mule(q6x10, q8y10), vec_mulo(q6x10, q8y10));
+ vector signed short qv20 = vec_add(vec_mule(q6x20, q8y20), vec_mulo(q6x20, q8y20));
+ vector signed short qv30 = vec_add(vec_mule(q6x30, q8y30), vec_mulo(q6x30, q8y30));
+ vector signed short qv01 = vec_add(vec_mule(q6x01, q8y01), vec_mulo(q6x01, q8y01));
+ vector signed short qv11 = vec_add(vec_mule(q6x11, q8y11), vec_mulo(q6x11, q8y11));
+ vector signed short qv21 = vec_add(vec_mule(q6x21, q8y21), vec_mulo(q6x21, q8y21));
+ vector signed short qv31 = vec_add(vec_mule(q6x31, q8y31), vec_mulo(q6x31, q8y31));
+
+ vector signed short vscales = vec_unpackh(vec_xl_len(qs, 8));
+ qs += 8;
+
+ vector signed short vs0 = vec_splat(vscales, 0);
+ vector signed short vs1 = vec_splat(vscales, 1);
+ vector signed short vs2 = vec_splat(vscales, 2);
+ vector signed short vs3 = vec_splat(vscales, 3);
+ vector signed short vs4 = vec_splat(vscales, 4);
+ vector signed short vs5 = vec_splat(vscales, 5);
+ vector signed short vs6 = vec_splat(vscales, 6);
+ vector signed short vs7 = vec_splat(vscales, 7);
+
+ vsumi0 = vec_msum(qv00, vs0, vsumi0);
+ vsumi1 = vec_msum(qv01, vs4, vsumi1);
+ vsumi2 = vec_msum(qv10, vs1, vsumi2);
+ vsumi3 = vec_msum(qv11, vs5, vsumi3);
+ vsumi4 = vec_msum(qv20, vs2, vsumi4);
+ vsumi5 = vec_msum(qv21, vs6, vsumi5);
+ vsumi6 = vec_msum(qv30, vs3, vsumi6);
+ vsumi7 = vec_msum(qv31, vs7, vsumi7);
+ }
+
+ vsumi0 = vec_add(vsumi0, vsumi4);
+ vsumi1 = vec_add(vsumi1, vsumi5);
+ vsumi2 = vec_add(vsumi2, vsumi6);
+ vsumi3 = vec_add(vsumi3, vsumi7);
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = vec_extract(vsumf0, 0);
+
+#elif defined __loongarch_asx
+
+ const __m256i m4 = __lasx_xvreplgr2vr_b(0xF);
+ const __m256i m2 = __lasx_xvreplgr2vr_b(3);
+ const __m256i m32s = __lasx_xvreplgr2vr_b(32);
+
+ __m256 acc = (__m256)__lasx_xvldi(0);
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q4 = x[i].ql;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const __m128i scales = __lsx_vld((const __m128i*)x[i].scales, 0);
+
+ __m256i sumi = __lasx_xvldi(0);
+
+ int is = 0;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ const __m128i scale_0 = lsx_shuffle_b(scales, get_scale_shuffle(is + 0));
+ const __m128i scale_1 = lsx_shuffle_b(scales, get_scale_shuffle(is + 1));
+ const __m128i scale_2 = lsx_shuffle_b(scales, get_scale_shuffle(is + 2));
+ const __m128i scale_3 = lsx_shuffle_b(scales, get_scale_shuffle(is + 3));
+ is += 4;
+
+ const __m256i q4bits1 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;
+ const __m256i q4bits2 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;
+ const __m256i q4bitsH = __lasx_xvld((const __m256i*)qh, 0); qh += 32;
+
+ const __m256i q4h_0 = __lasx_xvslli_h(__lasx_xvand_v(q4bitsH, m2), 4);
+ const __m256i q4h_1 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 2), m2), 4);
+ const __m256i q4h_2 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 4), m2), 4);
+ const __m256i q4h_3 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 6), m2), 4);
+
+ const __m256i q4_0 = __lasx_xvor_v(__lasx_xvand_v(q4bits1, m4), q4h_0);
+ const __m256i q4_1 = __lasx_xvor_v(__lasx_xvand_v(q4bits2, m4), q4h_1);
+ const __m256i q4_2 = __lasx_xvor_v(__lasx_xvand_v(__lasx_xvsrli_h(q4bits1, 4), m4), q4h_2);
+ const __m256i q4_3 = __lasx_xvor_v(__lasx_xvand_v(__lasx_xvsrli_h(q4bits2, 4), m4), q4h_3);
+
+ const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+
+ __m256i q8s_0 = lasx_maddubs_h(m32s, q8_0);
+ __m256i q8s_1 = lasx_maddubs_h(m32s, q8_1);
+ __m256i q8s_2 = lasx_maddubs_h(m32s, q8_2);
+ __m256i q8s_3 = lasx_maddubs_h(m32s, q8_3);
+
+ __m256i p16_0 = lasx_maddubs_h(q4_0, q8_0);
+ __m256i p16_1 = lasx_maddubs_h(q4_1, q8_1);
+ __m256i p16_2 = lasx_maddubs_h(q4_2, q8_2);
+ __m256i p16_3 = lasx_maddubs_h(q4_3, q8_3);
+
+ p16_0 = __lasx_xvsub_h(p16_0, q8s_0);
+ p16_1 = __lasx_xvsub_h(p16_1, q8s_1);
+ p16_2 = __lasx_xvsub_h(p16_2, q8s_2);
+ p16_3 = __lasx_xvsub_h(p16_3, q8s_3);
+
+ p16_0 = lasx_madd_h(lasx_ext8_16(scale_0), p16_0);
+ p16_1 = lasx_madd_h(lasx_ext8_16(scale_1), p16_1);
+ p16_2 = lasx_madd_h(lasx_ext8_16(scale_2), p16_2);
+ p16_3 = lasx_madd_h(lasx_ext8_16(scale_3), p16_3);
+
+ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1));
+ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_2, p16_3));
+ }
+
+ acc = __lasx_xvfmadd_s((__m256)__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);
+ }
+
+ *s = hsum_float_8(acc);
+
+#else
+
+ int8_t aux8[QK_K];
+ int16_t aux16[8];
+ float sums [8];
+ int32_t aux32[8];
+ memset(sums, 0, 8*sizeof(float));
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t * restrict q4 = x[i].ql;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
+ memset(aux32, 0, 8*sizeof(int32_t));
+ int8_t * restrict a = aux8;
+ for (int j = 0; j < QK_K; j += 128) {
+ for (int l = 0; l < 32; ++l) {
+ a[l + 0] = (int8_t)((q4[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;
+ a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;
+ a[l + 64] = (int8_t)((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;
+ a[l + 96] = (int8_t)((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;
+ }
+ a += 128;
+ q4 += 64;
+ qh += 32;
+ }
+ a = aux8;
+ int is = 0;
+ for (int j = 0; j < QK_K/16; ++j) {
+ int scale = x[i].scales[is++];
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ }
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
+ }
+ for (int l = 0; l < 8; ++l) sumf += sums[l];
+ *s = sumf;
+#endif
+}
+
+#if defined (__AVX__) || defined (__AVX2__) || defined (__ARM_NEON) || defined (__POWER9_VECTOR__) || defined(__loongarch_asx)
+static const int8_t keven_signs_q2xs[1024] = {
+ 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1,
+ 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1,
+ 1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, -1,
+ 1, 1, -1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, 1,
+ 1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, -1,
+ 1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1,
+ 1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, 1,
+ 1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, -1,
+ 1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, -1,
+ 1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, 1,
+ 1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, 1,
+ 1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1,
+ 1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, 1,
+ 1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, -1,
+ 1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, -1,
+ 1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1,
+ 1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, -1,
+ 1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, 1,
+ 1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, 1,
+ 1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, -1,
+ 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, 1,
+ 1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, -1,
+ 1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, -1,
+ 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1,
+ 1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, 1,
+ 1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, -1,
+ 1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, -1,
+ 1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, 1,
+ 1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1,
+ 1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, 1,
+ 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, 1,
+ 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1,
+};
+#endif
+
+void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_iq2_xxs * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#if defined(__ARM_NEON)
+
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ uint32_t aux32[4];
+ const uint8_t * aux8 = (const uint8_t *)aux32;
+
+ ggml_int8x16x4_t q2u;
+ ggml_int8x16x4_t q2s;
+ ggml_int8x16x4_t q8b;
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint16_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ float sumf1 = 0, sumf2 = 0;
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;
+ memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;
+ q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 0])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 1])));
+ q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 2])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 3])));
+ q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 8])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 9])));
+ q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[10])), vld1_s8((const void *)(iq2xxs_grid + aux8[11])));
+ q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 7) & 127))));
+ q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 21) & 127))));
+ q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[3] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[3] >> 7) & 127))));
+ q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[3] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[3] >> 21) & 127))));
+ q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]);
+ q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]);
+ q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]);
+ q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]);
+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]), q2u.val[1], q8b.val[1]);
+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]), q2u.val[3], q8b.val[3]);
+ sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[1] >> 28));
+ sumf2 += vaddvq_s32(p2) * (0.5f + (aux32[3] >> 28));
+ }
+ sumf += d*(sumf1 + sumf2);
+ }
+ *s = 0.25f * sumf;
+
+#elif defined(__AVX2__)
+
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ uint32_t aux32[4];
+ const uint8_t * aux8 = (const uint8_t *)aux32;
+
+ __m256 accumf = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint16_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ __m256i sumi1 = _mm256_setzero_si256();
+ __m256i sumi2 = _mm256_setzero_si256();
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;
+ const __m256i q2_1 = _mm256_set_epi64x(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);
+ const __m256i q2_2 = _mm256_set_epi64x(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);
+ const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],
+ signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);
+ const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127],
+ signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]);
+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);
+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);
+ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);
+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);
+ const uint16_t ls1 = aux32[1] >> 28;
+ const uint16_t ls2 = aux32[3] >> 28;
+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));
+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));
+ sumi1 = _mm256_add_epi32(sumi1, p1);
+ sumi2 = _mm256_add_epi32(sumi2, p2);
+ }
+
+ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);
+
+ }
+
+ *s = 0.125f * hsum_float_8(accumf);
+
+#elif defined(__AVX__)
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ uint32_t aux32[4];
+ const uint8_t * aux8 = (const uint8_t *)aux32;
+
+ __m256 accumf = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint16_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ __m128i sumi1_0 = _mm_setzero_si128();
+ __m128i sumi1_1 = _mm_setzero_si128();
+ __m128i sumi2_0 = _mm_setzero_si128();
+ __m128i sumi2_1 = _mm_setzero_si128();
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;
+ const __m128i q2_1_0 = _mm_set_epi64x(iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);
+ const __m128i q2_1_1 = _mm_set_epi64x(iq2xxs_grid[aux8[3]], iq2xxs_grid[aux8[2]]);
+ const __m128i q2_2_0 = _mm_set_epi64x(iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);
+ const __m128i q2_2_1 = _mm_set_epi64x(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]]);
+ const __m128i s2_1_0 = _mm_set_epi64x(signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);
+ const __m128i s2_1_1 = _mm_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127]);
+ const __m128i s2_2_0 = _mm_set_epi64x(signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]);
+ const __m128i s2_2_1 = _mm_set_epi64x(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127]);
+ const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, s2_1_0);
+ const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, s2_1_1);
+ const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, s2_2_0);
+ const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, s2_2_1);
+ const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);
+ const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);
+ const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);
+ const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);
+ const uint16_t ls1 = aux32[1] >> 28;
+ const uint16_t ls2 = aux32[3] >> 28;
+ const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));
+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));
+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));
+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));
+ sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);
+ sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);
+ sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);
+ sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);
+ }
+
+ accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);
+
+ }
+
+ *s = 0.125f * hsum_float_8(accumf);
+
+#elif defined(__POWER9_VECTOR__)
+ const vector int v0 = vec_splats((int32_t)0);
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ for (int i = 0; i < nb; ++i) {
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
+ vector float vyd = vec_splats(y[i].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+ vector signed int vsumi2 = v0;
+ vector signed int vsumi3 = v0;
+
+ const uint16_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ for (int j = 0; j < QK_K/32; j += 2) {
+ __builtin_prefetch(q2, 0, 1);
+ __builtin_prefetch(q8, 0, 1);
+
+ uint32_t aux32[4];
+ const uint8_t * aux8 = (const uint8_t *)aux32;
+
+ memcpy(aux32, q2, 4*sizeof(uint32_t));
+ q2 += 8;
+
+ vector signed long long aux64x2_0 = {*(const int64_t *)(iq2xxs_grid + aux8[ 0]), *(const int64_t *)(iq2xxs_grid + aux8[ 1])};
+ vector signed long long aux64x2_1 = {*(const int64_t *)(iq2xxs_grid + aux8[ 2]), *(const int64_t *)(iq2xxs_grid + aux8[ 3])};
+ vector signed long long aux64x2_2 = {*(const int64_t *)(iq2xxs_grid + aux8[ 8]), *(const int64_t *)(iq2xxs_grid + aux8[ 9])};
+ vector signed long long aux64x2_3 = {*(const int64_t *)(iq2xxs_grid + aux8[10]), *(const int64_t *)(iq2xxs_grid + aux8[11])};
+
+ vector signed long long vsigns0 = {*(const int64_t *)(signs64 + ((aux32[1] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 7) & 127))};
+ vector signed long long vsigns1 = {*(const int64_t *)(signs64 + ((aux32[1] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 21) & 127))};
+ vector signed long long vsigns2 = {*(const int64_t *)(signs64 + ((aux32[3] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 7) & 127))};
+ vector signed long long vsigns3 = {*(const int64_t *)(signs64 + ((aux32[3] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 21) & 127))};
+
+ vector signed char q2x0 = (vector signed char)vec_mul((vector signed char)vsigns0, (vector signed char)aux64x2_0);
+ vector signed char q2x1 = (vector signed char)vec_mul((vector signed char)vsigns1, (vector signed char)aux64x2_1);
+ vector signed char q2x2 = (vector signed char)vec_mul((vector signed char)vsigns2, (vector signed char)aux64x2_2);
+ vector signed char q2x3 = (vector signed char)vec_mul((vector signed char)vsigns3, (vector signed char)aux64x2_3);
+
+ vector signed char q8y0 = vec_xl( 0, q8);
+ vector signed char q8y1 = vec_xl(16, q8);
+ vector signed char q8y2 = vec_xl(32, q8);
+ vector signed char q8y3 = vec_xl(48, q8);
+ q8 += 64;
+
+ vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));
+ vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));
+ vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));
+ vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));
+
+ const uint16_t ls0 = aux32[1] >> 28;
+ const uint16_t ls1 = aux32[3] >> 28;
+
+ vector signed short vscales01 = vec_splats((int16_t)(2*ls0+1));
+ vector signed short vscales23 = vec_splats((int16_t)(2*ls1+1));
+
+ vsumi0 = vec_msum(qv0, vscales01, vsumi0);
+ vsumi1 = vec_msum(qv1, vscales01, vsumi1);
+ vsumi2 = vec_msum(qv2, vscales23, vsumi2);
+ vsumi3 = vec_msum(qv3, vscales23, vsumi3);
+ }
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = 0.125f * vec_extract(vsumf0, 0);
+
+#elif defined(__loongarch_asx)
+
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ uint32_t aux32[4];
+ const uint8_t * aux8 = (const uint8_t *)aux32;
+
+ __m256 accumf = (__m256)__lasx_xvldi(0);
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint16_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ __m256i sumi1 = __lasx_xvldi(0);
+ __m256i sumi2 = __lasx_xvldi(0);
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;
+
+ const __m256i q2_1 = lasx_set_d(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);
+ const __m256i q2_2 = lasx_set_d(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);
+ const __m256i s2_1 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],
+ signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);
+ const __m256i s2_2 = lasx_set_d(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127],
+ signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]);
+ const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1);
+ const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2);
+ const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);
+ const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);
+ const uint16_t ls1 = aux32[1] >> 28;
+ const uint16_t ls2 = aux32[3] >> 28;
+ const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));
+ const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));
+ sumi1 = __lasx_xvadd_w(sumi1, p1);
+ sumi2 = __lasx_xvadd_w(sumi2, p2);
+ }
+
+ accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);
+ }
+
+ *s = 0.125f * hsum_float_8(accumf);
+
+#else
+
+ uint32_t aux32[2];
+ const uint8_t * aux8 = (const uint8_t *)aux32;
+
+ float sumf = 0.f;
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint16_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ int32_t bsum = 0;
+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
+ memcpy(aux32, q2, 2*sizeof(uint32_t));
+ q2 += 4;
+ const uint32_t ls = 2*(aux32[1] >> 28) + 1;
+ int32_t sumi = 0;
+ for (int l = 0; l < 4; ++l) {
+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);
+ const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];
+ for (int j = 0; j < 8; ++j) {
+ sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
+ }
+ q8 += 8;
+ }
+ bsum += sumi * ls;
+ }
+ sumf += d * bsum;
+ }
+ *s = 0.125f * sumf;
+#endif
+}
+
+void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_iq2_xs * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#if defined(__ARM_NEON)
+
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ ggml_int8x16x4_t q2u;
+ ggml_int8x16x4_t q2s;
+ ggml_int8x16x4_t q8b;
+
+ int32x4x4_t scales32;
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint16_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ const uint8x8_t scales8 = vld1_u8(x[i].scales);
+ const uint8x8_t scales_l = vand_u8(scales8, vdup_n_u8(0xf));
+ const uint8x8_t scales_h = vshr_n_u8(scales8, 4);
+ uint8x16_t scales = vcombine_u8(vzip1_u8(scales_l, scales_h), vzip2_u8(scales_l, scales_h));
+ scales = vaddq_u8(vshlq_n_u8(scales, 1), vdupq_n_u8(1));
+ const uint16x8_t scales1 = vmovl_u8(vget_low_u8(scales));
+ const uint16x8_t scales2 = vmovl_u8(vget_high_u8(scales));
+ scales32.val[0] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales1)));
+ scales32.val[1] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales1)));
+ scales32.val[2] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales2)));
+ scales32.val[3] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales2)));
+ int32x4_t sumi = vdupq_n_s32(0);
+ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) {
+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;
+ q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[0] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[1] & 511))));
+ q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[2] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[3] & 511))));
+ q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[4] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[5] & 511))));
+ q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[6] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[7] & 511))));
+ q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[0] >> 9))), vld1_s8((const void *)(signs64 + (q2[1] >> 9))));
+ q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[2] >> 9))), vld1_s8((const void *)(signs64 + (q2[3] >> 9))));
+ q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[4] >> 9))), vld1_s8((const void *)(signs64 + (q2[5] >> 9))));
+ q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[6] >> 9))), vld1_s8((const void *)(signs64 + (q2[7] >> 9))));
+ q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]);
+ q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]);
+ q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]);
+ q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]);
+ const int32x4_t p1 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]);
+ const int32x4_t p2 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[1], q8b.val[1]);
+ const int32x4_t p3 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]);
+ const int32x4_t p4 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[3], q8b.val[3]);
+ const int32x4_t p = vpaddq_s32(vpaddq_s32(p1, p2), vpaddq_s32(p3, p4));
+ sumi = vmlaq_s32(sumi, p, scales32.val[ib64]);
+ q2 += 8;
+ }
+ sumf += d*vaddvq_s32(sumi);
+ }
+ *s = 0.125f * sumf;
+
+#elif defined(__AVX2__)
+
+ const __m256i mone = _mm256_set1_epi8(1);
+ static const char block_sign_shuffle_mask_1[32] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+ 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ };
+ static const char block_sign_shuffle_mask_2[32] = {
+ 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,
+ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,
+ };
+ static const uint8_t bit_selector_mask_bytes[32] = {
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ };
+
+ const __m256i bit_selector_mask = _mm256_loadu_si256((const __m256i*)bit_selector_mask_bytes);
+ const __m256i block_sign_shuffle_1 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_1);
+ const __m256i block_sign_shuffle_2 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_2);
+
+ static const uint8_t k_bit_helper[32] = {
+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,
+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,
+ };
+ const __m256i bit_helper = _mm256_loadu_si256((const __m256i*)k_bit_helper);
+ const __m256i m511 = _mm256_set1_epi16(511);
+ const __m128i m4 = _mm_set1_epi8(0xf);
+ const __m128i m1 = _mm_set1_epi8(1);
+
+ uint64_t aux64;
+
+ // somewhat hacky, but gives a significant boost in performance
+ __m256i aux_gindex;
+ const uint16_t * gindex = (const uint16_t *)&aux_gindex;
+
+ __m256 accumf = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint16_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ memcpy(&aux64, x[i].scales, 8);
+ __m128i stmp = _mm_set1_epi64x(aux64);
+ stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4));
+ const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1);
+
+ __m256i sumi1 = _mm256_setzero_si256();
+ __m256i sumi2 = _mm256_setzero_si256();
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {
+
+ const __m256i q2_data = _mm256_loadu_si256((const __m256i*)q2); q2 += 16;
+ aux_gindex = _mm256_and_si256(q2_data, m511);
+
+ const __m256i partial_sign_bits = _mm256_srli_epi16(q2_data, 9);
+ const __m256i partial_sign_bits_upper = _mm256_srli_epi16(q2_data, 13);
+ const __m256i partial_sign_bits_for_counting = _mm256_xor_si256(partial_sign_bits, partial_sign_bits_upper);
+
+ const __m256i odd_bits = _mm256_shuffle_epi8(bit_helper, partial_sign_bits_for_counting);
+ const __m256i full_sign_bits = _mm256_or_si256(partial_sign_bits, odd_bits);
+
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i q8_4 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+
+ const __m256i q2_1 = _mm256_set_epi64x(iq2xs_grid[gindex[ 3]], iq2xs_grid[gindex[ 2]],
+ iq2xs_grid[gindex[ 1]], iq2xs_grid[gindex[ 0]]);
+ const __m256i q2_2 = _mm256_set_epi64x(iq2xs_grid[gindex[ 7]], iq2xs_grid[gindex[ 6]],
+ iq2xs_grid[gindex[ 5]], iq2xs_grid[gindex[ 4]]);
+ const __m256i q2_3 = _mm256_set_epi64x(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]],
+ iq2xs_grid[gindex[ 9]], iq2xs_grid[gindex[ 8]]);
+ const __m256i q2_4 = _mm256_set_epi64x(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]],
+ iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);
+
+ const __m128i full_signs_l = _mm256_castsi256_si128(full_sign_bits);
+ const __m128i full_signs_h = _mm256_extractf128_si256(full_sign_bits, 1);
+ const __m256i full_signs_1 = MM256_SET_M128I(full_signs_l, full_signs_l);
+ const __m256i full_signs_2 = MM256_SET_M128I(full_signs_h, full_signs_h);
+
+ __m256i signs;
+ signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_1);
+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);
+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, _mm256_or_si256(signs, mone));
+
+ signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_2);
+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);
+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, _mm256_or_si256(signs, mone));
+
+ signs = _mm256_shuffle_epi8(full_signs_2, block_sign_shuffle_1);
+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);
+ const __m256i q8s_3 = _mm256_sign_epi8(q8_3, _mm256_or_si256(signs, mone));
+
+ signs = _mm256_shuffle_epi8(full_signs_2, block_sign_shuffle_2);
+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);
+ const __m256i q8s_4 = _mm256_sign_epi8(q8_4, _mm256_or_si256(signs, mone));
+
+ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);
+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);
+ const __m256i dot3 = _mm256_maddubs_epi16(q2_3, q8s_3);
+ const __m256i dot4 = _mm256_maddubs_epi16(q2_4, q8s_4);
+
+ const __m256i sc1 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0)));
+ const __m256i sc2 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1)));
+ const __m256i sc3 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+2)));
+ const __m256i sc4 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+3)));
+
+ sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot1, sc1));
+ sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot2, sc2));
+ sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot3, sc3));
+ sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot4, sc4));
+ }
+
+ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);
+
+ }
+
+ *s = 0.125f * hsum_float_8(accumf);
+
+#elif defined(__AVX__)
+ const __m128i mone = _mm_set1_epi8(1);
+ static const char block_sign_shuffle_mask_1[32] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+ 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ };
+ static const char block_sign_shuffle_mask_2[32] = {
+ 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,
+ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,
+ };
+ static const uint8_t bit_selector_mask_bytes[32] = {
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ };
+
+ const __m128i bit_selector_mask_0 = _mm_loadu_si128((const __m128i*)bit_selector_mask_bytes);
+ const __m128i bit_selector_mask_1 = _mm_loadu_si128((const __m128i*)bit_selector_mask_bytes + 1);
+ const __m128i block_sign_shuffle_1_0 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_1);
+ const __m128i block_sign_shuffle_1_1 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_1 + 1);
+ const __m128i block_sign_shuffle_2_0 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_2);
+ const __m128i block_sign_shuffle_2_1 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_2 + 1);
+
+ static const uint8_t k_bit_helper[32] = {
+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,
+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,
+ };
+ const __m128i bit_helper_0 = _mm_loadu_si128((const __m128i*)k_bit_helper);
+ const __m128i bit_helper_1 = _mm_loadu_si128((const __m128i*)k_bit_helper + 1);
+ const __m128i m511 = _mm_set1_epi16(511);
+ const __m128i m4 = _mm_set1_epi8(0xf);
+ const __m128i m1 = _mm_set1_epi8(1);
+
+ uint64_t aux64;
+
+ // somewhat hacky, but gives a significant boost in performance
+ __m256i aux_gindex;
+ const uint16_t * gindex = (const uint16_t *)&aux_gindex;
+
+ __m256 accumf = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint16_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ memcpy(&aux64, x[i].scales, 8);
+ __m128i stmp = _mm_set1_epi64x(aux64);
+ stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4));
+ const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1);
+
+ __m128i sumi1_0 = _mm_setzero_si128();
+ __m128i sumi1_1 = _mm_setzero_si128();
+ __m128i sumi2_0 = _mm_setzero_si128();
+ __m128i sumi2_1 = _mm_setzero_si128();
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {
+
+ const __m128i q2_data_0 = _mm_loadu_si128((const __m128i*)q2);
+ const __m128i q2_data_1 = _mm_loadu_si128((const __m128i*)q2 + 1); q2 += 16;
+ aux_gindex = MM256_SET_M128I(_mm_and_si128(q2_data_1, m511), _mm_and_si128(q2_data_0, m511));
+
+ const __m128i partial_sign_bits_0 = _mm_srli_epi16(q2_data_0, 9);
+ const __m128i partial_sign_bits_1 = _mm_srli_epi16(q2_data_1, 9);
+ const __m128i partial_sign_bits_upper_0 = _mm_srli_epi16(q2_data_0, 13);
+ const __m128i partial_sign_bits_upper_1 = _mm_srli_epi16(q2_data_1, 13);
+ const __m128i partial_sign_bits_for_counting_0 = _mm_xor_si128(partial_sign_bits_0, partial_sign_bits_upper_0);
+ const __m128i partial_sign_bits_for_counting_1 = _mm_xor_si128(partial_sign_bits_1, partial_sign_bits_upper_1);
+
+ const __m128i odd_bits_0 = _mm_shuffle_epi8(bit_helper_0, partial_sign_bits_for_counting_0);
+ const __m128i odd_bits_1 = _mm_shuffle_epi8(bit_helper_1, partial_sign_bits_for_counting_1);
+ const __m128i full_sign_bits_0 = _mm_or_si128(partial_sign_bits_0, odd_bits_0);
+ const __m128i full_sign_bits_1 = _mm_or_si128(partial_sign_bits_1, odd_bits_1);
+
+ const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_3_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_3_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_4_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_4_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+
+ const __m128i q2_1_0 = _mm_set_epi64x(iq2xs_grid[gindex[1]], iq2xs_grid[gindex[0]]);
+ const __m128i q2_1_1 = _mm_set_epi64x(iq2xs_grid[gindex[3]], iq2xs_grid[gindex[2]]);
+ const __m128i q2_2_0 = _mm_set_epi64x(iq2xs_grid[gindex[5]], iq2xs_grid[gindex[4]]);
+ const __m128i q2_2_1 = _mm_set_epi64x(iq2xs_grid[gindex[7]], iq2xs_grid[gindex[6]]);
+ const __m128i q2_3_0 = _mm_set_epi64x(iq2xs_grid[gindex[9]], iq2xs_grid[gindex[8]]);
+ const __m128i q2_3_1 = _mm_set_epi64x(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]]);
+ const __m128i q2_4_0 = _mm_set_epi64x(iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);
+ const __m128i q2_4_1 = _mm_set_epi64x(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]]);
+
+ // AVX2 full_signs_1 is full_sign_bits_0 here
+ // AVX2 full_signs_2 is full_sign_bits_1 here
+ __m128i signs_0, signs_1;
+ signs_0 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_1_0);
+ signs_1 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_1_1);
+ signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);
+ signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);
+ const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, _mm_or_si128(signs_0, mone));
+ const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, _mm_or_si128(signs_1, mone));
+
+ signs_0 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_2_0);
+ signs_1 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_2_1);
+ signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);
+ signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);
+ const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, _mm_or_si128(signs_0, mone));
+ const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, _mm_or_si128(signs_1, mone));
+
+ signs_0 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_1_0);
+ signs_1 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_1_1);
+ signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);
+ signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);
+ const __m128i q8s_3_0 = _mm_sign_epi8(q8_3_0, _mm_or_si128(signs_0, mone));
+ const __m128i q8s_3_1 = _mm_sign_epi8(q8_3_1, _mm_or_si128(signs_1, mone));
+
+ signs_0 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_2_0);
+ signs_1 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_2_1);
+ signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);
+ signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);
+ const __m128i q8s_4_0 = _mm_sign_epi8(q8_4_0, _mm_or_si128(signs_0, mone));
+ const __m128i q8s_4_1 = _mm_sign_epi8(q8_4_1, _mm_or_si128(signs_1, mone));
+
+ const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);
+ const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);
+ const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);
+ const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);
+ const __m128i dot3_0 = _mm_maddubs_epi16(q2_3_0, q8s_3_0);
+ const __m128i dot3_1 = _mm_maddubs_epi16(q2_3_1, q8s_3_1);
+ const __m128i dot4_0 = _mm_maddubs_epi16(q2_4_0, q8s_4_0);
+ const __m128i dot4_1 = _mm_maddubs_epi16(q2_4_1, q8s_4_1);
+
+ __m128i sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0));
+ const __m128i sc1_0 = _mm_cvtepi8_epi16(sc_tmp);
+ const __m128i sc1_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));
+ sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1));
+ const __m128i sc2_0 = _mm_cvtepi8_epi16(sc_tmp);
+ const __m128i sc2_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));
+ sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+2));
+ const __m128i sc3_0 = _mm_cvtepi8_epi16(sc_tmp);
+ const __m128i sc3_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));
+ sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+3));
+ const __m128i sc4_0 = _mm_cvtepi8_epi16(sc_tmp);
+ const __m128i sc4_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));
+
+ sumi1_0 = _mm_add_epi32(sumi1_0, _mm_madd_epi16(dot1_0, sc1_0));
+ sumi1_1 = _mm_add_epi32(sumi1_1, _mm_madd_epi16(dot1_1, sc1_1));
+ sumi2_0 = _mm_add_epi32(sumi2_0, _mm_madd_epi16(dot2_0, sc2_0));
+ sumi2_1 = _mm_add_epi32(sumi2_1, _mm_madd_epi16(dot2_1, sc2_1));
+ sumi1_0 = _mm_add_epi32(sumi1_0, _mm_madd_epi16(dot3_0, sc3_0));
+ sumi1_1 = _mm_add_epi32(sumi1_1, _mm_madd_epi16(dot3_1, sc3_1));
+ sumi2_0 = _mm_add_epi32(sumi2_0, _mm_madd_epi16(dot4_0, sc4_0));
+ sumi2_1 = _mm_add_epi32(sumi2_1, _mm_madd_epi16(dot4_1, sc4_1));
+ }
+
+ accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);
+
+ }
+
+ *s = 0.125f * hsum_float_8(accumf);
+
+#elif defined(__loongarch_asx)
+
+ const __m256i mone = __lasx_xvreplgr2vr_b(1);
+ static const char block_sign_shuffle_mask_1[32] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+ 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ };
+ static const char block_sign_shuffle_mask_2[32] = {
+ 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,
+ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,
+ };
+ static const uint8_t bit_selector_mask_bytes[32] = {
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ };
+
+ const __m256i bit_selector_mask = __lasx_xvld((const __m256i*)bit_selector_mask_bytes, 0);
+ const __m256i block_sign_shuffle_1 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_1, 0);
+ const __m256i block_sign_shuffle_2 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_2, 0);
+
+ static const uint8_t k_bit_helper[32] = {
+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,
+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,
+ };
+ const __m256i bit_helper = __lasx_xvld((const __m256i*)k_bit_helper, 0);
+ const __m256i m511 = __lasx_xvreplgr2vr_h(511);
+ const __m128i m4 = __lsx_vreplgr2vr_b(0xf);
+ const __m128i m1 = __lsx_vreplgr2vr_b(1);
+
+ uint64_t aux64;
+
+ // somewhat hacky, but gives a significant boost in performance
+ __m256i aux_gindex;
+ const uint16_t * gindex = (const uint16_t *)&aux_gindex;
+
+ __m256 accumf = (__m256)__lasx_xvldi(0);
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint16_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ memcpy(&aux64, x[i].scales, 8);
+ __m128i stmp = __lsx_vreplgr2vr_d(aux64);
+ stmp = __lsx_vilvl_b( __lsx_vand_v(__lsx_vsrli_h(stmp, 4), m4), __lsx_vand_v(stmp, m4));
+ const __m128i scales = __lsx_vadd_b(__lsx_vslli_h(stmp, 1), m1);
+
+ __m256i sumi1 = __lasx_xvldi(0);
+ __m256i sumi2 = __lasx_xvldi(0);
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {
+
+ const __m256i q2_data = __lasx_xvld((const __m256i*)q2, 0); q2 += 16;
+ aux_gindex = __lasx_xvand_v(q2_data, m511);
+
+ const __m256i partial_sign_bits = __lasx_xvsrli_h(q2_data, 9);
+ const __m256i partial_sign_bits_upper = __lasx_xvsrli_h(q2_data, 13);
+ const __m256i partial_sign_bits_for_counting = __lasx_xvxor_v(partial_sign_bits, partial_sign_bits_upper);
+
+ const __m256i odd_bits = lasx_shuffle_b(bit_helper, partial_sign_bits_for_counting);
+ const __m256i full_sign_bits = __lasx_xvor_v(partial_sign_bits, odd_bits);
+
+ const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ const __m256i q8_3 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ const __m256i q8_4 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+
+ const __m256i q2_1 = lasx_set_d(iq2xs_grid[gindex[ 3]], iq2xs_grid[gindex[ 2]],
+ iq2xs_grid[gindex[ 1]], iq2xs_grid[gindex[ 0]]);
+ const __m256i q2_2 = lasx_set_d(iq2xs_grid[gindex[ 7]], iq2xs_grid[gindex[ 6]],
+ iq2xs_grid[gindex[ 5]], iq2xs_grid[gindex[ 4]]);
+ const __m256i q2_3 = lasx_set_d(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]],
+ iq2xs_grid[gindex[ 9]], iq2xs_grid[gindex[ 8]]);
+ const __m256i q2_4 = lasx_set_d(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]],
+ iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);
+
+ const __m128i full_signs_l = lasx_extracti128(full_sign_bits, 0);
+ const __m128i full_signs_h = lasx_extracti128(full_sign_bits, 1);
+ const __m256i full_signs_1 = lasx_insertf128(full_signs_l, full_signs_l);
+ const __m256i full_signs_2 = lasx_insertf128(full_signs_h, full_signs_h);
+
+ __m256i signs;
+ signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_1);
+ signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);
+ const __m256i q8s_1 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_1);
+
+ signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_2);
+ signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);
+ const __m256i q8s_2 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_2);
+
+ signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_1);
+ signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);
+ const __m256i q8s_3 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_3);
+
+ signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_2);
+ signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);
+ const __m256i q8s_4 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_4);
+
+ const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);
+ const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);
+ const __m256i dot3 = lasx_maddubs_h(q2_3, q8s_3);
+ const __m256i dot4 = lasx_maddubs_h(q2_4, q8s_4);
+
+ const __m256i sc1 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+0)));
+ const __m256i sc2 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+1)));
+ const __m256i sc3 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+2)));
+ const __m256i sc4 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+3)));
+
+ sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot1, sc1));
+ sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot2, sc2));
+ sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot3, sc3));
+ sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot4, sc4));
+ }
+
+ accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);
+
+ }
+
+ *s = 0.125f * hsum_float_8(accumf);
+#elif defined(__POWER9_VECTOR__)
+ const vector int v0 = vec_splats((int32_t)0);
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ for (int i = 0; i < nb; ++i) {
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
+ vector float vyd = vec_splats(y[i].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+ vector signed int vsumi2 = v0;
+ vector signed int vsumi3 = v0;
+
+ const uint16_t * restrict q2 = x[i].qs;
+ const uint8_t * restrict sc = x[i].scales;
+ const int8_t * restrict q8 = y[i].qs;
+
+ for (int j = 0; j < QK_K/64; ++j) {
+ __builtin_prefetch(q2, 0, 1);
+ __builtin_prefetch(q8, 0, 1);
+
+ vector signed long long aux64x2_0 = {*(const int64_t *)(iq2xs_grid + (q2[0] & 511)), *(const int64_t *)(iq2xs_grid + (q2[1] & 511))};
+ vector signed long long aux64x2_1 = {*(const int64_t *)(iq2xs_grid + (q2[2] & 511)), *(const int64_t *)(iq2xs_grid + (q2[3] & 511))};
+ vector signed long long aux64x2_2 = {*(const int64_t *)(iq2xs_grid + (q2[4] & 511)), *(const int64_t *)(iq2xs_grid + (q2[5] & 511))};
+ vector signed long long aux64x2_3 = {*(const int64_t *)(iq2xs_grid + (q2[6] & 511)), *(const int64_t *)(iq2xs_grid + (q2[7] & 511))};
+
+ vector signed long long vsigns0 = {*(const int64_t *)(signs64 + ((q2[0] >> 9))), *(const int64_t *)(signs64 + ((q2[1] >> 9)))};
+ vector signed long long vsigns1 = {*(const int64_t *)(signs64 + ((q2[2] >> 9))), *(const int64_t *)(signs64 + ((q2[3] >> 9)))};
+ vector signed long long vsigns2 = {*(const int64_t *)(signs64 + ((q2[4] >> 9))), *(const int64_t *)(signs64 + ((q2[5] >> 9)))};
+ vector signed long long vsigns3 = {*(const int64_t *)(signs64 + ((q2[6] >> 9))), *(const int64_t *)(signs64 + ((q2[7] >> 9)))};
+ q2 += 8;
+
+ vector signed char q2x0 = (vector signed char)vec_mul((vector signed char)vsigns0, (vector signed char)aux64x2_0);
+ vector signed char q2x1 = (vector signed char)vec_mul((vector signed char)vsigns1, (vector signed char)aux64x2_1);
+ vector signed char q2x2 = (vector signed char)vec_mul((vector signed char)vsigns2, (vector signed char)aux64x2_2);
+ vector signed char q2x3 = (vector signed char)vec_mul((vector signed char)vsigns3, (vector signed char)aux64x2_3);
+
+ vector signed char q8y0 = vec_xl( 0, q8);
+ vector signed char q8y1 = vec_xl(16, q8);
+ vector signed char q8y2 = vec_xl(32, q8);
+ vector signed char q8y3 = vec_xl(48, q8);
+ q8 += 64;
+
+ vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));
+ vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));
+ vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));
+ vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));
+
+ const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);
+ const uint16_t ls1 = (uint16_t)(sc[0] >> 4);
+ const uint16_t ls2 = (uint16_t)(sc[1] & 0xf);
+ const uint16_t ls3 = (uint16_t)(sc[1] >> 4);
+ sc += 2;
+
+ vector signed short vscales0 = vec_splats((int16_t)(2*ls0+1));
+ vector signed short vscales1 = vec_splats((int16_t)(2*ls1+1));
+ vector signed short vscales2 = vec_splats((int16_t)(2*ls2+1));
+ vector signed short vscales3 = vec_splats((int16_t)(2*ls3+1));
+
+ vsumi0 = vec_msum(qv0, vscales0, vsumi0);
+ vsumi1 = vec_msum(qv1, vscales1, vsumi1);
+ vsumi2 = vec_msum(qv2, vscales2, vsumi2);
+ vsumi3 = vec_msum(qv3, vscales3, vsumi3);
+ }
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = 0.125f * vec_extract(vsumf0, 0);
+#else
+
+ float sumf = 0.f;
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint16_t * restrict q2 = x[i].qs;
+ const uint8_t * restrict sc = x[i].scales;
+ const int8_t * restrict q8 = y[i].qs;
+ int32_t bsum = 0;
+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
+ const uint16_t ls1 = 2*(sc[ib32] & 0xf) + 1;
+ const uint16_t ls2 = 2*(sc[ib32] >> 4) + 1;
+ int32_t sumi = 0;
+ for (int l = 0; l < 2; ++l) {
+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));
+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
+ for (int j = 0; j < 8; ++j) {
+ sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
+ }
+ q8 += 8;
+ }
+ bsum += sumi * ls1;
+ sumi = 0;
+ for (int l = 2; l < 4; ++l) {
+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));
+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
+ for (int j = 0; j < 8; ++j) {
+ sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
+ }
+ q8 += 8;
+ }
+ bsum += sumi * ls2;
+ q2 += 4;
+ }
+ sumf += d * bsum;
+ }
+ *s = 0.125f * sumf;
+#endif
+}
+
+void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_iq2_s * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#if defined(__ARM_NEON)
+
+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
+ };
+
+ static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};
+
+ const ggml_uint8x16x2_t mask1 = ggml_vld1q_u8_x2(k_mask1);
+ const uint8x16_t mask2 = vld1q_u8(k_mask2);
+ const uint8x16_t m1 = vdupq_n_u8(1);
+ const int32x4_t vzero = vdupq_n_s32(0);
+
+ uint8x16x2_t vs;
+ ggml_int8x16x4_t q2s;
+ ggml_int8x16x4_t q8b;
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+
+ const uint8_t * restrict qs = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const uint16_t * restrict signs = (const uint16_t *)(x[i].qs + QK_K/8);
+ const int8_t * restrict q8 = y[i].qs;
+
+ int sumi1 = 0, sumi2 = 0;
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;
+ q2s.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[0] | ((qh[ib32+0] << 8) & 0x300)))),
+ vld1_s8((const int8_t *)(iq2s_grid + (qs[1] | ((qh[ib32+0] << 6) & 0x300)))));
+ q2s.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[2] | ((qh[ib32+0] << 4) & 0x300)))),
+ vld1_s8((const int8_t *)(iq2s_grid + (qs[3] | ((qh[ib32+0] << 2) & 0x300)))));
+ q2s.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[4] | ((qh[ib32+1] << 8) & 0x300)))),
+ vld1_s8((const int8_t *)(iq2s_grid + (qs[5] | ((qh[ib32+1] << 6) & 0x300)))));
+ q2s.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[6] | ((qh[ib32+1] << 4) & 0x300)))),
+ vld1_s8((const int8_t *)(iq2s_grid + (qs[7] | ((qh[ib32+1] << 2) & 0x300)))));
+ qs += 8;
+
+ vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | ((uint32_t) signs[1] << 16)));
+ vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);
+ vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);
+ vs.val[0] = vceqq_u8(vs.val[0], mask2);
+ vs.val[1] = vceqq_u8(vs.val[1], mask2);
+
+ q2s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[0], m1)), q2s.val[0]);
+ q2s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[1], m1)), q2s.val[1]);
+
+ vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | ((uint32_t) signs[3] << 16)));
+ vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);
+ vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);
+ vs.val[0] = vceqq_u8(vs.val[0], mask2);
+ vs.val[1] = vceqq_u8(vs.val[1], mask2);
+
+ signs += 4;
+
+ q2s.val[2] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[0], m1)), q2s.val[2]);
+ q2s.val[3] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[1], m1)), q2s.val[3]);
+
+ const int32x4_t p1 = ggml_vdotq_s32(vzero, q2s.val[0], q8b.val[0]);
+ const int32x4_t p2 = ggml_vdotq_s32(vzero, q2s.val[1], q8b.val[1]);
+ const int32x4_t p3 = ggml_vdotq_s32(vzero, q2s.val[2], q8b.val[2]);
+ const int32x4_t p4 = ggml_vdotq_s32(vzero, q2s.val[3], q8b.val[3]);
+
+ sumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32+0] & 0xf));
+ sumi2 += vaddvq_s32(p2) * (1 + 2*(x[i].scales[ib32+0] >> 4));
+ sumi1 += vaddvq_s32(p3) * (1 + 2*(x[i].scales[ib32+1] & 0xf));
+ sumi2 += vaddvq_s32(p4) * (1 + 2*(x[i].scales[ib32+1] >> 4));
+ }
+ sumf += d*(sumi1 + sumi2);
+ }
+
+ *s = 0.125f * sumf;
+
+#elif defined(__AVX2__)
+
+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
+ };
+
+ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ };
+
+ const __m128i m4 = _mm_set1_epi8(0xf);
+ const __m128i m1 = _mm_set1_epi8(1);
+
+ const __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1);
+ const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2);
+
+ uint64_t aux64;
+
+ __m256 accumf = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict qs = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const uint16_t * restrict signs = (const uint16_t *)(x[i].qs + QK_K/8);
+ const int8_t * restrict q8 = y[i].qs;
+
+ memcpy(&aux64, x[i].scales, 8);
+ const __m128i scales8 = _mm_add_epi8(_mm_slli_epi16(_mm_and_si128(_mm_set_epi64x(aux64 >> 4, aux64), m4), 1), m1);
+ const __m256i scales16 = _mm256_cvtepi8_epi16(scales8); // 0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15
+
+ __m256i sumi1 = _mm256_setzero_si256();
+ __m256i sumi2 = _mm256_setzero_si256();
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i q2_1 = _mm256_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],
+ iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)],
+ iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],
+ iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);
+ const __m256i q2_2 = _mm256_set_epi64x(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],
+ iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)],
+ iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],
+ iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);
+ qs += 8;
+
+ __m256i aux256 = _mm256_set1_epi32(signs[0] | ((uint32_t) signs[1] << 16));
+ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);
+ const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2);
+ const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1);
+
+ aux256 = _mm256_set1_epi32(signs[2] | ((uint32_t) signs[3] << 16));
+ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);
+ const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2);
+ const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2);
+
+ signs += 4;
+
+ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1); // blocks 2*ib32+0, 2*ib32+1
+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2); // blocks 2*ib32+2, 2*ib32+3
+
+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32+0)));
+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32+1)));
+ sumi1 = _mm256_add_epi32(sumi1, p1);
+ sumi2 = _mm256_add_epi32(sumi2, p2);
+ }
+
+ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);
+
+ }
+
+ *s = 0.125f * hsum_float_8(accumf);
+
+#elif defined(__AVX__)
+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
+ };
+
+ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ };
+
+ const __m128i m4 = _mm_set1_epi8(0xf);
+ const __m128i m1 = _mm_set1_epi8(1);
+
+ const __m128i mask1_0 = _mm_loadu_si128((const __m128i*)k_mask1);
+ const __m128i mask1_1 = _mm_loadu_si128((const __m128i*)k_mask1 + 1);
+ const __m128i mask2_0 = _mm_loadu_si128((const __m128i*)k_mask2);
+ const __m128i mask2_1 = _mm_loadu_si128((const __m128i*)k_mask2 + 1);
+
+ uint64_t aux64;
+
+ __m256 accumf = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict qs = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const uint16_t * restrict signs = (const uint16_t *)(x[i].qs + QK_K/8);
+ const int8_t * restrict q8 = y[i].qs;
+
+ memcpy(&aux64, x[i].scales, 8);
+ const __m128i scales8 = _mm_add_epi8(_mm_slli_epi16(_mm_and_si128(_mm_set_epi64x(aux64 >> 4, aux64), m4), 1), m1);
+ const __m128i scales16_0 = _mm_cvtepi8_epi16(scales8);
+ const __m128i scales16_1 = _mm_cvtepi8_epi16(_mm_srli_si128(scales8, 8));
+
+ __m128i sumi1_0 = _mm_setzero_si128();
+ __m128i sumi1_1 = _mm_setzero_si128();
+ __m128i sumi2_0 = _mm_setzero_si128();
+ __m128i sumi2_1 = _mm_setzero_si128();
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q2_1_0 = _mm_set_epi64x(iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],
+ iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);
+ const __m128i q2_1_1 = _mm_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],
+ iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)]);
+ const __m128i q2_2_0 = _mm_set_epi64x(iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],
+ iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);
+ const __m128i q2_2_1 = _mm_set_epi64x(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],
+ iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)]);
+ qs += 8;
+
+ __m128i aux128_0 = _mm_set1_epi32(signs[0] | ((uint32_t) signs[1] << 16));
+ __m128i aux128_1 = aux128_0;
+ aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);
+ aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);
+ const __m128i s2_1_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);
+ const __m128i s2_1_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);
+ const __m128i q8s_1_0 = _mm_sub_epi8(_mm_xor_si128(s2_1_0, q8_1_0), s2_1_0);
+ const __m128i q8s_1_1 = _mm_sub_epi8(_mm_xor_si128(s2_1_1, q8_1_1), s2_1_1);
+
+ aux128_0 = _mm_set1_epi32(signs[2] | ((uint32_t) signs[3] << 16));
+ aux128_1 = aux128_0;
+ aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);
+ aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);
+ const __m128i s2_2_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);
+ const __m128i s2_2_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);
+ const __m128i q8s_2_0 = _mm_sub_epi8(_mm_xor_si128(s2_2_0, q8_2_0), s2_2_0);
+ const __m128i q8s_2_1 = _mm_sub_epi8(_mm_xor_si128(s2_2_1, q8_2_1), s2_2_1);
+
+ signs += 4;
+
+ const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);
+ const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);
+ const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);
+ const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);
+
+ const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_shuffle_epi8(scales16_0, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+0), 0)));
+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_shuffle_epi8(scales16_1, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+0), 1)));
+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_shuffle_epi8(scales16_0, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+1), 0)));
+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_shuffle_epi8(scales16_1, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+1), 1)));
+ sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);
+ sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);
+ sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);
+ sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);
+ }
+
+ accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);
+
+ }
+
+ *s = 0.125f * hsum_float_8(accumf);
+
+#elif defined(__POWER9_VECTOR__)
+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
+ };
+
+ static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};
+
+ const vector int v0 = vec_splats((int32_t)0);
+
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ const vector unsigned char mask0 = vec_xl( 0, k_mask1);
+ const vector unsigned char mask1 = vec_xl(16, k_mask1);
+ const vector signed char mask2 = (vector signed char)vec_xl( 0, k_mask2);
+
+ for (int i = 0; i < nb; ++i) {
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
+ vector float vyd = vec_splats(y[i].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+ vector signed int vsumi2 = v0;
+ vector signed int vsumi3 = v0;
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const uint16_t * restrict signs = (const uint16_t *)(x[i].qs + QK_K/8);
+ const uint8_t * restrict sc = x[i].scales;
+ const int8_t * restrict q8 = y[i].qs;
+
+ for (int j = 0; j < QK_K/32; j += 2) {
+ __builtin_prefetch(q2, 0, 1);
+ __builtin_prefetch(q8, 0, 1);
+
+ vector signed long long aux64x2_0 = {*(const int64_t *)(iq2s_grid + (q2[0] | ((qh[0] << 8) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[1] | ((qh[0] << 6) & 0x300)))};
+ vector signed long long aux64x2_1 = {*(const int64_t *)(iq2s_grid + (q2[2] | ((qh[0] << 4) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[3] | ((qh[0] << 2) & 0x300)))};
+ vector signed long long aux64x2_2 = {*(const int64_t *)(iq2s_grid + (q2[4] | ((qh[1] << 8) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[5] | ((qh[1] << 6) & 0x300)))};
+ vector signed long long aux64x2_3 = {*(const int64_t *)(iq2s_grid + (q2[6] | ((qh[1] << 4) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[7] | ((qh[1] << 2) & 0x300)))};
+ q2 += 8;
+ qh += 2;
+
+ vector signed char vsigns01 = (vector signed char)vec_splats(*(const uint32_t *)&signs[0]);
+ vector signed char vsigns23 = (vector signed char)vec_splats(*(const uint32_t *)&signs[2]);
+ signs += 4;
+
+ vector signed char vsigns0 = vec_perm(vsigns01, vsigns01, mask0);
+ vector signed char vsigns1 = vec_perm(vsigns01, vsigns01, mask1);
+ vector signed char vsigns2 = vec_perm(vsigns23, vsigns23, mask0);
+ vector signed char vsigns3 = vec_perm(vsigns23, vsigns23, mask1);
+
+ vsigns0 = (vector signed char)vec_cmpeq(vec_and(vsigns0, mask2), mask2);
+ vsigns1 = (vector signed char)vec_cmpeq(vec_and(vsigns1, mask2), mask2);
+ vsigns2 = (vector signed char)vec_cmpeq(vec_and(vsigns2, mask2), mask2);
+ vsigns3 = (vector signed char)vec_cmpeq(vec_and(vsigns3, mask2), mask2);
+
+ vector signed char q2x0 = vec_sub(vec_xor(vsigns0, (vector signed char)aux64x2_0), vsigns0);
+ vector signed char q2x1 = vec_sub(vec_xor(vsigns1, (vector signed char)aux64x2_1), vsigns1);
+ vector signed char q2x2 = vec_sub(vec_xor(vsigns2, (vector signed char)aux64x2_2), vsigns2);
+ vector signed char q2x3 = vec_sub(vec_xor(vsigns3, (vector signed char)aux64x2_3), vsigns3);
+
+ vector signed char q8y0 = vec_xl( 0, q8);
+ vector signed char q8y1 = vec_xl(16, q8);
+ vector signed char q8y2 = vec_xl(32, q8);
+ vector signed char q8y3 = vec_xl(48, q8);
+ q8 += 64;
+
+ vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));
+ vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));
+ vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));
+ vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));
+
+ const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);
+ const uint16_t ls1 = (uint16_t)(sc[0] >> 4);
+ const uint16_t ls2 = (uint16_t)(sc[1] & 0xf);
+ const uint16_t ls3 = (uint16_t)(sc[1] >> 4);
+ sc += 2;
+
+ vector signed short vscales0 = vec_splats((int16_t)(2*ls0+1));
+ vector signed short vscales1 = vec_splats((int16_t)(2*ls1+1));
+ vector signed short vscales2 = vec_splats((int16_t)(2*ls2+1));
+ vector signed short vscales3 = vec_splats((int16_t)(2*ls3+1));
+
+ vsumi0 = vec_msum(qv0, vscales0, vsumi0);
+ vsumi1 = vec_msum(qv1, vscales1, vsumi1);
+ vsumi2 = vec_msum(qv2, vscales2, vsumi2);
+ vsumi3 = vec_msum(qv3, vscales3, vsumi3);
+ }
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = 0.125f * vec_extract(vsumf0, 0);
+
+#elif defined(__loongarch_asx)
+
+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
+ };
+
+ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ };
+
+
+ const __m128i m4 = __lsx_vreplgr2vr_b(0xf);
+ const __m128i m1 = __lsx_vreplgr2vr_b(1);
+
+ const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0);
+ const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0);
+ uint64_t aux64;
+
+ __m256 accumf = (__m256)__lasx_xvldi(0);
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict qs = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const uint16_t * restrict signs = (const uint16_t *)(x[i].qs + QK_K/8);
+ const int8_t * restrict q8 = y[i].qs;
+
+ __m128i tmp1;
+ memcpy(&aux64, x[i].scales, 8);
+ tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64, 0);
+ tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64 >> 4, 1);
+ const __m128i scales8 = __lsx_vadd_b(__lsx_vslli_h(__lsx_vand_v(tmp1, m4), 1), m1);
+ const __m256i scales16 = lasx_ext8_16(scales8); // 0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15
+
+ __m256i sumi1 = __lasx_xvldi(0);
+ __m256i sumi2 = __lasx_xvldi(0);
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ const __m256i q2_1 = lasx_set_d(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],
+ iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)],
+ iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],
+ iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);
+ const __m256i q2_2 = lasx_set_d(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],
+ iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)],
+ iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],
+ iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);
+ qs += 8;
+
+ __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | ((uint32_t) signs[1] << 16));
+ aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);
+ const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2);
+ const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1);
+
+ aux256 = __lasx_xvreplgr2vr_w(signs[2] | ((uint32_t) signs[3] << 16));
+ aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);
+ const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2);
+ const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2);
+
+ signs += 4;
+
+ const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1); // blocks 2*ib32+0, 2*ib32+1
+ const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2); // blocks 2*ib32+2, 2*ib32+3
+
+ const __m256i p1 = lasx_madd_h(dot1, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+0)));
+ const __m256i p2 = lasx_madd_h(dot2, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+1)));
+ sumi1 = __lasx_xvadd_w(sumi1, p1);
+ sumi2 = __lasx_xvadd_w(sumi2, p2);
+ }
+
+ accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);
+ }
+
+ *s = 0.125f * hsum_float_8(accumf);
+
+#else
+
+ float sumf = 0;
+ for (int i = 0; i < nb; i++) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+ const uint8_t * signs = qs + QK_K/8;
+
+ int bsum = 0;
+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
+ int ls1 = 1 + 2*(x[i].scales[ib32] & 0xf);
+ int ls2 = 1 + 2*(x[i].scales[ib32] >> 4);
+ int sumi1 = 0, sumi2 = 0;
+ for (int l = 0; l < 2; ++l) {
+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));
+ for (int j = 0; j < 8; ++j) {
+ sumi1 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1);
+ }
+ q8 += 8;
+ }
+ for (int l = 2; l < 4; ++l) {
+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));
+ for (int j = 0; j < 8; ++j) {
+ sumi2 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1);
+ }
+ q8 += 8;
+ }
+ bsum += ls1 * sumi1 + ls2 * sumi2;
+ qs += 4;
+ signs += 4;
+ }
+
+ sumf += d * bsum;
+ }
+
+ *s = 0.125f * sumf;
+
+#endif
+
+}
+
+void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_iq3_xxs * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#if defined(__ARM_NEON)
+
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ uint32_t aux32[2];
+
+ ggml_int8x16x4_t q3s;
+ ggml_int8x16x4_t q8b;
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict gas = x[i].qs + QK_K/4;
+ const int8_t * restrict q8 = y[i].qs;
+ float sumf1 = 0, sumf2 = 0;
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;
+ memcpy(aux32, gas, 2*sizeof(uint32_t)); gas += 2*sizeof(uint32_t);
+ const uint32x4_t aux32x4_0 = ggml_vld1q_u32(iq3xxs_grid[q3[ 0]], iq3xxs_grid[q3[ 1]], iq3xxs_grid[q3[ 2]], iq3xxs_grid[q3[ 3]]);
+ const uint32x4_t aux32x4_1 = ggml_vld1q_u32(iq3xxs_grid[q3[ 4]], iq3xxs_grid[q3[ 5]], iq3xxs_grid[q3[ 6]], iq3xxs_grid[q3[ 7]]);
+ const uint32x4_t aux32x4_2 = ggml_vld1q_u32(iq3xxs_grid[q3[ 8]], iq3xxs_grid[q3[ 9]], iq3xxs_grid[q3[10]], iq3xxs_grid[q3[11]]);
+ const uint32x4_t aux32x4_3 = ggml_vld1q_u32(iq3xxs_grid[q3[12]], iq3xxs_grid[q3[13]], iq3xxs_grid[q3[14]], iq3xxs_grid[q3[15]]);
+ q3 += 16;
+ q3s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[0] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[0] >> 7) & 127))));
+ q3s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[0] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[0] >> 21) & 127))));
+ q3s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 7) & 127))));
+ q3s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 21) & 127))));
+ q3s.val[0] = vmulq_s8(q3s.val[0], vreinterpretq_s8_u32(aux32x4_0));
+ q3s.val[1] = vmulq_s8(q3s.val[1], vreinterpretq_s8_u32(aux32x4_1));
+ q3s.val[2] = vmulq_s8(q3s.val[2], vreinterpretq_s8_u32(aux32x4_2));
+ q3s.val[3] = vmulq_s8(q3s.val[3], vreinterpretq_s8_u32(aux32x4_3));
+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[0], q8b.val[0]), q3s.val[1], q8b.val[1]);
+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[2], q8b.val[2]), q3s.val[3], q8b.val[3]);
+ sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[0] >> 28));
+ sumf2 += vaddvq_s32(p2) * (0.5f + (aux32[1] >> 28));
+ }
+ sumf += d*(sumf1 + sumf2);
+ }
+ *s = 0.5f * sumf;
+
+#elif defined(__AVX2__)
+
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ uint32_t aux32[2];
+
+ __m256 accumf = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict gas = x[i].qs + QK_K/4;
+ const int8_t * restrict q8 = y[i].qs;
+ __m256i sumi1 = _mm256_setzero_si256();
+ __m256i sumi2 = _mm256_setzero_si256();
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i q2_1 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],
+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);
+ q3 += 8;
+ const __m256i q2_2 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],
+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);
+ q3 += 8;
+ memcpy(aux32, gas, 8); gas += 8;
+ const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],
+ signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]);
+ const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],
+ signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);
+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);
+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);
+ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);
+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);
+ const uint16_t ls1 = aux32[0] >> 28;
+ const uint16_t ls2 = aux32[1] >> 28;
+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));
+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));
+ sumi1 = _mm256_add_epi32(sumi1, p1);
+ sumi2 = _mm256_add_epi32(sumi2, p2);
+ }
+
+ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);
+
+ }
+
+ *s = 0.25f * hsum_float_8(accumf);
+
+#elif defined(__AVX__)
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ uint32_t aux32[2];
+
+ __m256 accumf = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict gas = x[i].qs + QK_K/4;
+ const int8_t * restrict q8 = y[i].qs;
+ __m128i sumi1_0 = _mm_setzero_si128();
+ __m128i sumi1_1 = _mm_setzero_si128();
+ __m128i sumi2_0 = _mm_setzero_si128();
+ __m128i sumi2_1 = _mm_setzero_si128();
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q2_1_0 = _mm_set_epi32(iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);
+ const __m128i q2_1_1 = _mm_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]]);
+ q3 += 8;
+ const __m128i q2_2_0 = _mm_set_epi32(iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);
+ const __m128i q2_2_1 = _mm_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]]);
+ q3 += 8;
+ memcpy(aux32, gas, 8); gas += 8;
+ const __m128i s2_1_0 = _mm_set_epi64x(signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]);
+ const __m128i s2_1_1 = _mm_set_epi64x(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127]);
+ const __m128i s2_2_0 = _mm_set_epi64x(signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);
+ const __m128i s2_2_1 = _mm_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127]);
+ const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, s2_1_0);
+ const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, s2_1_1);
+ const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, s2_2_0);
+ const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, s2_2_1);
+ const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);
+ const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);
+ const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);
+ const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);
+ const uint16_t ls1 = aux32[0] >> 28;
+ const uint16_t ls2 = aux32[1] >> 28;
+ const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));
+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));
+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));
+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));
+ sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);
+ sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);
+ sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);
+ sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);
+ }
+
+ accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);
+
+ }
+
+ *s = 0.25f * hsum_float_8(accumf);
+
+#elif defined(__POWER9_VECTOR__)
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ const vector int v0 = vec_splats((int32_t)0);
+
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ for (int i = 0; i < nb; ++i) {
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
+ vector float vyd = vec_splats(y[i].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+ vector signed int vsumi2 = v0;
+ vector signed int vsumi3 = v0;
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint32_t * restrict signs = (const uint32_t *)(x[i].qs + QK_K/4);
+ const int8_t * restrict q8 = y[i].qs;
+
+#pragma GCC unroll 1
+ for (int j = 0; j < QK_K/32; j += 2) {
+ __builtin_prefetch(q3, 0, 1);
+ __builtin_prefetch(q8, 0, 1);
+
+ vector unsigned int aux32x4_0 = {iq3xxs_grid[q3[ 0]], iq3xxs_grid[q3[ 1]], iq3xxs_grid[q3[ 2]], iq3xxs_grid[q3[ 3]]};
+ vector unsigned int aux32x4_1 = {iq3xxs_grid[q3[ 4]], iq3xxs_grid[q3[ 5]], iq3xxs_grid[q3[ 6]], iq3xxs_grid[q3[ 7]]};
+ vector unsigned int aux32x4_2 = {iq3xxs_grid[q3[ 8]], iq3xxs_grid[q3[ 9]], iq3xxs_grid[q3[10]], iq3xxs_grid[q3[11]]};
+ vector unsigned int aux32x4_3 = {iq3xxs_grid[q3[12]], iq3xxs_grid[q3[13]], iq3xxs_grid[q3[14]], iq3xxs_grid[q3[15]]};
+ q3 += 16;
+
+ vector unsigned long long aux64x2_0 = {(uint64_t)(signs64[(signs[0] >> 0) & 127]), (uint64_t)(signs64[(signs[0] >> 7) & 127])};
+ vector unsigned long long aux64x2_1 = {(uint64_t)(signs64[(signs[0] >> 14) & 127]), (uint64_t)(signs64[(signs[0] >> 21) & 127])};
+ vector unsigned long long aux64x2_2 = {(uint64_t)(signs64[(signs[1] >> 0) & 127]), (uint64_t)(signs64[(signs[1] >> 7) & 127])};
+ vector unsigned long long aux64x2_3 = {(uint64_t)(signs64[(signs[1] >> 14) & 127]), (uint64_t)(signs64[(signs[1] >> 21) & 127])};
+
+ vector signed char q3x0 = vec_mul((vector signed char)aux64x2_0, (vector signed char)aux32x4_0);
+ vector signed char q3x1 = vec_mul((vector signed char)aux64x2_1, (vector signed char)aux32x4_1);
+ vector signed char q3x2 = vec_mul((vector signed char)aux64x2_2, (vector signed char)aux32x4_2);
+ vector signed char q3x3 = vec_mul((vector signed char)aux64x2_3, (vector signed char)aux32x4_3);
+
+ vector signed char q8y0 = vec_xl( 0, q8);
+ vector signed char q8y1 = vec_xl(16, q8);
+ vector signed char q8y2 = vec_xl(32, q8);
+ vector signed char q8y3 = vec_xl(48, q8);
+ q8 += 64;
+
+ vector signed short qv0 = vec_add(vec_mule(q3x0, q8y0), vec_mulo(q3x0, q8y0));
+ vector signed short qv1 = vec_add(vec_mule(q3x1, q8y1), vec_mulo(q3x1, q8y1));
+ vector signed short qv2 = vec_add(vec_mule(q3x2, q8y2), vec_mulo(q3x2, q8y2));
+ vector signed short qv3 = vec_add(vec_mule(q3x3, q8y3), vec_mulo(q3x3, q8y3));
+
+ const uint16_t ls0 = (uint16_t)(signs[0] >> 28);
+ const uint16_t ls1 = (uint16_t)(signs[1] >> 28);
+ signs += 2;
+
+ vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));
+ vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));
+
+ vsumi0 = vec_msum(qv0, vscales01, vsumi0);
+ vsumi1 = vec_msum(qv1, vscales01, vsumi1);
+ vsumi2 = vec_msum(qv2, vscales23, vsumi2);
+ vsumi3 = vec_msum(qv3, vscales23, vsumi3);
+ }
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = 0.25f * vec_extract(vsumf0, 0);
+
+#elif defined(__loongarch_asx)
+
+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
+
+ uint32_t aux32[2];
+
+ __m256 accumf = (__m256)__lasx_xvldi(0);
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict gas = x[i].qs + QK_K/4;
+ const int8_t * restrict q8 = y[i].qs;
+ __m256i sumi1 = __lasx_xvldi(0);
+ __m256i sumi2 = __lasx_xvldi(0);
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ const __m256i q2_1 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],
+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);
+ q3 += 8;
+ const __m256i q2_2 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],
+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);
+ q3 += 8;
+ memcpy(aux32, gas, 8); gas += 8;
+
+ const __m256i s2_1 = lasx_set_d(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],
+ signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]);
+ const __m256i s2_2 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],
+ signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);
+ const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1);
+ const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2);
+ const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);
+ const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);
+ const uint16_t ls1 = aux32[0] >> 28;
+ const uint16_t ls2 = aux32[1] >> 28;
+
+ const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));
+ const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));
+ sumi1 = __lasx_xvadd_w(sumi1, p1);
+ sumi2 = __lasx_xvadd_w(sumi2, p2);
+ }
+
+ accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);
+ }
+
+ *s = 0.25f * hsum_float_8(accumf);
+
+#else
+
+ uint32_t aux32;
+
+ float sumf = 0.f;
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict gas = x[i].qs + QK_K/4;
+ const int8_t * restrict q8 = y[i].qs;
+ int32_t bsum = 0;
+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
+ memcpy(&aux32, gas, sizeof(uint32_t)); gas += sizeof(uint32_t);
+ const uint32_t ls = 2*(aux32 >> 28) + 1;
+ int32_t sumi = 0;
+ for (int l = 0; l < 4; ++l) {
+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*l+0]);
+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*l+1]);
+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127];
+ for (int j = 0; j < 4; ++j) {
+ sumi += grid1[j] * q8[j+0] * (signs & kmask_iq2xs[j+0] ? -1 : 1);
+ sumi += grid2[j] * q8[j+4] * (signs & kmask_iq2xs[j+4] ? -1 : 1);
+ }
+ q8 += 8;
+ }
+ q3 += 8;
+ bsum += sumi * ls;
+ }
+ sumf += d * bsum;
+ }
+ *s = 0.25f * sumf;
+#endif
+}
+
+void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_iq3_s * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#if defined(__ARM_NEON)
+
+ typedef union {
+ uint16x8_t vec_index;
+ uint16_t index[8];
+ } vec_index_t;
+
+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
+ };
+
+ static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};
+
+ static const int16_t k_shift[8] = {8, 7, 6, 5, 4, 3, 2, 1};
+
+ const ggml_uint8x16x2_t mask1 = ggml_vld1q_u8_x2(k_mask1);
+ const uint8x16_t mask2 = vld1q_u8(k_mask2);
+
+ const int16x8_t hshift = vld1q_s16(k_shift);
+ const uint16x8_t m256 = vdupq_n_u16(256);
+ const uint8x16_t m1 = vdupq_n_u8(1);
+
+ uint8x16x2_t vs;
+ ggml_int8x16x4_t q3s;
+ ggml_int8x16x4_t q8b;
+ vec_index_t idx;
+
+ uint32_t scales32[2];
+ const uint8_t * scales8 = (const uint8_t *)scales32;
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict qs = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const uint16_t * restrict signs = (const uint16_t *)x[i].signs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ memcpy(scales32, x[i].scales, 4);
+ scales32[1] = (((scales32[0] >> 4) & 0x0f0f0f0f) << 1) | 0x01010101;
+ scales32[0] = ((scales32[0] & 0x0f0f0f0f) << 1) | 0x01010101;
+
+ int sumi1 = 0, sumi2 = 0;
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ const uint8x16_t idx_l = vld1q_u8(qs); qs += 16;
+ idx.vec_index = vorrq_u16(vmovl_u8(vget_low_u8 (idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+0]), hshift), m256));
+ const uint32x4_t aux32x4_0 = ggml_vld1q_u32(iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]],
+ iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]);
+ const uint32x4_t aux32x4_1 = ggml_vld1q_u32(iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]],
+ iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]);
+ idx.vec_index = vorrq_u16(vmovl_u8(vget_high_u8(idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+1]), hshift), m256));
+ const uint32x4_t aux32x4_2 = ggml_vld1q_u32(iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]],
+ iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]);
+ const uint32x4_t aux32x4_3 = ggml_vld1q_u32(iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]],
+ iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]);
+
+
+ vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | ((uint32_t) signs[1] << 16)));
+ vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);
+ vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);
+ vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1);
+ vs.val[1] = vorrq_u8(vceqq_u8(vs.val[1], mask2), m1);
+
+ q3s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vs.val[0]), vreinterpretq_s8_u32(aux32x4_0));
+ q3s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vs.val[1]), vreinterpretq_s8_u32(aux32x4_1));
+
+ vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | ((uint32_t) signs[3] << 16)));
+ vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);
+ vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);
+ vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1);
+ vs.val[1] = vorrq_u8(vceqq_u8(vs.val[1], mask2), m1);
+
+ signs += 4;
+
+ q3s.val[2] = vmulq_s8(vreinterpretq_s8_u8(vs.val[0]), vreinterpretq_s8_u32(aux32x4_2));
+ q3s.val[3] = vmulq_s8(vreinterpretq_s8_u8(vs.val[1]), vreinterpretq_s8_u32(aux32x4_3));
+
+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[0], q8b.val[0]), q3s.val[1], q8b.val[1]);
+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[2], q8b.val[2]), q3s.val[3], q8b.val[3]);
+
+ sumi1 += vaddvq_s32(p1) * scales8[ib32/2+0];
+ sumi2 += vaddvq_s32(p2) * scales8[ib32/2+4];
+ }
+ sumf += d*(sumi1 + sumi2);
+ }
+ *s = sumf;
+
+#elif defined(__AVX2__)
+
+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
+ };
+
+ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ };
+
+ const __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1);
+ const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2);
+
+ const __m256i idx_shift = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);
+ const __m256i idx_mask = _mm256_set1_epi32(256);
+
+ typedef union {
+ __m256i vec[2];
+ uint32_t index[16];
+ } index_t;
+
+ index_t idx;
+
+ __m256 accumf = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict qs = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const uint16_t * restrict signs = (const uint16_t *)x[i].signs;
+ const int8_t * restrict q8 = y[i].qs;
+ __m256i sumi1 = _mm256_setzero_si256();
+ __m256i sumi2 = _mm256_setzero_si256();
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i idx_l = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)qs)); qs += 16;
+ idx.vec[0] = _mm256_set1_epi32(qh[ib32+0]);
+ idx.vec[1] = _mm256_set1_epi32(qh[ib32+1]);
+ idx.vec[0] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[0], idx_shift), idx_mask);
+ idx.vec[1] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[1], idx_shift), idx_mask);
+ idx.vec[0] = _mm256_or_si256(idx.vec[0], _mm256_cvtepi16_epi32(_mm256_castsi256_si128(idx_l)));
+ idx.vec[1] = _mm256_or_si256(idx.vec[1], _mm256_cvtepi16_epi32(_mm256_extractf128_si256(idx_l, 1)));
+
+ // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange.
+ //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4);
+ //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4);
+ const __m256i q2_1 = _mm256_set_epi32(
+ iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]],
+ iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]
+ );
+ const __m256i q2_2 = _mm256_set_epi32(
+ iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]],
+ iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]]
+ );
+
+ __m256i aux256 = _mm256_set1_epi32(signs[0] | (signs[1] << 16));
+ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);
+ const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2);
+ const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1);
+
+ aux256 = _mm256_set1_epi32(signs[2] | (signs[3] << 16));
+ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);
+ const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2);
+ const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2);
+
+ signs += 4;
+
+ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);
+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);
+ const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;
+ const uint16_t ls2 = x[i].scales[ib32/2] >> 4;
+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));
+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));
+ sumi1 = _mm256_add_epi32(sumi1, p1);
+ sumi2 = _mm256_add_epi32(sumi2, p2);
+ }
+
+ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);
+
+ }
+
+ *s = hsum_float_8(accumf);
+
+#elif defined(__AVX__)
+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
+ };
+
+ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ };
+
+ const __m128i mask1_0 = _mm_loadu_si128((const __m128i*)k_mask1);
+ const __m128i mask1_1 = _mm_loadu_si128((const __m128i*)k_mask1 + 1);
+ const __m128i mask2_0 = _mm_loadu_si128((const __m128i*)k_mask2);
+ const __m128i mask2_1 = _mm_loadu_si128((const __m128i*)k_mask2 + 1);
+
+ const __m128i idx_mul_0 = _mm_set_epi32(32, 64, 128, 256);
+ const __m128i idx_mul_1 = _mm_set_epi32(2, 4, 8, 16);
+ const __m128i idx_mask = _mm_set1_epi32(256);
+
+ typedef union {
+ __m128i vec[4];
+ uint32_t index[16];
+ } index_t;
+
+ index_t idx;
+
+ __m256 accumf = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict qs = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const uint16_t * restrict signs = (const uint16_t *)x[i].signs;
+ const int8_t * restrict q8 = y[i].qs;
+ __m128i sumi1_0 = _mm_setzero_si128();
+ __m128i sumi1_1 = _mm_setzero_si128();
+ __m128i sumi2_0 = _mm_setzero_si128();
+ __m128i sumi2_1 = _mm_setzero_si128();
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i qs_tmp = _mm_loadu_si128((const __m128i *)qs);
+ const __m128i idx_l_0 = _mm_cvtepu8_epi16(qs_tmp);
+ const __m128i idx_l_1 = _mm_cvtepu8_epi16(_mm_srli_si128(qs_tmp, 8)); qs += 16;
+ idx.vec[0] = _mm_set1_epi32(qh[ib32+0]);
+ idx.vec[1] = idx.vec[0];
+ idx.vec[2] = _mm_set1_epi32(qh[ib32+1]);
+ idx.vec[3] = idx.vec[2];
+
+ idx.vec[0] = _mm_and_si128(_mm_mullo_epi32(idx.vec[0], idx_mul_0), idx_mask);
+ idx.vec[1] = _mm_and_si128(_mm_mullo_epi32(idx.vec[1], idx_mul_1), idx_mask);
+ idx.vec[2] = _mm_and_si128(_mm_mullo_epi32(idx.vec[2], idx_mul_0), idx_mask);
+ idx.vec[3] = _mm_and_si128(_mm_mullo_epi32(idx.vec[3], idx_mul_1), idx_mask);
+
+ idx.vec[0] = _mm_or_si128(idx.vec[0], _mm_cvtepi16_epi32(idx_l_0));
+ idx.vec[1] = _mm_or_si128(idx.vec[1], _mm_cvtepi16_epi32(_mm_srli_si128(idx_l_0, 8)));
+ idx.vec[2] = _mm_or_si128(idx.vec[2], _mm_cvtepi16_epi32(idx_l_1));
+ idx.vec[3] = _mm_or_si128(idx.vec[3], _mm_cvtepi16_epi32(_mm_srli_si128(idx_l_1, 8)));
+
+ const __m128i q2_1_0 = _mm_set_epi32(iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]);
+ const __m128i q2_1_1 = _mm_set_epi32(iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]]);
+ const __m128i q2_2_0 = _mm_set_epi32(iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[9]], iq3s_grid[idx.index[8]]);
+ const __m128i q2_2_1 = _mm_set_epi32(iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]]);
+
+ __m128i aux128_0 = _mm_set1_epi32(signs[0] | (signs[1] << 16));
+ __m128i aux128_1 = aux128_0;
+ aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);
+ aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);
+ const __m128i s2_1_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);
+ const __m128i s2_1_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);
+ const __m128i q8s_1_0 = _mm_sub_epi8(_mm_xor_si128(s2_1_0, q8_1_0), s2_1_0);
+ const __m128i q8s_1_1 = _mm_sub_epi8(_mm_xor_si128(s2_1_1, q8_1_1), s2_1_1);
+
+ aux128_0 = _mm_set1_epi32(signs[2] | (signs[3] << 16));
+ aux128_1 = aux128_0;
+ aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);
+ aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);
+ const __m128i s2_2_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);
+ const __m128i s2_2_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);
+ const __m128i q8s_2_0 = _mm_sub_epi8(_mm_xor_si128(s2_2_0, q8_2_0), s2_2_0);
+ const __m128i q8s_2_1 = _mm_sub_epi8(_mm_xor_si128(s2_2_1, q8_2_1), s2_2_1);
+
+ signs += 4;
+
+ const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);
+ const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);
+ const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);
+ const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);
+ const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;
+ const uint16_t ls2 = x[i].scales[ib32/2] >> 4;
+ const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));
+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));
+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));
+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));
+ sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);
+ sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);
+ sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);
+ sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);
+ }
+
+ accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);
+
+ }
+
+ *s = hsum_float_8(accumf);
+
+#elif defined(__POWER9_VECTOR__)
+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
+ };
+
+ static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};
+
+ const vector int v0 = vec_splats((int32_t)0);
+
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ const vector unsigned char mask0 = vec_xl( 0, k_mask1);
+ const vector unsigned char mask1 = vec_xl(16, k_mask1);
+ const vector signed char mask2 = (vector signed char)vec_xl( 0, k_mask2);
+
+ for (int i = 0; i < nb; ++i) {
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
+ vector float vyd = vec_splats(y[i].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const uint16_t * restrict signs = (const uint16_t *)(x[i].signs);
+ const uint8_t * restrict sc = x[i].scales;
+ const int8_t * restrict q8 = y[i].qs;
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+ vector signed int vsumi2 = v0;
+ vector signed int vsumi3 = v0;
+
+ for (int j = 0; j < QK_K/32; j += 2) {
+ __builtin_prefetch(q3, 0, 1);
+ __builtin_prefetch(q8, 0, 1);
+
+ vector unsigned int aux32x4_0 = {iq3s_grid[q3[ 0] | ((qh[0] << 8) & 256)], iq3s_grid[q3[ 1] | ((qh[0] << 7) & 256)],
+ iq3s_grid[q3[ 2] | ((qh[0] << 6) & 256)], iq3s_grid[q3[ 3] | ((qh[0] << 5) & 256)]};
+ vector unsigned int aux32x4_1 = {iq3s_grid[q3[ 4] | ((qh[0] << 4) & 256)], iq3s_grid[q3[ 5] | ((qh[0] << 3) & 256)],
+ iq3s_grid[q3[ 6] | ((qh[0] << 2) & 256)], iq3s_grid[q3[ 7] | ((qh[0] << 1) & 256)]};
+ vector unsigned int aux32x4_2 = {iq3s_grid[q3[ 8] | ((qh[1] << 8) & 256)], iq3s_grid[q3[ 9] | ((qh[1] << 7) & 256)],
+ iq3s_grid[q3[10] | ((qh[1] << 6) & 256)], iq3s_grid[q3[11] | ((qh[1] << 5) & 256)]};
+ vector unsigned int aux32x4_3 = {iq3s_grid[q3[12] | ((qh[1] << 4) & 256)], iq3s_grid[q3[13] | ((qh[1] << 3) & 256)],
+ iq3s_grid[q3[14] | ((qh[1] << 2) & 256)], iq3s_grid[q3[15] | ((qh[1] << 1) & 256)]};
+ q3 += 16;
+ qh += 2;
+
+ vector signed char vsigns01 = (vector signed char)vec_splats(*(const uint32_t *)&signs[0]);
+ vector signed char vsigns02 = (vector signed char)vec_splats(*(const uint32_t *)&signs[2]);
+ signs += 4;
+
+ vector signed char vsigns0 = vec_perm(vsigns01, vsigns01, mask0);
+ vector signed char vsigns1 = vec_perm(vsigns01, vsigns01, mask1);
+ vector signed char vsigns2 = vec_perm(vsigns02, vsigns02, mask0);
+ vector signed char vsigns3 = vec_perm(vsigns02, vsigns02, mask1);
+
+ vsigns0 = (vector signed char)vec_cmpeq(vec_and(vsigns0, mask2), mask2);
+ vsigns1 = (vector signed char)vec_cmpeq(vec_and(vsigns1, mask2), mask2);
+ vsigns2 = (vector signed char)vec_cmpeq(vec_and(vsigns2, mask2), mask2);
+ vsigns3 = (vector signed char)vec_cmpeq(vec_and(vsigns3, mask2), mask2);
+
+ vector signed char q3x0 = vec_sub(vec_xor(vsigns0, (vector signed char)aux32x4_0), vsigns0);
+ vector signed char q3x1 = vec_sub(vec_xor(vsigns1, (vector signed char)aux32x4_1), vsigns1);
+ vector signed char q3x2 = vec_sub(vec_xor(vsigns2, (vector signed char)aux32x4_2), vsigns2);
+ vector signed char q3x3 = vec_sub(vec_xor(vsigns3, (vector signed char)aux32x4_3), vsigns3);
+
+ vector signed char q8y0 = vec_xl( 0, q8);
+ vector signed char q8y1 = vec_xl(16, q8);
+ vector signed char q8y2 = vec_xl(32, q8);
+ vector signed char q8y3 = vec_xl(48, q8);
+ q8 += 64;
+
+ vector signed short qv0 = vec_add(vec_mule(q3x0, q8y0), vec_mulo(q3x0, q8y0));
+ vector signed short qv1 = vec_add(vec_mule(q3x1, q8y1), vec_mulo(q3x1, q8y1));
+ vector signed short qv2 = vec_add(vec_mule(q3x2, q8y2), vec_mulo(q3x2, q8y2));
+ vector signed short qv3 = vec_add(vec_mule(q3x3, q8y3), vec_mulo(q3x3, q8y3));
+
+ const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);
+ const uint16_t ls1 = (uint16_t)(sc[0] >> 4);
+ sc ++;
+
+ vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));
+ vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));
+
+ vsumi0 = vec_msum(qv0, vscales01, vsumi0);
+ vsumi1 = vec_msum(qv1, vscales01, vsumi1);
+ vsumi2 = vec_msum(qv2, vscales23, vsumi2);
+ vsumi3 = vec_msum(qv3, vscales23, vsumi3);
+ }
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = vec_extract(vsumf0, 0);
+
+#elif defined(__loongarch_asx)
+
+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
+ };
+
+ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ };
+
+ const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0);
+ const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0);
+
+ __m256i idx_shift = lasx_set_w(1, 2, 3, 4, 5, 6, 7, 8);
+ const __m256i idx_mask = __lasx_xvreplgr2vr_w(256);
+
+ typedef union {
+ __m256i vec[2];
+ uint32_t index[16];
+ } index_t;
+
+ index_t idx;
+
+ __m256 accumf = (__m256)__lasx_xvldi(0);
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict qs = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const uint16_t * restrict signs = (const uint16_t *)x[i].signs;
+ const int8_t * restrict q8 = y[i].qs;
+ __m256i sumi1 = __lasx_xvldi(0);
+ __m256i sumi2 = __lasx_xvldi(0);
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ const __m256i idx_l = lasx_extu8_16(__lsx_vld(qs, 0)); qs += 16;
+ idx.vec[0] = __lasx_xvreplgr2vr_w(qh[ib32+0]);
+ idx.vec[1] = __lasx_xvreplgr2vr_w(qh[ib32+1]);
+ idx.vec[0] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[0], idx_shift), idx_mask);
+ idx.vec[1] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[1], idx_shift), idx_mask);
+ idx.vec[0] = __lasx_xvor_v(idx.vec[0], lasx_ext16_32(lasx_extracti128(idx_l, 0)));
+ idx.vec[1] = __lasx_xvor_v(idx.vec[1], lasx_ext16_32(lasx_extracti128(idx_l, 1)));
+
+ // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange.
+ //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4);
+ //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4);
+ const __m256i q2_1 = lasx_set_w(
+ iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]],
+ iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]
+ );
+ const __m256i q2_2 = lasx_set_w(
+ iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]],
+ iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]]
+ );
+
+ __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | (signs[1] << 16));
+ aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);
+ const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2);
+ const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1);
+
+ aux256 = __lasx_xvreplgr2vr_w(signs[2] | (signs[3] << 16));
+ aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);
+ const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2);
+ const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2);
+
+ signs += 4;
+
+ const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);
+ const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);
+ const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;
+ const uint16_t ls2 = x[i].scales[ib32/2] >> 4;
+ const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));
+ const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));
+ sumi1 = __lasx_xvadd_w(sumi1, p1);
+ sumi2 = __lasx_xvadd_w(sumi2, p2);
+ }
+
+ accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);
+ }
+
+ *s = hsum_float_8(accumf);
+
+#else
+
+ float sumf = 0.f;
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * restrict qs = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const uint8_t * restrict signs = x[i].signs;
+ const int8_t * restrict q8 = y[i].qs;
+ int32_t bsum = 0;
+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
+ const uint32_t ls1 = 2*(x[i].scales[ib32/2] & 0xf) + 1;
+ const uint32_t ls2 = 2*(x[i].scales[ib32/2] >> 4) + 1;
+ int32_t sumi = 0;
+ for (int l = 0; l < 4; ++l) {
+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+0] << (8-2*l)) & 256)));
+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+0] << (7-2*l)) & 256)));
+ for (int j = 0; j < 4; ++j) {
+ sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);
+ sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);
+ }
+ q8 += 8;
+ }
+ qs += 8;
+ signs += 4;
+ bsum += sumi * ls1;
+ sumi = 0;
+ for (int l = 0; l < 4; ++l) {
+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+1] << (8-2*l)) & 256)));
+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+1] << (7-2*l)) & 256)));
+ for (int j = 0; j < 4; ++j) {
+ sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);
+ sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);
+ }
+ q8 += 8;
+ }
+ qs += 8;
+ signs += 4;
+ bsum += sumi * ls2;
+ }
+ sumf += d * bsum;
+ }
+ *s = sumf;
+#endif
+}
+
+
+#if defined(__AVX__)
+static inline __m128i mul_add_epi8_sse(const __m128i x, const __m128i y) {
+ const __m128i ax = _mm_sign_epi8(x, x);
+ const __m128i sy = _mm_sign_epi8(y, x);
+ return _mm_maddubs_epi16(ax, sy);
+}
+#endif
+
+#if defined(__AVX2__)
+static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {
+ const __m256i ax = _mm256_sign_epi8(x, x);
+ const __m256i sy = _mm256_sign_epi8(y, x);
+ return _mm256_maddubs_epi16(ax, sy);
+}
+#elif defined(__loongarch_asx)
+static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {
+ const __m256i ax = __lasx_xvsigncov_b(x, x);
+ const __m256i sy = __lasx_xvsigncov_b(x, y);
+ __m256i tmp1, tmp2, tmp3;
+ tmp1 = __lasx_xvmulwev_h_bu_b(ax, sy);
+ tmp2 = __lasx_xvmulwod_h_bu_b(ax, sy);
+ tmp3 = __lasx_xvadd_h(tmp1, tmp2);
+ return __lasx_xvsat_h(tmp3, 15);
+}
+#endif
+
+void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_iq1_s * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#if defined __ARM_NEON
+
+ ggml_int8x16x4_t q1b;
+ ggml_int8x16x4_t q8b;
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * qs = x[i].qs;
+ const uint16_t * qh = x[i].qh;
+
+ int sumi1 = 0, sumi2 = 0, sumi3 = 0;
+
+ for (int ib = 0; ib < QK_K/32; ib += 2) {
+
+ q1b.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[0] | ((qh[ib+0] << 8) & 0x700)))),
+ vld1_s8((const int8_t *)(iq1s_grid + (qs[1] | ((qh[ib+0] << 5) & 0x700)))));
+ q1b.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[2] | ((qh[ib+0] << 2) & 0x700)))),
+ vld1_s8((const int8_t *)(iq1s_grid + (qs[3] | ((qh[ib+0] >> 1) & 0x700)))));
+ q1b.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[4] | ((qh[ib+1] << 8) & 0x700)))),
+ vld1_s8((const int8_t *)(iq1s_grid + (qs[5] | ((qh[ib+1] << 5) & 0x700)))));
+ q1b.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[6] | ((qh[ib+1] << 2) & 0x700)))),
+ vld1_s8((const int8_t *)(iq1s_grid + (qs[7] | ((qh[ib+1] >> 1) & 0x700)))));
+ qs += 8;
+
+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[0], q8b.val[0]), q1b.val[1], q8b.val[1]);
+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[2], q8b.val[2]), q1b.val[3], q8b.val[3]);
+
+ const int ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;
+ const int ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;
+ sumi1 += vaddvq_s32(p1) * ls1;
+ sumi2 += vaddvq_s32(p2) * ls2;
+ sumi3 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * ls1 * (qh[ib+0] & 0x8000 ? -1 : 1)
+ + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * ls2 * (qh[ib+1] & 0x8000 ? -1 : 1);
+
+ }
+
+ sumf += y[i].d * GGML_FP16_TO_FP32(x[i].d) * (sumi1 + sumi2 + IQ1S_DELTA * sumi3);
+ }
+
+ *s = sumf;
+
+#elif defined __AVX2__
+
+ __m256 accum = _mm256_setzero_ps();
+ float accum1 = 0;
+ for (int i = 0; i < nb; ++i) {
+
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * qs = x[i].qs;
+ const uint16_t * qh = x[i].qh;
+
+ __m256i sumi = _mm256_setzero_si256();
+ int sumi1 = 0;
+ for (int ib = 0; ib < QK_K/32; ib += 2) {
+ const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)],
+ iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)]);
+ const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)],
+ iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)]);
+ qs += 8;
+ const __m256i q8b_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8b_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+
+ const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);
+ const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);
+ const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;
+ const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;
+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(ls1));
+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(ls2));
+
+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p1, p2));
+ sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1
+ + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;
+ }
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ accum = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi), accum);
+ accum1 += d * sumi1;
+
+ }
+
+ *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;
+
+#elif defined __AVX__
+ __m256 accum = _mm256_setzero_ps();
+ float accum1 = 0;
+ for (int i = 0; i < nb; ++i) {
+
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * qs = x[i].qs;
+ const uint16_t * qh = x[i].qh;
+
+ __m128i sumi1_0 = _mm_setzero_si128();
+ __m128i sumi1_1 = _mm_setzero_si128();
+ int sumi1 = 0;
+ for (int ib = 0; ib < QK_K/32; ib += 2) {
+ const __m128i q1b_1_0 = _mm_set_epi64x(iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)]);
+ const __m128i q1b_1_1 = _mm_set_epi64x(iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)]);
+ const __m128i q1b_2_0 = _mm_set_epi64x(iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)]);
+ const __m128i q1b_2_1 = _mm_set_epi64x(iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)]);
+ qs += 8;
+ const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+
+ const __m128i dot1_0 = mul_add_epi8_sse(q1b_1_0, q8b_1_0);
+ const __m128i dot1_1 = mul_add_epi8_sse(q1b_1_1, q8b_1_1);
+ const __m128i dot2_0 = mul_add_epi8_sse(q1b_2_0, q8b_2_0);
+ const __m128i dot2_1 = mul_add_epi8_sse(q1b_2_1, q8b_2_1);
+ const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;
+ const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;
+ const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(ls1));
+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(ls1));
+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(ls2));
+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(ls2));
+
+ sumi1_0 = _mm_add_epi32(sumi1_0, _mm_add_epi32(p1_0, p2_0));
+ sumi1_1 = _mm_add_epi32(sumi1_1, _mm_add_epi32(p1_1, p2_1));
+ sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1
+ + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;
+ }
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ accum = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(sumi1_1, sumi1_0))), accum);
+ accum1 += d * sumi1;
+
+ }
+
+ *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;
+
+#elif defined(__POWER9_VECTOR__)
+ const vector unsigned char v0 = vec_splats((unsigned char)0x0);
+ const vector unsigned short vsign = vec_splats((unsigned short)0x8000);
+
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ for (int i = 0; i < nb; ++i) {
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
+ vector float vyd = vec_splats(y[i].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector signed int vsumi0 = vec_splats((int32_t)0);
+ vector signed int vsumi1 = vec_splats((int32_t)0);
+ vector signed int vsumi2 = vec_splats((int32_t)0);
+ vector signed int vsumi3 = vec_splats((int32_t)0);
+ vector signed int vsumi8 = vec_splats((int32_t)0);
+
+ const uint8_t * restrict q1 = x[i].qs;
+ const uint16_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
+ const int16_t * restrict qs = y[i].bsums;
+
+ for (int j = 0; j < QK_K/32; j += 2) {
+ __builtin_prefetch(q1, 0, 1);
+ __builtin_prefetch(qh, 0, 1);
+ __builtin_prefetch(q8, 0, 1);
+
+ vector signed long long aux64x2_0 = {*(const int64_t *)(iq1s_grid + (q1[0] | ((qh[0] << 8) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[1] | ((qh[0] << 5) & 0x700)))};
+ vector signed long long aux64x2_1 = {*(const int64_t *)(iq1s_grid + (q1[2] | ((qh[0] << 2) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[3] | ((qh[0] >> 1) & 0x700)))};
+ vector signed long long aux64x2_2 = {*(const int64_t *)(iq1s_grid + (q1[4] | ((qh[1] << 8) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[5] | ((qh[1] << 5) & 0x700)))};
+ vector signed long long aux64x2_3 = {*(const int64_t *)(iq1s_grid + (q1[6] | ((qh[1] << 2) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[7] | ((qh[1] >> 1) & 0x700)))};
+ q1 += 8;
+
+ vector signed char q1x0 = (vector signed char)aux64x2_0;
+ vector signed char q1x1 = (vector signed char)aux64x2_1;
+ vector signed char q1x2 = (vector signed char)aux64x2_2;
+ vector signed char q1x3 = (vector signed char)aux64x2_3;
+
+ vector signed char q8y0 = vec_xl( 0, q8);
+ vector signed char q8y1 = vec_xl(16, q8);
+ vector signed char q8y2 = vec_xl(32, q8);
+ vector signed char q8y3 = vec_xl(48, q8);
+ q8 += 64;
+
+ vector signed short qv0 = vec_add(vec_mule(q1x0, q8y0), vec_mulo(q1x0, q8y0));
+ vector signed short qv1 = vec_add(vec_mule(q1x1, q8y1), vec_mulo(q1x1, q8y1));
+ vector signed short qv2 = vec_add(vec_mule(q1x2, q8y2), vec_mulo(q1x2, q8y2));
+ vector signed short qv3 = vec_add(vec_mule(q1x3, q8y3), vec_mulo(q1x3, q8y3));
+
+ const uint16_t ls0 = (uint16_t)((qh[0] >> 12) & 7);
+ const uint16_t ls1 = (uint16_t)((qh[1] >> 12) & 7);
+
+ vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));
+ vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));
+ vector signed short vscales = vec_sld(vscales23, vscales01, 8);
+
+ vsumi0 = vec_msum(qv0, vscales01, vsumi0);
+ vsumi1 = vec_msum(qv1, vscales01, vsumi1);
+ vsumi2 = vec_msum(qv2, vscales23, vsumi2);
+ vsumi3 = vec_msum(qv3, vscales23, vsumi3);
+
+ vector signed short q8ysums = vec_xl_len(qs, 8);
+ qs += 4;
+ q8ysums = vec_mergeh(q8ysums, (vector signed short)v0);
+
+ vector signed short qxh = (vector signed short)vec_sld(vec_splats(qh[1]), vec_splats(qh[0]), 8);
+ qh += 2;
+ vector __bool short vsel = vec_cmpge(qxh, (vector signed short)v0);
+
+ vector signed short q8ysum = vec_sel((vector signed short)vec_xor((vector unsigned short)q8ysums, vsign), q8ysums, vsel);
+
+ vsumi8 = vec_add(vec_mule(q8ysum, vscales), vsumi8);
+ }
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+
+ vsumf0 = vec_madd(vec_ctf(vsumi8, 0), vec_mul(vd, vec_splats(IQ1S_DELTA)), vsumf0);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = vec_extract(vsumf0, 0);
+
+#elif defined(__loongarch_asx)
+
+ __m256 accum = (__m256)__lasx_xvldi(0);
+ float accum1 = 0;
+ for (int i = 0; i < nb; ++i) {
+
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * qs = x[i].qs;
+ const uint16_t * qh = x[i].qh;
+
+ __m256i sumi = __lasx_xvldi(0);
+ int sumi1 = 0;
+ for (int ib = 0; ib < QK_K/32; ib += 2) {
+ __m256i q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)], 0);
+ q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], 1);
+ q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)], 2);
+ q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], 3);
+
+ __m256i q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)], 0);
+ q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], 1);
+ q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)], 2);
+ q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], 3);
+
+ qs += 8;
+ const __m256i q8b_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+ const __m256i q8b_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
+
+ const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);
+ const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);
+ const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;
+ const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;
+
+ __m256i tmp1, tmp5, tmp6;
+ tmp1 = __lasx_xvreplgr2vr_h(ls1);
+ tmp5 = __lasx_xvmulwev_w_h(dot1, tmp1);
+ tmp6 = __lasx_xvmulwod_w_h(dot1, tmp1);
+ const __m256i p1 = __lasx_xvadd_w(tmp5, tmp6);
+
+ tmp1 = __lasx_xvreplgr2vr_h(ls2);
+ tmp5 = __lasx_xvmulwev_w_h(dot2, tmp1);
+ tmp6 = __lasx_xvmulwod_w_h(dot2, tmp1);
+ const __m256i p2 = __lasx_xvadd_w(tmp5, tmp6);
+
+ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p1, p2));
+ sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1
+ + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;
+ }
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), accum);
+ accum1 += d * sumi1;
+ }
+
+ *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;
+
+#else
+
+ float sumf = 0;
+ for (int i = 0; i < nb; i++) {
+
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * qs = x[i].qs;
+ const uint16_t * qh = x[i].qh;
+
+ int sumi = 0, sumi1 = 0;
+ for (int ib = 0; ib < QK_K/32; ++ib) {
+ const int ls = 2*((qh[ib] >> 12) & 7) + 1;
+ const int delta = qh[ib] & 0x8000 ? -1 : 1;
+ int lsum = 0;
+ for (int l = 0; l < 4; ++l) {
+ const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8)));
+ for (int j = 0; j < 8; ++j) {
+ lsum += q8[j] * grid[j];
+ }
+ q8 += 8;
+ }
+ sumi += ls * lsum;
+ sumi1 += ls * delta * (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]);
+ qs += 4;
+ }
+
+ sumf += GGML_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1);
+ }
+
+ *s = sumf;
+
+#endif
+}
+
+void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_iq1_m * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+ iq1m_scale_t scale;
+
+#if defined __ARM_NEON
+ const int32x4_t mask = vdupq_n_s32(0x7);
+ const int32x4_t mone = vdupq_n_s32(1);
+ const int32x4_t mzero = vdupq_n_s32(0);
+
+ ggml_int8x16x4_t deltas;
+ deltas.val[0] = vcombine_s8(vdup_n_s8(+1), vdup_n_s8(+1));
+ deltas.val[1] = vcombine_s8(vdup_n_s8(-1), vdup_n_s8(+1));
+ deltas.val[2] = vcombine_s8(vdup_n_s8(+1), vdup_n_s8(-1));
+ deltas.val[3] = vcombine_s8(vdup_n_s8(-1), vdup_n_s8(-1));
+
+ ggml_int8x16x4_t q1b;
+ ggml_int8x16x4_t q8b;
+
+ uint32_t aux32;
+ const uint8_t * aux8 = (const uint8_t *)&aux32;
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+ const uint16_t * sc = (const uint16_t *)x[i].scales;
+
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+
+ int32x4_t sumi1 = mzero;
+ int32x4_t sumi2 = mzero;
+
+ for (int ib = 0; ib < QK_K/32; ib += 2) {
+
+ q1b.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[0] | ((qh[0] << 8) & 0x700)))),
+ vld1_s8((const int8_t *)(iq1s_grid + (qs[1] | ((qh[0] << 4) & 0x700)))));
+ q1b.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[2] | ((qh[1] << 8) & 0x700)))),
+ vld1_s8((const int8_t *)(iq1s_grid + (qs[3] | ((qh[1] << 4) & 0x700)))));
+ q1b.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[4] | ((qh[2] << 8) & 0x700)))),
+ vld1_s8((const int8_t *)(iq1s_grid + (qs[5] | ((qh[2] << 4) & 0x700)))));
+ q1b.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[6] | ((qh[3] << 8) & 0x700)))),
+ vld1_s8((const int8_t *)(iq1s_grid + (qs[7] | ((qh[3] << 4) & 0x700)))));
+
+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ const int32x4_t p1 = vpaddq_s32(ggml_vdotq_s32(mzero, q1b.val[0], q8b.val[0]), ggml_vdotq_s32(mzero, q1b.val[1], q8b.val[1]));
+ const int32x4_t p2 = vpaddq_s32(ggml_vdotq_s32(mzero, q1b.val[2], q8b.val[2]), ggml_vdotq_s32(mzero, q1b.val[3], q8b.val[3]));
+ const int32x4_t p12 = vpaddq_s32(p1, p2);
+
+ const uint32_t * qh32 = (const uint32_t *)qh; // we are 4-byte aligned, so we can do that
+ aux32 = ((qh32[0] >> 3) & 0x01010101) | ((qh32[0] >> 6) & 0x02020202);
+
+ const int32x4_t p3 = vpaddq_s32(ggml_vdotq_s32(mzero, deltas.val[aux8[0]], q8b.val[0]), ggml_vdotq_s32(mzero, deltas.val[aux8[1]], q8b.val[1]));
+ const int32x4_t p4 = vpaddq_s32(ggml_vdotq_s32(mzero, deltas.val[aux8[2]], q8b.val[2]), ggml_vdotq_s32(mzero, deltas.val[aux8[3]], q8b.val[3]));
+ const int32x4_t p34 = vpaddq_s32(p3, p4);
+
+ int32x4_t scales_4 = ggml_vld1q_u32(sc[ib/2] >> 0, sc[ib/2] >> 3, sc[ib/2] >> 6, sc[ib/2] >> 9);
+
+ scales_4 = vaddq_s32(vshlq_n_s32(vandq_s32(scales_4, mask), 1), mone);
+
+ sumi1 = vmlaq_s32(sumi1, scales_4, p12);
+ sumi2 = vmlaq_s32(sumi2, scales_4, p34);
+
+ qs += 8; qh += 4;
+
+ }
+
+ sumf += y[i].d * GGML_FP16_TO_FP32(scale.f16) * (vaddvq_s32(sumi1) + IQ1M_DELTA * vaddvq_s32(sumi2));
+ }
+
+ *s = sumf;
+
+#elif defined __AVX2__
+
+ const __m256i mask = _mm256_set1_epi16(0x7);
+ const __m256i mone = _mm256_set1_epi16(1);
+
+ __m256 accum1 = _mm256_setzero_ps();
+ __m256 accum2 = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+ const uint16_t * sc = (const uint16_t *)x[i].scales;
+
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+
+ __m256i sumi1 = _mm256_setzero_si256();
+ __m256i sumi2 = _mm256_setzero_si256();
+ for (int ib = 0; ib < QK_K/32; ib += 2) {
+ const __m256i q1b_1 = _mm256_set_epi64x(
+ iq1s_grid[qs[3] | (((uint16_t)qh[1] << 4) & 0x700)], iq1s_grid[qs[2] | (((uint16_t)qh[1] << 8) & 0x700)],
+ iq1s_grid[qs[1] | (((uint16_t)qh[0] << 4) & 0x700)], iq1s_grid[qs[0] | (((uint16_t)qh[0] << 8) & 0x700)]
+ );
+ const __m256i q1b_2 = _mm256_set_epi64x(
+ iq1s_grid[qs[7] | (((uint16_t)qh[3] << 4) & 0x700)], iq1s_grid[qs[6] | (((uint16_t)qh[3] << 8) & 0x700)],
+ iq1s_grid[qs[5] | (((uint16_t)qh[2] << 4) & 0x700)], iq1s_grid[qs[4] | (((uint16_t)qh[2] << 8) & 0x700)]
+ );
+ const __m256i q8b_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8b_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+
+ const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);
+ const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);
+
+ const __m256i delta1 = _mm256_set_epi64x(qh[1] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,
+ qh[1] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101,
+ qh[0] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,
+ qh[0] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);
+ const __m256i delta2 = _mm256_set_epi64x(qh[3] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,
+ qh[3] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101,
+ qh[2] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,
+ qh[2] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);
+
+ const __m256i dot3 = mul_add_epi8(delta1, q8b_1);
+ const __m256i dot4 = mul_add_epi8(delta2, q8b_2);
+
+ __m256i scale1 = MM256_SET_M128I(_mm_set1_epi16(sc[ib/2] >> 3), _mm_set1_epi16(sc[ib/2] >> 0));
+ __m256i scale2 = MM256_SET_M128I(_mm_set1_epi16(sc[ib/2] >> 9), _mm_set1_epi16(sc[ib/2] >> 6));
+
+ scale1 = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scale1, mask), 1), mone);
+ scale2 = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scale2, mask), 1), mone);
+ const __m256i p1 = _mm256_madd_epi16(dot1, scale1);
+ const __m256i p2 = _mm256_madd_epi16(dot2, scale2);
+ const __m256i p3 = _mm256_madd_epi16(dot3, scale1);
+ const __m256i p4 = _mm256_madd_epi16(dot4, scale2);
+
+ sumi1 = _mm256_add_epi32(sumi1, _mm256_add_epi32(p1, p2));
+ sumi2 = _mm256_add_epi32(sumi2, _mm256_add_epi32(p3, p4));
+
+ qs += 8; qh += 4;
+ }
+
+ const __m256 d = _mm256_set1_ps(y[i].d * GGML_FP16_TO_FP32(scale.f16));
+
+ accum1 = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi1), accum1);
+ accum2 = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi2), accum2);
+ }
+
+ *s = hsum_float_8(accum1) + IQ1M_DELTA * hsum_float_8(accum2);
+
+#elif defined __AVX__
+ const __m128i mask = _mm_set1_epi16(0x7);
+ const __m128i mone = _mm_set1_epi16(1);
+
+ __m256 accum1 = _mm256_setzero_ps();
+ __m256 accum2 = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+ const uint16_t * sc = (const uint16_t *)x[i].scales;
+
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+
+ __m128i sumi1_0 = _mm_setzero_si128();
+ __m128i sumi1_1 = _mm_setzero_si128();
+ __m128i sumi2_0 = _mm_setzero_si128();
+ __m128i sumi2_1 = _mm_setzero_si128();
+ for (int ib = 0; ib < QK_K/32; ib += 2) {
+ const __m128i q1b_1_0 = _mm_set_epi64x(
+ iq1s_grid[qs[1] | (((uint16_t)qh[0] << 4) & 0x700)], iq1s_grid[qs[0] | (((uint16_t)qh[0] << 8) & 0x700)]);
+ const __m128i q1b_1_1 = _mm_set_epi64x(
+ iq1s_grid[qs[3] | (((uint16_t)qh[1] << 4) & 0x700)], iq1s_grid[qs[2] | (((uint16_t)qh[1] << 8) & 0x700)]);
+ const __m128i q1b_2_0 = _mm_set_epi64x(
+ iq1s_grid[qs[5] | (((uint16_t)qh[2] << 4) & 0x700)], iq1s_grid[qs[4] | (((uint16_t)qh[2] << 8) & 0x700)]);
+ const __m128i q1b_2_1 = _mm_set_epi64x(
+ iq1s_grid[qs[7] | (((uint16_t)qh[3] << 4) & 0x700)], iq1s_grid[qs[6] | (((uint16_t)qh[3] << 8) & 0x700)]);
+ const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+
+ const __m128i dot1_0 = mul_add_epi8_sse(q1b_1_0, q8b_1_0);
+ const __m128i dot1_1 = mul_add_epi8_sse(q1b_1_1, q8b_1_1);
+ const __m128i dot2_0 = mul_add_epi8_sse(q1b_2_0, q8b_2_0);
+ const __m128i dot2_1 = mul_add_epi8_sse(q1b_2_1, q8b_2_1);
+
+ const __m128i delta1_0 = _mm_set_epi64x(qh[0] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,
+ qh[0] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);
+ const __m128i delta1_1 = _mm_set_epi64x(qh[1] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,
+ qh[1] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);
+ const __m128i delta2_0 = _mm_set_epi64x(qh[2] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,
+ qh[2] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);
+ const __m128i delta2_1 = _mm_set_epi64x(qh[3] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,
+ qh[3] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);
+
+ const __m128i dot3_0 = mul_add_epi8_sse(delta1_0, q8b_1_0);
+ const __m128i dot3_1 = mul_add_epi8_sse(delta1_1, q8b_1_1);
+ const __m128i dot4_0 = mul_add_epi8_sse(delta2_0, q8b_2_0);
+ const __m128i dot4_1 = mul_add_epi8_sse(delta2_1, q8b_2_1);
+
+ __m128i scale1_0 = _mm_set1_epi16(sc[ib/2] >> 0);
+ __m128i scale1_1 = _mm_set1_epi16(sc[ib/2] >> 3);
+ __m128i scale2_0 = _mm_set1_epi16(sc[ib/2] >> 6);
+ __m128i scale2_1 = _mm_set1_epi16(sc[ib/2] >> 9);
+
+ scale1_0 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale1_0, mask), 1), mone);
+ scale1_1 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale1_1, mask), 1), mone);
+ scale2_0 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale2_0, mask), 1), mone);
+ scale2_1 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale2_1, mask), 1), mone);
+ const __m128i p1_0 = _mm_madd_epi16(dot1_0, scale1_0);
+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, scale1_1);
+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, scale2_0);
+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, scale2_1);
+ const __m128i p3_0 = _mm_madd_epi16(dot3_0, scale1_0);
+ const __m128i p3_1 = _mm_madd_epi16(dot3_1, scale1_1);
+ const __m128i p4_0 = _mm_madd_epi16(dot4_0, scale2_0);
+ const __m128i p4_1 = _mm_madd_epi16(dot4_1, scale2_1);
+
+ sumi1_0 = _mm_add_epi32(sumi1_0, _mm_add_epi32(p1_0, p2_0));
+ sumi1_1 = _mm_add_epi32(sumi1_1, _mm_add_epi32(p1_1, p2_1));
+ sumi2_0 = _mm_add_epi32(sumi2_0, _mm_add_epi32(p3_0, p4_0));
+ sumi2_1 = _mm_add_epi32(sumi2_1, _mm_add_epi32(p3_1, p4_1));
+
+ qs += 8; qh += 4;
+ }
+
+ const __m256 d = _mm256_set1_ps(y[i].d * GGML_FP16_TO_FP32(scale.f16));
+
+ accum1 = _mm256_add_ps(_mm256_mul_ps(d, _mm256_cvtepi32_ps(MM256_SET_M128I(sumi1_1, sumi1_0))), accum1);
+ accum2 = _mm256_add_ps(_mm256_mul_ps(d, _mm256_cvtepi32_ps(MM256_SET_M128I(sumi2_1, sumi2_0))), accum2);
+ }
+
+ *s = hsum_float_8(accum1) + IQ1M_DELTA * hsum_float_8(accum2);
+
+#else
+
+ int sum1[2], sum2[2], delta[4];
+
+ float sumf = 0;
+ for (int i = 0; i < nb; i++) {
+
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+ const uint16_t * sc = (const uint16_t *)x[i].scales;
+
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+
+ int sumi1 = 0, sumi2 = 0;
+ for (int ib = 0; ib < QK_K/32; ++ib) {
+ delta[0] = qh[0] & 0x08 ? -1 : 1;
+ delta[1] = qh[0] & 0x80 ? -1 : 1;
+ delta[2] = qh[1] & 0x08 ? -1 : 1;
+ delta[3] = qh[1] & 0x80 ? -1 : 1;
+ sum1[0] = sum1[1] = sum2[0] = sum2[1] = 0;
+ for (int l = 0; l < 4; ++l) {
+ const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((uint16_t)qh[l/2] << (8 - 4*(l%2))) & 0x700)));
+ int lsum1 = 0, lsum2 = 0;
+ for (int j = 0; j < 8; ++j) {
+ lsum1 += q8[j] * grid[j];
+ lsum2 += q8[j];
+ }
+ q8 += 8;
+ sum1[l/2] += lsum1;
+ sum2[l/2] += lsum2*delta[l];
+ }
+
+ const int ls1 = 2*((sc[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1;
+ const int ls2 = 2*((sc[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1;
+
+ sumi1 += sum1[0] * ls1 + sum1[1] * ls2;
+ sumi2 += sum2[0] * ls1 + sum2[1] * ls2;
+ qs += 4;
+ qh += 2;
+ }
+
+ sumf += GGML_FP16_TO_FP32(scale.f16) * y[i].d * (sumi1 + IQ1M_DELTA * sumi2);
+ }
+
+ *s = sumf;
+
+#endif
+}
+
+void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+ assert(n % QK4_NL == 0);
+ static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same");
+
+ const block_iq4_nl * restrict x = vx;
+ const block_q8_0 * restrict y = vy;
+
+ const int nb = n / QK4_NL;
+
+ int ib = 0;
+ float sumf = 0;
+
+#if defined __ARM_NEON
+ const int8x16_t values = vld1q_s8(kvalues_iq4nl);
+ const uint8x16_t m4b = vdupq_n_u8(0x0f);
+ uint8x16x2_t q4bits;
+ int8x16x4_t q4b;
+ int8x16x4_t q8b;
+ int32x4_t prod_1, prod_2;
+
+ for (; ib + 1 < nb; ib += 2) {
+
+ q4bits.val[0] = vld1q_u8(x[ib + 0].qs);
+ q4bits.val[1] = vld1q_u8(x[ib + 1].qs);
+ q8b.val[0] = vld1q_s8(y[ib + 0].qs);
+ q8b.val[1] = vld1q_s8(y[ib + 0].qs + 16);
+ q8b.val[2] = vld1q_s8(y[ib + 1].qs);
+ q8b.val[3] = vld1q_s8(y[ib + 1].qs + 16);
+
+ q4b.val[0] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[0], m4b));
+ q4b.val[1] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4));
+ q4b.val[2] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[1], m4b));
+ q4b.val[3] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4));
+
+ prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]);
+ prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]);
+
+ sumf +=
+ GGML_FP16_TO_FP32(x[ib+0].d) * GGML_FP16_TO_FP32(y[ib + 0].d) * vaddvq_s32(prod_1) +
+ GGML_FP16_TO_FP32(x[ib+1].d) * GGML_FP16_TO_FP32(y[ib + 1].d) * vaddvq_s32(prod_2);
+ }
+
+#elif defined __AVX2__
+
+ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);
+ const __m128i m4b = _mm_set1_epi8(0x0f);
+ const __m256i mone = _mm256_set1_epi16(1);
+
+ __m256 accum1 = _mm256_setzero_ps();
+ __m256 accum2 = _mm256_setzero_ps();
+ for (; ib + 1 < nb; ib += 2) {
+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)x[ib + 0].qs);
+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[ib + 1].qs);
+ const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[ib + 0].qs);
+ const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[ib + 1].qs);
+ const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),
+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));
+ const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),
+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));
+ const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);
+ const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);
+ const __m256i p_1 = _mm256_madd_epi16(p16_1, mone);
+ const __m256i p_2 = _mm256_madd_epi16(p16_2, mone);
+ accum1 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(y[ib + 0].d)*GGML_FP16_TO_FP32(x[ib + 0].d)),
+ _mm256_cvtepi32_ps(p_1), accum1);
+ accum2 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(y[ib + 1].d)*GGML_FP16_TO_FP32(x[ib + 1].d)),
+ _mm256_cvtepi32_ps(p_2), accum2);
+ }
+
+ sumf = hsum_float_8(_mm256_add_ps(accum1, accum2));
+
+#elif defined __AVX__
+ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);
+ const __m128i m4b = _mm_set1_epi8(0x0f);
+ const __m128i mone = _mm_set1_epi16(1);
+
+ __m256 accum1 = _mm256_setzero_ps();
+ __m256 accum2 = _mm256_setzero_ps();
+ for (; ib + 1 < nb; ib += 2) {
+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)x[ib + 0].qs);
+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);
+ const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs);
+ const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs + 1);
+ const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);
+ const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);
+
+ const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b));
+ const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b));
+ const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b));
+ const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b));
+ const __m128i p16_1_0 = mul_add_epi8_sse(q4b_1_0, q8b_1_0);
+ const __m128i p16_1_1 = mul_add_epi8_sse(q4b_1_1, q8b_1_1);
+ const __m128i p16_2_0 = mul_add_epi8_sse(q4b_2_0, q8b_2_0);
+ const __m128i p16_2_1 = mul_add_epi8_sse(q4b_2_1, q8b_2_1);
+ const __m128i p_1_0 = _mm_madd_epi16(p16_1_0, mone);
+ const __m128i p_1_1 = _mm_madd_epi16(p16_1_1, mone);
+ const __m128i p_2_0 = _mm_madd_epi16(p16_2_0, mone);
+ const __m128i p_2_1 = _mm_madd_epi16(p16_2_1, mone);
+ accum1 = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(y[ib + 0].d)*GGML_FP16_TO_FP32(x[ib + 0].d)),
+ _mm256_cvtepi32_ps(MM256_SET_M128I(p_1_1, p_1_0))), accum1);
+ accum2 = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(y[ib + 1].d)*GGML_FP16_TO_FP32(x[ib + 1].d)),
+ _mm256_cvtepi32_ps(MM256_SET_M128I(p_2_1, p_2_0))), accum2);
+ }
+
+ sumf = hsum_float_8(_mm256_add_ps(accum1, accum2));
+
+#elif defined(__POWER9_VECTOR__)
+ const vector signed char lowMask = vec_splats((signed char)0xF);
+ const vector signed int v0 = vec_splats((int32_t)0);
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
+
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+
+ const vector signed char values = vec_xl( 0, kvalues_iq4nl);
+
+#pragma GCC unroll 4
+ for (; ib < nb; ++ib) {
+ __builtin_prefetch(x[ib].qs, 0, 1);
+ __builtin_prefetch(y[ib].qs, 0, 1);
+
+
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ib].d));
+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[ib].d));
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);
+ vector signed char q4x0 = vec_and(qxs, lowMask);
+ vector signed char q4x1 = vec_sr(qxs, v4);
+
+ q4x0 = vec_perm(values, values, (vector unsigned char)q4x0);
+ q4x1 = vec_perm(values, values, (vector unsigned char)q4x1);
+
+ vector signed char q8y0 = vec_xl( 0, y[ib].qs);
+ vector signed char q8y1 = vec_xl(16, y[ib].qs);
+
+ vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));
+ vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+
+ vsumi0 = vec_sum4s(qv0, vsumi0);
+ vsumi1 = vec_sum4s(qv1, vsumi1);
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ sumf = vec_extract(vsumf0, 0);
+
+#elif defined (__loongarch_asx)
+
+ const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0);
+ const __m128i m4b = __lsx_vreplgr2vr_b(0x0f);
+ const __m256i mone = __lasx_xvreplgr2vr_h(1);
+
+ __m256 accum1 = (__m256)__lasx_xvldi(0);
+ __m256 accum2 = (__m256)__lasx_xvldi(0);
+ for (; ib + 1 < nb; ib += 2) {
+ const __m128i q4bits_1 = __lsx_vld((const __m128i*)x[ib + 0].qs, 0);
+ const __m128i q4bits_2 = __lsx_vld((const __m128i*)x[ib + 1].qs, 0);
+ const __m256i q8b_1 = __lasx_xvld((const __m256i *)y[ib + 0].qs, 0);
+ const __m256i q8b_2 = __lasx_xvld((const __m256i *)y[ib + 1].qs, 0);
+ const __m256i q4b_1 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_1, 4), m4b)),
+ lsx_shuffle_b(values128, __lsx_vand_v(q4bits_1, m4b)));
+ const __m256i q4b_2 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_2, 4), m4b)),
+ lsx_shuffle_b(values128, __lsx_vand_v(q4bits_2, m4b)));
+ const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);
+ const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);
+ const __m256i p_1 = lasx_madd_h(p16_1, mone);
+ const __m256i p_2 = lasx_madd_h(p16_2, mone);
+ accum1 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(y[ib + 0].d)*GGML_FP16_TO_FP32(x[ib + 0].d)),
+ __lasx_xvffint_s_w(p_1), accum1);
+ accum2 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(y[ib + 1].d)*GGML_FP16_TO_FP32(x[ib + 1].d)),
+ __lasx_xvffint_s_w(p_2), accum2);
+ }
+
+ sumf = hsum_float_8(__lasx_xvfadd_s(accum1, accum2));
+
+#endif
+ for (; ib < nb; ++ib) {
+ const float d = GGML_FP16_TO_FP32(y[ib].d)*GGML_FP16_TO_FP32(x[ib].d);
+ int sumi1 = 0, sumi2 = 0;
+ for (int j = 0; j < QK4_NL/2; ++j) {
+ sumi1 += y[ib].qs[j+ 0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];
+ sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >> 4];
+ }
+ sumf += d * (sumi1 + sumi2);
+ }
+ *s = sumf;
+}
+
+void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+ assert(n % QK_K == 0);
+
+ const block_iq4_xs * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#if defined __ARM_NEON
+ const int8x16_t values = vld1q_s8(kvalues_iq4nl);
+ const uint8x16_t m4b = vdupq_n_u8(0x0f);
+ ggml_uint8x16x2_t q4bits;
+ ggml_int8x16x4_t q4b;
+ ggml_int8x16x4_t q8b;
+ int32x4_t prod_1, prod_2;
+
+ float sumf = 0;
+
+ for (int ibl = 0; ibl < nb; ++ibl) {
+
+ const int8_t * q8 = y[ibl].qs;
+ const uint8_t * q4 = x[ibl].qs;
+ uint16_t h = x[ibl].scales_h;
+
+ int sumi1 = 0, sumi2 = 0;
+ for (int ib = 0; ib < QK_K/64; ++ib) {
+
+ q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;
+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ q4b.val[0] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[0], m4b));
+ q4b.val[1] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4));
+ q4b.val[2] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[1], m4b));
+ q4b.val[3] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4));
+
+ prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]);
+ prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]);
+
+ int ls1 = ((x[ibl].scales_l[ib] & 0xf) | ((h << 4) & 0x30)) - 32;
+ int ls2 = ((x[ibl].scales_l[ib] >> 4) | ((h << 2) & 0x30)) - 32;
+ h >>= 4;
+ sumi1 += vaddvq_s32(prod_1) * ls1;
+ sumi2 += vaddvq_s32(prod_2) * ls2;
+
+ }
+
+ sumf += GGML_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2);
+ }
+
+ *s = sumf;
+
+#elif defined __AVX2__
+
+ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);
+ const __m128i m4b = _mm_set1_epi8(0x0f);
+
+ __m256 accum = _mm256_setzero_ps();
+ for (int ibl = 0; ibl < nb; ++ibl) {
+ const uint8_t * qs = x[ibl].qs;
+ const int8_t * q8 = y[ibl].qs;
+ uint16_t sh = x[ibl].scales_h;
+ __m256i sumi1 = _mm256_setzero_si256();
+ __m256i sumi2 = _mm256_setzero_si256();
+ for (int ib = 0; ib < QK_K/32; ib += 2) {
+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)qs); qs += 16;
+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)qs); qs += 16;
+ const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;
+ const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),
+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));
+ const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),
+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));
+ const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);
+ const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);
+ const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;
+ const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32;
+ sh >>= 4;
+ const __m256i p_1 = _mm256_madd_epi16(p16_1, _mm256_set1_epi16(ls1));
+ const __m256i p_2 = _mm256_madd_epi16(p16_2, _mm256_set1_epi16(ls2));
+ sumi1 = _mm256_add_epi32(p_1, sumi1);
+ sumi2 = _mm256_add_epi32(p_2, sumi2);
+ }
+ accum = _mm256_fmadd_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(x[ibl].d)*y[ibl].d),
+ _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accum);
+ }
+
+ *s = hsum_float_8(accum);
+
+#elif defined __AVX__
+ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);
+ const __m128i m4b = _mm_set1_epi8(0x0f);
+
+ __m256 accum = _mm256_setzero_ps();
+ for (int ibl = 0; ibl < nb; ++ibl) {
+ const uint8_t * qs = x[ibl].qs;
+ const int8_t * q8 = y[ibl].qs;
+ uint16_t sh = x[ibl].scales_h;
+ __m128i sumi1_0 = _mm_setzero_si128();
+ __m128i sumi1_1 = _mm_setzero_si128();
+ __m128i sumi2_0 = _mm_setzero_si128();
+ __m128i sumi2_1 = _mm_setzero_si128();
+ for (int ib = 0; ib < QK_K/32; ib += 2) {
+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)qs); qs += 16;
+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)qs); qs += 16;
+ const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;
+ const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b));
+ const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b));
+ const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b));
+ const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b));
+ const __m128i p16_1_0 = mul_add_epi8_sse(q4b_1_0, q8b_1_0);
+ const __m128i p16_1_1 = mul_add_epi8_sse(q4b_1_1, q8b_1_1);
+ const __m128i p16_2_0 = mul_add_epi8_sse(q4b_2_0, q8b_2_0);
+ const __m128i p16_2_1 = mul_add_epi8_sse(q4b_2_1, q8b_2_1);
+ const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;
+ const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32;
+ sh >>= 4;
+ const __m128i p_1_0 = _mm_madd_epi16(p16_1_0, _mm_set1_epi16(ls1));
+ const __m128i p_1_1 = _mm_madd_epi16(p16_1_1, _mm_set1_epi16(ls1));
+ const __m128i p_2_0 = _mm_madd_epi16(p16_2_0, _mm_set1_epi16(ls2));
+ const __m128i p_2_1 = _mm_madd_epi16(p16_2_1, _mm_set1_epi16(ls2));
+ sumi1_0 = _mm_add_epi32(p_1_0, sumi1_0);
+ sumi1_1 = _mm_add_epi32(p_1_1, sumi1_1);
+ sumi2_0 = _mm_add_epi32(p_2_0, sumi2_0);
+ sumi2_1 = _mm_add_epi32(p_2_1, sumi2_1);
+ }
+ __m128i sumi12_0 = _mm_add_epi32(sumi1_0, sumi2_0);
+ __m128i sumi12_1 = _mm_add_epi32(sumi1_1, sumi2_1);
+ accum = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(x[ibl].d)*y[ibl].d),
+ _mm256_cvtepi32_ps(MM256_SET_M128I(sumi12_1, sumi12_0))), accum);
+ }
+
+ *s = hsum_float_8(accum);
+
+#elif defined(__POWER9_VECTOR__)
+ const vector signed char lowMask = vec_splats((signed char)0xF);
+ const vector int v0 = vec_splats((int32_t)0);
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
+
+ vector float vsumf0 = vec_splats(0.0f);
+ vector float vsumf1 = vec_splats(0.0f);
+ vector float vsumf2 = vec_splats(0.0f);
+ vector float vsumf3 = vec_splats(0.0f);
+
+ const vector signed char values = vec_xl( 0, kvalues_iq4nl);
+
+ for (int ibl = 0; ibl < nb; ++ibl) {
+
+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ibl].d));
+ vector float vyd = vec_splats(y[ibl].d);
+ vector float vd = vec_mul(vxd, vyd);
+
+ vector signed int vsumi0 = v0;
+ vector signed int vsumi1 = v0;
+ vector signed int vsumi2 = v0;
+ vector signed int vsumi3 = v0;
+
+ uint16_t h = x[ibl].scales_h;
+
+ const uint8_t * restrict q4 = x[ibl].qs;
+ const uint8_t * restrict sc = x[ibl].scales_l;
+ const int8_t * restrict q8 = y[ibl].qs;
+
+ for (int ib = 0; ib < QK_K/64; ib ++ ) {
+ __builtin_prefetch(q4, 0, 1);
+ __builtin_prefetch(q8, 0, 1);
+
+ vector signed char qxs0 = (vector signed char)vec_xl( 0, q4);
+ vector signed char qxs1 = (vector signed char)vec_xl(16, q4);
+ q4 += 32;
+
+ vector signed char q4x00 = (vector signed char)vec_and(qxs0, lowMask);
+ vector signed char q4x01 = (vector signed char)vec_sr(qxs0, v4);
+ vector signed char q4x10 = (vector signed char)vec_and(qxs1, lowMask);
+ vector signed char q4x11 = (vector signed char)vec_sr(qxs1, v4);
+
+ q4x00 = vec_perm(values, values, (vector unsigned char)q4x00);
+ q4x01 = vec_perm(values, values, (vector unsigned char)q4x01);
+ q4x10 = vec_perm(values, values, (vector unsigned char)q4x10);
+ q4x11 = vec_perm(values, values, (vector unsigned char)q4x11);
+
+ vector signed char q8y0 = vec_xl( 0, q8);
+ vector signed char q8y1 = vec_xl(16, q8);
+ vector signed char q8y2 = vec_xl(32, q8);
+ vector signed char q8y3 = vec_xl(48, q8);
+ q8 += 64;
+
+ vector signed short qv0 = vec_add(vec_mule(q4x00, q8y0), vec_mulo(q4x00, q8y0));
+ vector signed short qv1 = vec_add(vec_mule(q4x01, q8y1), vec_mulo(q4x01, q8y1));
+ vector signed short qv2 = vec_add(vec_mule(q4x10, q8y2), vec_mulo(q4x10, q8y2));
+ vector signed short qv3 = vec_add(vec_mule(q4x11, q8y3), vec_mulo(q4x11, q8y3));
+
+ const uint16_t ls0 = (uint16_t)(((sc[0] & 0xf) | ((h << 4) & 0x30)) - 32);
+ const uint16_t ls1 = (uint16_t)(((sc[0] >> 4) | ((h << 2) & 0x30)) - 32);
+ h >>= 4;
+ sc ++;
+
+ vector signed short vscales01 = vec_splats((int16_t)ls0);
+ vector signed short vscales23 = vec_splats((int16_t)ls1);
+
+ vsumi0 = vec_msum(qv0, vscales01, vsumi0);
+ vsumi1 = vec_msum(qv1, vscales01, vsumi1);
+ vsumi2 = vec_msum(qv2, vscales23, vsumi2);
+ vsumi3 = vec_msum(qv3, vscales23, vsumi3);
+ }
+
+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);
+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);
+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);
+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);
+ }
+
+ vsumf0 = vec_add(vsumf0, vsumf2);
+ vsumf1 = vec_add(vsumf1, vsumf3);
+
+ vsumf0 = vec_add(vsumf0, vsumf1);
+
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+
+ *s = vec_extract(vsumf0, 0);
+
+#elif defined(__loongarch_asx)
+
+ const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0);
+ const __m128i m4b = __lsx_vreplgr2vr_b(0x0f);
+
+ __m256 accum = (__m256)__lasx_xvldi(0);
+ __m256i tmp1;
+ __m128i tmp0, tmp2, tmp3, tmp4, mask_8f, mask;
+
+ mask_8f = __lsx_vreplgr2vr_b(0x8f);
+ for (int ibl = 0; ibl < nb; ++ibl) {
+ const uint8_t * qs = x[ibl].qs;
+ const int8_t * q8 = y[ibl].qs;
+ uint16_t sh = x[ibl].scales_h;
+ __m256i sumi1 = __lasx_xvldi(0);
+ __m256i sumi2 = __lasx_xvldi(0);
+ __m128i zero = __lsx_vldi(0);
+ for (int ib = 0; ib < QK_K/32; ib += 2) {
+ const __m128i q4bits_1 = __lsx_vld((const __m128i*)qs, 0); qs += 16;
+ const __m128i q4bits_2 = __lsx_vld((const __m128i*)qs, 0); qs += 16;
+ const __m256i q8b_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ const __m256i q8b_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
+ tmp2 = __lsx_vand_v(__lsx_vand_v(__lsx_vsrli_h(q4bits_1, 4), m4b), mask_8f);
+ tmp0 = __lsx_vori_b(tmp2, 0x10);
+ mask = __lsx_vsle_b(zero, tmp2);
+ tmp3 = __lsx_vand_v(tmp0, mask);
+ tmp3 = __lsx_vshuf_b(values128, zero, tmp3);
+
+ tmp2 = __lsx_vand_v(__lsx_vand_v(q4bits_1, m4b), mask_8f);
+ tmp0 = __lsx_vori_b(tmp2, 0x10);
+ mask = __lsx_vsle_b(zero, tmp2);
+ tmp4 = __lsx_vand_v(tmp0, mask);
+ tmp4 = __lsx_vshuf_b(values128, zero, tmp4);
+
+ const __m256i q4b_1 = lasx_insertf128(tmp3, tmp4);
+
+ tmp2 = __lsx_vand_v(__lsx_vand_v(__lsx_vsrli_h(q4bits_2, 4), m4b), mask_8f);
+ tmp0 = __lsx_vori_b(tmp2, 0x10);
+ mask = __lsx_vsle_b(zero, tmp2);
+ tmp3 = __lsx_vand_v(tmp0, mask);
+ tmp3 = __lsx_vshuf_b(values128, zero, tmp3);
+
+ tmp2 = __lsx_vand_v(__lsx_vand_v(q4bits_2, m4b), mask_8f);
+ tmp0 = __lsx_vori_b(tmp2, 0x10);
+ mask = __lsx_vsle_b(zero, tmp2);
+ tmp4 = __lsx_vand_v(tmp0, mask);
+ tmp4 = __lsx_vshuf_b(values128, zero, tmp4);
+
+ const __m256i q4b_2 = lasx_insertf128(tmp3, tmp4);
+
+ const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);
+ const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);
+ const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;
+ const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32;
+ sh >>= 4;
+ __m256i tmp5, tmp6;
+ tmp1 = __lasx_xvreplgr2vr_h(ls1);
+ tmp5 = __lasx_xvmulwev_w_h(p16_1, tmp1);
+ tmp6 = __lasx_xvmulwod_w_h(p16_1, tmp1);
+ const __m256i p_1 = __lasx_xvadd_w(tmp5, tmp6);
+ tmp1 = __lasx_xvreplgr2vr_h(ls2);
+ tmp5 = __lasx_xvmulwev_w_h(p16_2, tmp1);
+ tmp6 = __lasx_xvmulwod_w_h(p16_2, tmp1);
+ const __m256i p_2 = __lasx_xvadd_w(tmp5, tmp6);
+ sumi1 = __lasx_xvadd_w(p_1, sumi1);
+ sumi2 = __lasx_xvadd_w(p_2, sumi2);
+ }
+ accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[ibl].d)*y[ibl].d),
+ __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accum);
+ }
+
+ *s = hsum_float_8(accum);
+
+#else
+ float sumf = 0;
+ for (int ibl = 0; ibl < nb; ++ibl) {
+ const float d4d8 = GGML_FP16_TO_FP32(x[ibl].d) * y[ibl].d;
+ uint16_t h = x[ibl].scales_h;
+ const uint8_t * qs = x[ibl].qs;
+ const int8_t * q8 = y[ibl].qs;
+ for (int ib = 0; ib < QK_K/32; ib += 2) {
+ const uint8_t ls1 = (x[ibl].scales_l[ib/2] & 0xf) | ((h << 4) & 0x30);
+ const uint8_t ls2 = (x[ibl].scales_l[ib/2] >> 4) | ((h << 2) & 0x30);
+ h >>= 4;
+ const float d1 = d4d8*(ls1 - 32);
+ const float d2 = d4d8*(ls2 - 32);
+ int sumi1 = 0, sumi2 = 0;
+ for (int j = 0; j < 16; ++j) {
+ sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf];
+ sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >> 4];
+ }
+ sumf += d1 * (sumi1 + sumi2);
+ qs += 16;
+ q8 += 32;
+ sumi1 = sumi2 = 0;
+ for (int j = 0; j < 16; ++j) {
+ sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf];
+ sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >> 4];
+ }
+ sumf += d2 * (sumi1 + sumi2);
+ qs += 16;
+ q8 += 32;
+ }
+ }
+ *s = sumf;
+#endif
+}
+
+// ================================ IQ2 quantization =============================================
+
+typedef struct {
+ uint64_t * grid;
+ int * map;
+ uint16_t * neighbours;
+} iq2_entry_t;
+
+static iq2_entry_t iq2_data[4] = {
+ {NULL, NULL, NULL},
+ {NULL, NULL, NULL},
+ {NULL, NULL, NULL},
+ {NULL, NULL, NULL},
+};
+
+static inline int iq2_data_index(enum ggml_type type) {
+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S);
+ return type == GGML_TYPE_IQ2_XXS ? 0 :
+ type == GGML_TYPE_IQ2_XS ? 1 :
+ type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? 2 : 3;
+}
+
+static inline int iq2_grid_size(enum ggml_type type) {
+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S);
+ return type == GGML_TYPE_IQ2_XXS ? 256 :
+ type == GGML_TYPE_IQ2_XS ? 512 :
+ type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? NGRID_IQ1S : 1024;
+}
+
+static int iq2_compare_func(const void * left, const void * right) {
+ const int * l = (const int *)left;
+ const int * r = (const int *)right;
+ return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0;
+}
+
+void iq2xs_init_impl(enum ggml_type type) {
+ const int gindex = iq2_data_index(type);
+ const int grid_size = iq2_grid_size(type);
+ if (iq2_data[gindex].grid) {
+ return;
+ }
+ static const uint16_t kgrid_2bit_256[256] = {
+ 0, 2, 5, 8, 10, 17, 20, 32, 34, 40, 42, 65, 68, 80, 88, 97,
+ 100, 128, 130, 138, 162, 257, 260, 272, 277, 320, 388, 408, 512, 514, 546, 642,
+ 1025, 1028, 1040, 1057, 1060, 1088, 1090, 1096, 1120, 1153, 1156, 1168, 1188, 1280, 1282, 1288,
+ 1312, 1350, 1385, 1408, 1425, 1545, 1552, 1600, 1668, 1700, 2048, 2053, 2056, 2068, 2088, 2113,
+ 2116, 2128, 2130, 2184, 2308, 2368, 2562, 2580, 4097, 4100, 4112, 4129, 4160, 4192, 4228, 4240,
+ 4245, 4352, 4360, 4384, 4432, 4442, 4480, 4644, 4677, 5120, 5128, 5152, 5157, 5193, 5248, 5400,
+ 5474, 5632, 5654, 6145, 6148, 6160, 6208, 6273, 6400, 6405, 6560, 6737, 8192, 8194, 8202, 8260,
+ 8289, 8320, 8322, 8489, 8520, 8704, 8706, 9217, 9220, 9232, 9280, 9302, 9472, 9537, 9572, 9872,
+ 10248, 10272, 10388, 10820, 16385, 16388, 16400, 16408, 16417, 16420, 16448, 16456, 16470, 16480, 16513, 16516,
+ 16528, 16640, 16672, 16737, 16768, 16773, 16897, 16912, 16968, 16982, 17000, 17408, 17416, 17440, 17536, 17561,
+ 17682, 17700, 17920, 18433, 18436, 18448, 18496, 18501, 18688, 18776, 18785, 18818, 19013, 19088, 20480, 20488,
+ 20497, 20505, 20512, 20608, 20616, 20740, 20802, 20900, 21137, 21648, 21650, 21770, 22017, 22100, 22528, 22545,
+ 22553, 22628, 22848, 23048, 24580, 24592, 24640, 24680, 24832, 24917, 25112, 25184, 25600, 25605, 25872, 25874,
+ 25988, 26690, 32768, 32770, 32778, 32833, 32898, 33028, 33048, 33088, 33297, 33793, 33796, 33808, 33813, 33856,
+ 33888, 34048, 34118, 34196, 34313, 34368, 34400, 34818, 35076, 35345, 36868, 36880, 36900, 36928, 37025, 37142,
+ 37248, 37445, 37888, 37922, 37956, 38225, 39041, 39200, 40962, 41040, 41093, 41225, 41472, 42008, 43088, 43268,
+ };
+ static const uint16_t kgrid_2bit_512[512] = {
+ 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70,
+ 73, 80, 82, 85, 88, 97, 100, 128, 130, 133, 136, 145, 148, 153, 160, 257,
+ 260, 262, 265, 272, 274, 277, 280, 282, 289, 292, 320, 322, 325, 328, 337, 340,
+ 352, 360, 385, 388, 400, 512, 514, 517, 520, 529, 532, 544, 577, 580, 592, 597,
+ 640, 650, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1088, 1090, 1093, 1096,
+ 1105, 1108, 1110, 1120, 1153, 1156, 1168, 1280, 1282, 1285, 1288, 1297, 1300, 1312, 1345, 1348,
+ 1360, 1377, 1408, 1537, 1540, 1552, 1574, 1600, 1602, 1668, 2048, 2050, 2053, 2056, 2058, 2065,
+ 2068, 2080, 2085, 2113, 2116, 2128, 2136, 2176, 2208, 2218, 2305, 2308, 2320, 2368, 2433, 2441,
+ 2560, 2592, 2600, 2710, 2720, 4097, 4100, 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4160,
+ 4162, 4165, 4168, 4177, 4180, 4192, 4202, 4225, 4228, 4240, 4352, 4354, 4357, 4360, 4369, 4372,
+ 4384, 4417, 4420, 4432, 4480, 4500, 4502, 4609, 4612, 4614, 4624, 4672, 4704, 5120, 5122, 5125,
+ 5128, 5137, 5140, 5152, 5185, 5188, 5193, 5200, 5220, 5248, 5377, 5380, 5392, 5440, 5632, 5652,
+ 5705, 6145, 6148, 6160, 6162, 6208, 6228, 6278, 6400, 6405, 6502, 6737, 6825, 8192, 8194, 8197,
+ 8200, 8202, 8209, 8212, 8224, 8257, 8260, 8272, 8320, 8352, 8449, 8452, 8464, 8512, 8520, 8549,
+ 8704, 8738, 8832, 8872, 9217, 9220, 9232, 9257, 9280, 9472, 9537, 9554, 9625, 9729, 9754, 9894,
+ 10240, 10248, 10250, 10272, 10325, 10376, 10402, 10600, 10640, 10760, 10784, 10882, 10888, 10890, 16385, 16388,
+ 16390, 16393, 16400, 16402, 16405, 16408, 16417, 16420, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16480,
+ 16485, 16513, 16516, 16528, 16640, 16642, 16645, 16648, 16657, 16660, 16672, 16705, 16708, 16720, 16768, 16773,
+ 16802, 16897, 16900, 16912, 16914, 16937, 16960, 17408, 17410, 17413, 17416, 17425, 17428, 17433, 17440, 17473,
+ 17476, 17488, 17536, 17556, 17665, 17668, 17680, 17700, 17728, 17818, 17920, 17930, 17988, 18000, 18433, 18436,
+ 18448, 18496, 18501, 18516, 18530, 18688, 18705, 18756, 18768, 18793, 18948, 20480, 20482, 20485, 20488, 20497,
+ 20500, 20512, 20520, 20545, 20548, 20560, 20608, 20737, 20740, 20752, 20757, 20800, 20802, 20992, 21060, 21162,
+ 21505, 21508, 21520, 21537, 21568, 21600, 21633, 21665, 21760, 21768, 21888, 21896, 22049, 22120, 22177, 22528,
+ 22548, 22593, 22608, 22681, 22810, 22848, 22850, 23173, 24577, 24580, 24592, 24640, 24660, 24674, 24710, 24745,
+ 24832, 25124, 25162, 25234, 25600, 25622, 25872, 25920, 25925, 26020, 26625, 26730, 26917, 27142, 27220, 27234,
+ 32768, 32770, 32773, 32776, 32785, 32788, 32800, 32810, 32833, 32836, 32848, 32896, 32898, 32936, 32938, 33025,
+ 33028, 33030, 33040, 33088, 33105, 33113, 33280, 33312, 33408, 33410, 33440, 33448, 33793, 33796, 33808, 33810,
+ 33813, 33856, 33888, 33929, 34048, 34116, 34213, 34328, 34410, 34816, 34824, 34853, 34906, 34944, 34946, 34984,
+ 35078, 35362, 35456, 35464, 35478, 35496, 36865, 36868, 36880, 36928, 36950, 36996, 37120, 37154, 37220, 37462,
+ 37513, 37888, 37893, 37956, 37968, 37976, 38185, 38288, 38290, 38465, 38993, 39078, 39241, 39445, 39520, 40960,
+ 40962, 40968, 40970, 40992, 41002, 41120, 41297, 41305, 41382, 41472, 41474, 41480, 41514, 41600, 41632, 42048,
+ 42133, 42597, 42648, 43018, 43040, 43042, 43048, 43168, 43176, 43268, 43396, 43398, 43560, 43562, 43665, 43690,
+ };
+ static const uint16_t kgrid_1bit_2048[NGRID_IQ1S] = {
+ 0, 2, 5, 8, 10, 17, 21, 32, 34, 40, 42, 69, 81, 84, 86, 101,
+ 128, 130, 136, 138, 149, 160, 162, 168, 170, 260, 261, 273, 276, 278, 281, 282,
+ 293, 321, 326, 329, 338, 341, 346, 353, 356, 358, 360, 389, 401, 404, 406, 421,
+ 512, 514, 520, 522, 533, 544, 546, 552, 554, 581, 593, 601, 612, 617, 640, 642,
+ 648, 650, 657, 661, 665, 672, 674, 680, 682, 1041, 1044, 1046, 1061, 1089, 1097, 1109,
+ 1114, 1124, 1125, 1169, 1177, 1189, 1281, 1284, 1285, 1286, 1301, 1304, 1306, 1321, 1344, 1349,
+ 1354, 1360, 1361, 1364, 1365, 1366, 1369, 1376, 1378, 1381, 1384, 1386, 1409, 1425, 1429, 1432,
+ 1434, 1441, 1444, 1445, 1446, 1449, 1556, 1561, 1601, 1604, 1616, 1618, 1621, 1624, 1632, 1633,
+ 1638, 1641, 1669, 1681, 1684, 1689, 2048, 2050, 2056, 2058, 2069, 2080, 2082, 2088, 2090, 2117,
+ 2129, 2134, 2149, 2176, 2178, 2184, 2186, 2197, 2208, 2210, 2216, 2218, 2309, 2321, 2324, 2329,
+ 2340, 2341, 2369, 2384, 2385, 2389, 2401, 2404, 2409, 2449, 2452, 2454, 2457, 2469, 2560, 2562,
+ 2568, 2570, 2581, 2592, 2594, 2600, 2602, 2629, 2641, 2649, 2657, 2661, 2688, 2690, 2693, 2696,
+ 2698, 2709, 2720, 2722, 2728, 2730, 4112, 4113, 4116, 4121, 4132, 4133, 4161, 4164, 4176, 4181,
+ 4184, 4193, 4196, 4197, 4201, 4241, 4244, 4246, 4257, 4261, 4353, 4356, 4358, 4361, 4368, 4370,
+ 4373, 4376, 4385, 4388, 4393, 4421, 4426, 4432, 4433, 4434, 4436, 4437, 4438, 4441, 4448, 4453,
+ 4484, 4498, 4501, 4513, 4516, 4625, 4628, 4630, 4645, 4672, 4678, 4681, 4690, 4693, 4696, 4698,
+ 4708, 4710, 4741, 4753, 4756, 4758, 4773, 5121, 5126, 5129, 5140, 5141, 5144, 5145, 5153, 5158,
+ 5185, 5189, 5190, 5192, 5194, 5201, 5204, 5205, 5206, 5209, 5218, 5221, 5224, 5252, 5257, 5264,
+ 5268, 5269, 5272, 5273, 5274, 5281, 5284, 5285, 5289, 5378, 5381, 5386, 5393, 5396, 5397, 5398,
+ 5401, 5408, 5410, 5413, 5416, 5418, 5441, 5444, 5445, 5446, 5457, 5458, 5460, 5461, 5462, 5465,
+ 5466, 5473, 5476, 5477, 5478, 5481, 5504, 5506, 5508, 5509, 5512, 5514, 5520, 5521, 5524, 5525,
+ 5526, 5529, 5530, 5536, 5538, 5541, 5633, 5636, 5637, 5638, 5653, 5654, 5656, 5658, 5665, 5670,
+ 5696, 5698, 5700, 5701, 5704, 5706, 5713, 5717, 5718, 5720, 5721, 5729, 5732, 5733, 5736, 5737,
+ 5738, 5766, 5770, 5778, 5781, 5796, 5801, 6161, 6166, 6181, 6209, 6212, 6214, 6217, 6224, 6229,
+ 6232, 6234, 6240, 6241, 6244, 6246, 6249, 6277, 6289, 6292, 6309, 6416, 6418, 6421, 6426, 6433,
+ 6437, 6466, 6468, 6469, 6472, 6481, 6484, 6485, 6486, 6489, 6490, 6496, 6501, 6506, 6537, 6545,
+ 6546, 6549, 6552, 6561, 6566, 6569, 6665, 6678, 6692, 6694, 6724, 6726, 6729, 6736, 6738, 6741,
+ 6744, 6753, 6758, 6761, 6789, 6801, 6806, 6810, 8192, 8194, 8200, 8202, 8213, 8224, 8226, 8229,
+ 8232, 8234, 8261, 8273, 8281, 8289, 8293, 8320, 8322, 8328, 8330, 8341, 8352, 8354, 8357, 8360,
+ 8362, 8453, 8465, 8468, 8473, 8485, 8514, 8516, 8521, 8533, 8536, 8538, 8545, 8548, 8549, 8550,
+ 8581, 8592, 8598, 8601, 8613, 8705, 8712, 8714, 8721, 8725, 8736, 8738, 8744, 8746, 8773, 8785,
+ 8790, 8793, 8805, 8833, 8840, 8842, 8849, 8853, 8864, 8866, 8872, 8874, 9221, 9236, 9238, 9241,
+ 9253, 9284, 9285, 9286, 9289, 9298, 9301, 9304, 9306, 9318, 9349, 9361, 9364, 9369, 9377, 9381,
+ 9481, 9493, 9505, 9513, 9536, 9541, 9544, 9553, 9556, 9557, 9561, 9570, 9573, 9576, 9609, 9616,
+ 9620, 9621, 9624, 9626, 9633, 9636, 9638, 9641, 9733, 9744, 9746, 9753, 9765, 9793, 9801, 9813,
+ 9824, 9825, 9833, 9860, 9862, 9872, 9882, 10240, 10242, 10248, 10250, 10261, 10272, 10274, 10280, 10282,
+ 10309, 10321, 10324, 10341, 10368, 10370, 10376, 10378, 10400, 10402, 10408, 10410, 10505, 10513, 10516, 10521,
+ 10533, 10566, 10569, 10578, 10581, 10593, 10596, 10598, 10601, 10629, 10640, 10646, 10649, 10660, 10661, 10752,
+ 10754, 10760, 10762, 10784, 10786, 10792, 10794, 10821, 10833, 10838, 10841, 10853, 10880, 10882, 10888, 10890,
+ 10901, 10912, 10914, 10920, 10922, 16389, 16401, 16406, 16421, 16457, 16466, 16469, 16472, 16474, 16481, 16484,
+ 16486, 16532, 16537, 16545, 16550, 16640, 16641, 16644, 16646, 16649, 16658, 16661, 16662, 16664, 16666, 16673,
+ 16678, 16681, 16709, 16712, 16714, 16721, 16724, 16725, 16726, 16729, 16730, 16741, 16744, 16746, 16769, 16772,
+ 16774, 16784, 16786, 16789, 16800, 16801, 16802, 16901, 16913, 16916, 16918, 16933, 16961, 16978, 16981, 16986,
+ 16996, 17001, 17033, 17044, 17061, 17409, 17429, 17433, 17449, 17477, 17480, 17482, 17489, 17492, 17493, 17494,
+ 17505, 17506, 17509, 17512, 17514, 17537, 17542, 17545, 17552, 17554, 17557, 17568, 17569, 17577, 17665, 17666,
+ 17669, 17674, 17681, 17684, 17685, 17686, 17689, 17696, 17701, 17706, 17729, 17732, 17733, 17734, 17737, 17744,
+ 17745, 17748, 17749, 17750, 17752, 17753, 17761, 17764, 17765, 17766, 17769, 17794, 17796, 17797, 17800, 17809,
+ 17812, 17813, 17814, 17817, 17818, 17829, 17832, 17834, 17921, 17925, 17929, 17940, 17941, 17944, 17946, 17953,
+ 17956, 17961, 17984, 17986, 17989, 17992, 18000, 18001, 18002, 18005, 18006, 18009, 18018, 18021, 18024, 18049,
+ 18053, 18058, 18068, 18069, 18081, 18084, 18086, 18437, 18449, 18453, 18458, 18469, 18498, 18505, 18512, 18517,
+ 18520, 18529, 18532, 18534, 18537, 18565, 18577, 18580, 18582, 18585, 18597, 18689, 18693, 18694, 18698, 18704,
+ 18708, 18709, 18712, 18721, 18724, 18726, 18752, 18757, 18762, 18769, 18770, 18772, 18773, 18774, 18777, 18784,
+ 18786, 18789, 18790, 18794, 18822, 18825, 18834, 18837, 18838, 18840, 18849, 18852, 18854, 18857, 18966, 19012,
+ 19014, 19017, 19029, 19032, 19034, 19044, 19049, 19092, 19109, 20481, 20484, 20485, 20486, 20489, 20498, 20501,
+ 20506, 20513, 20516, 20521, 20544, 20549, 20552, 20561, 20564, 20565, 20566, 20569, 20581, 20584, 20614, 20617,
+ 20629, 20632, 20640, 20641, 20646, 20649, 20741, 20744, 20745, 20746, 20753, 20756, 20757, 20758, 20760, 20761,
+ 20768, 20773, 20774, 20776, 20778, 20801, 20804, 20805, 20806, 20809, 20816, 20817, 20818, 20820, 20821, 20822,
+ 20824, 20825, 20826, 20833, 20836, 20837, 20838, 20841, 20866, 20869, 20881, 20884, 20885, 20886, 20889, 20896,
+ 20901, 20906, 20993, 20998, 21010, 21013, 21018, 21025, 21028, 21058, 21061, 21066, 21073, 21076, 21077, 21078,
+ 21081, 21090, 21093, 21125, 21136, 21138, 21141, 21145, 21146, 21156, 21508, 21509, 21521, 21524, 21525, 21526,
+ 21528, 21529, 21537, 21541, 21544, 21546, 21569, 21572, 21573, 21574, 21577, 21578, 21584, 21585, 21588, 21589,
+ 21590, 21592, 21593, 21594, 21601, 21602, 21604, 21605, 21606, 21609, 21632, 21640, 21642, 21649, 21652, 21653,
+ 21654, 21657, 21665, 21668, 21669, 21674, 21761, 21762, 21764, 21765, 21766, 21769, 21776, 21777, 21778, 21780,
+ 21781, 21782, 21785, 21786, 21793, 21796, 21797, 21798, 21801, 21824, 21825, 21826, 21828, 21829, 21830, 21832,
+ 21833, 21840, 21841, 21842, 21844, 21845, 21846, 21848, 21849, 21850, 21856, 21857, 21860, 21861, 21862, 21864,
+ 21865, 21866, 21889, 21892, 21893, 21897, 21898, 21904, 21905, 21908, 21909, 21910, 21912, 21913, 21921, 21924,
+ 21925, 21926, 21929, 22016, 22017, 22018, 22020, 22022, 22024, 22025, 22033, 22036, 22037, 22040, 22041, 22048,
+ 22049, 22050, 22052, 22053, 22054, 22056, 22057, 22081, 22085, 22086, 22088, 22089, 22090, 22096, 22097, 22098,
+ 22100, 22101, 22102, 22104, 22105, 22106, 22113, 22116, 22117, 22121, 22146, 22149, 22150, 22152, 22153, 22154,
+ 22161, 22165, 22170, 22178, 22181, 22182, 22184, 22185, 22532, 22533, 22534, 22537, 22544, 22549, 22552, 22561,
+ 22570, 22597, 22600, 22602, 22609, 22612, 22613, 22614, 22616, 22617, 22624, 22626, 22628, 22629, 22658, 22665,
+ 22672, 22674, 22677, 22680, 22689, 22697, 22785, 22786, 22789, 22794, 22801, 22804, 22805, 22806, 22809, 22821,
+ 22849, 22852, 22853, 22854, 22857, 22864, 22865, 22866, 22868, 22869, 22870, 22872, 22873, 22874, 22881, 22884,
+ 22885, 22886, 22889, 22913, 22917, 22921, 22929, 22932, 22933, 22934, 22936, 22937, 22949, 23044, 23048, 23061,
+ 23066, 23072, 23077, 23078, 23081, 23109, 23112, 23113, 23121, 23125, 23126, 23128, 23129, 23138, 23141, 23144,
+ 23146, 23169, 23178, 23186, 23189, 23190, 23192, 23194, 23201, 24581, 24596, 24598, 24601, 24613, 24644, 24656,
+ 24661, 24662, 24664, 24666, 24673, 24676, 24678, 24681, 24705, 24726, 24741, 24833, 24836, 24838, 24841, 24850,
+ 24853, 24865, 24866, 24870, 24873, 24901, 24905, 24913, 24917, 24918, 24921, 24933, 24934, 24938, 24964, 24970,
+ 24978, 24981, 24993, 24998, 25001, 25105, 25110, 25113, 25152, 25153, 25158, 25173, 25174, 25176, 25184, 25221,
+ 25233, 25238, 25253, 25617, 25618, 25621, 25622, 25626, 25633, 25638, 25641, 25664, 25666, 25669, 25672, 25674,
+ 25681, 25684, 25685, 25686, 25689, 25690, 25696, 25698, 25701, 25732, 25733, 25737, 25744, 25746, 25748, 25749,
+ 25750, 25752, 25754, 25761, 25764, 25769, 25861, 25864, 25866, 25873, 25877, 25878, 25881, 25924, 25925, 25926,
+ 25929, 25936, 25937, 25940, 25941, 25942, 25945, 25953, 25956, 25957, 25958, 25961, 25990, 25993, 25994, 26001,
+ 26005, 26006, 26009, 26010, 26018, 26021, 26022, 26024, 26114, 26121, 26133, 26144, 26150, 26152, 26153, 26176,
+ 26181, 26184, 26186, 26193, 26196, 26197, 26198, 26200, 26202, 26208, 26213, 26216, 26240, 26242, 26245, 26250,
+ 26260, 26262, 26264, 26265, 26272, 26276, 26278, 26282, 26646, 26649, 26661, 26689, 26706, 26709, 26714, 26721,
+ 26729, 26757, 26769, 26776, 26790, 26881, 26884, 26896, 26901, 26913, 26916, 26918, 26921, 26944, 26945, 26949,
+ 26950, 26952, 26961, 26964, 26965, 26966, 26969, 26976, 26981, 26986, 27010, 27012, 27018, 27029, 27041, 27044,
+ 27045, 27049, 27153, 27158, 27160, 27201, 27204, 27209, 27216, 27221, 27224, 27226, 27236, 27237, 27241, 27270,
+ 27284, 27288, 27290, 27302, 32768, 32770, 32776, 32778, 32800, 32802, 32808, 32810, 32837, 32848, 32849, 32852,
+ 32854, 32857, 32869, 32896, 32898, 32904, 32906, 32917, 32928, 32930, 32936, 32938, 33029, 33041, 33044, 33046,
+ 33049, 33061, 33089, 33092, 33097, 33104, 33106, 33109, 33110, 33112, 33113, 33124, 33126, 33129, 33157, 33161,
+ 33172, 33174, 33177, 33189, 33280, 33282, 33288, 33290, 33301, 33312, 33314, 33320, 33322, 33361, 33364, 33369,
+ 33381, 33408, 33410, 33416, 33418, 33429, 33440, 33442, 33448, 33450, 33812, 33817, 33857, 33860, 33873, 33877,
+ 33882, 33889, 33892, 33897, 33940, 33945, 34049, 34057, 34066, 34069, 34074, 34086, 34089, 34112, 34113, 34117,
+ 34120, 34129, 34132, 34133, 34134, 34137, 34138, 34149, 34150, 34152, 34154, 34177, 34180, 34182, 34185, 34192,
+ 34194, 34197, 34200, 34214, 34321, 34326, 34329, 34341, 34369, 34372, 34377, 34378, 34384, 34389, 34393, 34394,
+ 34401, 34406, 34410, 34437, 34449, 34458, 34468, 34816, 34818, 34824, 34826, 34837, 34848, 34850, 34856, 34858,
+ 34881, 34885, 34897, 34900, 34905, 34917, 34921, 34944, 34946, 34952, 34954, 34965, 34976, 34978, 34984, 34986,
+ 35077, 35078, 35089, 35092, 35094, 35109, 35137, 35140, 35142, 35145, 35152, 35154, 35157, 35162, 35169, 35172,
+ 35205, 35222, 35225, 35237, 35328, 35330, 35336, 35338, 35349, 35360, 35362, 35368, 35370, 35397, 35409, 35412,
+ 35414, 35456, 35458, 35464, 35466, 35477, 35488, 35490, 35496, 35498, 36869, 36881, 36886, 36888, 36889, 36901,
+ 36929, 36934, 36937, 36949, 36952, 36954, 36969, 36970, 36997, 37009, 37012, 37014, 37017, 37029, 37121, 37124,
+ 37126, 37129, 37136, 37141, 37144, 37146, 37153, 37156, 37158, 37161, 37184, 37189, 37200, 37201, 37204, 37205,
+ 37206, 37209, 37218, 37221, 37252, 37254, 37266, 37269, 37272, 37281, 37284, 37286, 37289, 37381, 37393, 37396,
+ 37401, 37413, 37444, 37446, 37449, 37456, 37458, 37461, 37464, 37478, 37481, 37509, 37524, 37526, 37545, 37889,
+ 37892, 37894, 37904, 37909, 37912, 37926, 37952, 37962, 37969, 37972, 37973, 37974, 37976, 37977, 37984, 37985,
+ 37986, 37989, 38020, 38022, 38034, 38036, 38037, 38040, 38049, 38057, 38144, 38149, 38152, 38154, 38160, 38161,
+ 38164, 38165, 38166, 38169, 38177, 38181, 38185, 38186, 38209, 38212, 38213, 38214, 38217, 38224, 38225, 38226,
+ 38228, 38229, 38230, 38232, 38233, 38234, 38241, 38244, 38245, 38246, 38249, 38273, 38277, 38280, 38289, 38290,
+ 38292, 38293, 38294, 38297, 38298, 38304, 38306, 38309, 38312, 38314, 38401, 38404, 38416, 38421, 38425, 38432,
+ 38438, 38441, 38469, 38472, 38473, 38481, 38482, 38485, 38486, 38489, 38501, 38504, 38530, 38532, 38537, 38538,
+ 38546, 38548, 38549, 38564, 38566, 38569, 38917, 38934, 38937, 38949, 38977, 38982, 38992, 38994, 38997, 38998,
+ 39002, 39012, 39013, 39045, 39057, 39062, 39065, 39077, 39172, 39174, 39177, 39184, 39186, 39189, 39192, 39194,
+ 39200, 39201, 39204, 39206, 39232, 39234, 39237, 39240, 39242, 39249, 39252, 39253, 39254, 39257, 39266, 39269,
+ 39270, 39274, 39297, 39300, 39312, 39314, 39317, 39322, 39329, 39334, 39429, 39445, 39461, 39492, 39494, 39497,
+ 39504, 39509, 39512, 39521, 39557, 39569, 39572, 39573, 39574, 40960, 40962, 40968, 40970, 40981, 40992, 40994,
+ 41000, 41002, 41029, 41041, 41044, 41046, 41049, 41088, 41090, 41096, 41098, 41109, 41120, 41122, 41128, 41130,
+ 41221, 41225, 41233, 41236, 41238, 41241, 41242, 41286, 41289, 41297, 41301, 41304, 41306, 41313, 41316, 41349,
+ 41360, 41362, 41366, 41369, 41474, 41480, 41482, 41488, 41497, 41506, 41512, 41514, 41541, 41553, 41558, 41561,
+ 41573, 41600, 41602, 41608, 41610, 41621, 41632, 41634, 41640, 41642, 42009, 42021, 42049, 42052, 42064, 42068,
+ 42069, 42072, 42074, 42081, 42085, 42086, 42088, 42089, 42117, 42246, 42249, 42256, 42258, 42261, 42264, 42278,
+ 42281, 42306, 42309, 42321, 42324, 42325, 42326, 42329, 42341, 42346, 42369, 42372, 42373, 42374, 42377, 42386,
+ 42389, 42392, 42501, 42513, 42518, 42522, 42529, 42533, 42564, 42566, 42570, 42578, 42581, 42582, 42584, 42592,
+ 42594, 42630, 42640, 42645, 42646, 42649, 42657, 42660, 42662, 43008, 43010, 43016, 43018, 43040, 43042, 43048,
+ 43050, 43089, 43092, 43094, 43097, 43136, 43138, 43144, 43146, 43157, 43168, 43170, 43176, 43178, 43269, 43284,
+ 43289, 43297, 43301, 43329, 43344, 43349, 43354, 43361, 43366, 43369, 43408, 43414, 43520, 43522, 43528, 43530,
+ 43552, 43554, 43560, 43562, 43601, 43604, 43606, 43648, 43650, 43656, 43658, 43669, 43680, 43682, 43688, 43690,
+ };
+ static const uint16_t kgrid_2bit_1024[1024] = {
+ 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70,
+ 73, 80, 82, 85, 88, 97, 100, 102, 105, 128, 130, 133, 136, 145, 148, 160,
+ 165, 170, 257, 260, 262, 265, 272, 274, 277, 280, 289, 292, 320, 322, 325, 328,
+ 337, 340, 342, 345, 352, 357, 360, 385, 388, 400, 402, 405, 417, 420, 512, 514,
+ 517, 520, 529, 532, 544, 554, 577, 580, 582, 585, 592, 597, 640, 645, 650, 660,
+ 674, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1062, 1065, 1088, 1090, 1093,
+ 1096, 1098, 1105, 1108, 1110, 1113, 1120, 1122, 1125, 1153, 1156, 1158, 1161, 1168, 1173, 1176,
+ 1185, 1188, 1280, 1282, 1285, 1288, 1290, 1297, 1300, 1302, 1305, 1312, 1317, 1320, 1345, 1348,
+ 1350, 1353, 1360, 1362, 1365, 1368, 1377, 1380, 1408, 1410, 1413, 1416, 1425, 1428, 1440, 1537,
+ 1540, 1542, 1545, 1552, 1557, 1600, 1605, 1608, 1617, 1620, 1632, 1665, 1668, 1680, 2048, 2050,
+ 2053, 2056, 2065, 2068, 2070, 2073, 2080, 2085, 2090, 2113, 2116, 2118, 2121, 2128, 2130, 2133,
+ 2136, 2145, 2148, 2176, 2181, 2196, 2218, 2305, 2308, 2320, 2322, 2325, 2328, 2337, 2368, 2373,
+ 2376, 2385, 2388, 2400, 2433, 2448, 2560, 2577, 2580, 2594, 2600, 2602, 2640, 2713, 4097, 4100,
+ 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4134, 4160, 4162, 4165, 4168, 4177, 4180, 4182,
+ 4185, 4192, 4194, 4197, 4200, 4225, 4228, 4230, 4240, 4245, 4248, 4257, 4260, 4352, 4354, 4357,
+ 4360, 4362, 4369, 4372, 4374, 4377, 4384, 4386, 4389, 4392, 4417, 4420, 4422, 4425, 4432, 4434,
+ 4437, 4440, 4449, 4452, 4480, 4482, 4485, 4488, 4497, 4500, 4609, 4612, 4617, 4624, 4629, 4641,
+ 4644, 4672, 4677, 4689, 4692, 4737, 4740, 4752, 5120, 5122, 5125, 5128, 5137, 5140, 5142, 5145,
+ 5152, 5157, 5160, 5185, 5188, 5190, 5193, 5200, 5202, 5205, 5208, 5217, 5220, 5248, 5250, 5253,
+ 5256, 5265, 5268, 5280, 5377, 5380, 5382, 5385, 5392, 5394, 5397, 5400, 5409, 5412, 5440, 5442,
+ 5445, 5448, 5457, 5460, 5472, 5505, 5508, 5520, 5632, 5637, 5640, 5649, 5652, 5664, 5697, 5700,
+ 5712, 5760, 5802, 6145, 6148, 6150, 6153, 6160, 6165, 6168, 6177, 6208, 6210, 6213, 6216, 6225,
+ 6228, 6240, 6273, 6276, 6400, 6402, 6405, 6408, 6417, 6420, 6432, 6465, 6468, 6480, 6505, 6562,
+ 6660, 6672, 6720, 6742, 8192, 8194, 8197, 8200, 8209, 8212, 8214, 8217, 8224, 8229, 8234, 8257,
+ 8260, 8272, 8274, 8277, 8292, 8320, 8330, 8340, 8362, 8449, 8452, 8464, 8466, 8469, 8481, 8512,
+ 8514, 8517, 8529, 8532, 8544, 8577, 8580, 8592, 8704, 8714, 8738, 8744, 8746, 8772, 8784, 8840,
+ 8842, 8872, 9217, 9220, 9222, 9225, 9232, 9237, 9240, 9249, 9252, 9280, 9282, 9285, 9288, 9297,
+ 9300, 9312, 9345, 9348, 9360, 9472, 9477, 9480, 9489, 9492, 9504, 9537, 9540, 9552, 9574, 9600,
+ 9729, 9732, 9744, 9792, 9817, 10240, 10245, 10257, 10260, 10305, 10308, 10320, 10378, 10410, 10497, 10500,
+ 10512, 10645, 10762, 10786, 10852, 10888, 10890, 16385, 16388, 16390, 16393, 16400, 16402, 16405, 16408, 16410,
+ 16417, 16420, 16422, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16470, 16473, 16480, 16482, 16485, 16513,
+ 16516, 16528, 16533, 16536, 16545, 16548, 16640, 16642, 16645, 16648, 16657, 16660, 16662, 16665, 16672, 16674,
+ 16677, 16705, 16708, 16710, 16713, 16720, 16722, 16725, 16728, 16737, 16740, 16768, 16770, 16773, 16776, 16785,
+ 16788, 16800, 16897, 16900, 16912, 16914, 16917, 16920, 16932, 16960, 16965, 16968, 16977, 16980, 16992, 17025,
+ 17028, 17408, 17410, 17413, 17416, 17418, 17425, 17428, 17430, 17433, 17440, 17442, 17445, 17448, 17473, 17476,
+ 17478, 17481, 17488, 17490, 17493, 17496, 17505, 17508, 17536, 17538, 17541, 17544, 17553, 17556, 17568, 17665,
+ 17668, 17670, 17673, 17680, 17682, 17685, 17688, 17697, 17700, 17728, 17730, 17733, 17736, 17745, 17748, 17760,
+ 17770, 17793, 17796, 17808, 17920, 17922, 17925, 17928, 17937, 17940, 17952, 17985, 17988, 18000, 18048, 18085,
+ 18433, 18436, 18441, 18448, 18450, 18453, 18456, 18465, 18468, 18496, 18498, 18501, 18504, 18513, 18516, 18528,
+ 18564, 18576, 18688, 18690, 18693, 18696, 18705, 18708, 18720, 18753, 18756, 18768, 18816, 18838, 18945, 18948,
+ 18960, 19008, 20480, 20482, 20485, 20488, 20497, 20500, 20502, 20505, 20512, 20514, 20517, 20520, 20545, 20548,
+ 20550, 20553, 20560, 20562, 20565, 20568, 20577, 20580, 20608, 20610, 20613, 20616, 20625, 20628, 20737, 20740,
+ 20742, 20745, 20752, 20754, 20757, 20760, 20769, 20772, 20800, 20802, 20805, 20808, 20817, 20820, 20832, 20865,
+ 20868, 20880, 20992, 20997, 21000, 21009, 21012, 21024, 21057, 21060, 21072, 21097, 21120, 21505, 21508, 21510,
+ 21513, 21520, 21522, 21525, 21528, 21537, 21540, 21568, 21570, 21573, 21576, 21585, 21588, 21600, 21633, 21636,
+ 21648, 21760, 21762, 21765, 21768, 21777, 21780, 21792, 21825, 21828, 21840, 21888, 22017, 22020, 22032, 22054,
+ 22080, 22528, 22530, 22533, 22536, 22545, 22548, 22560, 22593, 22596, 22608, 22618, 22656, 22785, 22788, 22800,
+ 22848, 23040, 23065, 23173, 23208, 24577, 24580, 24582, 24592, 24594, 24597, 24600, 24609, 24612, 24640, 24645,
+ 24648, 24657, 24660, 24672, 24708, 24720, 24832, 24834, 24837, 24840, 24849, 24852, 24864, 24897, 24900, 24912,
+ 24960, 24985, 25092, 25104, 25152, 25174, 25249, 25600, 25605, 25608, 25617, 25620, 25632, 25665, 25668, 25680,
+ 25728, 25857, 25860, 25872, 25920, 25930, 25960, 26002, 26112, 26260, 26625, 26628, 26640, 26725, 26776, 26880,
+ 26922, 27202, 27297, 32768, 32770, 32773, 32776, 32785, 32788, 32793, 32800, 32805, 32833, 32836, 32848, 32850,
+ 32853, 32856, 32865, 32896, 32901, 32913, 32916, 33025, 33028, 33033, 33040, 33042, 33045, 33048, 33057, 33060,
+ 33088, 33090, 33093, 33096, 33105, 33108, 33153, 33156, 33168, 33193, 33280, 33285, 33290, 33297, 33300, 33345,
+ 33348, 33360, 33793, 33796, 33798, 33801, 33808, 33810, 33813, 33816, 33825, 33856, 33858, 33861, 33864, 33873,
+ 33876, 33888, 33921, 33924, 33936, 34048, 34050, 34053, 34056, 34065, 34068, 34080, 34113, 34116, 34128, 34176,
+ 34186, 34305, 34308, 34320, 34345, 34368, 34816, 34821, 34833, 34836, 34881, 34884, 34896, 34978, 35073, 35076,
+ 35136, 35173, 35362, 35416, 35418, 35458, 35490, 36865, 36868, 36873, 36880, 36882, 36885, 36888, 36900, 36928,
+ 36930, 36933, 36936, 36945, 36948, 36960, 36993, 36996, 37008, 37120, 37125, 37137, 37140, 37185, 37188, 37200,
+ 37210, 37377, 37380, 37392, 37440, 37542, 37888, 37890, 37893, 37896, 37905, 37908, 37920, 37953, 37956, 37968,
+ 38016, 38038, 38145, 38148, 38160, 38208, 38296, 38305, 38400, 38470, 38500, 38913, 38916, 38928, 38950, 38976,
+ 39081, 39168, 39241, 39250, 39568, 40960, 40965, 40970, 40980, 40994, 41002, 41025, 41028, 41040, 41122, 41130,
+ 41280, 41317, 41474, 41482, 41506, 41512, 41514, 41602, 41608, 41610, 41640, 41985, 41988, 42000, 42048, 42121,
+ 42148, 42240, 42265, 42577, 43018, 43048, 43170, 43348, 43398, 43528, 43530, 43552, 43554, 43560, 43656, 43690,
+ };
+
+ const int kmap_size = 43692;
+ //const int nwant = type == GGML_TYPE_IQ1_S ? 3 : 2;
+ const int nwant = type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? 3 : type == GGML_TYPE_IQ2_S ? 1 : 2;
+ const uint16_t * kgrid = type == GGML_TYPE_IQ2_XXS ? kgrid_2bit_256 :
+ type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 :
+ type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? kgrid_1bit_2048 : kgrid_2bit_1024;
+ uint64_t * kgrid_q2xs;
+ int * kmap_q2xs;
+ uint16_t * kneighbors_q2xs;
+
+ //printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);
+ uint64_t * the_grid = (uint64_t *)malloc(grid_size*sizeof(uint64_t));
+ for (int k = 0; k < grid_size; ++k) {
+ int8_t * pos = (int8_t *)(the_grid + k);
+ for (int i = 0; i < 8; ++i) {
+ int l = (kgrid[k] >> 2*i) & 0x3;
+ pos[i] = 2*l + 1;
+ }
+ }
+ kgrid_q2xs = the_grid;
+ iq2_data[gindex].grid = the_grid;
+ kmap_q2xs = (int *)malloc(kmap_size*sizeof(int));
+ iq2_data[gindex].map = kmap_q2xs;
+ for (int i = 0; i < kmap_size; ++i) kmap_q2xs[i] = -1;
+ uint64_t aux64;
+ uint8_t * aux8 = (uint8_t *)&aux64;
+ for (int i = 0; i < grid_size; ++i) {
+ aux64 = kgrid_q2xs[i];
+ uint16_t index = 0;
+ for (int k=0; k<8; ++k) {
+ uint16_t q = (aux8[k] - 1)/2;
+ index |= (q << 2*k);
+ }
+ kmap_q2xs[index] = i;
+ }
+ int8_t pos[8];
+ int * dist2 = (int *)malloc(2*grid_size*sizeof(int));
+ int num_neighbors = 0, num_not_in_map = 0;
+ for (int i = 0; i < kmap_size; ++i) {
+ if (kmap_q2xs[i] >= 0) continue;
+ ++num_not_in_map;
+ for (int k = 0; k < 8; ++k) {
+ int l = (i >> 2*k) & 0x3;
+ pos[k] = 2*l + 1;
+ }
+ for (int j = 0; j < grid_size; ++j) {
+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + j);
+ int d2 = 0;
+ for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);
+ dist2[2*j+0] = d2;
+ dist2[2*j+1] = j;
+ }
+ qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);
+ int n = 0; int d2 = dist2[0];
+ int nhave = 1;
+ for (int j = 0; j < grid_size; ++j) {
+ if (dist2[2*j] > d2) {
+ if (nhave == nwant) break;
+ d2 = dist2[2*j];
+ ++nhave;
+ }
+ ++n;
+ }
+ num_neighbors += n;
+ }
+ //printf("%s: %d neighbours in total\n", __func__, num_neighbors);
+ kneighbors_q2xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));
+ iq2_data[gindex].neighbours = kneighbors_q2xs;
+ int counter = 0;
+ for (int i = 0; i < kmap_size; ++i) {
+ if (kmap_q2xs[i] >= 0) continue;
+ for (int k = 0; k < 8; ++k) {
+ int l = (i >> 2*k) & 0x3;
+ pos[k] = 2*l + 1;
+ }
+ for (int j = 0; j < grid_size; ++j) {
+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + j);
+ int d2 = 0;
+ for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);
+ dist2[2*j+0] = d2;
+ dist2[2*j+1] = j;
+ }
+ qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);
+ kmap_q2xs[i] = -(counter + 1);
+ int d2 = dist2[0];
+ uint16_t * start = &kneighbors_q2xs[counter++];
+ int n = 0, nhave = 1;
+ for (int j = 0; j < grid_size; ++j) {
+ if (dist2[2*j] > d2) {
+ if (nhave == nwant) break;
+ d2 = dist2[2*j];
+ ++nhave;
+ }
+ kneighbors_q2xs[counter++] = dist2[2*j+1];
+ ++n;
+ }
+ *start = n;
+ }
+ free(dist2);
+}
+
+void iq2xs_free_impl(enum ggml_type type) {
+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S);
+ const int gindex = iq2_data_index(type);
+ if (iq2_data[gindex].grid) {
+ free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL;
+ free(iq2_data[gindex].map); iq2_data[gindex].map = NULL;
+ free(iq2_data[gindex].neighbours); iq2_data[gindex].neighbours = NULL;
+ }
+}
+
+static int iq2_find_best_neighbour(const uint16_t * restrict neighbours, const uint64_t * restrict grid,
+ const float * restrict xval, const float * restrict weight, float scale, int8_t * restrict L) {
+ int num_neighbors = neighbours[0];
+ GGML_ASSERT(num_neighbors > 0);
+ float best_d2 = FLT_MAX;
+ int grid_index = -1;
+ for (int j = 1; j <= num_neighbors; ++j) {
+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);
+ float d2 = 0;
+ for (int i = 0; i < 8; ++i) {
+ float q = pg[i];
+ float diff = scale*q - xval[i];
+ d2 += weight[i]*diff*diff;
+ }
+ if (d2 < best_d2) {
+ best_d2 = d2; grid_index = neighbours[j];
+ }
+ }
+ GGML_ASSERT(grid_index >= 0);
+ const int8_t * pg = (const int8_t *)(grid + grid_index);
+ for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;
+ return grid_index;
+}
+
+static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict vy, int64_t n, const float * restrict quant_weights) {
+
+ const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS);
+
+ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;
+ const int * kmap_q2xs = iq2_data[gindex].map;
+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;
+
+ GGML_ASSERT(quant_weights && "missing quantization weights");
+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(n%QK_K == 0);
+
+ const int kMaxQ = 3;
+
+ const int64_t nbl = n/QK_K;
+
+ block_iq2_xxs * y = vy;
+
+ float scales[QK_K/32];
+ float weight[32];
+ float xval[32];
+ int8_t L[32];
+ int8_t Laux[32];
+ float waux[32];
+ uint8_t block_signs[4];
+ uint32_t q2[2*(QK_K/32)];
+
+ for (int ibl = 0; ibl < nbl; ++ibl) {
+
+ y[ibl].d = GGML_FP32_TO_FP16(0.f);
+ memset(q2, 0, QK_K/4);
+
+ float max_scale = 0;
+
+ const float * xbl = x + QK_K*ibl;
+ float sumx2 = 0;
+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];
+ float sigma2 = sumx2/QK_K;
+
+ for (int ib = 0; ib < QK_K/32; ++ib) {
+ const float * xb = xbl + 32*ib;
+ const float * qw = quant_weights + QK_K*ibl + 32*ib;
+ for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);
+ for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]);
+ for (int k = 0; k < 4; ++k) {
+ int nflip = 0;
+ uint8_t s = 0;
+ for (int i = 0; i < 8; ++i) {
+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];
+ else {
+ xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);
+ }
+ }
+ if (nflip%2) {
+ int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];
+ for (int i = 1; i < 8; ++i) {
+ float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];
+ if (ax < min) {
+ min = ax; imin = i;
+ }
+ }
+ xval[8*k+imin] = -xval[8*k+imin];
+ s ^= (1 << imin);
+ }
+ block_signs[k] = s & 127;
+ }
+ float max = xval[0];
+ for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]);
+ if (max < GROUP_MAX_EPS) {
+ scales[ib] = 0;
+ memset(L, 0, 32);
+ continue;
+ }
+ float scale = make_qp_quants(32, kMaxQ+1, xval, (uint8_t*)L, weight);
+ float eff_max = scale*kMaxQ;
+ float best = 0;
+ for (int is = -6; is <= 6; ++is) {
+ float id = (2*kMaxQ-1+is*0.1f)/eff_max;
+ float this_scale = 1/id;
+ for (int k = 0; k < 4; ++k) {
+ for (int i = 0; i < 8; ++i) {
+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));
+ Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));
+ }
+ uint16_t u = 0;
+ for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);
+ int grid_index = kmap_q2xs[u];
+ if (grid_index < 0) {
+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
+ }
+ }
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < 32; ++i) {
+ float w = weight[i];
+ float q = 2*Laux[i] + 1;
+ sumqx += w*xval[i]*q;
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
+ scale = sumqx/sumq2; best = scale*sumqx;
+ memcpy(L, Laux, 32);
+ }
+ }
+ if (scale > 0) {
+ float id = 1/scale;
+ for (int k = 0; k < 4; ++k) {
+ uint16_t u = 0;
+ for (int i = 0; i < 8; ++i) {
+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));
+ l = MAX(0, MIN(kMaxQ-1, l));
+ u |= (l << 2*i);
+ }
+ int grid_index = kmap_q2xs[u];
+ if (grid_index < 0) {
+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
+ }
+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index);
+ for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2;
+ }
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < 32; ++i) {
+ float w = weight[i];
+ float q = 2*L[i] + 1;
+ sumqx += w*xval[i]*q;
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0) scale = sumqx/sumq2;
+ }
+ if (scale < 0) {
+ // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale)
+ // and correspondingly flip quant signs.
+ scale = -scale;
+ for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127;
+ }
+ for (int k = 0; k < 4; ++k) {
+ uint16_t u = 0;
+ for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);
+ int grid_index = kmap_q2xs[u];
+ if (grid_index < 0) {
+ printf("Oops: found point %u not on grid:", u);
+ for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);
+ printf("\n");
+ GGML_ABORT("fatal error");
+ }
+ q2[2*ib+0] |= ((uint32_t) grid_index << 8*k);
+ q2[2*ib+1] |= (block_signs[k] << 7*k);
+ }
+ GGML_ASSERT(scale >= 0);
+ scales[ib] = scale;
+ max_scale = MAX(max_scale, scale);
+ }
+
+ if (!max_scale) {
+ memset(y[ibl].qs, 0, QK_K/4);
+ continue;
+ }
+
+ float d = max_scale/31;
+ y[ibl].d = GGML_FP32_TO_FP16(d);
+ float id = 1/d;
+ for (int ib = 0; ib < QK_K/32; ++ib) {
+ int l = nearest_int(0.5f*(id*scales[ib]-1));
+ l = MAX(0, MIN(15, l));
+ q2[2*ib+1] |= ((uint32_t)l << 28);
+ }
+ memcpy(y[ibl].qs, q2, QK_K/4);
+ }
+}
+
+static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict vy, int64_t n, const float * restrict quant_weights) {
+
+ const int gindex = iq2_data_index(GGML_TYPE_IQ2_XS);
+
+ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;
+ const int * kmap_q2xs = iq2_data[gindex].map;
+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;
+
+ GGML_ASSERT(quant_weights && "missing quantization weights");
+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(n%QK_K == 0);
+
+ const int kMaxQ = 3;
+
+ const int64_t nbl = n/QK_K;
+
+ block_iq2_xs * y = vy;
+
+ float scales[QK_K/16];
+ float weight[16];
+ float xval[16];
+ int8_t L[16];
+ int8_t Laux[16];
+ float waux[16];
+ bool is_on_grid[2];
+ bool is_on_grid_aux[2];
+ uint8_t block_signs[2];
+ uint16_t q2[2*(QK_K/16)];
+
+ for (int ibl = 0; ibl < nbl; ++ibl) {
+
+ y[ibl].d = GGML_FP32_TO_FP16(0.f);
+ memset(q2, 0, QK_K/4);
+ memset(y[ibl].scales, 0, QK_K/32);
+
+ float max_scale = 0;
+
+ const float * xbl = x + QK_K*ibl;
+ float sumx2 = 0;
+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];
+ float sigma2 = sumx2/QK_K;
+
+ for (int ib = 0; ib < QK_K/16; ++ib) {
+ const float * xb = xbl + 16*ib;
+ const float * qw = quant_weights + QK_K*ibl + 16*ib;
+ for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);
+ for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]);
+ for (int k = 0; k < 2; ++k) {
+ int nflip = 0;
+ uint8_t s = 0;
+ for (int i = 0; i < 8; ++i) {
+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];
+ else {
+ xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);
+ }
+ }
+ if (nflip%2) {
+ int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];
+ for (int i = 1; i < 8; ++i) {
+ float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];
+ if (ax < min) {
+ min = ax; imin = i;
+ }
+ }
+ xval[8*k+imin] = -xval[8*k+imin];
+ s ^= (1 << imin);
+ }
+ block_signs[k] = s & 127;
+ }
+ float max = xval[0];
+ for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);
+ if (max < GROUP_MAX_EPS) {
+ scales[ib] = 0;
+ memset(L, 0, 16);
+ continue;
+ }
+ float best = 0;
+ float scale = max/(2*kMaxQ-1);
+ is_on_grid[0] = is_on_grid[1] = true;
+ for (int is = -9; is <= 9; ++is) {
+ float id = (2*kMaxQ-1+is*0.1f)/max;
+ float this_scale = 1/id;
+ for (int k = 0; k < 2; ++k) {
+ for (int i = 0; i < 8; ++i) {
+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));
+ Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));
+ }
+ uint16_t u = 0;
+ for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);
+ int grid_index = kmap_q2xs[u];
+ is_on_grid_aux[k] = true;
+ if (grid_index < 0) {
+ is_on_grid_aux[k] = false;
+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
+ }
+ }
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < 16; ++i) {
+ float w = weight[i];
+ float q = 2*Laux[i] + 1;
+ sumqx += w*xval[i]*q;
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
+ scale = sumqx/sumq2; best = scale*sumqx;
+ for (int i = 0; i < 16; ++i) L[i] = Laux[i];
+ for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k];
+ }
+ }
+ int n_not_ongrid = 0;
+ for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid;
+ if (n_not_ongrid > 0 && scale > 0) {
+ float id = 1/scale;
+ for (int k = 0; k < 2; ++k) {
+ if (is_on_grid[k]) continue;
+ uint16_t u = 0;
+ for (int i = 0; i < 8; ++i) {
+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));
+ l = MAX(0, MIN(kMaxQ-1, l));
+ u |= (l << 2*i);
+ L[8*k + i] = l;
+ }
+ int grid_index = kmap_q2xs[u];
+ if (grid_index < 0) {
+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
+ }
+ }
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < 16; ++i) {
+ float w = weight[i];
+ float q = 2*L[i] + 1;
+ sumqx += w*xval[i]*q;
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0) scale = sumqx/sumq2;
+ }
+ if (scale < 0) {
+ scale = -scale;
+ for (int k = 0; k < 2; ++k) block_signs[k] = (~block_signs[k]) & 127;
+ }
+ for (int k = 0; k < 2; ++k) {
+ uint16_t u = 0;
+ for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);
+ int grid_index = kmap_q2xs[u];
+ if (grid_index < 0) {
+ printf("Oops: found point %u not on grid:", u);
+ for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);
+ printf("\n");
+ GGML_ABORT("fatal error");
+ }
+ q2[2*ib+k] = grid_index | (block_signs[k] << 9);
+ }
+ GGML_ASSERT(scale >= 0);
+ scales[ib] = scale;
+ max_scale = MAX(max_scale, scale);
+ }
+
+ if (!max_scale) {
+ memset(y[ibl].qs, 0, QK_K/4);
+ continue;
+ }
+
+ float d = max_scale/31;
+ y[ibl].d = GGML_FP32_TO_FP16(d);
+ float id = 1/d;
+ for (int ib = 0; ib < QK_K/16; ++ib) {
+ int l = nearest_int(0.5f*(id*scales[ib]-1));
+ l = MAX(0, MIN(15, l));
+ if (ib%2 == 0) y[ibl].scales[ib/2] = l;
+ else y[ibl].scales[ib/2] |= (l << 4);
+ }
+ memcpy(y[ibl].qs, q2, QK_K/4);
+
+ }
+}
+
+size_t quantize_iq2_xxs(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ GGML_ASSERT(n_per_row%QK_K == 0);
+ int64_t nblock = n_per_row/QK_K;
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_iq2_xxs_impl(src, qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += nblock*sizeof(block_iq2_xxs);
+ }
+ return nrow * nblock * sizeof(block_iq2_xxs);
+}
+
+size_t quantize_iq2_xs(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ GGML_ASSERT(n_per_row%QK_K == 0);
+ int64_t nblock = n_per_row/QK_K;
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_iq2_xs_impl(src, qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += nblock*sizeof(block_iq2_xs);
+ }
+ return nrow * nblock * sizeof(block_iq2_xs);
+}
+
+//
+// ============================================= 3-bit using D4 lattice
+//
+
+typedef struct {
+ uint32_t * grid;
+ int * map;
+ uint16_t * neighbours;
+} iq3_entry_t;
+
+static iq3_entry_t iq3_data[2] = {
+ {NULL, NULL, NULL},
+ {NULL, NULL, NULL},
+};
+
+static inline int iq3_data_index(int grid_size) {
+ (void)grid_size;
+ GGML_ASSERT(grid_size == 256 || grid_size == 512);
+ return grid_size == 256 ? 0 : 1;
+}
+
+static int iq3_compare_func(const void * left, const void * right) {
+ const int * l = (const int *)left;
+ const int * r = (const int *)right;
+ return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0;
+}
+
+void iq3xs_init_impl(int grid_size) {
+ const int gindex = iq3_data_index(grid_size);
+ if (iq3_data[gindex].grid) {
+ return;
+ }
+ static const uint16_t kgrid_256[256] = {
+ 0, 2, 4, 9, 11, 15, 16, 18, 25, 34, 59, 61, 65, 67, 72, 74,
+ 81, 85, 88, 90, 97, 108, 120, 128, 130, 132, 137, 144, 146, 153, 155, 159,
+ 169, 175, 189, 193, 199, 200, 202, 213, 248, 267, 287, 292, 303, 315, 317, 321,
+ 327, 346, 362, 413, 436, 456, 460, 462, 483, 497, 513, 515, 520, 522, 529, 531,
+ 536, 538, 540, 551, 552, 576, 578, 585, 592, 594, 641, 643, 648, 650, 657, 664,
+ 698, 704, 706, 720, 729, 742, 758, 769, 773, 808, 848, 852, 870, 889, 901, 978,
+ 992, 1024, 1026, 1033, 1035, 1040, 1042, 1046, 1049, 1058, 1089, 1091, 1093, 1096, 1098, 1105,
+ 1112, 1139, 1143, 1144, 1152, 1154, 1161, 1167, 1168, 1170, 1183, 1184, 1197, 1217, 1224, 1228,
+ 1272, 1276, 1309, 1323, 1347, 1367, 1377, 1404, 1473, 1475, 1486, 1509, 1537, 1544, 1546, 1553,
+ 1555, 1576, 1589, 1594, 1600, 1602, 1616, 1625, 1636, 1638, 1665, 1667, 1672, 1685, 1706, 1722,
+ 1737, 1755, 1816, 1831, 1850, 1856, 1862, 1874, 1901, 1932, 1950, 1971, 2011, 2032, 2052, 2063,
+ 2077, 2079, 2091, 2095, 2172, 2192, 2207, 2208, 2224, 2230, 2247, 2277, 2308, 2345, 2356, 2389,
+ 2403, 2424, 2501, 2504, 2506, 2520, 2570, 2593, 2616, 2624, 2630, 2646, 2669, 2700, 2714, 2746,
+ 2754, 2795, 2824, 2835, 2839, 2874, 2882, 2905, 2984, 3028, 3042, 3092, 3108, 3110, 3124, 3153,
+ 3185, 3215, 3252, 3288, 3294, 3364, 3397, 3434, 3483, 3523, 3537, 3587, 3589, 3591, 3592, 3610,
+ 3626, 3670, 3680, 3722, 3749, 3754, 3776, 3789, 3803, 3824, 3857, 3873, 3904, 3906, 3924, 3992,
+ };
+ static const uint16_t kgrid_512[512] = {
+ 0, 1, 2, 5, 7, 8, 9, 10, 12, 14, 16, 17, 21, 27, 32, 34,
+ 37, 39, 41, 43, 48, 50, 57, 60, 63, 64, 65, 66, 68, 72, 73, 77,
+ 80, 83, 87, 89, 93, 100, 113, 117, 122, 128, 129, 133, 135, 136, 139, 142,
+ 145, 149, 152, 156, 162, 165, 167, 169, 171, 184, 187, 195, 201, 205, 208, 210,
+ 217, 219, 222, 228, 232, 234, 247, 249, 253, 256, 267, 271, 273, 276, 282, 288,
+ 291, 297, 312, 322, 324, 336, 338, 342, 347, 353, 357, 359, 374, 379, 390, 393,
+ 395, 409, 426, 441, 448, 450, 452, 464, 466, 470, 475, 488, 492, 512, 513, 514,
+ 516, 520, 521, 523, 525, 527, 528, 530, 537, 540, 542, 556, 558, 561, 570, 576,
+ 577, 579, 582, 584, 588, 593, 600, 603, 609, 616, 618, 632, 638, 640, 650, 653,
+ 655, 656, 660, 666, 672, 675, 685, 688, 698, 705, 708, 711, 712, 715, 721, 727,
+ 728, 732, 737, 754, 760, 771, 773, 778, 780, 793, 795, 802, 806, 808, 812, 833,
+ 840, 843, 849, 856, 858, 873, 912, 916, 919, 932, 934, 961, 963, 968, 970, 977,
+ 989, 993, 1010, 1016, 1024, 1025, 1027, 1029, 1031, 1032, 1034, 1036, 1038, 1041, 1043, 1047,
+ 1048, 1050, 1057, 1059, 1061, 1064, 1066, 1079, 1080, 1083, 1085, 1088, 1090, 1096, 1099, 1103,
+ 1106, 1109, 1113, 1116, 1122, 1129, 1153, 1156, 1159, 1169, 1171, 1176, 1183, 1185, 1195, 1199,
+ 1209, 1212, 1216, 1218, 1221, 1225, 1234, 1236, 1241, 1243, 1250, 1256, 1270, 1281, 1287, 1296,
+ 1299, 1306, 1309, 1313, 1338, 1341, 1348, 1353, 1362, 1375, 1376, 1387, 1400, 1408, 1410, 1415,
+ 1425, 1453, 1457, 1477, 1481, 1494, 1496, 1507, 1512, 1538, 1545, 1547, 1549, 1551, 1554, 1561,
+ 1563, 1565, 1570, 1572, 1575, 1577, 1587, 1593, 1601, 1603, 1605, 1612, 1617, 1619, 1632, 1648,
+ 1658, 1662, 1664, 1674, 1680, 1690, 1692, 1704, 1729, 1736, 1740, 1745, 1747, 1751, 1752, 1761,
+ 1763, 1767, 1773, 1787, 1795, 1801, 1806, 1810, 1817, 1834, 1840, 1844, 1857, 1864, 1866, 1877,
+ 1882, 1892, 1902, 1915, 1934, 1953, 1985, 1987, 2000, 2002, 2013, 2048, 2052, 2058, 2064, 2068,
+ 2071, 2074, 2081, 2088, 2104, 2114, 2119, 2121, 2123, 2130, 2136, 2141, 2147, 2153, 2157, 2177,
+ 2179, 2184, 2189, 2193, 2203, 2208, 2223, 2226, 2232, 2244, 2249, 2251, 2256, 2258, 2265, 2269,
+ 2304, 2306, 2324, 2335, 2336, 2361, 2373, 2375, 2385, 2418, 2443, 2460, 2480, 2504, 2509, 2520,
+ 2531, 2537, 2562, 2568, 2572, 2578, 2592, 2596, 2599, 2602, 2614, 2620, 2625, 2627, 2629, 2634,
+ 2641, 2650, 2682, 2688, 2697, 2707, 2712, 2718, 2731, 2754, 2759, 2760, 2775, 2788, 2793, 2805,
+ 2811, 2817, 2820, 2832, 2842, 2854, 2890, 2902, 2921, 2923, 2978, 3010, 3012, 3026, 3081, 3083,
+ 3085, 3097, 3099, 3120, 3136, 3152, 3159, 3188, 3210, 3228, 3234, 3245, 3250, 3256, 3264, 3276,
+ 3281, 3296, 3349, 3363, 3378, 3392, 3395, 3420, 3440, 3461, 3488, 3529, 3531, 3584, 3588, 3591,
+ 3600, 3602, 3614, 3616, 3628, 3634, 3650, 3657, 3668, 3683, 3685, 3713, 3716, 3720, 3726, 3729,
+ 3736, 3753, 3778, 3802, 3805, 3819, 3841, 3845, 3851, 3856, 3880, 3922, 3938, 3970, 3993, 4032,
+ };
+
+ const int kmap_size = 4096;
+ const int nwant = grid_size == 256 ? 2 : 3;
+ const uint16_t * kgrid = grid_size == 256 ? kgrid_256 : kgrid_512;
+ uint32_t * kgrid_q3xs;
+ int * kmap_q3xs;
+ uint16_t * kneighbors_q3xs;
+
+ //printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);
+ uint32_t * the_grid = (uint32_t *)malloc(grid_size*sizeof(uint32_t));
+ for (int k = 0; k < grid_size; ++k) {
+ int8_t * pos = (int8_t *)(the_grid + k);
+ for (int i = 0; i < 4; ++i) {
+ int l = (kgrid[k] >> 3*i) & 0x7;
+ pos[i] = 2*l + 1;
+ }
+ }
+ kgrid_q3xs = the_grid;
+ iq3_data[gindex].grid = the_grid;
+ kmap_q3xs = (int *)malloc(kmap_size*sizeof(int));
+ iq3_data[gindex].map = kmap_q3xs;
+ for (int i = 0; i < kmap_size; ++i) kmap_q3xs[i] = -1;
+ uint32_t aux32;
+ uint8_t * aux8 = (uint8_t *)&aux32;
+ for (int i = 0; i < grid_size; ++i) {
+ aux32 = kgrid_q3xs[i];
+ uint16_t index = 0;
+ for (int k=0; k<4; ++k) {
+ uint16_t q = (aux8[k] - 1)/2;
+ index |= (q << 3*k);
+ }
+ kmap_q3xs[index] = i;
+ }
+ int8_t pos[4];
+ int * dist2 = (int *)malloc(2*grid_size*sizeof(int));
+ int num_neighbors = 0, num_not_in_map = 0;
+ for (int i = 0; i < kmap_size; ++i) {
+ if (kmap_q3xs[i] >= 0) continue;
+ ++num_not_in_map;
+ for (int k = 0; k < 4; ++k) {
+ int l = (i >> 3*k) & 0x7;
+ pos[k] = 2*l + 1;
+ }
+ for (int j = 0; j < grid_size; ++j) {
+ const int8_t * pg = (const int8_t *)(kgrid_q3xs + j);
+ int d2 = 0;
+ for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);
+ dist2[2*j+0] = d2;
+ dist2[2*j+1] = j;
+ }
+ qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func);
+ int n = 0; int d2 = dist2[0];
+ int nhave = 1;
+ for (int j = 0; j < grid_size; ++j) {
+ if (dist2[2*j] > d2) {
+ if (nhave == nwant) break;
+ d2 = dist2[2*j];
+ ++nhave;
+ }
+ ++n;
+ }
+ num_neighbors += n;
+ }
+ //printf("%s: %d neighbours in total\n", __func__, num_neighbors);
+ kneighbors_q3xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));
+ iq3_data[gindex].neighbours = kneighbors_q3xs;
+ int counter = 0;
+ for (int i = 0; i < kmap_size; ++i) {
+ if (kmap_q3xs[i] >= 0) continue;
+ for (int k = 0; k < 4; ++k) {
+ int l = (i >> 3*k) & 0x7;
+ pos[k] = 2*l + 1;
+ }
+ for (int j = 0; j < grid_size; ++j) {
+ const int8_t * pg = (const int8_t *)(kgrid_q3xs + j);
+ int d2 = 0;
+ for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);
+ dist2[2*j+0] = d2;
+ dist2[2*j+1] = j;
+ }
+ qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func);
+ kmap_q3xs[i] = -(counter + 1);
+ int d2 = dist2[0];
+ uint16_t * start = &kneighbors_q3xs[counter++];
+ int n = 0, nhave = 1;
+ for (int j = 0; j < grid_size; ++j) {
+ if (dist2[2*j] > d2) {
+ if (nhave == nwant) break;
+ d2 = dist2[2*j];
+ ++nhave;
+ }
+ kneighbors_q3xs[counter++] = dist2[2*j+1];
+ ++n;
+ }
+ *start = n;
+ }
+ free(dist2);
+}
+
+void iq3xs_free_impl(int grid_size) {
+ GGML_ASSERT(grid_size == 256 || grid_size == 512);
+ const int gindex = iq3_data_index(grid_size);
+ if (iq3_data[gindex].grid) {
+ free(iq3_data[gindex].grid); iq3_data[gindex].grid = NULL;
+ free(iq3_data[gindex].map); iq3_data[gindex].map = NULL;
+ free(iq3_data[gindex].neighbours); iq3_data[gindex].neighbours = NULL;
+ }
+}
+
+static int iq3_find_best_neighbour(const uint16_t * restrict neighbours, const uint32_t * restrict grid,
+ const float * restrict xval, const float * restrict weight, float scale, int8_t * restrict L) {
+ int num_neighbors = neighbours[0];
+ GGML_ASSERT(num_neighbors > 0);
+ float best_d2 = FLT_MAX;
+ int grid_index = -1;
+ for (int j = 1; j <= num_neighbors; ++j) {
+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);
+ float d2 = 0;
+ for (int i = 0; i < 4; ++i) {
+ float q = pg[i];
+ float diff = scale*q - xval[i];
+ d2 += weight[i]*diff*diff;
+ }
+ if (d2 < best_d2) {
+ best_d2 = d2; grid_index = neighbours[j];
+ }
+ }
+ GGML_ASSERT(grid_index >= 0);
+ const int8_t * pg = (const int8_t *)(grid + grid_index);
+ for (int i = 0; i < 4; ++i) L[i] = (pg[i] - 1)/2;
+ return grid_index;
+}
+
+static void quantize_row_iq3_xxs_impl(int grid_size, const float * restrict x, void * restrict vy, int64_t n,
+ const float * restrict quant_weights) {
+
+ const int gindex = iq3_data_index(grid_size);
+
+ const uint32_t * kgrid_q3xs = iq3_data[gindex].grid;
+ const int * kmap_q3xs = iq3_data[gindex].map;
+ const uint16_t * kneighbors_q3xs = iq3_data[gindex].neighbours;
+
+ //GGML_ASSERT(quant_weights && "missing quantization weights");
+ GGML_ASSERT(kgrid_q3xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kmap_q3xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kneighbors_q3xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(n%QK_K == 0);
+
+ const int kMaxQ = 8;
+
+ const int64_t nbl = n/QK_K;
+
+ ggml_fp16_t * dh;
+ uint8_t * qs;
+ int block_size;
+ if (grid_size == 256) {
+ block_iq3_xxs * y = vy;
+ dh = &y->d;
+ qs = y->qs;
+ block_size = sizeof(block_iq3_xxs);
+ } else {
+ block_iq3_s * y = vy;
+ dh = &y->d;
+ qs = y->qs;
+ block_size = sizeof(block_iq3_s);
+ }
+ int quant_size = block_size - sizeof(ggml_fp16_t);
+
+ float scales[QK_K/32];
+ float weight[32];
+ float xval[32];
+ int8_t L[32];
+ int8_t Laux[32];
+ float waux[32];
+ bool is_on_grid[8];
+ bool is_on_grid_aux[8];
+ uint8_t block_signs[8];
+ uint8_t q3[3*(QK_K/8)+QK_K/32];
+ uint32_t * scales_and_signs = (uint32_t *)(q3 + QK_K/4);
+ uint8_t * qh = q3 + 3*(QK_K/8);
+
+ for (int ibl = 0; ibl < nbl; ++ibl) {
+
+ dh[0] = GGML_FP32_TO_FP16(0.f);
+ memset(q3, 0, 3*QK_K/8+QK_K/32);
+
+ float max_scale = 0;
+
+ const float * xbl = x + QK_K*ibl;
+ float sumx2 = 0;
+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];
+ float sigma2 = 2*sumx2/QK_K;
+
+ for (int ib = 0; ib < QK_K/32; ++ib) {
+ const float * xb = xbl + 32*ib;
+ if (quant_weights) {
+ const float * qw = quant_weights + QK_K*ibl + 32*ib;
+ for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);
+ } else {
+ for (int i = 0; i < 32; ++i) weight[i] = xb[i]*xb[i];
+ }
+ for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]);
+ for (int k = 0; k < 4; ++k) {
+ int nflip = 0;
+ uint8_t s = 0;
+ for (int i = 0; i < 8; ++i) {
+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];
+ else {
+ xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);
+ }
+ }
+ if (nflip%2) {
+ int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];
+ for (int i = 1; i < 8; ++i) {
+ float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];
+ if (ax < min) {
+ min = ax; imin = i;
+ }
+ }
+ xval[8*k+imin] = -xval[8*k+imin];
+ s ^= (1 << imin);
+ }
+ block_signs[k] = s & 127;
+ }
+ float max = xval[0];
+ for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]);
+ if (max < GROUP_MAX_EPS_IQ3_XXS) {
+ scales[ib] = 0;
+ memset(L, 0, 32);
+ continue;
+ }
+ float best = 0;
+ float scale = max/(2*kMaxQ-1);
+ for (int is = -15; is <= 15; ++is) {
+ float id = (2*kMaxQ-1+is*0.2f)/max;
+ float this_scale = 1/id;
+ for (int k = 0; k < 8; ++k) {
+ for (int i = 0; i < 4; ++i) {
+ int l = nearest_int(0.5f*(id*xval[4*k+i]-1));
+ Laux[4*k+i] = MAX(0, MIN(kMaxQ-1, l));
+ }
+ uint16_t u = 0;
+ for (int i = 0; i < 4; ++i) u |= (Laux[4*k+i] << 3*i);
+ int grid_index = kmap_q3xs[u];
+ is_on_grid_aux[k] = true;
+ if (grid_index < 0) {
+ is_on_grid_aux[k] = false;
+ const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;
+ grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, this_scale, Laux + 4*k);
+ }
+ }
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < 32; ++i) {
+ float w = weight[i];
+ float q = 2*Laux[i] + 1;
+ sumqx += w*xval[i]*q;
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
+ scale = sumqx/sumq2; best = scale*sumqx;
+ for (int i = 0; i < 32; ++i) L[i] = Laux[i];
+ for (int k = 0; k < 8; ++k) is_on_grid[k] = is_on_grid_aux[k];
+ }
+ }
+ int n_not_ongrid = 0;
+ for (int k = 0; k < 8; ++k) if (!is_on_grid[k]) ++n_not_ongrid;
+ if (n_not_ongrid > 0 && scale > 0) {
+ float id = 1/scale;
+ for (int k = 0; k < 8; ++k) {
+ if (is_on_grid[k]) continue;
+ uint16_t u = 0;
+ for (int i = 0; i < 4; ++i) {
+ int l = nearest_int(0.5f*(id*xval[4*k+i]-1));
+ l = MAX(0, MIN(kMaxQ-1, l));
+ u |= (l << 3*i);
+ }
+ int grid_index = kmap_q3xs[u];
+ if (grid_index < 0) {
+ const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;
+ grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, scale, L + 4*k);
+ }
+ const int8_t * pg = (const int8_t *)(kgrid_q3xs + grid_index);
+ for (int i = 0; i < 4; ++i) L[4*k+i] = (pg[i] - 1)/2;
+ }
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < 32; ++i) {
+ float w = weight[i];
+ float q = 2*L[i] + 1;
+ sumqx += w*xval[i]*q;
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0) scale = sumqx/sumq2;
+ }
+ if (scale < 0) {
+ // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale)
+ // and correspondingly flip quant signs.
+ scale = -scale;
+ for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127;
+ }
+ for (int k = 0; k < 8; ++k) {
+ uint16_t u = 0;
+ for (int i = 0; i < 4; ++i) u |= (L[4*k+i] << 3*i);
+ int grid_index = kmap_q3xs[u];
+ if (grid_index < 0) {
+ printf("Oops: found point %u not on grid:", u);
+ for (int i = 0; i < 4; ++i) printf(" %d", L[4*k+i]);
+ printf("\n");
+ GGML_ABORT("fatal error");
+ }
+ if (grid_size == 256) {
+ q3[8*ib+k] = grid_index;
+ } else {
+ q3[8*ib+k] = grid_index & 255;
+ qh[ib] |= ((grid_index >> 8) << k);
+ }
+
+ }
+ scales_and_signs[ib] = block_signs[0] | (block_signs[1] << 7) | (block_signs[2] << 14) | (block_signs[3] << 21);
+ GGML_ASSERT(scale >= 0);
+ scales[ib] = scale;
+ max_scale = MAX(max_scale, scale);
+ }
+
+ if (!max_scale) {
+ memset(qs, 0, quant_size);
+ dh += block_size/sizeof(ggml_fp16_t);
+ qs += block_size;
+ continue;
+ }
+
+ float d = max_scale/31;
+ dh[0] = GGML_FP32_TO_FP16(d * 1.0125f); // small improvement via this fudge factor
+ float id = 1/d;
+ for (int ib = 0; ib < QK_K/32; ++ib) {
+ int l = nearest_int(0.5f*(id*scales[ib]-1));
+ l = MAX(0, MIN(15, l));
+ scales_and_signs[ib] |= ((uint32_t)l << 28);
+ }
+ memcpy(qs, q3, quant_size);
+
+ dh += block_size/sizeof(ggml_fp16_t);
+ qs += block_size;
+
+ }
+}
+
+size_t quantize_iq3_xxs(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ GGML_ASSERT(n_per_row%QK_K == 0);
+ int64_t nblock = n_per_row/QK_K;
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_iq3_xxs_impl(256, src, qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += nblock*sizeof(block_iq3_xxs);
+ }
+ return nrow * nblock * sizeof(block_iq3_xxs);
+}
+
+void quantize_row_iq3_xxs(const float * restrict x, void * restrict vy, int64_t k) {
+ assert(k % QK_K == 0);
+ block_iq3_xxs * restrict y = vy;
+ quantize_row_iq3_xxs_ref(x, y, k);
+}
+
+void quantize_row_iq3_xxs_ref(const float * restrict x, block_iq3_xxs * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ quantize_row_iq3_xxs_impl(256, x, y, k, NULL);
+}
+
+static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, void * restrict vy, int n,
+ const float * restrict quant_weights,
+ float * scales,
+ float * weight,
+ float * xval,
+ int8_t * L,
+ int8_t * Laux,
+ float * waux,
+ bool * is_on_grid,
+ bool * is_on_grid_aux,
+ uint8_t * block_signs) {
+
+ const int gindex = iq3_data_index(512);
+
+ const uint32_t * kgrid_q3xs = iq3_data[gindex].grid;
+ const int * kmap_q3xs = iq3_data[gindex].map;
+ const uint16_t * kneighbors_q3xs = iq3_data[gindex].neighbours;
+
+ //GGML_ASSERT(quant_weights && "missing quantization weights");
+ GGML_ASSERT(kgrid_q3xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kmap_q3xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kneighbors_q3xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(n%QK_K == 0);
+
+ const int kMaxQ = 8;
+
+ const int64_t nbl = n/QK_K;
+
+ block_iq3_s * y = vy;
+
+ const int bs4 = block_size/4;
+ const int bs8 = block_size/8;
+
+ for (int ibl = 0; ibl < nbl; ++ibl) {
+
+ memset(&y[ibl], 0, sizeof(block_iq3_s));
+ y[ibl].d = GGML_FP32_TO_FP16(0.f);
+
+ uint8_t * qs = y[ibl].qs;
+ uint8_t * qh = y[ibl].qh;
+ uint8_t * signs = y[ibl].signs;
+
+ float max_scale = 0;
+
+ const float * xbl = x + QK_K*ibl;
+ float sumx2 = 0;
+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];
+ float sigma2 = 2*sumx2/QK_K;
+
+ for (int ib = 0; ib < QK_K/block_size; ++ib) {
+ const float * xb = xbl + block_size*ib;
+ if (quant_weights) {
+ const float * qw = quant_weights + QK_K*ibl + block_size*ib;
+ for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);
+ } else {
+ for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i];
+ }
+ for (int i = 0; i < block_size; ++i) waux[i] = sqrtf(weight[i]);
+ for (int k = 0; k < bs8; ++k) {
+ uint8_t s = 0;
+ for (int i = 0; i < 8; ++i) {
+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];
+ else {
+ xval[8*k + i] = -xb[8*k + i]; s |= (1 << i);
+ }
+ }
+ block_signs[k] = s;
+ }
+ float max = xval[0];
+ for (int i = 1; i < block_size; ++i) max = MAX(max, xval[i]);
+ if (!max) {
+ scales[ib] = 0;
+ continue;
+ }
+ float best = 0;
+ float scale = max/(2*kMaxQ-1);
+ for (int k = 0; k < bs4; ++k) is_on_grid[k] = false;
+ for (int is = -9; is <= 9; ++is) {
+ float id = (2*kMaxQ-1+is*0.2f)/max;
+ float this_scale = 1/id;
+ for (int k = 0; k < bs4; ++k) {
+ for (int i = 0; i < 4; ++i) {
+ int l = nearest_int(0.5f*(id*xval[4*k+i]-1));
+ Laux[4*k+i] = MAX(0, MIN(kMaxQ-1, l));
+ }
+ uint16_t u = 0;
+ for (int i = 0; i < 4; ++i) u |= (Laux[4*k+i] << 3*i);
+ int grid_index = kmap_q3xs[u];
+ is_on_grid_aux[k] = true;
+ if (grid_index < 0) {
+ is_on_grid_aux[k] = false;
+ const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;
+ grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, this_scale, Laux + 4*k);
+ }
+ }
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < block_size; ++i) {
+ float w = weight[i];
+ float q = 2*Laux[i] + 1;
+ sumqx += w*xval[i]*q;
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
+ scale = sumqx/sumq2; best = scale*sumqx;
+ for (int i = 0; i < block_size; ++i) L[i] = Laux[i];
+ for (int k = 0; k < bs4; ++k) is_on_grid[k] = is_on_grid_aux[k];
+ }
+ }
+ int n_not_ongrid = 0;
+ for (int k = 0; k < bs4; ++k) if (!is_on_grid[k]) ++n_not_ongrid;
+ if (n_not_ongrid > 0 && scale > 0) {
+ float id = 1/scale;
+ for (int k = 0; k < bs4; ++k) {
+ //if (is_on_grid[k]) continue;
+ uint16_t u = 0;
+ for (int i = 0; i < 4; ++i) {
+ int l = nearest_int(0.5f*(id*xval[4*k+i]-1));
+ l = MAX(0, MIN(kMaxQ-1, l));
+ u |= (l << 3*i);
+ }
+ int grid_index = kmap_q3xs[u];
+ if (grid_index < 0) {
+ const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;
+ grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, scale, L + 4*k);
+ }
+ const int8_t * pg = (const int8_t *)(kgrid_q3xs + grid_index);
+ for (int i = 0; i < 4; ++i) L[4*k+i] = (pg[i] - 1)/2;
+ }
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < block_size; ++i) {
+ float w = weight[i];
+ float q = 2*L[i] + 1;
+ sumqx += w*xval[i]*q;
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0) scale = sumqx/sumq2;
+ }
+ if (scale < 0) {
+ // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale)
+ // and correspondingly flip quant signs.
+ scale = -scale;
+ for (int k = 0; k < bs8; ++k) block_signs[k] = ~block_signs[k];
+ }
+ for (int k = 0; k < bs4; ++k) {
+ uint16_t u = 0;
+ for (int i = 0; i < 4; ++i) u |= (L[4*k+i] << 3*i);
+ int grid_index = kmap_q3xs[u];
+ if (grid_index < 0) {
+ printf("Oops: found point %u not on grid:", u);
+ for (int i = 0; i < 4; ++i) printf(" %d", L[4*k+i]);
+ printf("\n");
+ GGML_ABORT("fatal error");
+ }
+ qs[k] = grid_index & 255;
+ qh[(ib*bs4+k)/8] |= ((grid_index >> 8) << ((ib*bs4+k)%8));
+ }
+ qs += bs4;
+ for (int k = 0; k < bs8; ++k) signs[k] = block_signs[k];
+ signs += bs8;
+ GGML_ASSERT(scale >= 0);
+ scales[ib] = scale;
+ max_scale = MAX(max_scale, scale);
+ }
+
+ if (!max_scale) {
+ continue;
+ }
+
+ float d = max_scale/31;
+ y[ibl].d = GGML_FP32_TO_FP16(d * 1.033f);
+ float id = 1/d;
+ for (int ib = 0; ib < QK_K/block_size; ib += 2) {
+ int l1 = nearest_int(0.5f*(id*scales[ib+0]-1));
+ l1 = MAX(0, MIN(15, l1));
+ int l2 = nearest_int(0.5f*(id*scales[ib+1]-1));
+ l2 = MAX(0, MIN(15, l2));
+ y[ibl].scales[ib/2] = l1 | (l2 << 4);
+ }
+
+ }
+}
+
+#define IQ3S_BLOCK_SIZE 32
+size_t quantize_iq3_s(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ GGML_ASSERT(n_per_row%QK_K == 0);
+ int64_t nblock = n_per_row/QK_K;
+ float scales[QK_K/IQ3S_BLOCK_SIZE];
+ float weight[IQ3S_BLOCK_SIZE];
+ float xval[IQ3S_BLOCK_SIZE];
+ int8_t L[IQ3S_BLOCK_SIZE];
+ int8_t Laux[IQ3S_BLOCK_SIZE];
+ float waux[IQ3S_BLOCK_SIZE];
+ bool is_on_grid[IQ3S_BLOCK_SIZE/4];
+ bool is_on_grid_aux[IQ3S_BLOCK_SIZE/4];
+ uint8_t block_signs[IQ3S_BLOCK_SIZE/8];
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_iq3_s_impl(IQ3S_BLOCK_SIZE, src, qrow, n_per_row, quant_weights,
+ scales, weight, xval, L, Laux, waux, is_on_grid, is_on_grid_aux, block_signs);
+ src += n_per_row;
+ qrow += nblock*sizeof(block_iq3_s);
+ }
+ return nrow * nblock * sizeof(block_iq3_s);
+}
+
+void quantize_row_iq3_s(const float * restrict x, void * restrict vy, int64_t k) {
+ assert(k % QK_K == 0);
+ block_iq3_s * restrict y = vy;
+ quantize_row_iq3_s_ref(x, y, k);
+}
+
+void quantize_row_iq3_s_ref(const float * restrict x, block_iq3_s * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ quantize_iq3_s(x, y, 1, k, NULL);
+}
+
+
+// =================================== 1.5 bpw ===================================================
+
+static int iq1_find_best_neighbour(const uint16_t * restrict neighbours, const uint64_t * restrict grid,
+ const float * restrict xval, const float * restrict weight, float * scale, int8_t * restrict L, int ngrid) {
+ int num_neighbors = neighbours[0];
+ GGML_ASSERT(num_neighbors > 0);
+ float best_score = -FLT_MAX;
+ int grid_index = -1;
+ for (int j = 1; j <= num_neighbors; ++j) {
+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < 8; ++i) {
+ float q = (pg[i] - 3)/2;
+ float w = weight[i];
+ sumqx += w*q*xval[i];
+ sumq2 += w*q*q;
+ }
+ if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {
+ *scale = sumqx/sumq2; best_score = *scale * sumqx;
+ grid_index = neighbours[j];
+ }
+ }
+ if (grid_index < 0) {
+ for (int i = 0; i < ngrid; ++i) {
+ const int8_t * grid_i = (const int8_t *)(grid + i);
+ float sumqx = 0, sumq2 = 0;
+ for (int j = 0; j < 8; ++j) {
+ float w = weight[j];
+ float q = (grid_i[j] - 3)/2;
+ sumqx += w*q*xval[j];
+ sumq2 += w*q*q;
+ }
+ if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {
+ *scale = sumqx/sumq2; best_score = *scale*sumqx;
+ grid_index = i;
+ }
+ }
+ }
+ if (grid_index < 0) {
+ printf("Oops, did not find grid point\n");
+ printf("Have %d neighbours\n", num_neighbors);
+ for (int j = 1; j <= num_neighbors; ++j) {
+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < 8; ++i) {
+ float q = (pg[i] - 3)/2;
+ float w = weight[i];
+ sumqx += w*q*xval[i];
+ sumq2 += w*q*q;
+ }
+ printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2);
+ }
+ }
+ GGML_ASSERT(grid_index >= 0);
+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ *scale *= 1.05f; // This is a fudge factor. Don't ask me why it improves the result.
+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ const int8_t * pg = (const int8_t *)(grid + grid_index);
+ for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;
+ return grid_index;
+}
+
+static int iq1_find_best_neighbour2(const uint16_t * restrict neighbours, const uint64_t * restrict grid,
+ const float * restrict xval, const float * restrict weight, float scale, const float * restrict xg, int8_t * restrict L, int ngrid) {
+ int num_neighbors = neighbours[0];
+ GGML_ASSERT(num_neighbors > 0);
+ float best_score = FLT_MAX;
+ int grid_index = -1;
+ for (int j = 1; j <= num_neighbors; ++j) {
+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);
+ float d2 = 0;
+ for (int i = 0; i < 8; ++i) {
+ float q = xg[(pg[i] - 1)/2];
+ float w = weight[i];
+ float diff = scale*q - xval[i];
+ d2 += w*diff*diff;
+ }
+ if (d2 < best_score) {
+ best_score = d2;
+ grid_index = neighbours[j];
+ }
+ }
+ if (grid_index < 0) {
+ for (int i = 0; i < ngrid; ++i) {
+ const int8_t * grid_i = (const int8_t *)(grid + i);
+ float d2 = 0;
+ for (int j = 0; j < 8; ++j) {
+ float w = weight[j];
+ float q = xg[(grid_i[j] - 1)/2];
+ float diff = scale*q - xval[i];
+ d2 += w*diff*diff;
+ }
+ if (d2 < best_score) {
+ best_score = d2;
+ grid_index = i;
+ }
+ }
+ }
+ if (grid_index < 0) {
+ printf("Oops, did not find grid point\n");
+ printf("Have %d neighbours\n", num_neighbors);
+ for (int j = 1; j <= num_neighbors; ++j) {
+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < 8; ++i) {
+ float q = xg[(pg[i] - 1)/2];
+ float w = weight[i];
+ sumqx += w*q*xval[i];
+ sumq2 += w*q*q;
+ }
+ printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2);
+ }
+ }
+ GGML_ASSERT(grid_index >= 0);
+ const int8_t * pg = (const int8_t *)(grid + grid_index);
+ for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;
+ return grid_index;
+}
+
+static int iq1_sort_helper(const void * left, const void * right) {
+ const float * l = left;
+ const float * r = right;
+ return *l < *r ? -1 : *l > *r ? 1 : 0;
+}
+
+#define IQ1S_BLOCK_SIZE 32
+#define IQ1M_BLOCK_SIZE 16
+static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy, int64_t n, const float * restrict quant_weights,
+ float * scales,
+ float * weight,
+ float * sumx,
+ float * sumw,
+ float * pairs,
+ int8_t * L,
+ uint16_t * index,
+ int8_t * shifts) {
+
+ const int gindex = iq2_data_index(GGML_TYPE_IQ1_S);
+
+ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;
+ const int * kmap_q2xs = iq2_data[gindex].map;
+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;
+
+ GGML_ASSERT(quant_weights && "missing quantization weights");
+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(n%QK_K == 0);
+
+ block_iq1_s * y = vy;
+
+ const int64_t nbl = n/QK_K;
+
+ const int block_size = IQ1S_BLOCK_SIZE;
+
+ const float x_p[3] = {-1 + IQ1S_DELTA, IQ1S_DELTA, 1 + IQ1S_DELTA};
+ const float x_m[3] = {-1 - IQ1S_DELTA, -IQ1S_DELTA, 1 - IQ1S_DELTA};
+
+
+ int * idx = (int *)(pairs + 1);
+
+ for (int ibl = 0; ibl < nbl; ++ibl) {
+
+ y[ibl].d = GGML_FP32_TO_FP16(0.f);
+ memset(y[ibl].qs, 0, QK_K/8);
+ memset(y[ibl].qh, 0, QK_K/16);
+
+ float max_scale = 0;
+
+ const float * xbl = x + QK_K*ibl;
+ float sumx2 = 0;
+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];
+ float sigma2 = 2*sumx2/QK_K;
+
+ for (int ib = 0; ib < QK_K/block_size; ++ib) {
+ const float * xb = xbl + block_size*ib;
+ const float * qw = quant_weights + QK_K*ibl + block_size*ib;
+ for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);
+ float max = fabsf(xb[0]);
+ for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]));
+ if (max < GROUP_MAX_EPS_IQ1_S) {
+ scales[ib] = 0;
+ memset(L, 1, block_size);
+ continue;
+ }
+ // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem.
+ // With just 3 allowed quant values (-1, 0, 1), we can search exhaustively for the two
+ // boundaries that split the weights xb[i] into 3 groups. To do so, we sort the weights
+ // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and
+ // Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale
+ // for each possible and score for each split.
+ for (int j = 0; j < block_size; ++j) {
+ pairs[2*j] = xb[j];
+ idx[2*j] = j;
+ }
+ qsort(pairs, block_size, 2*sizeof(float), iq1_sort_helper);
+ {
+ sumx[0] = sumw[0] = 0;
+ for (int j = 0; j < block_size; ++j) {
+ int i = idx[2*j];
+ sumx[j+1] = sumx[j] + weight[i]*xb[i];
+ sumw[j+1] = sumw[j] + weight[i];
+ }
+ }
+ float best_score = -FLT_MIN, scale = max;
+ int besti1 = -1, besti2 = -1, best_shift = 0;
+ for (int i1 = 0; i1 <= block_size; ++i1) {
+ for (int i2 = i1; i2 <= block_size; ++i2) {
+ float sumqx = (sumx[i1] - sumx[0])*x_p[0] + (sumx[i2] - sumx[i1])*x_p[1] + (sumx[block_size] - sumx[i2])*x_p[2];
+ float sumq2 = (sumw[i1] - sumw[0])*x_p[0]*x_p[0] + (sumw[i2] - sumw[i1])*x_p[1]*x_p[1] + (sumw[block_size] - sumw[i2])*x_p[2]*x_p[2];
+ if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {
+ scale = sumqx/sumq2; best_score = scale*sumqx;
+ besti1 = i1; besti2 = i2; best_shift = 1;
+ }
+ sumqx = (sumx[i1] - sumx[0])*x_m[0] + (sumx[i2] - sumx[i1])*x_m[1] + (sumx[block_size] - sumx[i2])*x_m[2];
+ sumq2 = (sumw[i1] - sumw[0])*x_m[0]*x_m[0] + (sumw[i2] - sumw[i1])*x_m[1]*x_m[1] + (sumw[block_size] - sumw[i2])*x_m[2]*x_m[2];
+ if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {
+ scale = sumqx/sumq2; best_score = scale*sumqx;
+ besti1 = i1; besti2 = i2; best_shift = -1;
+ }
+ }
+ }
+ GGML_ASSERT(besti1 >= 0 && besti2 >= 0 && best_shift != 0);
+ for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0;
+ for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1;
+ for (int j = besti2; j < block_size; ++j) L[idx[2*j]] = 2;
+ if (scale < 0) {
+ for (int j = 0; j < block_size; ++j) L[j] = 2 - L[j];
+ scale = -scale; best_shift = -best_shift;
+ }
+ bool all_on_grid = true;
+ const float * xx = best_shift == 1 ? x_p : x_m;
+ for (int k = 0; k < block_size/8; ++k) {
+ uint16_t u = 0;
+ for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j);
+ int grid_index = kmap_q2xs[u];
+ if (grid_index < 0) {
+ all_on_grid = false;
+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
+ grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, xx, L + 8*k, NGRID_IQ1S);
+ GGML_ASSERT(grid_index >= 0);
+ }
+ index[k] = grid_index;
+ }
+ if (!all_on_grid) {
+ float sumqx = 0, sumq2 = 0;
+ for (int k = 0; k < block_size/8; ++k) {
+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + index[k]);
+ for (int j = 0; j < 8; ++j) {
+ float w = weight[8*k + j];
+ float q = xx[(pg[j] - 1)/2];
+ sumqx += w*q*xb[8*k+j];
+ sumq2 += w*q*q;
+ }
+ }
+ if (sumqx > 0 && sumq2 > 0) scale = sumqx/sumq2;
+ }
+ uint16_t h = 0;
+ for (int k = 0; k < block_size/8; ++k) {
+ y[ibl].qs[(block_size/8)*ib + k] = index[k] & 255;
+ h |= (index[k] >> 8) << 3*k;
+ }
+ y[ibl].qh[ib] = h;
+ GGML_ASSERT(scale >= 0);
+ scales[ib] = scale;
+ shifts[ib] = best_shift;
+ max_scale = MAX(max_scale, scale);
+ }
+
+ if (!max_scale) {
+ continue;
+ }
+
+ float d = max_scale/15;
+ y[ibl].d = GGML_FP32_TO_FP16(d*1.125f); // 1.125f is another fudge factor. Don't ask me why it is needed.
+ float id = 1/d;
+ for (int ib = 0; ib < QK_K/block_size; ++ib) {
+ int l = nearest_int(0.5f*(id*scales[ib]-1));
+ l = MAX(0, MIN(7, l));
+ if (shifts[ib] == -1) l |= 8;
+ y[ibl].qh[ib] |= (l << 12);
+ }
+ }
+}
+
+size_t quantize_iq1_s(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ GGML_ASSERT(n_per_row%QK_K == 0);
+ float scales[QK_K/IQ1S_BLOCK_SIZE];
+ float weight[IQ1S_BLOCK_SIZE];
+ int8_t L[IQ1S_BLOCK_SIZE];
+ float sumx[IQ1S_BLOCK_SIZE+1];
+ float sumw[IQ1S_BLOCK_SIZE+1];
+ float pairs[2*IQ1S_BLOCK_SIZE];
+ uint16_t index[IQ1S_BLOCK_SIZE/8];
+ int8_t shifts[QK_K/IQ1S_BLOCK_SIZE];
+ int64_t nblock = n_per_row/QK_K;
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_iq1_s_impl(src, qrow, n_per_row, quant_weights, scales, weight, sumx, sumw, pairs, L, index, shifts);
+ src += n_per_row;
+ qrow += nblock*sizeof(block_iq1_s);
+ }
+ return nrow * nblock * sizeof(block_iq1_s);
+}
+
+static void quantize_row_iq1_m_impl(const float * restrict x, void * restrict vy, int64_t n, const float * restrict quant_weights,
+ float * scales,
+ float * weight,
+ float * pairs,
+ int8_t * L,
+ uint16_t * index,
+ int8_t * shifts) {
+
+ const int gindex = iq2_data_index(GGML_TYPE_IQ1_M);
+
+ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;
+ const int * kmap_q2xs = iq2_data[gindex].map;
+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;
+
+ //GGML_ASSERT(quant_weights && "missing quantization weights");
+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(n%QK_K == 0);
+
+ block_iq1_m * y = vy;
+
+ const int64_t nbl = n/QK_K;
+
+ const int block_size = IQ1M_BLOCK_SIZE;
+
+ const float x_p[3] = {-1 + IQ1M_DELTA, IQ1M_DELTA, 1 + IQ1M_DELTA};
+ const float x_m[3] = {-1 - IQ1M_DELTA, -IQ1M_DELTA, 1 - IQ1M_DELTA};
+ const uint8_t masks[4] = {0x00, 0x80, 0x08, 0x88};
+
+ int * idx = (int *)(pairs + 1);
+
+ float sumqx[4], sumq2[4];
+
+ iq1m_scale_t s;
+ const float * xx;
+
+ for (int ibl = 0; ibl < nbl; ++ibl) {
+ memset(y[ibl].qs, 0, QK_K/8);
+ memset(y[ibl].qh, 0, QK_K/16);
+ memset(y[ibl].scales, 0, QK_K/32);
+
+ float max_scale = 0;
+
+ const float * xbl = x + QK_K*ibl;
+ float sumx2 = 0;
+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];
+ float sigma2 = 2*sumx2/QK_K;
+
+ for (int ib = 0; ib < QK_K/block_size; ++ib) {
+ const float * xb = xbl + block_size*ib;
+ if (quant_weights) {
+ const float * qw = quant_weights + QK_K*ibl + block_size*ib;
+ for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);
+ } else {
+ for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i];
+ }
+ float max = fabsf(xb[0]);
+ for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]));
+ if (max < GROUP_MAX_EPS_IQ1_M) {
+ scales[ib] = 0;
+ memset(L, 1, block_size);
+ continue;
+ }
+ // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem.
+ // With just 3 allowed quant values (-1, 0, 1), we can search exhaustively for the two
+ // boundaries that split the weights xb[i] into 3 groups. To do so, we sort the weights
+ // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and
+ // Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale
+ // for each possible and score for each split.
+ for (int j = 0; j < block_size; ++j) {
+ pairs[2*j] = xb[j];
+ idx[2*j] = j;
+ }
+ qsort(pairs, block_size, 2*sizeof(float), iq1_sort_helper);
+ float best_score = -FLT_MIN, scale = max;
+ int besti1 = -1, besti2 = -1, best_k = -1;
+ // 0: +, +
+ // 1: +, -
+ // 2: -, +
+ // 3: -, -
+ for (int i1 = 0; i1 <= block_size; ++i1) {
+ for (int i2 = i1; i2 <= block_size; ++i2) {
+ memset(sumqx, 0, 4*sizeof(float));
+ memset(sumq2, 0, 4*sizeof(float));
+ for (int j = 0; j < i1; ++j) {
+ int i = idx[2*j];
+ if (i < block_size/2) {
+ sumqx[0] += weight[i]*x_p[0]*xb[i];
+ sumqx[1] += weight[i]*x_p[0]*xb[i];
+ sumqx[2] += weight[i]*x_m[0]*xb[i];
+ sumqx[3] += weight[i]*x_m[0]*xb[i];
+ sumq2[0] += weight[i]*x_p[0]*x_p[0];
+ sumq2[1] += weight[i]*x_p[0]*x_p[0];
+ sumq2[2] += weight[i]*x_m[0]*x_m[0];
+ sumq2[3] += weight[i]*x_m[0]*x_m[0];
+ } else {
+ sumqx[0] += weight[i]*x_p[0]*xb[i];
+ sumqx[2] += weight[i]*x_p[0]*xb[i];
+ sumqx[1] += weight[i]*x_m[0]*xb[i];
+ sumqx[3] += weight[i]*x_m[0]*xb[i];
+ sumq2[0] += weight[i]*x_p[0]*x_p[0];
+ sumq2[2] += weight[i]*x_p[0]*x_p[0];
+ sumq2[1] += weight[i]*x_m[0]*x_m[0];
+ sumq2[3] += weight[i]*x_m[0]*x_m[0];
+ }
+ }
+ for (int j = i1; j < i2; ++j) {
+ int i = idx[2*j];
+ if (i < block_size/2) {
+ sumqx[0] += weight[i]*x_p[1]*xb[i];
+ sumqx[1] += weight[i]*x_p[1]*xb[i];
+ sumqx[2] += weight[i]*x_m[1]*xb[i];
+ sumqx[3] += weight[i]*x_m[1]*xb[i];
+ sumq2[0] += weight[i]*x_p[1]*x_p[1];
+ sumq2[1] += weight[i]*x_p[1]*x_p[1];
+ sumq2[2] += weight[i]*x_m[1]*x_m[1];
+ sumq2[3] += weight[i]*x_m[1]*x_m[1];
+ } else {
+ sumqx[0] += weight[i]*x_p[1]*xb[i];
+ sumqx[2] += weight[i]*x_p[1]*xb[i];
+ sumqx[1] += weight[i]*x_m[1]*xb[i];
+ sumqx[3] += weight[i]*x_m[1]*xb[i];
+ sumq2[0] += weight[i]*x_p[1]*x_p[1];
+ sumq2[2] += weight[i]*x_p[1]*x_p[1];
+ sumq2[1] += weight[i]*x_m[1]*x_m[1];
+ sumq2[3] += weight[i]*x_m[1]*x_m[1];
+ }
+ }
+ for (int j = i2; j < block_size; ++j) {
+ int i = idx[2*j];
+ if (i < block_size/2) {
+ sumqx[0] += weight[i]*x_p[2]*xb[i];
+ sumqx[1] += weight[i]*x_p[2]*xb[i];
+ sumqx[2] += weight[i]*x_m[2]*xb[i];
+ sumqx[3] += weight[i]*x_m[2]*xb[i];
+ sumq2[0] += weight[i]*x_p[2]*x_p[2];
+ sumq2[1] += weight[i]*x_p[2]*x_p[2];
+ sumq2[2] += weight[i]*x_m[2]*x_m[2];
+ sumq2[3] += weight[i]*x_m[2]*x_m[2];
+ } else {
+ sumqx[0] += weight[i]*x_p[2]*xb[i];
+ sumqx[2] += weight[i]*x_p[2]*xb[i];
+ sumqx[1] += weight[i]*x_m[2]*xb[i];
+ sumqx[3] += weight[i]*x_m[2]*xb[i];
+ sumq2[0] += weight[i]*x_p[2]*x_p[2];
+ sumq2[2] += weight[i]*x_p[2]*x_p[2];
+ sumq2[1] += weight[i]*x_m[2]*x_m[2];
+ sumq2[3] += weight[i]*x_m[2]*x_m[2];
+ }
+ }
+ for (int k = 0; k < 4; ++k) {
+ if (sumq2[k] > 0 && sumqx[k]*sumqx[k] > best_score*sumq2[k]) {
+ scale = sumqx[k]/sumq2[k]; best_score = scale*sumqx[k];
+ besti1 = i1; besti2 = i2; best_k = k;
+ }
+ }
+ }
+ }
+ GGML_ASSERT(besti1 >= 0 && besti2 >= 0 && best_k >= 0);
+ for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0;
+ for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1;
+ for (int j = besti2; j < block_size; ++j) L[idx[2*j]] = 2;
+ if (scale < 0) {
+ for (int j = 0; j < block_size; ++j) L[j] = 2 - L[j];
+ scale = -scale;
+ best_k = best_k == 0 ? 3 : best_k == 1 ? 2 : best_k == 2 ? 1 : 0;
+ }
+ bool all_on_grid = true;
+ for (int k = 0; k < block_size/8; ++k) {
+ if (k == 0) xx = best_k < 2 ? x_p : x_m;
+ else xx = best_k%2 == 0 ? x_p : x_m;
+ uint16_t u = 0;
+ for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j);
+ int grid_index = kmap_q2xs[u];
+ if (grid_index < 0) {
+ all_on_grid = false;
+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
+ grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, xx, L + 8*k, NGRID_IQ1S);
+ GGML_ASSERT(grid_index >= 0);
+ }
+ index[k] = grid_index;
+ }
+ if (!all_on_grid) {
+ float sumqx_f = 0, sumq2_f = 0;
+ for (int k = 0; k < block_size/8; ++k) {
+ if (k == 0) xx = best_k < 2 ? x_p : x_m;
+ else xx = best_k%2 == 0 ? x_p : x_m;
+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + index[k]);
+ for (int j = 0; j < 8; ++j) {
+ float w = weight[8*k + j];
+ float q = xx[(pg[j] - 1)/2];
+ sumqx_f += w*q*xb[8*k+j];
+ sumq2_f += w*q*q;
+ }
+ }
+ if (sumqx_f > 0 && sumq2_f > 0) scale = sumqx_f/sumq2_f;
+ }
+ y[ibl].qs[2*ib + 0] = index[0] & 255;
+ y[ibl].qs[2*ib + 1] = index[1] & 255;
+ y[ibl].qh[ib] = (index[0] >> 8) | ((index[1] >> 8) << 4);
+ GGML_ASSERT(scale >= 0);
+ scales[ib] = scale;
+ shifts[ib] = best_k;
+ max_scale = MAX(max_scale, scale);
+ }
+
+ if (!max_scale) {
+ continue;
+ }
+
+ uint16_t * sc = (uint16_t *)y[ibl].scales;
+ float d = max_scale/15;
+ float id = 1/d;
+ float sumqx_f = 0, sumq2_f = 0;
+ for (int ib = 0; ib < QK_K/block_size; ++ib) {
+ int l = nearest_int(0.5f*(id*scales[ib+0]-1));
+ l = MAX(0, MIN(7, l));
+ sc[ib/4] |= (l << 3*(ib%4));
+ y[ibl].qh[ib] |= masks[shifts[ib]];
+ const float * xb = xbl + block_size*ib;
+ if (quant_weights) {
+ const float * qw = quant_weights + QK_K*ibl + block_size*ib;
+ for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);
+ } else {
+ for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i];
+ }
+ for (int k = 0; k < block_size/8; ++k) {
+ if (k == 0) xx = shifts[ib] < 2 ? x_p : x_m;
+ else xx = shifts[ib]%2 == 0 ? x_p : x_m;
+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + y[ibl].qs[2*ib+k] + ((y[ibl].qh[ib] << (8 - 4*k)) & 0x700));
+ for (int j = 0; j < 8; ++j) {
+ float w = weight[8*k + j];
+ float q = xx[(pg[j] - 1)/2]*(2*l+1);
+ sumqx_f += w*q*xb[8*k+j];
+ sumq2_f += w*q*q;
+ }
+ }
+ }
+ if (sumq2_f > 0) d = sumqx_f/sumq2_f;
+ s.f16 = GGML_FP32_TO_FP16(d*1.1125f); // 1.1125f is another fudge factor. Don't ask me why it is needed.
+ sc[0] |= ((s.u16 & 0x000f) << 12);
+ sc[1] |= ((s.u16 & 0x00f0) << 8);
+ sc[2] |= ((s.u16 & 0x0f00) << 4);
+ sc[3] |= ((s.u16 & 0xf000) << 0);
+ }
+}
+
+size_t quantize_iq1_m(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ GGML_ASSERT(n_per_row%QK_K == 0);
+ float scales[QK_K/IQ1M_BLOCK_SIZE];
+ float weight[IQ1M_BLOCK_SIZE];
+ int8_t L[IQ1M_BLOCK_SIZE];
+ float pairs[2*IQ1M_BLOCK_SIZE];
+ uint16_t index[IQ1M_BLOCK_SIZE/8];
+ int8_t shifts[QK_K/IQ1M_BLOCK_SIZE];
+ int64_t nblock = n_per_row/QK_K;
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_iq1_m_impl(src, qrow, n_per_row, quant_weights, scales, weight, pairs, L, index, shifts);
+ src += n_per_row;
+ qrow += nblock*sizeof(block_iq1_m);
+ }
+ return nrow * nblock * sizeof(block_iq1_m);
+}
+
+// ============================ 4-bit non-linear quants
+
+static inline int best_index_int8(int n, const int8_t * val, float x) {
+ if (x <= val[0]) return 0;
+ if (x >= val[n-1]) return n-1;
+ int ml = 0, mu = n-1;
+ while (mu-ml > 1) {
+ int mav = (ml+mu)/2;
+ if (x < val[mav]) mu = mav; else ml = mav;
+ }
+ return x - val[mu-1] < val[mu] - x ? mu-1 : mu;
+}
+
+static void quantize_row_iq4_nl_impl(const int super_block_size, const int block_size, const float * restrict x,
+ ggml_fp16_t * dh, uint8_t * q4, uint16_t * scales_h, uint8_t * scales_l,
+ float * scales, float * weight, uint8_t * L,
+ const int8_t * values,
+ const float * quant_weights,
+ const int ntry) {
+
+ float sigma2 = 0;
+ for (int j = 0; j < super_block_size; ++j) sigma2 += x[j]*x[j];
+ sigma2 *= 2.f/super_block_size;
+
+ memset(q4, 0, super_block_size/2);
+ dh[0] = GGML_FP32_TO_FP16(0.f);
+
+ float max_scale = 0, amax_scale = 0;
+ for (int ib = 0; ib < super_block_size/block_size; ++ib) {
+ const float * xb = x + ib*block_size;
+ uint8_t * Lb = L + ib*block_size;
+ if (quant_weights) {
+ const float * qw = quant_weights + ib*block_size;
+ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
+ } else {
+ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j];
+ }
+ float amax = 0, max = 0;
+ for (int j = 0; j < block_size; ++j) {
+ float ax = fabsf(xb[j]);
+ if (ax > amax) {
+ amax = ax; max = xb[j];
+ }
+ }
+ if (amax < GROUP_MAX_EPS) {
+ scales[ib] = 0;
+ continue;
+ }
+ float d = ntry > 0 ? -max/values[0] : max/values[0];
+ float id = 1/d;
+ float sumqx = 0, sumq2 = 0;
+ for (int j = 0; j < block_size; ++j) {
+ float al = id*xb[j];
+ int l = best_index_int8(16, values, al);
+ Lb[j] = l;
+ float q = values[l];
+ float w = weight[j];
+ sumqx += w*q*xb[j];
+ sumq2 += w*q*q;
+ }
+ d = sumqx/sumq2;
+ float best = d*sumqx;
+ for (int itry = -ntry; itry <= ntry; ++itry) {
+ id = (itry + values[0])/max;
+ sumqx = sumq2 = 0;
+ for (int j = 0; j < block_size; ++j) {
+ float al = id*xb[j];
+ int l = best_index_int8(16, values, al);
+ float q = values[l];
+ float w = weight[j];
+ sumqx += w*q*xb[j];
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
+ d = sumqx/sumq2; best = d * sumqx;
+ }
+ }
+ scales[ib] = d;
+ float abs_d = fabsf(d);
+ if (abs_d > amax_scale) {
+ amax_scale = abs_d; max_scale = d;
+ }
+ }
+
+ if (super_block_size/block_size > 1) {
+ int nb = super_block_size/block_size;
+ memset(scales_h, 0, ((nb+7)/8)*sizeof(uint16_t));
+ float d = -max_scale/32;
+ dh[0] = GGML_FP32_TO_FP16(d);
+ float id = d ? 1/d : 0.f;
+ for (int ib = 0; ib < super_block_size/block_size; ++ib) {
+ int l = nearest_int(id*scales[ib]);
+ l = MAX(-32, MIN(31, l));
+ float dl = d * l;
+ float idl = dl ? 1/dl : 0.f;
+ uint8_t * Lb = L + ib*block_size;
+ const float * xb = x + ib*block_size;
+ for (int j = 0; j < block_size; ++j) {
+ Lb[j] = best_index_int8(16, values, idl*xb[j]);
+ }
+ l += 32;
+ uint8_t l_l = l & 0xf;
+ uint8_t l_h = l >> 4;
+ if (ib%2 == 0) scales_l[ib/2] = l_l;
+ else scales_l[ib/2] |= (l_l << 4);
+ scales_h[ib/8] |= (l_h << 2*(ib%8));
+ }
+ } else {
+ dh[0] = GGML_FP32_TO_FP16(scales[0]);
+ if (ntry > 0) {
+ float id = scales[0] ? 1/scales[0] : 0;
+ for (int j = 0; j < super_block_size; ++j) {
+ L[j] = best_index_int8(16, values, id*x[j]);
+ }
+ }
+ }
+
+ for (int i = 0; i < super_block_size/32; ++i) {
+ for (int j = 0; j < 16; ++j) {
+ q4[16*i + j] = L[32*i + j] | (L[32*i + 16 + j] << 4);
+ }
+ }
+}
+
+size_t quantize_iq4_nl(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ GGML_ASSERT(n_per_row%QK4_NL == 0);
+ int64_t nblock = n_per_row/QK4_NL;
+ char * qrow = (char *)dst;
+ uint8_t L[QK4_NL];
+ float weight[QK4_NL];
+ uint16_t unused_h;
+ uint8_t * unused_l = NULL;
+ float scale;
+ for (int64_t row = 0; row < nrow; ++row) {
+ block_iq4_nl * iq4 = (block_iq4_nl *)qrow;
+ for (int ibl = 0; ibl < nblock; ++ibl) {
+ const float * qw = quant_weights ? quant_weights + QK4_NL*ibl : NULL;
+ quantize_row_iq4_nl_impl(QK4_NL, 32, src + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l,
+ &scale, weight, L, kvalues_iq4nl, qw, 7);
+ }
+ src += n_per_row;
+ qrow += nblock*sizeof(block_iq4_nl);
+ }
+ return nrow * nblock * sizeof(block_iq4_nl);
+}
+
+void quantize_row_iq4_nl(const float * restrict x, void * restrict vy, int64_t k) {
+ GGML_ASSERT(k%QK4_NL == 0);
+ int64_t nblock = k/QK4_NL;
+ uint8_t L[QK4_NL];
+ float weight[QK4_NL];
+ uint16_t unused_h;
+ uint8_t * unused_l = NULL;
+ float scale;
+ block_iq4_nl * iq4 = (block_iq4_nl *)vy;
+ for (int ibl = 0; ibl < nblock; ++ibl) {
+ quantize_row_iq4_nl_impl(QK4_NL, 32, x + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l,
+ &scale, weight, L, kvalues_iq4nl, NULL, -1);
+ }
+}
+
+void quantize_row_iq4_nl_ref(const float * restrict x, block_iq4_nl * restrict y, int64_t k) {
+ assert(k % QK4_NL == 0);
+ quantize_row_iq4_nl(x, y, k);
+}
+
+size_t quantize_iq4_xs(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ GGML_ASSERT(n_per_row%QK_K == 0);
+ int64_t nblock = n_per_row/QK_K;
+ char * qrow = (char *)dst;
+ uint8_t L[QK_K];
+ float weight[32];
+ float scales[QK_K/32];
+ for (int64_t row = 0; row < nrow; ++row) {
+ block_iq4_xs * iq4 = (block_iq4_xs *)qrow;
+ for (int ibl = 0; ibl < nblock; ++ibl) {
+ const float * qw = quant_weights ? quant_weights + QK_K*ibl : NULL;
+ quantize_row_iq4_nl_impl(QK_K, 32, src + QK_K*ibl, &iq4[ibl].d, iq4[ibl].qs, &iq4[ibl].scales_h, iq4[ibl].scales_l,
+ scales, weight, L, kvalues_iq4nl, qw, 7);
+ }
+ src += n_per_row;
+ qrow += nblock*sizeof(block_iq4_xs);
+ }
+ return nrow * nblock * sizeof(block_iq4_xs);
+}
+
+void quantize_row_iq4_xs(const float * restrict x, void * restrict vy, int64_t k) {
+ assert(k % QK_K == 0);
+ block_iq4_xs * restrict y = vy;
+ quantize_row_iq4_xs_ref(x, y, k);
+}
+
+void quantize_row_iq4_xs_ref(const float * restrict x, block_iq4_xs * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ quantize_iq4_xs(x, y, 1, k, NULL);
+}
+
+// =============================== 2.5625 bpw
+
+static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy, int64_t n, const float * restrict quant_weights) {
+
+ const int gindex = iq2_data_index(GGML_TYPE_IQ2_S);
+
+ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;
+ const int * kmap_q2xs = iq2_data[gindex].map;
+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;
+
+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");
+ GGML_ASSERT(n%QK_K == 0);
+
+ const int kMaxQ = 3;
+
+ const int64_t nbl = n/QK_K;
+
+ block_iq2_s * y = vy;
+
+ float scales[QK_K/16];
+ float weight[16];
+ float xval[16];
+ int8_t L[16];
+ int8_t Laux[16];
+ float waux[16];
+ bool is_on_grid[2];
+ bool is_on_grid_aux[2];
+ uint8_t block_signs[2];
+
+ for (int ibl = 0; ibl < nbl; ++ibl) {
+
+ memset(&y[ibl], 0, sizeof(block_iq2_s));
+ y[ibl].d = GGML_FP32_TO_FP16(0.f);
+
+ float max_scale = 0;
+
+ const float * xbl = x + QK_K*ibl;
+ float sumx2 = 0;
+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];
+ float sigma2 = 2*sumx2/QK_K;
+
+ for (int ib = 0; ib < QK_K/16; ++ib) {
+ const float * xb = xbl + 16*ib;
+ if (quant_weights) {
+ const float * qw = quant_weights + QK_K*ibl + 16*ib;
+ for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);
+ } else {
+ for (int i = 0; i < 16; ++i) weight[i] = 0.25f*sigma2 + xb[i]*xb[i];
+ }
+ for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]);
+ for (int k = 0; k < 2; ++k) {
+ uint8_t s = 0;
+ for (int i = 0; i < 8; ++i) {
+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];
+ else {
+ xval[8*k + i] = -xb[8*k + i]; s |= (1 << i);
+ }
+ }
+ block_signs[k] = s;
+ }
+ float max = xval[0];
+ for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);
+ if (max < GROUP_MAX_EPS_IQ2_S) {
+ scales[ib] = 0;
+ continue;
+ }
+ float best = 0;
+ float scale = max/(2*kMaxQ-1);
+ is_on_grid[0] = is_on_grid[1] = true;
+ for (int is = -9; is <= 9; ++is) {
+ float id = (2*kMaxQ-1+is*0.1f)/max;
+ float this_scale = 1/id;
+ for (int k = 0; k < 2; ++k) {
+ for (int i = 0; i < 8; ++i) {
+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));
+ Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));
+ }
+ uint16_t u = 0;
+ for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);
+ int grid_index = kmap_q2xs[u];
+ is_on_grid_aux[k] = true;
+ if (grid_index < 0) {
+ is_on_grid_aux[k] = false;
+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
+ }
+ }
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < 16; ++i) {
+ float w = weight[i];
+ float q = 2*Laux[i] + 1;
+ sumqx += w*xval[i]*q;
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
+ scale = sumqx/sumq2; best = scale*sumqx;
+ for (int i = 0; i < 16; ++i) L[i] = Laux[i];
+ for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k];
+ }
+ }
+ int n_not_ongrid = 0;
+ for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid;
+ if (n_not_ongrid > 0 && scale > 0) {
+ float id = 1/scale;
+ for (int k = 0; k < 2; ++k) {
+ if (is_on_grid[k]) continue;
+ uint16_t u = 0;
+ for (int i = 0; i < 8; ++i) {
+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));
+ l = MAX(0, MIN(kMaxQ-1, l));
+ u |= (l << 2*i);
+ L[8*k + i] = l;
+ }
+ int grid_index = kmap_q2xs[u];
+ if (grid_index < 0) {
+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
+ }
+ }
+ float sumqx = 0, sumq2 = 0;
+ for (int i = 0; i < 16; ++i) {
+ float w = weight[i];
+ float q = 2*L[i] + 1;
+ sumqx += w*xval[i]*q;
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0) scale = sumqx/sumq2;
+ }
+ if (scale < 0) {
+ scale = -scale;
+ for (int k = 0; k < 2; ++k) block_signs[k] = ~block_signs[k];
+ }
+ for (int k = 0; k < 2; ++k) {
+ uint16_t u = 0;
+ for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);
+ int grid_index = kmap_q2xs[u];
+ if (grid_index < 0) {
+ printf("Oops: found point %u not on grid:", u);
+ for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);
+ printf("\n");
+ GGML_ABORT("fatal error");
+ }
+ const int i8 = 2*ib + k;
+ y[ibl].qs[i8] = grid_index & 255;
+ y[ibl].qh[i8/4] |= ((grid_index >> 8) << 2*(i8%4));
+ y[ibl].qs[QK_K/8 + i8] = block_signs[k];
+ }
+ GGML_ASSERT(scale >= 0);
+ scales[ib] = scale;
+ max_scale = MAX(max_scale, scale);
+ }
+
+ if (!max_scale) {
+ continue;
+ }
+
+ float d = max_scale/31;
+ y[ibl].d = GGML_FP32_TO_FP16(d * 0.9875f);
+ float id = 1/d;
+ for (int ib = 0; ib < QK_K/16; ++ib) {
+ int l = nearest_int(0.5f*(id*scales[ib]-1));
+ l = MAX(0, MIN(15, l));
+ if (ib%2 == 0) y[ibl].scales[ib/2] = l;
+ else y[ibl].scales[ib/2] |= (l << 4);
+ }
+ }
+}
+
+size_t quantize_iq2_s(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+ GGML_ASSERT(n_per_row%QK_K == 0);
+ int64_t nblock = n_per_row/QK_K;
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrow; ++row) {
+ quantize_row_iq2_s_impl(src, qrow, n_per_row, quant_weights);
+ src += n_per_row;
+ qrow += nblock*sizeof(block_iq2_s);
+ }
+ return nrow * nblock * sizeof(block_iq2_s);
+}
+
+void quantize_row_iq2_s_ref(const float * restrict x, block_iq2_s * restrict y, int64_t k) {
+ assert(k % QK_K == 0);
+ quantize_iq2_s(x, y, 1, k, NULL);
+}
+
+void quantize_row_iq2_s(const float * restrict x, void * restrict vy, int64_t k) {
+ assert(k % QK_K == 0);
+ block_iq2_s * restrict y = vy;
+ quantize_row_iq2_s_ref(x, y, k);
+}
+
+static bool validate_float(float f, size_t i) {
+ if (isinf(f)) {
+ fprintf(stderr, "ggml_validate_row_data: found inf value at block %zu\n", i);
+ return false;
+ }
+
+ if (isnan(f)) {
+ fprintf(stderr, "ggml_validate_row_data: found nan value at block %zu\n", i);
+ return false;
+ }
+
+ return true;
+}
+
+static bool isinf_fp16(ggml_fp16_t f) {
+ return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) == 0;
+}
+
+static bool isnan_fp16(ggml_fp16_t f) {
+ return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) != 0;
+}
+
+static bool validate_fp16(ggml_fp16_t f, size_t i) {
+ if (isinf_fp16(f)) {
+ fprintf(stderr, "ggml_validate_row_data: found inf value at block %zu\n", i);
+ return false;
+ }
+
+ if (isnan_fp16(f)) {
+ fprintf(stderr, "ggml_validate_row_data: found nan value at block %zu\n", i);
+ return false;
+ }
+
+ return true;
+}
+
+#define VALIDATE_ROW_DATA_D_F16_IMPL(type, data, nb) \
+ const type * q = (const type *) (data); \
+ for (size_t i = 0; i < (nb); ++i) { \
+ if (!validate_fp16(q[i].d, i)) { \
+ return false; \
+ } \
+ }
+
+#define VALIDATE_ROW_DATA_DM_F16_IMPL(type, data, nb, d, m) \
+ const type * q = (const type *) (data); \
+ for (size_t i = 0; i < (nb); ++i) { \
+ if (!validate_fp16(q[i].d, i) || !validate_fp16(q[i].m, i)) { \
+ return false; \
+ } \
+ }
+
+#define VALIDATE_ROW_DATA_DVEC_F16_IMPL(type, data, nb, nr) \
+ const type * q = (const type *) (data); \
+ for (size_t i = 0; i < (nb); ++i) { \
+ for (size_t j = 0; j < (nr); ++j) { \
+ if (!validate_fp16(q[i].d[j], i)) { \
+ return false; \
+ } \
+ } \
+ }
+
+bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes) {
+ if (type < 0 || type >= GGML_TYPE_COUNT) {
+ fprintf(stderr, "%s: invalid type %d\n", __func__, type);
+ return false;
+ }
+
+ if (nbytes % ggml_type_size(type) != 0) {
+ fprintf(stderr, "%s: invalid size %zu for type %s (type size = %zu)\n", __func__, nbytes, ggml_type_name(type), ggml_type_size(type));
+ return false;
+ }
+
+ const size_t nb = nbytes/ggml_type_size(type);
+
+ switch (type) {
+ case GGML_TYPE_BF16:
+ {
+ int nans = 0;
+ int infs = 0;
+ const unsigned short * f = (const unsigned short *) data;
+ for (size_t i = 0; i < nb; ++i) {
+ nans += (f[i] & 0x7fff) > 0x7f80;
+ infs += (f[i] & 0x7fff) == 0x7f80;
+ }
+ if (nans) {
+ fprintf(stderr, "%s: found %d NaNs in row of %zu BF16 values\n", __func__, nans, nb);
+ return false;
+ }
+ if (infs) {
+ fprintf(stderr, "%s: found %d infinities in row of %zu BF16 values\n", __func__, infs, nb);
+ return false;
+ }
+ } break;
+ case GGML_TYPE_F16:
+ {
+ const ggml_fp16_t * f = (const ggml_fp16_t *) data;
+ size_t i = 0;
+#if defined(__AVX2__)
+ for (; i + 15 < nb; i += 16) {
+ __m256i v = _mm256_loadu_si256((const __m256i *)(f + i));
+ __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi16(0x7c00));
+ __m256i cmp = _mm256_cmpeq_epi16(vexp, _mm256_set1_epi16(0x7c00));
+ int mask = _mm256_movemask_epi8(cmp);
+ if (mask) {
+ for (size_t j = 0; j < 16; ++j) {
+ if (!validate_fp16(f[i + j], i + j)) {
+ return false;
+ }
+ }
+ GGML_UNREACHABLE();
+ }
+ }
+#elif defined(__ARM_NEON)
+ for (; i + 7 < nb; i += 8) {
+ uint16x8_t v = vld1q_u16(f + i);
+ uint16x8_t vexp = vandq_u16(v, vdupq_n_u16(0x7c00));
+ uint16x8_t cmp = vceqq_u16(vexp, vdupq_n_u16(0x7c00));
+ uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(cmp, 4)), 0);
+ if (mask) {
+ for (size_t j = 0; j < 8; ++j) {
+ if (!validate_fp16(f[i + j], i + j)) {
+ return false;
+ }
+ }
+ GGML_UNREACHABLE();
+ }
+ }
+#endif
+ for (; i < nb; ++i) {
+ if (!validate_fp16(f[i], i)) {
+ return false;
+ }
+ }
+ } break;
+ case GGML_TYPE_F32:
+ {
+ const float * f = (const float *) data;
+ size_t i = 0;
+#if defined(__AVX2__)
+ for (; i + 7 < nb; i += 8) {
+ __m256i v = _mm256_loadu_si256((const __m256i *)(f + i));
+ __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi32(0x7f800000));
+ __m256i cmp = _mm256_cmpeq_epi32(vexp, _mm256_set1_epi32(0x7f800000));
+ int mask = _mm256_movemask_epi8(cmp);
+ if (mask) {
+ for (size_t j = 0; j < 8; ++j) {
+ if (!validate_float(f[i + j], i + j)) {
+ return false;
+ }
+ }
+ GGML_UNREACHABLE();
+ }
+ }
+#elif defined(__ARM_NEON)
+ for (; i + 3 < nb; i += 4) {
+ uint32x4_t v = vld1q_u32((const uint32_t *)f + i);
+ uint32x4_t vexp = vandq_u32(v, vdupq_n_u32(0x7f800000));
+ uint32x4_t cmp = vceqq_u32(vexp, vdupq_n_u32(0x7f800000));
+ uint64_t mask = vget_lane_u64(vreinterpret_u64_u16(vshrn_n_u32(cmp, 8)), 0);
+ if (mask) {
+ for (size_t j = 0; j < 4; ++j) {
+ if (!validate_float(f[i + j], i + j)) {
+ return false;
+ }
+ }
+ GGML_UNREACHABLE();
+ }
+ }
+#endif
+ for (; i < nb; ++i) {
+ if (!validate_float(f[i], i)) {
+ return false;
+ }
+ }
+ } break;
+ case GGML_TYPE_F64:
+ {
+ const double * f = (const double *) data;
+ for (size_t i = 0; i < nb; ++i) {
+ if (!validate_float(f[i], i)) {
+ return false;
+ }
+ }
+ } break;
+ case GGML_TYPE_Q4_0:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_0, data, nb);
+ } break;
+ case GGML_TYPE_Q4_1:
+ {
+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_1, data, nb, d, m);
+ } break;
+ case GGML_TYPE_Q5_0:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q5_0, data, nb);
+ } break;
+ case GGML_TYPE_Q5_1:
+ {
+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_1, data, nb, d, m);
+ } break;
+ case GGML_TYPE_Q8_0:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q8_0, data, nb);
+ } break;
+ case GGML_TYPE_Q2_K:
+ {
+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q2_K, data, nb, d, dmin);
+ } break;
+ case GGML_TYPE_Q3_K:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q3_K, data, nb);
+ } break;
+ case GGML_TYPE_Q4_K:
+ {
+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_K, data, nb, d, dmin);
+ } break;
+ case GGML_TYPE_Q5_K:
+ {
+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_K, data, nb, d, dmin);
+ } break;
+ case GGML_TYPE_Q6_K:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q6_K, data, nb);
+ } break;
+ case GGML_TYPE_Q8_K:
+ {
+ const block_q8_K * q = (const block_q8_K *) data;
+ for (size_t i = 0; i < nb; ++i) {
+ if (!validate_float(q[i].d, i)) {
+ return false;
+ }
+ }
+ } break;
+ case GGML_TYPE_IQ1_S:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_s, data, nb);
+ } break;
+ case GGML_TYPE_IQ1_M:
+ {
+ const block_iq1_m * q = (const block_iq1_m *) data;
+ for (size_t i = 0; i < nb; ++i) {
+ iq1m_scale_t scale;
+ const uint16_t * sc = (const uint16_t *)q[i].scales;
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+ if (!validate_fp16(scale.f16, i)) {
+ return false;
+ }
+ }
+ } break;
+ case GGML_TYPE_IQ2_XXS:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xxs, data, nb);
+ } break;
+ case GGML_TYPE_IQ2_XS:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xs, data, nb);
+ } break;
+ case GGML_TYPE_IQ2_S:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_s, data, nb);
+ } break;
+ case GGML_TYPE_IQ3_XXS:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_xxs, data, nb);
+ } break;
+
+ case GGML_TYPE_IQ3_S:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_s, data, nb);
+ } break;
+ case GGML_TYPE_IQ4_XS:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_xs, data, nb);
+ } break;
+ case GGML_TYPE_IQ4_NL:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_nl, data, nb);
+ } break;
+ case GGML_TYPE_Q4_0_4_4:
+ case GGML_TYPE_Q4_0_4_8:
+ {
+ VALIDATE_ROW_DATA_DVEC_F16_IMPL(block_q4_0x4, data, nbytes / sizeof(block_q4_0x4), 4);
+ } break;
+ case GGML_TYPE_Q4_0_8_8:
+ {
+ VALIDATE_ROW_DATA_DVEC_F16_IMPL(block_q4_0x8, data, nbytes / sizeof(block_q4_0x8), 8);
+ } break;
+
+ case GGML_TYPE_I8:
+ case GGML_TYPE_I16:
+ case GGML_TYPE_I32:
+ case GGML_TYPE_I64:
+ // nothing to validate
+ break;
+ default:
+ {
+ fprintf(stderr, "%s: invalid type %d\n", __func__, type);
+ return false;
+ }
+ }
+
+ return true;
+}
diff --git a/llama/ggml-quants.h b/llama/ggml-quants.h
new file mode 100644
index 00000000..fa25ebab
--- /dev/null
+++ b/llama/ggml-quants.h
@@ -0,0 +1,162 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#define GGML_COMMON_DECL_C
+#include "ggml-common.h"
+
+#include "ggml.h"
+
+// GGML internal header
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Quantization
+void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
+void quantize_row_q4_1_ref(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_1_ref(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_0_ref(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_1_ref(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
+
+void quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
+void quantize_row_q3_K_ref(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
+void quantize_row_q4_K_ref(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_K_ref(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
+void quantize_row_q6_K_ref(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_K_ref(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
+
+void quantize_row_iq3_xxs_ref(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq4_nl_ref (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq4_xs_ref (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq3_s_ref (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
+
+void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq3_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq2_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+// Dequantization
+void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+//void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq1_m (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+// Dot product
+void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")
+size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq2_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq1_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq1_m (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+void iq2xs_init_impl(enum ggml_type type);
+void iq2xs_free_impl(enum ggml_type type);
+void iq3xs_init_impl(int grid_size);
+void iq3xs_free_impl(int grid_size);
+
+#if defined(__ARM_FEATURE_SVE)
+extern int ggml_sve_cnt_b;
+#endif
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama/ggml.c b/llama/ggml.c
new file mode 100644
index 00000000..0a64e875
--- /dev/null
+++ b/llama/ggml.c
@@ -0,0 +1,23365 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define _CRT_SECURE_NO_DEPRECATE // Disables ridiculous "unsafe" warnings on Windows
+#define _USE_MATH_DEFINES // For M_PI on MSVC
+
+#include "ggml-impl.h"
+#include "ggml-quants.h"
+#include "ggml.h"
+#include "ggml-aarch64.h"
+
+#if defined(_MSC_VER) || defined(__MINGW32__)
+#include <malloc.h> // using malloc.h with MSC/MINGW
+#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
+#include <alloca.h>
+#endif
+
+#include <assert.h>
+#include <errno.h>
+#include <time.h>
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <float.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <signal.h>
+#if defined(__gnu_linux__)
+#include <syscall.h>
+#endif
+
+#ifdef GGML_USE_OPENMP
+#include <omp.h>
+#endif
+
+#ifdef GGML_USE_METAL
+#include <unistd.h>
+#endif
+
+#if defined(__ARM_FEATURE_SVE)
+int ggml_sve_cnt_b = 0;
+#endif
+#if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_MATMUL_INT8)
+#undef GGML_USE_LLAMAFILE
+#endif
+
+#ifdef GGML_USE_LLAMAFILE
+#include <llamafile/sgemm.h>
+#endif
+
+#if defined(_MSC_VER)
+// disable "possible loss of data" to avoid hundreds of casts
+// we should just be careful :)
+#pragma warning(disable: 4244 4267)
+
+// disable POSIX deprecation warnings
+// these functions are never going away, anyway
+#pragma warning(disable: 4996)
+
+// unreachable code because of multiple instances of code after GGML_ABORT
+#pragma warning(disable: 4702)
+#endif
+
+#if defined(_WIN32)
+
+#define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+ #define NOMINMAX
+#endif
+#include <windows.h>
+
+#if !defined(__clang__)
+typedef volatile LONG atomic_int;
+typedef atomic_int atomic_bool;
+typedef atomic_int atomic_flag;
+
+#define ATOMIC_FLAG_INIT 0
+
+typedef enum {
+ memory_order_relaxed,
+ memory_order_consume,
+ memory_order_acquire,
+ memory_order_release,
+ memory_order_acq_rel,
+ memory_order_seq_cst
+} memory_order;
+
+static void atomic_store(atomic_int * ptr, LONG val) {
+ InterlockedExchange(ptr, val);
+}
+static void atomic_store_explicit(atomic_int * ptr, LONG val, memory_order mo) {
+ // TODO: add support for explicit memory order
+ InterlockedExchange(ptr, val);
+}
+static LONG atomic_load(atomic_int * ptr) {
+ return InterlockedCompareExchange(ptr, 0, 0);
+}
+static LONG atomic_load_explicit(atomic_int * ptr, memory_order mo) {
+ // TODO: add support for explicit memory order
+ return InterlockedCompareExchange(ptr, 0, 0);
+}
+static LONG atomic_fetch_add(atomic_int * ptr, LONG inc) {
+ return InterlockedExchangeAdd(ptr, inc);
+}
+static LONG atomic_fetch_add_explicit(atomic_int * ptr, LONG inc, memory_order mo) {
+ // TODO: add support for explicit memory order
+ return InterlockedExchangeAdd(ptr, inc);
+}
+static atomic_bool atomic_flag_test_and_set(atomic_flag * ptr) {
+ return InterlockedExchange(ptr, 1);
+}
+static void atomic_flag_clear(atomic_flag * ptr) {
+ InterlockedExchange(ptr, 0);
+}
+#else // clang
+#include <stdatomic.h>
+#endif
+
+typedef HANDLE pthread_t;
+
+typedef DWORD thread_ret_t;
+static int pthread_create(pthread_t * out, void * unused, thread_ret_t(*func)(void *), void * arg) {
+ (void) unused;
+ HANDLE handle = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) func, arg, 0, NULL);
+ if (handle == NULL)
+ {
+ return EAGAIN;
+ }
+
+ *out = handle;
+ return 0;
+}
+
+static int pthread_join(pthread_t thread, void * unused) {
+ (void) unused;
+ int ret = (int) WaitForSingleObject(thread, INFINITE);
+ CloseHandle(thread);
+ return ret;
+}
+
+static int sched_yield (void) {
+ Sleep (0);
+ return 0;
+}
+#else
+
+#include <pthread.h>
+#include <stdatomic.h>
+#include <sched.h>
+#if defined(__FreeBSD__)
+#include <pthread_np.h>
+#endif
+
+typedef void * thread_ret_t;
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#endif
+
+typedef pthread_t ggml_thread_t;
+
+#ifdef GGML_USE_CPU_HBM
+#include <hbwmalloc.h>
+#endif
+
+#if defined(__APPLE__)
+#include <TargetConditionals.h>
+#endif
+
+#if (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)) && \
+ (!defined(TARGET_OS_TV) && !defined(TARGET_OS_WATCH))
+
+#include <sys/wait.h>
+
+#if defined(__ANDROID__)
+#include <unwind.h>
+#include <dlfcn.h>
+#include <stdio.h>
+
+struct backtrace_state {
+ void ** current;
+ void ** end;
+};
+
+static _Unwind_Reason_Code unwind_callback(struct _Unwind_Context* context, void* arg) {
+ struct backtrace_state * state = (struct backtrace_state *)arg;
+ uintptr_t pc = _Unwind_GetIP(context);
+ if (pc) {
+ if (state->current == state->end) {
+ return _URC_END_OF_STACK;
+ } else {
+ *state->current++ = (void*)pc;
+ }
+ }
+ return _URC_NO_REASON;
+}
+
+static void ggml_print_backtrace_symbols(void) {
+ const int max = 100;
+ void* buffer[max];
+
+ struct backtrace_state state = {buffer, buffer + max};
+ _Unwind_Backtrace(unwind_callback, &state);
+
+ int count = state.current - buffer;
+
+ for (int idx = 0; idx < count; ++idx) {
+ const void * addr = buffer[idx];
+ const char * symbol = "";
+
+ Dl_info info;
+ if (dladdr(addr, &info) && info.dli_sname) {
+ symbol = info.dli_sname;
+ }
+
+ fprintf(stderr, "%d: %p %s\n", idx, addr, symbol);
+ }
+}
+#elif defined(__linux__) && defined(__GLIBC__)
+#include <execinfo.h>
+static void ggml_print_backtrace_symbols(void) {
+ void * trace[100];
+ int nptrs = backtrace(trace, sizeof(trace)/sizeof(trace[0]));
+ backtrace_symbols_fd(trace, nptrs, STDERR_FILENO);
+}
+#else
+static void ggml_print_backtrace_symbols(void) {
+ // platform not supported
+}
+#endif
+
+static void ggml_print_backtrace(void) {
+ char attach[32];
+ snprintf(attach, sizeof(attach), "attach %d", getpid());
+ int pid = fork();
+ if (pid == 0) {
+ // try gdb
+ execlp("gdb", "gdb", "--batch",
+ "-ex", "set style enabled on",
+ "-ex", attach,
+ "-ex", "bt -frame-info source-and-location",
+ "-ex", "detach",
+ "-ex", "quit",
+ (char *) NULL);
+ // try lldb
+ execlp("lldb", "lldb", "--batch",
+ "-o", "bt",
+ "-o", "quit",
+ "-p", attach,
+ (char *) NULL);
+ exit(EXIT_FAILURE);
+ } else {
+ int wstatus;
+ waitpid(pid, &wstatus, 0);
+ if (WIFEXITED(wstatus)) {
+ if (WEXITSTATUS(wstatus) == EXIT_FAILURE) {
+ // gdb failed, fallback to backtrace_symbols
+ ggml_print_backtrace_symbols();
+ }
+ }
+ }
+}
+#else
+static void ggml_print_backtrace(void) {
+ // platform not supported
+}
+#endif
+
+void ggml_abort(const char * file, int line, const char * fmt, ...) {
+ fflush(stdout);
+
+ fprintf(stderr, "%s:%d: ", file, line);
+
+ va_list args;
+ va_start(args, fmt);
+ vfprintf(stderr, fmt, args);
+ va_end(args);
+
+ fprintf(stderr, "\n");
+
+ ggml_print_backtrace();
+ abort();
+}
+
+#define GGML_DEBUG 0
+#define GGML_GELU_FP16
+#define GGML_GELU_QUICK_FP16
+
+#define GGML_SOFT_MAX_UNROLL 4
+#define GGML_VEC_DOT_UNROLL 2
+#define GGML_VEC_MAD_UNROLL 32
+
+//
+// logging
+//
+
+#if (GGML_DEBUG >= 1)
+#define GGML_PRINT_DEBUG(...) printf(__VA_ARGS__)
+#else
+#define GGML_PRINT_DEBUG(...)
+#endif
+
+#if (GGML_DEBUG >= 5)
+#define GGML_PRINT_DEBUG_5(...) printf(__VA_ARGS__)
+#else
+#define GGML_PRINT_DEBUG_5(...)
+#endif
+
+#if (GGML_DEBUG >= 10)
+#define GGML_PRINT_DEBUG_10(...) printf(__VA_ARGS__)
+#else
+#define GGML_PRINT_DEBUG_10(...)
+#endif
+
+#define GGML_PRINT(...) printf(__VA_ARGS__)
+
+//
+// end of logging block
+//
+
+#ifdef GGML_USE_ACCELERATE
+// uncomment to use vDSP for soft max computation
+// note: not sure if it is actually faster
+//#define GGML_SOFT_MAX_ACCELERATE
+#endif
+
+#if defined(_MSC_VER) || defined(__MINGW32__)
+#define GGML_ALIGNED_MALLOC(size) _aligned_malloc(size, GGML_MEM_ALIGN)
+#define GGML_ALIGNED_FREE(ptr) _aligned_free(ptr)
+#else
+inline static void * ggml_aligned_malloc(size_t size) {
+ if (size == 0) {
+ GGML_PRINT("WARNING: Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n");
+ return NULL;
+ }
+ void * aligned_memory = NULL;
+#ifdef GGML_USE_CPU_HBM
+ int result = hbw_posix_memalign(&aligned_memory, 16, size);
+#elif GGML_USE_METAL
+ int result = posix_memalign(&aligned_memory, sysconf(_SC_PAGESIZE), size);
+#else
+ int result = posix_memalign(&aligned_memory, GGML_MEM_ALIGN, size);
+#endif
+ if (result != 0) {
+ // Handle allocation failure
+ const char *error_desc = "unknown allocation error";
+ switch (result) {
+ case EINVAL:
+ error_desc = "invalid alignment value";
+ break;
+ case ENOMEM:
+ error_desc = "insufficient memory";
+ break;
+ }
+ GGML_PRINT("%s: %s (attempted to allocate %6.2f MB)\n", __func__, error_desc, size/(1024.0*1024.0));
+ GGML_ABORT("fatal error");
+ return NULL;
+ }
+ return aligned_memory;
+}
+#define GGML_ALIGNED_MALLOC(size) ggml_aligned_malloc(size)
+#ifdef GGML_USE_CPU_HBM
+#define GGML_ALIGNED_FREE(ptr) if(NULL != ptr) hbw_free(ptr)
+#else
+#define GGML_ALIGNED_FREE(ptr) free(ptr)
+#endif
+#endif
+
+inline static void * ggml_malloc(size_t size) {
+ if (size == 0) {
+ GGML_PRINT("WARNING: Behavior may be unexpected when allocating 0 bytes for ggml_malloc!\n");
+ return NULL;
+ }
+ void * result = malloc(size);
+ if (result == NULL) {
+ GGML_PRINT("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0));
+ GGML_ABORT("fatal error");
+ }
+ return result;
+}
+
+// calloc
+inline static void * ggml_calloc(size_t num, size_t size) {
+ if (num == 0 || size == 0) {
+ GGML_PRINT("WARNING: Behavior may be unexpected when allocating 0 bytes for ggml_calloc!\n");
+ return NULL;
+ }
+ void * result = calloc(num, size);
+ if (result == NULL) {
+ GGML_PRINT("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0));
+ GGML_ABORT("fatal error");
+ }
+ return result;
+}
+
+#define GGML_MALLOC(size) ggml_malloc(size)
+#define GGML_CALLOC(num, size) ggml_calloc(num, size)
+
+#define GGML_FREE(ptr) free(ptr)
+
+#define UNUSED GGML_UNUSED
+#define SWAP(x, y, T) do { T SWAP = x; (x) = y; (y) = SWAP; } while (0)
+
+#if defined(GGML_USE_ACCELERATE)
+#include <Accelerate/Accelerate.h>
+#endif
+
+// floating point type used to accumulate sums
+typedef double ggml_float;
+
+#undef MIN
+#undef MAX
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+//
+// global data
+//
+
+// precomputed gelu table for f16 (128 KB)
+static ggml_fp16_t ggml_table_gelu_f16[1 << 16];
+
+// precomputed quick gelu table for f16 (128 KB)
+static ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16];
+
+// precomputed f32 table for f16 (256 KB) (ggml-impl.h)
+float ggml_table_f32_f16[1 << 16];
+
+GGML_CALL const char * ggml_status_to_string(enum ggml_status status) {
+ switch (status) {
+ case GGML_STATUS_ALLOC_FAILED: return "GGML status: error (failed to allocate memory)";
+ case GGML_STATUS_FAILED: return "GGML status: error (operation failed)";
+ case GGML_STATUS_SUCCESS: return "GGML status: success";
+ case GGML_STATUS_ABORTED: return "GGML status: warning (operation aborted)";
+ }
+
+ return "GGML status: unknown";
+}
+
+float ggml_fp16_to_fp32(ggml_fp16_t x) {
+#define ggml_fp16_to_fp32 do_not_use__ggml_fp16_to_fp32__in_ggml
+ return GGML_FP16_TO_FP32(x);
+}
+
+ggml_fp16_t ggml_fp32_to_fp16(float x) {
+#define ggml_fp32_to_fp16 do_not_use__ggml_fp32_to_fp16__in_ggml
+ return GGML_FP32_TO_FP16(x);
+}
+
+float ggml_bf16_to_fp32(ggml_bf16_t x) {
+#define ggml_bf16_to_fp32 do_not_use__ggml_bf16_to_fp32__in_ggml
+ return GGML_BF16_TO_FP32(x); // it just left shifts
+}
+
+ggml_bf16_t ggml_fp32_to_bf16(float x) {
+#define ggml_fp32_to_bf16 do_not_use__ggml_fp32_to_bf16__in_ggml
+ return GGML_FP32_TO_BF16(x);
+}
+
+void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {
+ for (int64_t i = 0; i < n; i++) {
+ y[i] = GGML_FP16_TO_FP32(x[i]);
+ }
+}
+
+void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {
+ int64_t i = 0;
+#if defined(__F16C__)
+ for (; i + 7 < n; i += 8) {
+ __m256 x_vec = _mm256_loadu_ps(x + i);
+ __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
+ _mm_storeu_si128((__m128i *)(y + i), y_vec);
+ }
+ for(; i + 3 < n; i += 4) {
+ __m128 x_vec = _mm_loadu_ps(x + i);
+ __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
+ _mm_storel_epi64((__m128i *)(y + i), y_vec);
+ }
+#endif
+ for (; i < n; i++) {
+ y[i] = GGML_FP32_TO_FP16(x[i]);
+ }
+}
+
+void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {
+ int64_t i = 0;
+#if defined(__AVX512F__)
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(y + i,
+ _mm512_castsi512_ps(
+ _mm512_slli_epi32(
+ _mm512_cvtepu16_epi32(
+ _mm256_loadu_si256(
+ (const __m256i *)(x + i))),
+ 16)));
+ }
+#elif defined(__AVX2__)
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(y + i,
+ _mm256_castsi256_ps(
+ _mm256_slli_epi32(
+ _mm256_cvtepu16_epi32(
+ _mm_loadu_si128(
+ (const __m128i *)(x + i))),
+ 16)));
+ }
+#endif
+ for (; i < n; i++) {
+ y[i] = GGML_BF16_TO_FP32(x[i]);
+ }
+}
+
+void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) {
+ for (int i = 0; i < n; i++) {
+ y[i] = ggml_compute_fp32_to_bf16(x[i]);
+ }
+}
+
+void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {
+ int i = 0;
+#if defined(__AVX512BF16__)
+ // subnormals are flushed to zero on this platform
+ for (; i + 32 <= n; i += 32) {
+ _mm512_storeu_si512(
+ (__m512i *)(y + i),
+ m512i(_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),
+ _mm512_loadu_ps(x + i))));
+ }
+#endif
+ for (; i < n; i++) {
+ y[i] = GGML_FP32_TO_BF16(x[i]);
+ }
+}
+
+bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {
+ return memcmp(guid_a, guid_b, sizeof(ggml_guid)) == 0;
+}
+
+//
+// timing
+//
+
+#if defined(_MSC_VER) || defined(__MINGW32__)
+static int64_t timer_freq, timer_start;
+void ggml_time_init(void) {
+ LARGE_INTEGER t;
+ QueryPerformanceFrequency(&t);
+ timer_freq = t.QuadPart;
+
+ // The multiplication by 1000 or 1000000 below can cause an overflow if timer_freq
+ // and the uptime is high enough.
+ // We subtract the program start time to reduce the likelihood of that happening.
+ QueryPerformanceCounter(&t);
+ timer_start = t.QuadPart;
+}
+int64_t ggml_time_ms(void) {
+ LARGE_INTEGER t;
+ QueryPerformanceCounter(&t);
+ return ((t.QuadPart-timer_start) * 1000) / timer_freq;
+}
+int64_t ggml_time_us(void) {
+ LARGE_INTEGER t;
+ QueryPerformanceCounter(&t);
+ return ((t.QuadPart-timer_start) * 1000000) / timer_freq;
+}
+#else
+void ggml_time_init(void) {}
+int64_t ggml_time_ms(void) {
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (int64_t)ts.tv_sec*1000 + (int64_t)ts.tv_nsec/1000000;
+}
+
+int64_t ggml_time_us(void) {
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (int64_t)ts.tv_sec*1000000 + (int64_t)ts.tv_nsec/1000;
+}
+#endif
+
+int64_t ggml_cycles(void) {
+ return clock();
+}
+
+int64_t ggml_cycles_per_ms(void) {
+ return CLOCKS_PER_SEC/1000;
+}
+
+//
+// cross-platform UTF-8 file paths
+//
+
+#ifdef _WIN32
+static wchar_t * ggml_mbstowcs(const char * mbs) {
+ int wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, NULL, 0);
+ if (!wlen) {
+ errno = EINVAL;
+ return NULL;
+ }
+
+ wchar_t * wbuf = GGML_MALLOC(wlen * sizeof(wchar_t));
+ wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, wbuf, wlen);
+ if (!wlen) {
+ GGML_FREE(wbuf);
+ errno = EINVAL;
+ return NULL;
+ }
+
+ return wbuf;
+}
+#endif
+
+FILE * ggml_fopen(const char * fname, const char * mode) {
+#ifdef _WIN32
+ FILE * file = NULL;
+
+ // convert fname (UTF-8)
+ wchar_t * wfname = ggml_mbstowcs(fname);
+ if (wfname) {
+ // convert mode (ANSI)
+ wchar_t * wmode = GGML_MALLOC((strlen(mode) + 1) * sizeof(wchar_t));
+ wchar_t * wmode_p = wmode;
+ do {
+ *wmode_p++ = (wchar_t)*mode;
+ } while (*mode++);
+
+ // open file
+ file = _wfopen(wfname, wmode);
+
+ GGML_FREE(wfname);
+ GGML_FREE(wmode);
+ }
+
+ return file;
+#else
+ return fopen(fname, mode);
+#endif
+}
+
+//
+// cache line
+//
+
+#if defined(__cpp_lib_hardware_interference_size)
+#define CACHE_LINE_SIZE hardware_destructive_interference_size
+#else
+#if defined(__POWER9_VECTOR__)
+#define CACHE_LINE_SIZE 128
+#else
+#define CACHE_LINE_SIZE 64
+#endif
+#endif
+
+static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);
+
+static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
+static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
+static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
+
+static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {
+ [GGML_TYPE_I8] = {
+ .type_name = "i8",
+ .blck_size = 1,
+ .type_size = sizeof(int8_t),
+ .is_quantized = false,
+ },
+ [GGML_TYPE_I16] = {
+ .type_name = "i16",
+ .blck_size = 1,
+ .type_size = sizeof(int16_t),
+ .is_quantized = false,
+ },
+ [GGML_TYPE_I32] = {
+ .type_name = "i32",
+ .blck_size = 1,
+ .type_size = sizeof(int32_t),
+ .is_quantized = false,
+ },
+ [GGML_TYPE_I64] = {
+ .type_name = "i64",
+ .blck_size = 1,
+ .type_size = sizeof(int64_t),
+ .is_quantized = false,
+ },
+ [GGML_TYPE_F64] = {
+ .type_name = "f64",
+ .blck_size = 1,
+ .type_size = sizeof(double),
+ .is_quantized = false,
+ .nrows = 1,
+ },
+ [GGML_TYPE_F32] = {
+ .type_name = "f32",
+ .blck_size = 1,
+ .type_size = sizeof(float),
+ .is_quantized = false,
+ .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f32,
+ .vec_dot_type = GGML_TYPE_F32,
+ .nrows = 1,
+ },
+ [GGML_TYPE_F16] = {
+ .type_name = "f16",
+ .blck_size = 1,
+ .type_size = sizeof(ggml_fp16_t),
+ .is_quantized = false,
+ .to_float = (ggml_to_float_t) ggml_fp16_to_fp32_row,
+ .from_float = (ggml_from_float_t) ggml_fp32_to_fp16_row,
+ .from_float_ref = (ggml_from_float_t) ggml_fp32_to_fp16_row,
+ .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16,
+ .vec_dot_type = GGML_TYPE_F16,
+ .nrows = 1,
+ },
+ [GGML_TYPE_Q4_0] = {
+ .type_name = "q4_0",
+ .blck_size = QK4_0,
+ .type_size = sizeof(block_q4_0),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_q4_0,
+ .from_float = quantize_row_q4_0,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q4_0_ref,
+ .vec_dot = ggml_vec_dot_q4_0_q8_0,
+ .vec_dot_type = GGML_TYPE_Q8_0,
+#if defined (__ARM_FEATURE_MATMUL_INT8)
+ .nrows = 2,
+#else
+ .nrows = 1,
+#endif
+ },
+ [GGML_TYPE_Q4_1] = {
+ .type_name = "q4_1",
+ .blck_size = QK4_1,
+ .type_size = sizeof(block_q4_1),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_q4_1,
+ .from_float = quantize_row_q4_1,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q4_1_ref,
+ .vec_dot = ggml_vec_dot_q4_1_q8_1,
+ .vec_dot_type = GGML_TYPE_Q8_1,
+#if defined (__ARM_FEATURE_MATMUL_INT8)
+ .nrows = 2,
+#else
+ .nrows = 1,
+#endif
+ },
+ [4] = { // GGML_TYPE_Q4_2
+ .type_name = "DEPRECATED",
+ .blck_size = 0,
+ .type_size = 0,
+ .is_quantized = false,
+ .to_float = NULL,
+ .from_float = NULL,
+ .from_float_ref = NULL,
+ .vec_dot = NULL,
+ .vec_dot_type = GGML_TYPE_COUNT,
+ .nrows = 1,
+ },
+ [5] = { // GGML_TYPE_Q4_3
+ .type_name = "DEPRECATED",
+ .blck_size = 0,
+ .type_size = 0,
+ .is_quantized = false,
+ .to_float = NULL,
+ .from_float = NULL,
+ .from_float_ref = NULL,
+ .vec_dot = NULL,
+ .vec_dot_type = GGML_TYPE_COUNT,
+ .nrows = 1,
+ },
+ [GGML_TYPE_Q5_0] = {
+ .type_name = "q5_0",
+ .blck_size = QK5_0,
+ .type_size = sizeof(block_q5_0),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_q5_0,
+ .from_float = quantize_row_q5_0,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q5_0_ref,
+ .vec_dot = ggml_vec_dot_q5_0_q8_0,
+ .vec_dot_type = GGML_TYPE_Q8_0,
+ .nrows = 1,
+ },
+ [GGML_TYPE_Q5_1] = {
+ .type_name = "q5_1",
+ .blck_size = QK5_1,
+ .type_size = sizeof(block_q5_1),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_q5_1,
+ .from_float = quantize_row_q5_1,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q5_1_ref,
+ .vec_dot = ggml_vec_dot_q5_1_q8_1,
+ .vec_dot_type = GGML_TYPE_Q8_1,
+ .nrows = 1,
+ },
+ [GGML_TYPE_Q8_0] = {
+ .type_name = "q8_0",
+ .blck_size = QK8_0,
+ .type_size = sizeof(block_q8_0),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_q8_0,
+ .from_float = quantize_row_q8_0,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q8_0_ref,
+ .from_float_to_mat = quantize_mat_q8_0,
+ .vec_dot = ggml_vec_dot_q8_0_q8_0,
+ .vec_dot_type = GGML_TYPE_Q8_0,
+#if defined (__ARM_FEATURE_MATMUL_INT8)
+ .nrows = 2,
+#else
+ .nrows = 1,
+#endif
+ },
+ [GGML_TYPE_Q8_1] = {
+ .type_name = "q8_1",
+ .blck_size = QK8_1,
+ .type_size = sizeof(block_q8_1),
+ .is_quantized = true,
+ .from_float = quantize_row_q8_1,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q8_1_ref,
+ .vec_dot_type = GGML_TYPE_Q8_1,
+ .nrows = 1,
+ },
+ [GGML_TYPE_Q2_K] = {
+ .type_name = "q2_K",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_q2_K),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_q2_K,
+ .from_float = quantize_row_q2_K,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q2_K_ref,
+ .vec_dot = ggml_vec_dot_q2_K_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_Q3_K] = {
+ .type_name = "q3_K",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_q3_K),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_q3_K,
+ .from_float = quantize_row_q3_K,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q3_K_ref,
+ .vec_dot = ggml_vec_dot_q3_K_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_Q4_K] = {
+ .type_name = "q4_K",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_q4_K),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_q4_K,
+ .from_float = quantize_row_q4_K,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q4_K_ref,
+ .vec_dot = ggml_vec_dot_q4_K_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_Q5_K] = {
+ .type_name = "q5_K",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_q5_K),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_q5_K,
+ .from_float = quantize_row_q5_K,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q5_K_ref,
+ .vec_dot = ggml_vec_dot_q5_K_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_Q6_K] = {
+ .type_name = "q6_K",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_q6_K),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_q6_K,
+ .from_float = quantize_row_q6_K,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q6_K_ref,
+ .vec_dot = ggml_vec_dot_q6_K_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_IQ2_XXS] = {
+ .type_name = "iq2_xxs",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_iq2_xxs),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_iq2_xxs,
+ .from_float = NULL,
+ .from_float_ref = NULL,
+ .vec_dot = ggml_vec_dot_iq2_xxs_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_IQ2_XS] = {
+ .type_name = "iq2_xs",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_iq2_xs),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_iq2_xs,
+ .from_float = NULL,
+ .from_float_ref = NULL,
+ .vec_dot = ggml_vec_dot_iq2_xs_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_IQ3_XXS] = {
+ .type_name = "iq3_xxs",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_iq3_xxs),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_iq3_xxs,
+ .from_float = quantize_row_iq3_xxs,
+ .from_float_ref = (ggml_from_float_t)quantize_row_iq3_xxs_ref,
+ .vec_dot = ggml_vec_dot_iq3_xxs_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_IQ3_S] = {
+ .type_name = "iq3_s",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_iq3_s),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_iq3_s,
+ .from_float = quantize_row_iq3_s,
+ .from_float_ref = (ggml_from_float_t)quantize_row_iq3_s_ref,
+ .vec_dot = ggml_vec_dot_iq3_s_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_IQ2_S] = {
+ .type_name = "iq2_s",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_iq2_s),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_iq2_s,
+ .from_float = quantize_row_iq2_s,
+ .from_float_ref = (ggml_from_float_t)quantize_row_iq2_s_ref,
+ .vec_dot = ggml_vec_dot_iq2_s_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_IQ1_S] = {
+ .type_name = "iq1_s",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_iq1_s),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_iq1_s,
+ .from_float = NULL,
+ .from_float_ref = NULL,
+ .vec_dot = ggml_vec_dot_iq1_s_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_IQ1_M] = {
+ .type_name = "iq1_m",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_iq1_m),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_iq1_m,
+ .from_float = NULL,
+ .from_float_ref = NULL,
+ .vec_dot = ggml_vec_dot_iq1_m_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_IQ4_NL] = {
+ .type_name = "iq4_nl",
+ .blck_size = QK4_NL,
+ .type_size = sizeof(block_iq4_nl),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_iq4_nl,
+ .from_float = quantize_row_iq4_nl,
+ .from_float_ref = (ggml_from_float_t)quantize_row_iq4_nl_ref,
+ .vec_dot = ggml_vec_dot_iq4_nl_q8_0,
+ .vec_dot_type = GGML_TYPE_Q8_0,
+ .nrows = 1,
+ },
+ [GGML_TYPE_IQ4_XS] = {
+ .type_name = "iq4_xs",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_iq4_xs),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_iq4_xs,
+ .from_float = quantize_row_iq4_xs,
+ .from_float_ref = (ggml_from_float_t)quantize_row_iq4_xs_ref,
+ .vec_dot = ggml_vec_dot_iq4_xs_q8_K,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ .nrows = 1,
+ },
+ [GGML_TYPE_Q8_K] = {
+ .type_name = "q8_K",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_q8_K),
+ .is_quantized = true,
+ .from_float = quantize_row_q8_K,
+ },
+ [GGML_TYPE_BF16] = {
+ .type_name = "bf16",
+ .blck_size = 1,
+ .type_size = sizeof(ggml_bf16_t),
+ .is_quantized = false,
+ .to_float = (ggml_to_float_t) ggml_bf16_to_fp32_row,
+ .from_float = (ggml_from_float_t) ggml_fp32_to_bf16_row,
+ .from_float_ref = (ggml_from_float_t) ggml_fp32_to_bf16_row_ref,
+ .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_bf16,
+ .vec_dot_type = GGML_TYPE_BF16,
+ .nrows = 1,
+ },
+ [GGML_TYPE_Q4_0_4_4] = {
+ .type_name = "q4_0_4x4",
+ .blck_size = QK4_0,
+ .blck_size_interleave = 4,
+ .type_size = sizeof(block_q4_0),
+ .is_quantized = true,
+ .to_float = NULL,
+ .from_float = NULL,
+ .from_float_ref = NULL,
+ .vec_dot = NULL,
+ .vec_dot_type = GGML_TYPE_Q8_0,
+ .nrows = 1,
+ .ncols = 4,
+ .gemv = ggml_gemv_q4_0_4x4_q8_0,
+ .gemm = ggml_gemm_q4_0_4x4_q8_0,
+ },
+ [GGML_TYPE_Q4_0_4_8] = {
+ .type_name = "q4_0_4x8",
+ .blck_size = QK4_0,
+ .blck_size_interleave = 8,
+ .type_size = sizeof(block_q4_0),
+ .is_quantized = true,
+ .to_float = NULL,
+ .from_float = NULL,
+ .from_float_ref = NULL,
+ .vec_dot = NULL,
+ .vec_dot_type = GGML_TYPE_Q8_0,
+ .nrows = 1,
+ .ncols = 4,
+ .gemv = ggml_gemv_q4_0_4x8_q8_0,
+ .gemm = ggml_gemm_q4_0_4x8_q8_0,
+ },
+ [GGML_TYPE_Q4_0_8_8] = {
+ .type_name = "q4_0_8x8",
+ .blck_size = QK4_0,
+ .blck_size_interleave = 8,
+ .type_size = sizeof(block_q4_0),
+ .is_quantized = true,
+ .to_float = NULL,
+ .from_float = NULL,
+ .from_float_ref = NULL,
+ .vec_dot = NULL,
+ .vec_dot_type = GGML_TYPE_Q8_0,
+ .nrows = 1,
+ .ncols = 8,
+ .gemv = ggml_gemv_q4_0_8x8_q8_0,
+ .gemm = ggml_gemm_q4_0_8x8_q8_0,
+ }
+};
+
+// For internal test use
+ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type) {
+ GGML_ASSERT(type < GGML_TYPE_COUNT);
+ return type_traits[type];
+}
+
+//
+// simd mappings
+//
+
+// we define a common set of C macros which map to specific intrinsics based on the current architecture
+// we then implement the fundamental computation operations below using only these macros
+// adding support for new architectures requires to define the corresponding SIMD macros
+//
+// GGML_F32_STEP / GGML_F16_STEP
+// number of elements to process in a single step
+//
+// GGML_F32_EPR / GGML_F16_EPR
+// number of elements to fit in a single register
+//
+
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)
+
+#define GGML_SIMD
+
+// F32 NEON
+
+#define GGML_F32_STEP 16
+#define GGML_F32_EPR 4
+
+#define GGML_F32x4 float32x4_t
+#define GGML_F32x4_ZERO vdupq_n_f32(0.0f)
+#define GGML_F32x4_SET1(x) vdupq_n_f32(x)
+#define GGML_F32x4_LOAD vld1q_f32
+#define GGML_F32x4_STORE vst1q_f32
+#define GGML_F32x4_FMA(a, b, c) vfmaq_f32(a, b, c)
+#define GGML_F32x4_ADD vaddq_f32
+#define GGML_F32x4_MUL vmulq_f32
+#define GGML_F32x4_REDUCE_ONE(x) vaddvq_f32(x)
+#define GGML_F32x4_REDUCE(res, x) \
+{ \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f32(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f32(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f32(x[i], x[offset+i]); \
+ } \
+ res = GGML_F32x4_REDUCE_ONE(x[0]); \
+}
+
+#define GGML_F32_VEC GGML_F32x4
+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x4_STORE
+#define GGML_F32_VEC_FMA GGML_F32x4_FMA
+#define GGML_F32_VEC_ADD GGML_F32x4_ADD
+#define GGML_F32_VEC_MUL GGML_F32x4_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
+
+// F16 NEON
+
+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+ #define GGML_F16_STEP 32
+ #define GGML_F16_EPR 8
+
+ #define GGML_F16x8 float16x8_t
+ #define GGML_F16x8_ZERO vdupq_n_f16(0.0f)
+ #define GGML_F16x8_SET1(x) vdupq_n_f16(x)
+ #define GGML_F16x8_LOAD(x) vld1q_f16((const ggml_fp16_internal_t *)(x))
+ #define GGML_F16x8_STORE vst1q_f16
+ #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)
+ #define GGML_F16x8_ADD vaddq_f16
+ #define GGML_F16x8_MUL vmulq_f16
+ #define GGML_F16x8_REDUCE(res, x) \
+ do { \
+ int offset = GGML_F16_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f16(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f16(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f16(x[i], x[offset+i]); \
+ } \
+ const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 (x[0])); \
+ const float32x4_t t1 = vcvt_f32_f16(vget_high_f16(x[0])); \
+ res = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1)); \
+ } while (0)
+
+ #define GGML_F16_VEC GGML_F16x8
+ #define GGML_F16_VEC_ZERO GGML_F16x8_ZERO
+ #define GGML_F16_VEC_SET1 GGML_F16x8_SET1
+ #define GGML_F16_VEC_LOAD(p, i) GGML_F16x8_LOAD(p)
+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((ggml_fp16_internal_t *)(p), r[i])
+ #define GGML_F16_VEC_FMA GGML_F16x8_FMA
+ #define GGML_F16_VEC_ADD GGML_F16x8_ADD
+ #define GGML_F16_VEC_MUL GGML_F16x8_MUL
+ #define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE
+#else
+ // if FP16 vector arithmetic is not supported, we use FP32 instead
+ // and take advantage of the vcvt_ functions to convert to/from FP16
+
+ #define GGML_F16_STEP 16
+ #define GGML_F16_EPR 4
+
+ #define GGML_F32Cx4 float32x4_t
+ #define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f)
+ #define GGML_F32Cx4_SET1(x) vdupq_n_f32(x)
+ #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const ggml_fp16_internal_t *)(x)))
+ #define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y))
+ #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)
+ #define GGML_F32Cx4_ADD vaddq_f32
+ #define GGML_F32Cx4_MUL vmulq_f32
+ #define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE
+
+ #define GGML_F16_VEC GGML_F32Cx4
+ #define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO
+ #define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1
+ #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)
+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((ggml_fp16_internal_t *)(p), r[i])
+ #define GGML_F16_VEC_FMA GGML_F32Cx4_FMA
+ #define GGML_F16_VEC_ADD GGML_F32Cx4_ADD
+ #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL
+ #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE
+#endif
+
+#elif defined(__AVX512F__)
+
+#define GGML_SIMD
+
+// F32 AVX512
+
+#define GGML_F32_STEP 64
+#define GGML_F32_EPR 16
+
+#define GGML_F32x16 __m512
+#define GGML_F32x16_ZERO _mm512_setzero_ps()
+#define GGML_F32x16_SET1(x) _mm512_set1_ps(x)
+#define GGML_F32x16_LOAD _mm512_loadu_ps
+#define GGML_F32x16_STORE _mm512_storeu_ps
+// _mm512_fmadd_ps is defined in AVX512F so no guard is required
+#define GGML_F32x16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)
+#define GGML_F32x16_ADD _mm512_add_ps
+#define GGML_F32x16_MUL _mm512_mul_ps
+#define GGML_F32x16_REDUCE(res, x) \
+do { \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ res = _mm512_reduce_add_ps(x[0]); \
+} while (0)
+
+// TODO: is this optimal ?
+
+#define GGML_F32_VEC GGML_F32x16
+#define GGML_F32_VEC_ZERO GGML_F32x16_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x16_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x16_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x16_STORE
+#define GGML_F32_VEC_FMA GGML_F32x16_FMA
+#define GGML_F32_VEC_ADD GGML_F32x16_ADD
+#define GGML_F32_VEC_MUL GGML_F32x16_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE
+
+// F16 AVX512
+
+// F16 AVX
+
+#define GGML_F16_STEP 64
+#define GGML_F16_EPR 16
+
+// AVX512 has FP16 extension (AVX512_FP16) but I don't have it on my machine so I use FP32 instead
+
+#define GGML_F32Cx16 __m512
+#define GGML_F32Cx16_ZERO _mm512_setzero_ps()
+#define GGML_F32Cx16_SET1(x) _mm512_set1_ps(x)
+
+// unlike _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F
+// so F16C guard isn't required
+#define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x)))
+#define GGML_F32Cx16_STORE(x, y) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0))
+
+#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)
+#define GGML_F32Cx16_ADD _mm512_add_ps
+#define GGML_F32Cx16_MUL _mm512_mul_ps
+#define GGML_F32Cx16_REDUCE(res, x) \
+do { \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ res = _mm512_reduce_add_ps(x[0]); \
+} while (0)
+
+#define GGML_F16_VEC GGML_F32Cx16
+#define GGML_F16_VEC_ZERO GGML_F32Cx16_ZERO
+#define GGML_F16_VEC_SET1 GGML_F32Cx16_SET1
+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx16_LOAD(p)
+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx16_STORE(p, r[i])
+#define GGML_F16_VEC_FMA GGML_F32Cx16_FMA
+#define GGML_F16_VEC_ADD GGML_F32Cx16_ADD
+#define GGML_F16_VEC_MUL GGML_F32Cx16_MUL
+#define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE
+
+#elif defined(__AVX__)
+
+#define GGML_SIMD
+
+// F32 AVX
+
+#define GGML_F32_STEP 32
+#define GGML_F32_EPR 8
+
+#define GGML_F32x8 __m256
+#define GGML_F32x8_ZERO _mm256_setzero_ps()
+#define GGML_F32x8_SET1(x) _mm256_set1_ps(x)
+#define GGML_F32x8_LOAD _mm256_loadu_ps
+#define GGML_F32x8_STORE _mm256_storeu_ps
+#if defined(__FMA__)
+ #define GGML_F32x8_FMA(a, b, c) _mm256_fmadd_ps(b, c, a)
+#else
+ #define GGML_F32x8_FMA(a, b, c) _mm256_add_ps(_mm256_mul_ps(b, c), a)
+#endif
+#define GGML_F32x8_ADD _mm256_add_ps
+#define GGML_F32x8_MUL _mm256_mul_ps
+#define GGML_F32x8_REDUCE(res, x) \
+do { \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \
+ } \
+ const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \
+ _mm256_extractf128_ps(x[0], 1)); \
+ const __m128 t1 = _mm_hadd_ps(t0, t0); \
+ res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \
+} while (0)
+// TODO: is this optimal ?
+
+#define GGML_F32_VEC GGML_F32x8
+#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x8_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x8_STORE
+#define GGML_F32_VEC_FMA GGML_F32x8_FMA
+#define GGML_F32_VEC_ADD GGML_F32x8_ADD
+#define GGML_F32_VEC_MUL GGML_F32x8_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE
+
+// F16 AVX
+
+#define GGML_F16_STEP 32
+#define GGML_F16_EPR 8
+
+// F16 arithmetic is not supported by AVX, so we use F32 instead
+
+#define GGML_F32Cx8 __m256
+#define GGML_F32Cx8_ZERO _mm256_setzero_ps()
+#define GGML_F32Cx8_SET1(x) _mm256_set1_ps(x)
+
+#if defined(__F16C__)
+// the _mm256_cvt intrinsics require F16C
+#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))
+#define GGML_F32Cx8_STORE(x, y) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0))
+#else
+static inline __m256 __avx_f32cx8_load(ggml_fp16_t *x) {
+ float tmp[8];
+
+ for (int i = 0; i < 8; i++) {
+ tmp[i] = GGML_FP16_TO_FP32(x[i]);
+ }
+
+ return _mm256_loadu_ps(tmp);
+}
+static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {
+ float arr[8];
+
+ _mm256_storeu_ps(arr, y);
+
+ for (int i = 0; i < 8; i++)
+ x[i] = GGML_FP32_TO_FP16(arr[i]);
+}
+#define GGML_F32Cx8_LOAD(x) __avx_f32cx8_load(x)
+#define GGML_F32Cx8_STORE(x, y) __avx_f32cx8_store(x, y)
+#endif
+
+#define GGML_F32Cx8_FMA GGML_F32x8_FMA
+#define GGML_F32Cx8_ADD _mm256_add_ps
+#define GGML_F32Cx8_MUL _mm256_mul_ps
+#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE
+
+#define GGML_F16_VEC GGML_F32Cx8
+#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO
+#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1
+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p)
+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])
+#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA
+#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD
+#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL
+#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE
+
+#elif defined(__POWER9_VECTOR__)
+
+#define GGML_SIMD
+
+// F32 POWER9
+
+#define GGML_F32_STEP 32
+#define GGML_F32_EPR 4
+
+#define GGML_F32x4 vector float
+#define GGML_F32x4_ZERO 0.0f
+#define GGML_F32x4_SET1 vec_splats
+#define GGML_F32x4_LOAD(p) vec_xl(0, p)
+#define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p)
+#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a)
+#define GGML_F32x4_ADD vec_add
+#define GGML_F32x4_MUL vec_mul
+#define GGML_F32x4_REDUCE(res, x) \
+{ \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vec_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vec_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vec_add(x[i], x[offset+i]); \
+ } \
+ res = vec_extract(x[0], 0) + \
+ vec_extract(x[0], 1) + \
+ vec_extract(x[0], 2) + \
+ vec_extract(x[0], 3); \
+}
+
+#define GGML_F32_VEC GGML_F32x4
+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x4_STORE
+#define GGML_F32_VEC_FMA GGML_F32x4_FMA
+#define GGML_F32_VEC_ADD GGML_F32x4_ADD
+#define GGML_F32_VEC_MUL GGML_F32x4_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
+
+// F16 POWER9
+#define GGML_F16_STEP GGML_F32_STEP
+#define GGML_F16_EPR GGML_F32_EPR
+#define GGML_F16_VEC GGML_F32x4
+#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO
+#define GGML_F16_VEC_SET1 GGML_F32x4_SET1
+#define GGML_F16_VEC_FMA GGML_F32x4_FMA
+#define GGML_F16_VEC_ADD GGML_F32x4_ADD
+#define GGML_F16_VEC_MUL GGML_F32x4_MUL
+#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE
+// Use vec_xl, not vec_ld, in case the load address is not aligned.
+#define GGML_F16_VEC_LOAD(p, i) (i & 0x1) ? \
+ vec_extract_fp32_from_shorth(vec_xl(0, p - GGML_F16_EPR)) : \
+ vec_extract_fp32_from_shortl(vec_xl(0, p))
+#define GGML_ENDIAN_BYTE(i) ((unsigned char *)&(uint16_t){1})[i]
+#define GGML_F16_VEC_STORE(p, r, i) \
+ if (i & 0x1) \
+ vec_xst(vec_pack_to_short_fp32(r[i - GGML_ENDIAN_BYTE(1)], \
+ r[i - GGML_ENDIAN_BYTE(0)]), \
+ 0, p - GGML_F16_EPR)
+
+#elif defined(__wasm_simd128__)
+
+#define GGML_SIMD
+
+// F32 WASM
+
+#define GGML_F32_STEP 16
+#define GGML_F32_EPR 4
+
+#define GGML_F32x4 v128_t
+#define GGML_F32x4_ZERO wasm_f32x4_splat(0.0f)
+#define GGML_F32x4_SET1(x) wasm_f32x4_splat(x)
+#define GGML_F32x4_LOAD wasm_v128_load
+#define GGML_F32x4_STORE wasm_v128_store
+#define GGML_F32x4_FMA(a, b, c) wasm_f32x4_add(wasm_f32x4_mul(b, c), a)
+#define GGML_F32x4_ADD wasm_f32x4_add
+#define GGML_F32x4_MUL wasm_f32x4_mul
+#define GGML_F32x4_REDUCE(res, x) \
+{ \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ res = wasm_f32x4_extract_lane(x[0], 0) + \
+ wasm_f32x4_extract_lane(x[0], 1) + \
+ wasm_f32x4_extract_lane(x[0], 2) + \
+ wasm_f32x4_extract_lane(x[0], 3); \
+}
+
+#define GGML_F32_VEC GGML_F32x4
+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x4_STORE
+#define GGML_F32_VEC_FMA GGML_F32x4_FMA
+#define GGML_F32_VEC_ADD GGML_F32x4_ADD
+#define GGML_F32_VEC_MUL GGML_F32x4_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
+
+// F16 WASM
+
+#define GGML_F16_STEP 16
+#define GGML_F16_EPR 4
+
+inline static v128_t __wasm_f16x4_load(const ggml_fp16_t * p) {
+ float tmp[4];
+
+ tmp[0] = GGML_FP16_TO_FP32(p[0]);
+ tmp[1] = GGML_FP16_TO_FP32(p[1]);
+ tmp[2] = GGML_FP16_TO_FP32(p[2]);
+ tmp[3] = GGML_FP16_TO_FP32(p[3]);
+
+ return wasm_v128_load(tmp);
+}
+
+inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {
+ float tmp[4];
+
+ wasm_v128_store(tmp, x);
+
+ p[0] = GGML_FP32_TO_FP16(tmp[0]);
+ p[1] = GGML_FP32_TO_FP16(tmp[1]);
+ p[2] = GGML_FP32_TO_FP16(tmp[2]);
+ p[3] = GGML_FP32_TO_FP16(tmp[3]);
+}
+
+#define GGML_F16x4 v128_t
+#define GGML_F16x4_ZERO wasm_f32x4_splat(0.0f)
+#define GGML_F16x4_SET1(x) wasm_f32x4_splat(x)
+#define GGML_F16x4_LOAD(x) __wasm_f16x4_load(x)
+#define GGML_F16x4_STORE(x, y) __wasm_f16x4_store(x, y)
+#define GGML_F16x4_FMA GGML_F32x4_FMA
+#define GGML_F16x4_ADD wasm_f32x4_add
+#define GGML_F16x4_MUL wasm_f32x4_mul
+#define GGML_F16x4_REDUCE(res, x) \
+{ \
+ int offset = GGML_F16_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ res = wasm_f32x4_extract_lane(x[0], 0) + \
+ wasm_f32x4_extract_lane(x[0], 1) + \
+ wasm_f32x4_extract_lane(x[0], 2) + \
+ wasm_f32x4_extract_lane(x[0], 3); \
+}
+
+#define GGML_F16_VEC GGML_F16x4
+#define GGML_F16_VEC_ZERO GGML_F16x4_ZERO
+#define GGML_F16_VEC_SET1 GGML_F16x4_SET1
+#define GGML_F16_VEC_LOAD(p, i) GGML_F16x4_LOAD(p)
+#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x4_STORE(p, r[i])
+#define GGML_F16_VEC_FMA GGML_F16x4_FMA
+#define GGML_F16_VEC_ADD GGML_F16x4_ADD
+#define GGML_F16_VEC_MUL GGML_F16x4_MUL
+#define GGML_F16_VEC_REDUCE GGML_F16x4_REDUCE
+
+#elif defined(__SSE3__)
+
+#define GGML_SIMD
+
+// F32 SSE
+
+#define GGML_F32_STEP 32
+#define GGML_F32_EPR 4
+
+#define GGML_F32x4 __m128
+#define GGML_F32x4_ZERO _mm_setzero_ps()
+#define GGML_F32x4_SET1(x) _mm_set1_ps(x)
+#define GGML_F32x4_LOAD _mm_loadu_ps
+#define GGML_F32x4_STORE _mm_storeu_ps
+#if defined(__FMA__)
+ // TODO: Does this work?
+ #define GGML_F32x4_FMA(a, b, c) _mm_fmadd_ps(b, c, a)
+#else
+ #define GGML_F32x4_FMA(a, b, c) _mm_add_ps(_mm_mul_ps(b, c), a)
+#endif
+#define GGML_F32x4_ADD _mm_add_ps
+#define GGML_F32x4_MUL _mm_mul_ps
+#define GGML_F32x4_REDUCE(res, x) \
+{ \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm_add_ps(x[i], x[offset+i]); \
+ } \
+ const __m128 t0 = _mm_hadd_ps(x[0], x[0]); \
+ res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t0, t0)); \
+}
+// TODO: is this optimal ?
+
+#define GGML_F32_VEC GGML_F32x4
+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x4_STORE
+#define GGML_F32_VEC_FMA GGML_F32x4_FMA
+#define GGML_F32_VEC_ADD GGML_F32x4_ADD
+#define GGML_F32_VEC_MUL GGML_F32x4_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
+
+// F16 SSE
+
+#define GGML_F16_STEP 32
+#define GGML_F16_EPR 4
+
+static inline __m128 __sse_f16x4_load(ggml_fp16_t *x) {
+ float tmp[4];
+
+ tmp[0] = GGML_FP16_TO_FP32(x[0]);
+ tmp[1] = GGML_FP16_TO_FP32(x[1]);
+ tmp[2] = GGML_FP16_TO_FP32(x[2]);
+ tmp[3] = GGML_FP16_TO_FP32(x[3]);
+
+ return _mm_loadu_ps(tmp);
+}
+
+static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) {
+ float arr[4];
+
+ _mm_storeu_ps(arr, y);
+
+ x[0] = GGML_FP32_TO_FP16(arr[0]);
+ x[1] = GGML_FP32_TO_FP16(arr[1]);
+ x[2] = GGML_FP32_TO_FP16(arr[2]);
+ x[3] = GGML_FP32_TO_FP16(arr[3]);
+}
+
+#define GGML_F32Cx4 __m128
+#define GGML_F32Cx4_ZERO _mm_setzero_ps()
+#define GGML_F32Cx4_SET1(x) _mm_set1_ps(x)
+#define GGML_F32Cx4_LOAD(x) __sse_f16x4_load(x)
+#define GGML_F32Cx4_STORE(x, y) __sse_f16x4_store(x, y)
+#define GGML_F32Cx4_FMA GGML_F32x4_FMA
+#define GGML_F32Cx4_ADD _mm_add_ps
+#define GGML_F32Cx4_MUL _mm_mul_ps
+#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE
+
+#define GGML_F16_VEC GGML_F32Cx4
+#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO
+#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1
+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)
+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])
+#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA
+#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD
+#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL
+#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE
+
+#elif defined(__loongarch_asx)
+
+#define GGML_SIMD
+
+// F32 LASX
+#define GGML_F32_STEP 32
+#define GGML_F32_EPR 8
+
+#define GGML_F32x8 __m256
+#define GGML_F32x8_ZERO (__m256)__lasx_xvldi(0)
+#define GGML_F32x8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x))
+#define GGML_F32x8_LOAD(x) (__m256)__lasx_xvld((x), 0)
+#define GGML_F32x8_STORE(x,y) __lasx_xvst((y), (x), 0)
+#define GGML_F32x8_FMA(a, b, c) __lasx_xvfmadd_s(b, c, a)
+#define GGML_F32x8_ADD __lasx_xvfadd_s
+#define GGML_F32x8_MUL __lasx_xvfmul_s
+#define GGML_F32x8_REDUCE(res, x) \
+do { \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \
+ } \
+ float *tmp_p = (float *)&x[0]; \
+ res = tmp_p[0] + tmp_p[1] + tmp_p[2] + tmp_p[3] + tmp_p[4] + tmp_p[5] + tmp_p[6] + tmp_p[7]; \
+} while (0)
+// TODO: is this optimal ?
+
+#define GGML_F32_VEC GGML_F32x8
+#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x8_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x8_STORE
+#define GGML_F32_VEC_FMA GGML_F32x8_FMA
+#define GGML_F32_VEC_ADD GGML_F32x8_ADD
+#define GGML_F32_VEC_MUL GGML_F32x8_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE
+
+// F16 LASX
+
+#define GGML_F16_STEP 32
+#define GGML_F16_EPR 8
+
+// F16 arithmetic is not supported by AVX, so we use F32 instead
+
+#define GGML_F32Cx8 __m256
+#define GGML_F32Cx8_ZERO (__m256)__lasx_xvldi(0)
+#define GGML_F32Cx8_SET1(x) (__m256)__lasx_xvreplgr2vr_w((x))
+
+static inline __m256 __lasx_f32cx8_load(const ggml_fp16_t * x) {
+ float tmp[8];
+
+ for (int i = 0; i < 8; i++) {
+ tmp[i] = GGML_FP16_TO_FP32(x[i]);
+ }
+
+ return (__m256)__lasx_xvld(tmp, 0);
+}
+static inline void __lasx_f32cx8_store(ggml_fp16_t * x, __m256 y) {
+ float arr[8];
+
+ __lasx_xvst(y, arr, 0);
+
+ for (int i = 0; i < 8; i++) {
+ x[i] = GGML_FP32_TO_FP16(arr[i]);
+ }
+}
+#define GGML_F32Cx8_LOAD(x) __lasx_f32cx8_load(x)
+#define GGML_F32Cx8_STORE(x, y) __lasx_f32cx8_store(x, y)
+
+#define GGML_F32Cx8_FMA GGML_F32x8_FMA
+#define GGML_F32Cx8_ADD __lasx_xvfadd_s
+#define GGML_F32Cx8_MUL __lasx_xvfmul_s
+#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE
+
+#define GGML_F16_VEC GGML_F32Cx8
+#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO
+#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1
+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p)
+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])
+#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA
+#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD
+#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL
+#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE
+
+#elif defined(__loongarch_sx)
+
+#define GGML_SIMD
+
+// F32 LSX
+
+#define GGML_F32_STEP 32
+#define GGML_F32_EPR 4
+
+#define GGML_F32x4 __m128
+#define GGML_F32x4_ZERO __lsx_vldi(0)
+#define GGML_F32x4_SET1(x) __lsx_vinsgr2vr_w(__lsx_vldi(0),(x), 0)
+#define GGML_F32x4_LOAD(x) __lsx_vld((x), 0)
+#define GGML_F32x4_STORE((x),(y)) __lsx_vst((y), (x), 0)
+#define GGML_F32x4_FMA(a, b, c) __lsx_vfmadd_s(b, c, a)
+#define GGML_F32x4_ADD __lsx_vfadd_s
+#define GGML_F32x4_MUL __lsx_vfmul_s
+#define GGML_F32x4_REDUCE(res, x) \
+{ \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \
+ } \
+ __m128i tmp = __lsx_vsrli_d((__m128i)x[0], 32); \
+ tmp = (__m128i)__lsx_vfadd_s((__m128)tmp, x[0]); \
+ tmp = __lsx_vpickev_w(__lsx_vldi(0), tmp); \
+ const __m128 t0 = __lsx_vshuf4i_w(tmp, 0x88); \
+ tmp = __lsx_vsrli_d((__m128i)t0, 32); \
+ tmp = (__m128i)__lsx_vfadd_s((__m128)tmp, t0); \
+ tmp = __lsx_vpickev_w(__lsx_vldi(0), tmp); \
+ res = (ggml_float) __lsx_vpickve2gr_w(__lsx_vshuf4i_w(tmp, 0x88), 0); \
+}
+
+#define GGML_F32_VEC GGML_F32x4
+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x4_STORE
+#define GGML_F32_VEC_FMA GGML_F32x4_FMA
+#define GGML_F32_VEC_ADD GGML_F32x4_ADD
+#define GGML_F32_VEC_MUL GGML_F32x4_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
+
+// F16 LSX
+
+#define GGML_F16_STEP 32
+#define GGML_F16_EPR 4
+
+static inline __m128 __lsx_f16x4_load(const ggml_fp16_t * x) {
+ float tmp[4];
+
+ tmp[0] = GGML_FP16_TO_FP32(x[0]);
+ tmp[1] = GGML_FP16_TO_FP32(x[1]);
+ tmp[2] = GGML_FP16_TO_FP32(x[2]);
+ tmp[3] = GGML_FP16_TO_FP32(x[3]);
+
+ return __lsx_vld(tmp, 0);
+}
+
+static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) {
+ float arr[4];
+
+ __lsx_vst(y, arr, 0);
+
+ x[0] = GGML_FP32_TO_FP16(arr[0]);
+ x[1] = GGML_FP32_TO_FP16(arr[1]);
+ x[2] = GGML_FP32_TO_FP16(arr[2]);
+ x[3] = GGML_FP32_TO_FP16(arr[3]);
+}
+
+#define GGML_F32Cx4 __m128
+#define GGML_F32Cx4_ZERO __lsx_vldi(0)
+#define GGML_F32Cx4_SET1(x) __lsx_vinsgr2vr_w(__lsx_vldi(0),(x), 0)
+#define GGML_F32Cx4_LOAD(x) __lsx_f16x4_load(x)
+#define GGML_F32Cx4_STORE(x, y) __lsx_f16x4_store(x, y)
+#define GGML_F32Cx4_FMA GGML_F32x4_FMA
+#define GGML_F32Cx4_ADD __lsx_vfadd_s
+#define GGML_F32Cx4_MUL __lsx_vfmul_s
+#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE
+
+#define GGML_F16_VEC GGML_F32Cx4
+#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO
+#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1
+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)
+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])
+#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA
+#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD
+#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL
+#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE
+
+#endif
+
+// GGML_F32_ARR / GGML_F16_ARR
+// number of registers to use per step
+#ifdef GGML_SIMD
+#define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR)
+#define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR)
+#endif
+
+//
+// ggml context
+//
+
+struct ggml_context {
+ size_t mem_size;
+ void* mem_buffer;
+ bool mem_buffer_owned;
+ bool no_alloc;
+ bool no_alloc_save; // this is used to save the no_alloc state when using scratch buffers
+
+ int n_objects;
+
+ struct ggml_object * objects_begin;
+ struct ggml_object * objects_end;
+
+ struct ggml_scratch scratch;
+ struct ggml_scratch scratch_save;
+};
+
+struct ggml_context_container {
+ bool used;
+
+ struct ggml_context context;
+};
+
+//
+// Threading defs
+//
+
+typedef pthread_t ggml_thread_t;
+
+#if defined(_WIN32)
+
+typedef CONDITION_VARIABLE ggml_cond_t;
+typedef SRWLOCK ggml_mutex_t;
+
+#define ggml_mutex_init(m) InitializeSRWLock(m)
+#define ggml_mutex_destroy(m)
+#define ggml_mutex_lock(m) AcquireSRWLockExclusive(m)
+#define ggml_mutex_unlock(m) ReleaseSRWLockExclusive(m)
+#define ggml_mutex_lock_shared(m) AcquireSRWLockShared(m)
+#define ggml_mutex_unlock_shared(m) ReleaseSRWLockShared(m)
+
+#define ggml_cond_init(c) InitializeConditionVariable(c)
+#define ggml_cond_destroy(c)
+#define ggml_cond_wait(c, m) SleepConditionVariableSRW(c, m, INFINITE, CONDITION_VARIABLE_LOCKMODE_SHARED)
+#define ggml_cond_broadcast(c) WakeAllConditionVariable(c)
+
+#define ggml_thread_create pthread_create
+#define ggml_thread_join pthread_join
+
+#else
+
+typedef pthread_cond_t ggml_cond_t;
+typedef pthread_mutex_t ggml_mutex_t;
+
+#define ggml_mutex_init(m) pthread_mutex_init(m, NULL)
+#define ggml_mutex_destroy(m) pthread_mutex_destroy(m)
+#define ggml_mutex_lock(m) pthread_mutex_lock(m)
+#define ggml_mutex_unlock(m) pthread_mutex_unlock(m)
+#define ggml_mutex_lock_shared(m) pthread_mutex_lock(m)
+#define ggml_mutex_unlock_shared(m) pthread_mutex_unlock(m)
+
+#define ggml_lock_init(x) UNUSED(x)
+#define ggml_lock_destroy(x) UNUSED(x)
+#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))
+#define ggml_lock_lock(x) _mm_pause()
+#else
+#define ggml_lock_lock(x) UNUSED(x)
+#endif
+#define ggml_lock_unlock(x) UNUSED(x)
+
+#define GGML_LOCK_INITIALIZER 0
+#define ggml_cond_init(c) pthread_cond_init(c, NULL)
+#define ggml_cond_destroy(c) pthread_cond_destroy(c)
+#define ggml_cond_wait(c, m) pthread_cond_wait(c, m)
+#define ggml_cond_broadcast(c) pthread_cond_broadcast(c)
+
+#define ggml_thread_create pthread_create
+#define ggml_thread_join pthread_join
+
+#endif
+
+// Threadpool def
+struct ggml_threadpool {
+ ggml_mutex_t mutex; // mutex for cond.var
+ ggml_cond_t cond; // cond.var for waiting for new work
+
+ struct ggml_cgraph * cgraph;
+ struct ggml_cplan * cplan;
+
+ // synchronization primitives
+ atomic_int n_graph; // incremented when there is work to be done (i.e each graph)
+ atomic_int n_barrier;
+ atomic_int n_barrier_passed;
+ atomic_int current_chunk; // currently processing chunk during Mat_Mul, shared between all the threads.
+
+ // these are atomic as an annotation for thread-sanitizer
+ atomic_bool stop; // Used for stopping the threadpool altogether
+ atomic_bool pause; // Used for pausing the threadpool or individual threads
+
+ struct ggml_compute_state * workers; // per thread state
+ int n_threads_max; // number of threads in the pool
+ int n_threads_cur; // number of threads used in the current graph
+
+ int32_t prio; // Scheduling priority
+ uint32_t poll; // Polling level (0 - no polling)
+
+ enum ggml_status ec;
+};
+
+// Per-thread state
+struct ggml_compute_state {
+#ifndef GGML_USE_OPENMP
+ ggml_thread_t thrd;
+ bool cpumask[GGML_MAX_N_THREADS];
+ int last_graph;
+ bool pending;
+#endif
+ struct ggml_threadpool * threadpool;
+ int ith;
+};
+
+struct ggml_compute_params {
+ // ith = thread index, nth = number of threads
+ int ith, nth;
+
+ // work buffer for all threads
+ size_t wsize;
+ void * wdata;
+
+ struct ggml_threadpool * threadpool;
+};
+
+//
+// fundamental operations
+//
+
+inline static void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+
+inline static void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+
+inline static void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+
+inline static void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+
+inline static void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+
+inline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; }
+inline static void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v) { for (int i = 0; i < n; ++i) z[i] = x[i] + v; }
+inline static void ggml_vec_acc_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] += x[i]; }
+inline static void ggml_vec_acc1_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] += v; }
+inline static void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] - y[i]; }
+inline static void ggml_vec_set_f32 (const int n, float * x, const float v) { for (int i = 0; i < n; ++i) x[i] = v; }
+inline static void ggml_vec_cpy_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; }
+inline static void ggml_vec_neg_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = -x[i]; }
+inline static void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]*y[i]; }
+inline static void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; }
+
+static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+#if defined(GGML_SIMD)
+ float sumf = 0.0f;
+ const int np = (n & ~(GGML_F32_STEP - 1));
+
+ GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };
+
+ GGML_F32_VEC ax[GGML_F32_ARR];
+ GGML_F32_VEC ay[GGML_F32_ARR];
+
+ for (int i = 0; i < np; i += GGML_F32_STEP) {
+ for (int j = 0; j < GGML_F32_ARR; j++) {
+ ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);
+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
+
+ sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);
+ }
+ }
+
+ // reduce sum0..sum3 to sum0
+ GGML_F32_VEC_REDUCE(sumf, sum);
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ sumf += x[i]*y[i];
+ }
+#else
+ // scalar
+ ggml_float sumf = 0.0;
+ for (int i = 0; i < n; ++i) {
+ sumf += (ggml_float)(x[i]*y[i]);
+ }
+#endif
+
+ *s = sumf;
+}
+
+static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+ int i = 0;
+ ggml_float sumf = 0;
+
+#if defined(__AVX512BF16__)
+ __m512 c1 = _mm512_setzero_ps();
+ __m512 c2 = _mm512_setzero_ps();
+ for (; i + 64 <= n; i += 64) {
+ c1 = _mm512_dpbf16_ps(c1, m512bh(_mm512_loadu_si512((x + i))),
+ m512bh(_mm512_loadu_si512((y + i))));
+ c2 = _mm512_dpbf16_ps(c2, m512bh(_mm512_loadu_si512((x + i + 32))),
+ m512bh(_mm512_loadu_si512((y + i + 32))));
+ }
+ sumf += (ggml_float)_mm512_reduce_add_ps(c1);
+ sumf += (ggml_float)_mm512_reduce_add_ps(c2);
+
+#elif defined(__AVX512F__)
+#define LOAD(p) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)(p))), 16))
+ __m512 c1 = _mm512_setzero_ps();
+ __m512 c2 = _mm512_setzero_ps();
+ for (; i + 32 <= n; i += 32) {
+ c1 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i), LOAD(y + i)), c1);
+ c2 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c2);
+ }
+ sumf += (ggml_float)_mm512_reduce_add_ps(c1);
+ sumf += (ggml_float)_mm512_reduce_add_ps(c2);
+
+#undef LOAD
+#elif defined(__AVX2__)
+#define LOAD(p) _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16))
+ __m256 c1 = _mm256_setzero_ps();
+ __m256 c2 = _mm256_setzero_ps();
+ __m256 c3 = _mm256_setzero_ps();
+ __m256 c4 = _mm256_setzero_ps();
+ for (; i + 32 <= n; i += 32) {
+ c1 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i), LOAD(y + i)), c1);
+ c2 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 8), LOAD(y + i + 8)), c2);
+ c3 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c3);
+ c4 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 24), LOAD(y + i + 24)), c4);
+ }
+ __m128 g;
+ c1 = _mm256_add_ps(_mm256_add_ps(c1, c3),
+ _mm256_add_ps(c2, c4));
+ g = _mm_add_ps(_mm256_extractf128_ps(c1, 1),
+ _mm256_castps256_ps128(c1));
+ g = _mm_add_ps(g, _mm_movehl_ps(g, g));
+ g = _mm_add_ss(g, _mm_movehdup_ps(g));
+ sumf += (ggml_float)_mm_cvtss_f32(g);
+
+#undef LOAD
+#endif
+
+ for (; i < n; ++i) {
+ sumf += (ggml_float)(GGML_BF16_TO_FP32(x[i]) *
+ GGML_BF16_TO_FP32(y[i]));
+ }
+ *s = sumf;
+}
+
+static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ ggml_float sumf = 0.0;
+
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F16_STEP - 1));
+
+ GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO };
+
+ GGML_F16_VEC ax[GGML_F16_ARR];
+ GGML_F16_VEC ay[GGML_F16_ARR];
+
+ for (int i = 0; i < np; i += GGML_F16_STEP) {
+ for (int j = 0; j < GGML_F16_ARR; j++) {
+ ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);
+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
+
+ sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]);
+ }
+ }
+
+ // reduce sum0..sum3 to sum0
+ GGML_F16_VEC_REDUCE(sumf, sum);
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));
+ }
+#else
+ for (int i = 0; i < n; ++i) {
+ sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));
+ }
+#endif
+
+ *s = sumf;
+}
+
+// compute GGML_VEC_DOT_UNROLL dot products at once
+// xs - x row stride in bytes
+inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y) {
+ ggml_float sumf[GGML_VEC_DOT_UNROLL] = { 0.0 };
+
+ ggml_fp16_t * restrict x[GGML_VEC_DOT_UNROLL];
+
+ for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {
+ x[i] = (ggml_fp16_t *) ((char *) xv + i*xs);
+ }
+
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F16_STEP - 1));
+
+ GGML_F16_VEC sum[GGML_VEC_DOT_UNROLL][GGML_F16_ARR] = { { GGML_F16_VEC_ZERO } };
+
+ GGML_F16_VEC ax[GGML_F16_ARR];
+ GGML_F16_VEC ay[GGML_F16_ARR];
+
+ for (int i = 0; i < np; i += GGML_F16_STEP) {
+ for (int j = 0; j < GGML_F16_ARR; j++) {
+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
+
+ for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {
+ ax[j] = GGML_F16_VEC_LOAD(x[k] + i + j*GGML_F16_EPR, j);
+
+ sum[k][j] = GGML_F16_VEC_FMA(sum[k][j], ax[j], ay[j]);
+ }
+ }
+ }
+
+ // reduce sum0..sum3 to sum0
+ for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {
+ GGML_F16_VEC_REDUCE(sumf[k], sum[k]);
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {
+ sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));
+ }
+ }
+#else
+ for (int i = 0; i < n; ++i) {
+ for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {
+ sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));
+ }
+ }
+#endif
+
+ for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {
+ s[i] = sumf[i];
+ }
+}
+
+inline static void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) {
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F32_STEP - 1));
+
+ GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);
+
+ GGML_F32_VEC ax[GGML_F32_ARR];
+ GGML_F32_VEC ay[GGML_F32_ARR];
+
+ for (int i = 0; i < np; i += GGML_F32_STEP) {
+ for (int j = 0; j < GGML_F32_ARR; j++) {
+ ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);
+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
+ ay[j] = GGML_F32_VEC_FMA(ay[j], ax[j], vx);
+
+ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);
+ }
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ y[i] += x[i]*v;
+ }
+#else
+ // scalar
+ for (int i = 0; i < n; ++i) {
+ y[i] += x[i]*v;
+ }
+#endif
+}
+
+inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) {
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F16_STEP - 1));
+
+ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);
+
+ GGML_F16_VEC ax[GGML_F16_ARR];
+ GGML_F16_VEC ay[GGML_F16_ARR];
+
+ for (int i = 0; i < np; i += GGML_F16_STEP) {
+ for (int j = 0; j < GGML_F16_ARR; j++) {
+ ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);
+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
+ ay[j] = GGML_F16_VEC_FMA(ay[j], ax[j], vx);
+
+ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);
+ }
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);
+ }
+#else
+ // scalar
+ for (int i = 0; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);
+ }
+#endif
+}
+
+// xs and vs are byte strides of x and v
+inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv) {
+
+ const float * restrict x[GGML_VEC_MAD_UNROLL];
+ const float * restrict v[GGML_VEC_MAD_UNROLL];
+
+ for (int i = 0; i < GGML_VEC_MAD_UNROLL; ++i) {
+ x[i] = (const float *) ((const char *) xv + i*xs);
+ v[i] = (const float *) ((const char *) vv + i*vs);
+ }
+
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F32_STEP - 1));
+
+ GGML_F32_VEC vx[GGML_VEC_MAD_UNROLL];
+
+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
+ vx[k] = GGML_F32_VEC_SET1(v[k][0]);
+ }
+
+ GGML_F32_VEC ax[GGML_VEC_MAD_UNROLL][GGML_F32_ARR];
+ GGML_F32_VEC ay[GGML_F32_ARR];
+
+ for (int i = 0; i < np; i += GGML_F32_STEP) {
+ for (int j = 0; j < GGML_F32_ARR; j++) {
+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
+
+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
+ ax[k][j] = GGML_F32_VEC_LOAD(x[k] + i + j*GGML_F32_EPR);
+ ay[j] = GGML_F32_VEC_FMA(ay[j], ax[k][j], vx[k]);
+ }
+
+ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);
+ }
+ }
+
+ // leftovers
+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
+ for (int i = np; i < n; ++i) {
+ y[i] += x[k][i]*v[k][0];
+ }
+ }
+#else
+ // scalar
+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
+ for (int i = 0; i < n; ++i) {
+ y[i] += x[k][i]*v[k][0];
+ }
+ }
+#endif
+}
+
+//inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] *= v; }
+inline static void ggml_vec_scale_f32(const int n, float * y, const float v) {
+#if defined(GGML_USE_ACCELERATE)
+ vDSP_vsmul(y, 1, &v, y, 1, n);
+#elif defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F32_STEP - 1));
+
+ GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);
+
+ GGML_F32_VEC ay[GGML_F32_ARR];
+
+ for (int i = 0; i < np; i += GGML_F32_STEP) {
+ for (int j = 0; j < GGML_F32_ARR; j++) {
+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
+ ay[j] = GGML_F32_VEC_MUL(ay[j], vx);
+
+ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);
+ }
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ y[i] *= v;
+ }
+#else
+ // scalar
+ for (int i = 0; i < n; ++i) {
+ y[i] *= v;
+ }
+#endif
+}
+
+inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F16_STEP - 1));
+
+ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);
+
+ GGML_F16_VEC ay[GGML_F16_ARR];
+
+ for (int i = 0; i < np; i += GGML_F16_STEP) {
+ for (int j = 0; j < GGML_F16_ARR; j++) {
+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
+ ay[j] = GGML_F16_VEC_MUL(ay[j], vx);
+
+ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);
+ }
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);
+ }
+#else
+ // scalar
+ for (int i = 0; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);
+ }
+#endif
+}
+
+inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); }
+inline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }
+inline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }
+inline static void ggml_vec_log_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = logf(x[i]); }
+inline static void ggml_vec_sin_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sinf(x[i]); }
+inline static void ggml_vec_cos_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = cosf(x[i]); }
+inline static void ggml_vec_abs_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fabsf(x[i]); }
+inline static void ggml_vec_sgn_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : ((x[i] < 0.f) ? -1.f : 0.f); }
+inline static void ggml_vec_step_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : 0.f; }
+inline static void ggml_vec_tanh_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = tanhf(x[i]); }
+inline static void ggml_vec_elu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : expm1f(x[i]); }
+inline static void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }
+inline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }
+inline static void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = 1.f / (1.f + expf(-x[i])); }
+// TODO: optimize performance
+inline static void ggml_vec_hardswish_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }
+inline static void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }
+inline static void ggml_vec_exp_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = expf(x[i]); }
+
+static const float GELU_COEF_A = 0.044715f;
+static const float GELU_QUICK_COEF = -1.702f;
+static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
+
+inline static float ggml_gelu_f32(float x) {
+ return 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));
+}
+
+inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
+ const uint16_t * i16 = (const uint16_t *) x;
+ for (int i = 0; i < n; ++i) {
+ y[i] = ggml_table_gelu_f16[i16[i]];
+ }
+}
+
+#ifdef GGML_GELU_FP16
+inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
+ uint16_t t;
+ for (int i = 0; i < n; ++i) {
+ if (x[i] <= -10.0f) {
+ y[i] = 0.0f;
+ } else if (x[i] >= 10.0f) {
+ y[i] = x[i];
+ } else {
+ ggml_fp16_t fp16 = GGML_FP32_TO_FP16(x[i]);
+ memcpy(&t, &fp16, sizeof(uint16_t));
+ y[i] = GGML_FP16_TO_FP32(ggml_table_gelu_f16[t]);
+ }
+ }
+}
+#else
+inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
+ for (int i = 0; i < n; ++i) {
+ y[i] = ggml_gelu_f32(x[i]);
+ }
+}
+#endif
+
+inline static float ggml_gelu_quick_f32(float x) {
+ return x*(1.0f/(1.0f+expf(GELU_QUICK_COEF*x)));
+}
+
+//inline static void ggml_vec_gelu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
+// const uint16_t * i16 = (const uint16_t *) x;
+// for (int i = 0; i < n; ++i) {
+// y[i] = ggml_table_gelu_quick_f16[i16[i]];
+// }
+//}
+
+#ifdef GGML_GELU_QUICK_FP16
+inline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {
+ uint16_t t;
+ for (int i = 0; i < n; ++i) {
+ ggml_fp16_t fp16 = GGML_FP32_TO_FP16(x[i]);
+ memcpy(&t, &fp16, sizeof(uint16_t));
+ y[i] = GGML_FP16_TO_FP32(ggml_table_gelu_quick_f16[t]);
+ }
+}
+#else
+inline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {
+ for (int i = 0; i < n; ++i) {
+ y[i] = ggml_gelu_quick_f32(x[i]);
+ }
+}
+#endif
+
+// Sigmoid Linear Unit (SiLU) function
+inline static float ggml_silu_f32(float x) {
+ return x/(1.0f + expf(-x));
+}
+
+#if __FINITE_MATH_ONLY__
+#error "some routines in ggml.c require non-finite math arithmetics -- pass -fno-finite-math-only to the compiler to fix"
+#error "ref: https://github.com/ggerganov/llama.cpp/pull/7154#issuecomment-2143844461"
+#endif
+
+#if defined(__ARM_NEON) && defined(__aarch64__)
+
+// adapted from arm limited optimized routine
+// the maximum error is 1.45358 plus 0.5 ulps
+// numbers above 88.38 will flush to infinity
+// numbers beneath -103.97 will flush to zero
+inline static float32x4_t ggml_v_expf(float32x4_t x) {
+ const float32x4_t r = vdupq_n_f32(0x1.8p23f);
+ const float32x4_t z = vfmaq_f32(r, x, vdupq_n_f32(0x1.715476p+0f));
+ const float32x4_t n = vsubq_f32(z, r);
+ const float32x4_t b = vfmsq_f32(vfmsq_f32(x, n, vdupq_n_f32(0x1.62e4p-1f)), n,
+ vdupq_n_f32(0x1.7f7d1cp-20f));
+ const uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_f32(z), 23);
+ const float32x4_t k = vreinterpretq_f32_u32(vaddq_u32(e, vreinterpretq_u32_f32(vdupq_n_f32(1))));
+ const uint32x4_t c = vcagtq_f32(n, vdupq_n_f32(126));
+ const float32x4_t u = vmulq_f32(b, b);
+ const float32x4_t j = vfmaq_f32(
+ vmulq_f32(vdupq_n_f32(0x1.ffffecp-1f), b),
+ vfmaq_f32(vfmaq_f32(vdupq_n_f32(0x1.fffdb6p-2f), vdupq_n_f32(0x1.555e66p-3f), b),
+ vfmaq_f32(vdupq_n_f32(0x1.573e2ep-5f), vdupq_n_f32(0x1.0e4020p-7f), b), u), u);
+ if (!vpaddd_u64(vreinterpretq_u64_u32(c)))
+ return vfmaq_f32(k, j, k);
+ const uint32x4_t d = vandq_u32(vclezq_f32(n), vdupq_n_u32(0x82000000));
+ const float32x4_t s1 = vreinterpretq_f32_u32(vaddq_u32(d, vdupq_n_u32(0x7f000000)));
+ const float32x4_t s2 = vreinterpretq_f32_u32(vsubq_u32(e, d));
+ return vbslq_f32(vcagtq_f32(n, vdupq_n_f32(192)), vmulq_f32(s1, s1),
+ vbslq_f32(c, vmulq_f32(vfmaq_f32(s2, s2, j), s1), vfmaq_f32(k, k, j)));
+}
+
+// computes silu x/(1+exp(-x)) in single precision vector
+inline static float32x4_t ggml_v_silu(float32x4_t x) {
+ const float32x4_t one = vdupq_n_f32(1.0f);
+ const float32x4_t zero = vdupq_n_f32(0.0f);
+ const float32x4_t neg_x = vsubq_f32(zero, x);
+ const float32x4_t exp_neg_x = ggml_v_expf(neg_x);
+ const float32x4_t one_plus_exp_neg_x = vaddq_f32(one, exp_neg_x);
+ return vdivq_f32(x, one_plus_exp_neg_x);
+}
+
+#elif defined(__AVX512F__) && defined(__AVX512DQ__)
+
+// adapted from arm limited optimized routine
+// the maximum error is 1.45358 plus 0.5 ulps
+// numbers above 88.38 will flush to infinity
+// numbers beneath -103.97 will flush to zero
+inline static __m512 ggml_v_expf(__m512 x) {
+ const __m512 r = _mm512_set1_ps(0x1.8p23f);
+ const __m512 z = _mm512_fmadd_ps(x, _mm512_set1_ps(0x1.715476p+0f), r);
+ const __m512 n = _mm512_sub_ps(z, r);
+ const __m512 b =
+ _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.7f7d1cp-20f),
+ _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.62e4p-1f), x));
+ const __mmask16 d =
+ _mm512_cmp_ps_mask(_mm512_abs_ps(n), _mm512_set1_ps(192), _CMP_GT_OQ);
+ const __m512 u = _mm512_mul_ps(b, b);
+ const __m512 j = _mm512_fmadd_ps(
+ _mm512_fmadd_ps(_mm512_fmadd_ps(_mm512_set1_ps(0x1.0e4020p-7f), b,
+ _mm512_set1_ps(0x1.573e2ep-5f)),
+ u,
+ _mm512_fmadd_ps(_mm512_set1_ps(0x1.555e66p-3f), b,
+ _mm512_set1_ps(0x1.fffdb6p-2f))),
+ u,
+ _mm512_fmadd_ps(_mm512_set1_ps(0x1.ffffecp-1f), b, _mm512_set1_ps(1.0F)));
+ const __m512 res = _mm512_scalef_ps(j, n);
+ if (_mm512_kortestz(d, d))
+ return res;
+ const __m512 zero = _mm512_setzero_ps();
+ const __m512 alt = _mm512_mask_blend_ps(
+ _mm512_cmp_ps_mask(n, zero, _CMP_LE_OQ), _mm512_set1_ps(INFINITY), zero);
+ return _mm512_mask_blend_ps(d, res, alt);
+}
+
+// computes silu x/(1+exp(-x)) in single precision vector
+inline static __m512 ggml_v_silu(__m512 x) {
+ const __m512 one = _mm512_set1_ps(1);
+ const __m512 zero = _mm512_setzero_ps();
+ const __m512 neg_x = _mm512_sub_ps(zero, x);
+ const __m512 exp_neg_x = ggml_v_expf(neg_x);
+ const __m512 one_plus_exp_neg_x = _mm512_add_ps(one, exp_neg_x);
+ return _mm512_div_ps(x, one_plus_exp_neg_x);
+}
+
+#elif defined(__AVX2__) && defined(__FMA__)
+
+// adapted from arm limited optimized routine
+// the maximum error is 1.45358 plus 0.5 ulps
+// numbers above 88.38 will flush to infinity
+// numbers beneath -103.97 will flush to zero
+inline static __m256 ggml_v_expf(__m256 x) {
+ const __m256 r = _mm256_set1_ps(0x1.8p23f);
+ const __m256 z = _mm256_fmadd_ps(x, _mm256_set1_ps(0x1.715476p+0f), r);
+ const __m256 n = _mm256_sub_ps(z, r);
+ const __m256 b = _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.7f7d1cp-20f),
+ _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.62e4p-1f), x));
+ const __m256i e = _mm256_slli_epi32(_mm256_castps_si256(z), 23);
+ const __m256 k = _mm256_castsi256_ps(
+ _mm256_add_epi32(e, _mm256_castps_si256(_mm256_set1_ps(1))));
+ const __m256i c = _mm256_castps_si256(
+ _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n),
+ _mm256_set1_ps(126), _CMP_GT_OQ));
+ const __m256 u = _mm256_mul_ps(b, b);
+ const __m256 j = _mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_set1_ps(0x1.0e4020p-7f), b,
+ _mm256_set1_ps(0x1.573e2ep-5f)), u,
+ _mm256_fmadd_ps(_mm256_set1_ps(0x1.555e66p-3f), b,
+ _mm256_set1_ps(0x1.fffdb6p-2f))),
+ u, _mm256_mul_ps(_mm256_set1_ps(0x1.ffffecp-1f), b));
+ if (!_mm256_movemask_ps(_mm256_castsi256_ps(c)))
+ return _mm256_fmadd_ps(j, k, k);
+ const __m256i g = _mm256_and_si256(
+ _mm256_castps_si256(_mm256_cmp_ps(n, _mm256_setzero_ps(), _CMP_LE_OQ)),
+ _mm256_set1_epi32(0x82000000u));
+ const __m256 s1 =
+ _mm256_castsi256_ps(_mm256_add_epi32(g, _mm256_set1_epi32(0x7f000000u)));
+ const __m256 s2 = _mm256_castsi256_ps(_mm256_sub_epi32(e, g));
+ const __m256i d = _mm256_castps_si256(
+ _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n),
+ _mm256_set1_ps(192), _CMP_GT_OQ));
+ return _mm256_or_ps(
+ _mm256_and_ps(_mm256_castsi256_ps(d), _mm256_mul_ps(s1, s1)),
+ _mm256_andnot_ps(
+ _mm256_castsi256_ps(d),
+ _mm256_or_ps(
+ _mm256_and_ps(_mm256_castsi256_ps(c),
+ _mm256_mul_ps(_mm256_fmadd_ps(s2, j, s2), s1)),
+ _mm256_andnot_ps(_mm256_castsi256_ps(c), _mm256_fmadd_ps(k, j, k)))));
+}
+
+// computes silu x/(1+exp(-x)) in single precision vector
+inline static __m256 ggml_v_silu(__m256 x) {
+ const __m256 one = _mm256_set1_ps(1);
+ const __m256 zero = _mm256_setzero_ps();
+ const __m256 neg_x = _mm256_sub_ps(zero, x);
+ const __m256 exp_neg_x = ggml_v_expf(neg_x);
+ const __m256 one_plus_exp_neg_x = _mm256_add_ps(one, exp_neg_x);
+ return _mm256_div_ps(x, one_plus_exp_neg_x);
+}
+
+#elif defined(__SSE2__) // __AVX2__ / __ARM_NEON
+
+#if defined(__FMA__)
+#define MADD128(x, y, z) _mm_fmadd_ps(x, y, z)
+#define NMADD128(x, y, z) _mm_fnmadd_ps(x, y, z)
+#else
+#define MADD128(x, y, z) _mm_add_ps(_mm_mul_ps(x, y), z)
+#define NMADD128(x, y, z) _mm_sub_ps(z, _mm_mul_ps(x, y))
+#endif
+
+// adapted from arm limited optimized routine
+// the maximum error is 1.45358 plus 0.5 ulps
+// numbers above 88.38 will flush to infinity
+// numbers beneath -103.97 will flush to zero
+inline static __m128 ggml_v_expf(__m128 x) {
+ const __m128 r = _mm_set1_ps(0x1.8p23f);
+ const __m128 z = MADD128(x, _mm_set1_ps(0x1.715476p+0f), r);
+ const __m128 n = _mm_sub_ps(z, r);
+ const __m128 b =
+ NMADD128(n, _mm_set1_ps(0x1.7f7d1cp-20f), NMADD128(n, _mm_set1_ps(0x1.62e4p-1f), x));
+ const __m128i e = _mm_slli_epi32(_mm_castps_si128(z), 23);
+ const __m128 k = _mm_castsi128_ps(_mm_add_epi32(e, _mm_castps_si128(_mm_set1_ps(1))));
+ const __m128i c =
+ _mm_castps_si128(_mm_cmpgt_ps(_mm_andnot_ps(_mm_set1_ps(-0.f), n), _mm_set1_ps(126)));
+ const __m128 u = _mm_mul_ps(b, b);
+ const __m128 j =
+ MADD128(MADD128(MADD128(_mm_set1_ps(0x1.0e4020p-7f), b, _mm_set1_ps(0x1.573e2ep-5f)), u,
+ MADD128(_mm_set1_ps(0x1.555e66p-3f), b, _mm_set1_ps(0x1.fffdb6p-2f))),
+ u, _mm_mul_ps(_mm_set1_ps(0x1.ffffecp-1f), b));
+ if (!_mm_movemask_epi8(c))
+ return MADD128(j, k, k);
+ const __m128i g = _mm_and_si128(_mm_castps_si128(_mm_cmple_ps(n, _mm_setzero_ps())),
+ _mm_set1_epi32(0x82000000u));
+ const __m128 s1 = _mm_castsi128_ps(_mm_add_epi32(g, _mm_set1_epi32(0x7f000000u)));
+ const __m128 s2 = _mm_castsi128_ps(_mm_sub_epi32(e, g));
+ const __m128i d =
+ _mm_castps_si128(_mm_cmpgt_ps(_mm_andnot_ps(_mm_set1_ps(-0.f), n), _mm_set1_ps(192)));
+ return _mm_or_ps(
+ _mm_and_ps(_mm_castsi128_ps(d), _mm_mul_ps(s1, s1)),
+ _mm_andnot_ps(_mm_castsi128_ps(d),
+ _mm_or_ps(_mm_and_ps(_mm_castsi128_ps(c), _mm_mul_ps(MADD128(s2, j, s2), s1)),
+ _mm_andnot_ps(_mm_castsi128_ps(c), MADD128(k, j, k)))));
+}
+
+// computes silu x/(1+exp(-x)) in single precision vector
+inline static __m128 ggml_v_silu(__m128 x) {
+ const __m128 one = _mm_set1_ps(1);
+ const __m128 zero = _mm_setzero_ps();
+ const __m128 neg_x = _mm_sub_ps(zero, x);
+ const __m128 exp_neg_x = ggml_v_expf(neg_x);
+ const __m128 one_plus_exp_neg_x = _mm_add_ps(one, exp_neg_x);
+ return _mm_div_ps(x, one_plus_exp_neg_x);
+}
+
+#endif // __ARM_NEON / __AVX2__ / __SSE2__
+
+static void ggml_vec_silu_f32(const int n, float * y, const float * x) {
+ int i = 0;
+#if defined(__AVX512F__) && defined(__AVX512DQ__)
+ for (; i + 15 < n; i += 16) {
+ _mm512_storeu_ps(y + i, ggml_v_silu(_mm512_loadu_ps(x + i)));
+ }
+#elif defined(__AVX2__) && defined(__FMA__)
+ for (; i + 7 < n; i += 8) {
+ _mm256_storeu_ps(y + i, ggml_v_silu(_mm256_loadu_ps(x + i)));
+ }
+#elif defined(__SSE2__)
+ for (; i + 3 < n; i += 4) {
+ _mm_storeu_ps(y + i, ggml_v_silu(_mm_loadu_ps(x + i)));
+ }
+#elif defined(__ARM_NEON) && defined(__aarch64__)
+ for (; i + 3 < n; i += 4) {
+ vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i)));
+ }
+#endif
+ for (; i < n; ++i) {
+ y[i] = ggml_silu_f32(x[i]);
+ }
+}
+
+static ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {
+ int i = 0;
+ ggml_float sum = 0;
+#if defined(__AVX512F__) && defined(__AVX512DQ__)
+ for (; i + 15 < n; i += 16) {
+ __m512 val = ggml_v_expf(_mm512_sub_ps(_mm512_loadu_ps(x + i),
+ _mm512_set1_ps(max)));
+ _mm512_storeu_ps(y + i, val);
+ sum += (ggml_float)_mm512_reduce_add_ps(val);
+ }
+#elif defined(__AVX2__) && defined(__FMA__)
+ for (; i + 7 < n; i += 8) {
+ __m256 val = ggml_v_expf(_mm256_sub_ps(_mm256_loadu_ps(x + i),
+ _mm256_set1_ps(max)));
+ _mm256_storeu_ps(y + i, val);
+ __m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),
+ _mm256_castps256_ps128(val));
+ val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));
+ val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2));
+ sum += (ggml_float)_mm_cvtss_f32(val2);
+ }
+#elif defined(__SSE2__)
+ for (; i + 3 < n; i += 4) {
+ __m128 val = ggml_v_expf(_mm_sub_ps(_mm_loadu_ps(x + i),
+ _mm_set1_ps(max)));
+ _mm_storeu_ps(y + i, val);
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+ val = _mm_add_ps(val, _mm_movehl_ps(val, val));
+ val = _mm_add_ss(val, _mm_movehdup_ps(val));
+#else
+ __m128 tmp = _mm_shuffle_ps(val, val, _MM_SHUFFLE(2, 3, 0, 1));
+ val = _mm_add_ps(val, tmp);
+ tmp = _mm_movehl_ps(tmp, val);
+ val = _mm_add_ss(val, tmp);
+#endif
+ sum += (ggml_float)_mm_cvtss_f32(val);
+ }
+#elif defined(__ARM_NEON) && defined(__aarch64__)
+ for (; i + 3 < n; i += 4) {
+ float32x4_t val = ggml_v_expf(vsubq_f32(vld1q_f32(x + i),
+ vdupq_n_f32(max)));
+ vst1q_f32(y + i, val);
+ sum += (ggml_float)vaddvq_f32(val);
+ }
+#endif
+ for (; i < n; ++i) {
+ float val = expf(x[i] - max);
+ sum += (ggml_float)val;
+ y[i] = val;
+ }
+ return sum;
+}
+
+static ggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, float max) {
+ // log(soft_max) = log(soft_max_i / soft_max_sum) = log(soft_max_i) - log(soft_max_sum) = (logit_i - max) - log(soft_max_i)
+
+ int i = 0;
+ ggml_float sum = 0;
+ for (; i < n; ++i) {
+ float val = x[i] - max;
+ y[i] = val;
+ sum += (ggml_float)expf(val);
+ }
+ return sum = (ggml_float)logf(sum);
+}
+
+inline static float ggml_silu_backward_f32(float x, float dy) {
+ const float s = 1.0f/(1.0f + expf(-x));
+ return dy*s*(1.0f + x*(1.0f - s));
+}
+
+inline static void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) {
+ for (int i = 0; i < n; ++i) {
+ dx[i] = ggml_silu_backward_f32(x[i], dy[i]);
+ }
+}
+
+inline static void ggml_vec_sum_f32(const int n, float * s, const float * x) {
+#ifndef GGML_USE_ACCELERATE
+ ggml_float sum = 0.0;
+ for (int i = 0; i < n; ++i) {
+ sum += (ggml_float)x[i];
+ }
+ *s = sum;
+#else
+ vDSP_sve(x, 1, s, n);
+#endif
+}
+
+inline static void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x) {
+ ggml_float sum = 0.0;
+ for (int i = 0; i < n; ++i) {
+ sum += (ggml_float)x[i];
+ }
+ *s = sum;
+}
+
+inline static void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {
+ float sum = 0.0f;
+ for (int i = 0; i < n; ++i) {
+ sum += GGML_FP16_TO_FP32(x[i]);
+ }
+ *s = sum;
+}
+
+inline static void ggml_vec_sum_bf16_ggf(const int n, float * s, const ggml_bf16_t * x) {
+ float sum = 0.0f;
+ for (int i = 0; i < n; ++i) {
+ sum += GGML_BF16_TO_FP32(x[i]);
+ }
+ *s = sum;
+}
+
+inline static void ggml_vec_max_f32(const int n, float * s, const float * x) {
+#ifndef GGML_USE_ACCELERATE
+ float max = -INFINITY;
+ for (int i = 0; i < n; ++i) {
+ max = MAX(max, x[i]);
+ }
+ *s = max;
+#else
+ vDSP_maxv(x, 1, s, n);
+#endif
+}
+
+inline static void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {
+ ggml_vec_norm_f32(n, s, x);
+ *s = 1.f/(*s);
+}
+
+inline static void ggml_vec_argmax_f32(const int n, int * s, const float * x) {
+ float max = -INFINITY;
+ int idx = 0;
+ for (int i = 0; i < n; ++i) {
+ max = MAX(max, x[i]);
+ if (max == x[i]) { idx = i; }
+ }
+ *s = idx;
+}
+
+//
+// data types
+//
+
+static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
+ "NONE",
+
+ "DUP",
+ "ADD",
+ "ADD1",
+ "ACC",
+ "SUB",
+ "MUL",
+ "DIV",
+ "SQR",
+ "SQRT",
+ "LOG",
+ "SIN",
+ "COS",
+ "SUM",
+ "SUM_ROWS",
+ "MEAN",
+ "ARGMAX",
+ "REPEAT",
+ "REPEAT_BACK",
+ "CONCAT",
+ "SILU_BACK",
+ "NORM",
+ "RMS_NORM",
+ "RMS_NORM_BACK",
+ "GROUP_NORM",
+
+ "MUL_MAT",
+ "MUL_MAT_ID",
+ "OUT_PROD",
+
+ "SCALE",
+ "SET",
+ "CPY",
+ "CONT",
+ "RESHAPE",
+ "VIEW",
+ "PERMUTE",
+ "TRANSPOSE",
+ "GET_ROWS",
+ "GET_ROWS_BACK",
+ "DIAG",
+ "DIAG_MASK_INF",
+ "DIAG_MASK_ZERO",
+ "SOFT_MAX",
+ "SOFT_MAX_BACK",
+ "ROPE",
+ "ROPE_BACK",
+ "CLAMP",
+ "CONV_TRANSPOSE_1D",
+ "IM2COL",
+ "IM2COL_BACK",
+ "CONV_TRANSPOSE_2D",
+ "POOL_1D",
+ "POOL_2D",
+ "POOL_2D_BACK",
+ "UPSCALE",
+ "PAD",
+ "ARANGE",
+ "TIMESTEP_EMBEDDING",
+ "ARGSORT",
+ "LEAKY_RELU",
+
+ "FLASH_ATTN_EXT",
+ "FLASH_ATTN_BACK",
+ "SSM_CONV",
+ "SSM_SCAN",
+ "WIN_PART",
+ "WIN_UNPART",
+ "GET_REL_POS",
+ "ADD_REL_POS",
+ "RWKV_WKV",
+
+ "UNARY",
+
+ "MAP_UNARY",
+ "MAP_BINARY",
+
+ "MAP_CUSTOM1_F32",
+ "MAP_CUSTOM2_F32",
+ "MAP_CUSTOM3_F32",
+
+ "MAP_CUSTOM1",
+ "MAP_CUSTOM2",
+ "MAP_CUSTOM3",
+
+ "CROSS_ENTROPY_LOSS",
+ "CROSS_ENTROPY_LOSS_BACK",
+};
+
+static_assert(GGML_OP_COUNT == 79, "GGML_OP_COUNT != 79");
+
+static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
+ "none",
+
+ "x",
+ "x+y",
+ "x+y",
+ "view(x,nb,offset)+=y->x",
+ "x-y",
+ "x*y",
+ "x/y",
+ "x^2",
+ "√x",
+ "log(x)",
+ "sin(x)",
+ "cos(x)",
+ "Σx",
+ "Σx_k",
+ "Σx/n",
+ "argmax(x)",
+ "repeat(x)",
+ "repeat_back(x)",
+ "concat(x, y)",
+ "silu_back(x)",
+ "norm(x)",
+ "rms_norm(x)",
+ "rms_norm_back(x)",
+ "group_norm(x)",
+
+ "X*Y",
+ "X[i]*Y",
+ "X*Y",
+
+ "x*v",
+ "y-\\>view(x)",
+ "x-\\>y",
+ "cont(x)",
+ "reshape(x)",
+ "view(x)",
+ "permute(x)",
+ "transpose(x)",
+ "get_rows(x)",
+ "get_rows_back(x)",
+ "diag(x)",
+ "diag_mask_inf(x)",
+ "diag_mask_zero(x)",
+ "soft_max(x)",
+ "soft_max_back(x)",
+ "rope(x)",
+ "rope_back(x)",
+ "clamp(x)",
+ "conv_transpose_1d(x)",
+ "im2col(x)",
+ "im2col_back(x)",
+ "conv_transpose_2d(x)",
+ "pool_1d(x)",
+ "pool_2d(x)",
+ "pool_2d_back(x)",
+ "upscale(x)",
+ "pad(x)",
+ "arange(start, stop, step)",
+ "timestep_embedding(timesteps, dim, max_period)",
+ "argsort(x)",
+ "leaky_relu(x)",
+
+ "flash_attn_ext(x)",
+ "flash_attn_back(x)",
+ "ssm_conv(x)",
+ "ssm_scan(x)",
+ "win_part(x)",
+ "win_unpart(x)",
+ "get_rel_pos(x)",
+ "add_rel_pos(x)",
+ "rwkv_wkv(k, v, r, tf, td, s)",
+
+ "unary(x)",
+
+ "f(x)",
+ "f(x,y)",
+
+ "custom_f32(x)",
+ "custom_f32(x,y)",
+ "custom_f32(x,y,z)",
+
+ "custom(x)",
+ "custom(x,y)",
+ "custom(x,y,z)",
+
+ "cross_entropy_loss(x,y)",
+ "cross_entropy_loss_back(x,y)",
+};
+
+static_assert(GGML_OP_COUNT == 79, "GGML_OP_COUNT != 79");
+
+static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
+
+
+static const char * GGML_UNARY_OP_NAME[GGML_UNARY_OP_COUNT] = {
+ "ABS",
+ "SGN",
+ "NEG",
+ "STEP",
+ "TANH",
+ "ELU",
+ "RELU",
+ "SIGMOID",
+ "GELU",
+ "GELU_QUICK",
+ "SILU",
+ "HARDSWISH",
+ "HARDSIGMOID",
+ "EXP",
+};
+
+static_assert(GGML_UNARY_OP_COUNT == 14, "GGML_UNARY_OP_COUNT != 14");
+
+
+static_assert(sizeof(struct ggml_object)%GGML_MEM_ALIGN == 0, "ggml_object size must be a multiple of GGML_MEM_ALIGN");
+static_assert(sizeof(struct ggml_tensor)%GGML_MEM_ALIGN == 0, "ggml_tensor size must be a multiple of GGML_MEM_ALIGN");
+
+// Helpers for polling loops
+#if defined(__aarch64__) && ( defined(__clang__) || defined(__GNUC__) )
+static inline void ggml_thread_cpu_relax(void) {
+ __asm__ volatile("yield" ::: "memory");
+}
+#elif defined(__x86_64__)
+static inline void ggml_thread_cpu_relax(void) {
+ _mm_pause();
+}
+#else
+static inline void ggml_thread_cpu_relax(void) {;}
+#endif
+
+//
+// NUMA support
+//
+
+#define GGML_NUMA_MAX_NODES 8
+#define GGML_NUMA_MAX_CPUS 512
+
+struct ggml_numa_node {
+ uint32_t cpus[GGML_NUMA_MAX_CPUS]; // hardware threads on this node
+ uint32_t n_cpus;
+};
+
+struct ggml_numa_nodes {
+ enum ggml_numa_strategy numa_strategy;
+ struct ggml_numa_node nodes[GGML_NUMA_MAX_NODES];
+ uint32_t n_nodes;
+ uint32_t total_cpus; // hardware threads on system
+ uint32_t current_node; // node on which main process is execting
+#if defined(__gnu_linux__)
+ cpu_set_t cpuset; // cpuset from numactl
+#else
+ uint32_t cpuset; // no NUMA support outside of Linux at this time. Use a portable datatype
+#endif
+};
+
+//
+// ggml state
+//
+
+struct ggml_state {
+ struct ggml_context_container contexts[GGML_MAX_CONTEXTS];
+ struct ggml_numa_nodes numa;
+};
+
+// global state
+static struct ggml_state g_state;
+static atomic_flag g_state_critical = ATOMIC_FLAG_INIT;
+
+// critical section via spin lock
+inline static void ggml_critical_section_start(void) {
+ while (atomic_flag_test_and_set(&g_state_critical)) {
+ // spin
+ sched_yield();
+ }
+}
+
+#ifdef GGML_USE_OPENMP
+static void ggml_barrier(struct ggml_threadpool * threadpool) {
+ if (threadpool->n_threads_cur == 1) {
+ return;
+ }
+
+ #pragma omp barrier
+}
+#else
+static void ggml_barrier(struct ggml_threadpool * threadpool) {
+ if (threadpool->n_threads_cur == 1) {
+ return;
+ }
+
+ atomic_int * n_barrier = &threadpool->n_barrier;
+ atomic_int * n_barrier_passed = &threadpool->n_barrier_passed;
+
+ int n_threads = threadpool->n_threads_cur;
+ int passed_old = atomic_load_explicit(n_barrier_passed, memory_order_relaxed);
+
+ if (atomic_fetch_add(n_barrier, 1) == n_threads - 1) {
+ // last thread
+ atomic_store(n_barrier, 0);
+ atomic_fetch_add_explicit(n_barrier_passed, 1, memory_order_relaxed);
+ } else {
+ // wait for other threads
+ while (true) {
+ if (atomic_load_explicit(n_barrier_passed, memory_order_relaxed) != passed_old) {
+ return;
+ }
+ ggml_thread_cpu_relax();
+ }
+ }
+}
+#endif
+
+// TODO: make this somehow automatically executed
+// some sort of "sentry" mechanism
+inline static void ggml_critical_section_end(void) {
+ atomic_flag_clear(&g_state_critical);
+}
+
+#if defined(__gnu_linux__)
+static cpu_set_t ggml_get_numa_affinity(void) {
+ cpu_set_t cpuset;
+ pthread_t thread;
+ thread = pthread_self();
+ CPU_ZERO(&cpuset);
+ pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
+ return cpuset;
+}
+#else
+static uint32_t ggml_get_numa_affinity(void) {
+ return 0; // no NUMA support
+}
+#endif
+
+void ggml_numa_init(enum ggml_numa_strategy numa_flag) {
+ if (g_state.numa.n_nodes > 0) {
+ fprintf(stderr, "ggml_numa_init: NUMA already initialized\n");
+
+ return;
+ }
+
+#if defined(__gnu_linux__)
+ struct stat st;
+ char path[256];
+ int rv;
+
+ // set numa scheme
+ g_state.numa.numa_strategy = numa_flag;
+
+ GGML_PRINT_DEBUG("numa strategy %u\n",g_state.numa.numa_strategy);
+
+ g_state.numa.cpuset = ggml_get_numa_affinity();
+
+ // enumerate nodes
+ while (g_state.numa.n_nodes < GGML_NUMA_MAX_NODES) {
+ rv = snprintf(path, sizeof(path), "/sys/devices/system/node/node%u", g_state.numa.n_nodes);
+ GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));
+ if (stat(path, &st) != 0) { break; }
+ ++g_state.numa.n_nodes;
+ }
+
+ // enumerate CPUs
+ while (g_state.numa.total_cpus < GGML_NUMA_MAX_CPUS) {
+ rv = snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%u", g_state.numa.total_cpus);
+ GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));
+ if (stat(path, &st) != 0) { break; }
+ ++g_state.numa.total_cpus;
+ }
+
+ GGML_PRINT_DEBUG("found %u numa nodes, %u CPUs\n", g_state.numa.n_nodes, g_state.numa.total_cpus);
+
+ // figure out which node we're on
+ uint current_cpu;
+ int getcpu_ret = 0;
+#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 28) || defined(__COSMOPOLITAN__)
+ getcpu_ret = getcpu(¤t_cpu, &g_state.numa.current_node);
+#else
+ // old glibc doesn't have a wrapper for this call. Fall back on direct syscall
+# if !defined(SYS_getcpu) && defined(SYS_get_cpu)
+# define SYS_getcpu SYS_get_cpu // some older glibc versions use this name
+# endif
+ getcpu_ret = syscall(SYS_getcpu, ¤t_cpu, &g_state.numa.current_node);
+#endif
+
+ if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1 || getcpu_ret != 0) {
+ g_state.numa.n_nodes = 0;
+ return;
+ }
+
+ GGML_PRINT_DEBUG("found our process on numa node %u, CPU %u\n", g_state.numa.current_node, current_cpu);
+
+ for (uint32_t n = 0; n < g_state.numa.n_nodes; ++n) {
+ struct ggml_numa_node * node = &g_state.numa.nodes[n];
+ GGML_PRINT_DEBUG("CPUs on node %u:", n);
+ node->n_cpus = 0;
+ for (uint32_t c = 0; c < g_state.numa.total_cpus; ++c) {
+ rv = snprintf(path, sizeof(path), "/sys/devices/system/node/node%u/cpu%u", n, c);
+ GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));
+ if (stat(path, &st) == 0) {
+ node->cpus[node->n_cpus++] = c;
+ GGML_PRINT_DEBUG(" %u", c);
+ }
+ }
+ GGML_PRINT_DEBUG("\n");
+ }
+
+ if (ggml_is_numa()) {
+ FILE *fptr = fopen("/proc/sys/kernel/numa_balancing", "r");
+ if (fptr != NULL) {
+ char buf[42];
+ if (fgets(buf, sizeof(buf), fptr) && strncmp(buf, "0\n", sizeof(buf)) != 0) {
+ GGML_PRINT("WARNING: /proc/sys/kernel/numa_balancing is enabled, this has been observed to impair performance\n");
+ }
+ fclose(fptr);
+ }
+ }
+#else
+ UNUSED(numa_flag);
+ // TODO
+#endif
+}
+
+bool ggml_is_numa(void) {
+ return g_state.numa.n_nodes > 1;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void ggml_print_object(const struct ggml_object * obj) {
+ GGML_PRINT(" - ggml_object: type = %d, offset = %zu, size = %zu, next = %p\n",
+ obj->type, obj->offs, obj->size, (const void *) obj->next);
+}
+
+void ggml_print_objects(const struct ggml_context * ctx) {
+ struct ggml_object * obj = ctx->objects_begin;
+
+ GGML_PRINT("%s: objects in context %p:\n", __func__, (const void *) ctx);
+
+ while (obj != NULL) {
+ ggml_print_object(obj);
+ obj = obj->next;
+ }
+
+ GGML_PRINT("%s: --- end ---\n", __func__);
+}
+
+GGML_CALL int64_t ggml_nelements(const struct ggml_tensor * tensor) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return tensor->ne[0]*tensor->ne[1]*tensor->ne[2]*tensor->ne[3];
+}
+
+GGML_CALL int64_t ggml_nrows(const struct ggml_tensor * tensor) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return tensor->ne[1]*tensor->ne[2]*tensor->ne[3];
+}
+
+GGML_CALL size_t ggml_nbytes(const struct ggml_tensor * tensor) {
+ size_t nbytes;
+ size_t blck_size = ggml_blck_size(tensor->type);
+ if (blck_size == 1) {
+ nbytes = ggml_type_size(tensor->type);
+ for (int i = 0; i < GGML_MAX_DIMS; ++i) {
+ nbytes += (tensor->ne[i] - 1)*tensor->nb[i];
+ }
+ }
+ else {
+ nbytes = tensor->ne[0]*tensor->nb[0]/blck_size;
+ for (int i = 1; i < GGML_MAX_DIMS; ++i) {
+ nbytes += (tensor->ne[i] - 1)*tensor->nb[i];
+ }
+ }
+
+ return nbytes;
+}
+
+size_t ggml_nbytes_pad(const struct ggml_tensor * tensor) {
+ return GGML_PAD(ggml_nbytes(tensor), GGML_MEM_ALIGN);
+}
+
+GGML_CALL int64_t ggml_blck_size(enum ggml_type type) {
+ return type_traits[type].blck_size;
+}
+
+GGML_CALL size_t ggml_type_size(enum ggml_type type) {
+ return type_traits[type].type_size;
+}
+
+GGML_CALL size_t ggml_row_size(enum ggml_type type, int64_t ne) {
+ assert(ne % ggml_blck_size(type) == 0);
+ return ggml_type_size(type)*ne/ggml_blck_size(type);
+}
+
+double ggml_type_sizef(enum ggml_type type) {
+ return ((double)(type_traits[type].type_size))/type_traits[type].blck_size;
+}
+
+GGML_CALL const char * ggml_type_name(enum ggml_type type) {
+ return type_traits[type].type_name;
+}
+
+GGML_CALL bool ggml_is_quantized(enum ggml_type type) {
+ return type_traits[type].is_quantized;
+}
+
+GGML_CALL const char * ggml_op_name(enum ggml_op op) {
+ return GGML_OP_NAME[op];
+}
+
+const char * ggml_op_symbol(enum ggml_op op) {
+ return GGML_OP_SYMBOL[op];
+}
+
+const char * ggml_unary_op_name(enum ggml_unary_op op) {
+ return GGML_UNARY_OP_NAME[op];
+}
+
+GGML_CALL const char * ggml_op_desc(const struct ggml_tensor * t) {
+ if (t->op == GGML_OP_UNARY) {
+ enum ggml_unary_op uop = ggml_get_unary_op(t);
+ return ggml_unary_op_name(uop);
+ }
+ return ggml_op_name(t->op);
+}
+
+GGML_CALL size_t ggml_element_size(const struct ggml_tensor * tensor) {
+ return ggml_type_size(tensor->type);
+}
+
+bool ggml_is_scalar(const struct ggml_tensor * tensor) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return tensor->ne[0] == 1 && tensor->ne[1] == 1 && tensor->ne[2] == 1 && tensor->ne[3] == 1;
+}
+
+bool ggml_is_vector(const struct ggml_tensor * tensor) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return tensor->ne[1] == 1 && tensor->ne[2] == 1 && tensor->ne[3] == 1;
+}
+
+bool ggml_is_matrix(const struct ggml_tensor * tensor) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return tensor->ne[2] == 1 && tensor->ne[3] == 1;
+}
+
+bool ggml_is_3d(const struct ggml_tensor * tensor) {
+ return tensor->ne[3] == 1;
+}
+
+int ggml_n_dims(const struct ggml_tensor * tensor) {
+ for (int i = GGML_MAX_DIMS - 1; i >= 1; --i) {
+ if (tensor->ne[i] > 1) {
+ return i + 1;
+ }
+ }
+ return 1;
+}
+
+static inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return (t0->ne[0] == t1->ne[0]) &&
+ (t1->ne[2]%t0->ne[2] == 0) && // verify t0 is broadcastable
+ (t1->ne[3]%t0->ne[3] == 0);
+}
+
+static inline bool ggml_can_out_prod(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return (t0->ne[1] == t1->ne[1]) &&
+ (t1->ne[2]%t0->ne[2] == 0) && // verify t0 is broadcastable
+ (t1->ne[3]%t0->ne[3] == 0);
+}
+
+enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) {
+ enum ggml_type wtype = GGML_TYPE_COUNT;
+
+ switch (ftype) {
+ case GGML_FTYPE_ALL_F32: wtype = GGML_TYPE_F32; break;
+ case GGML_FTYPE_MOSTLY_F16: wtype = GGML_TYPE_F16; break;
+ case GGML_FTYPE_MOSTLY_BF16: wtype = GGML_TYPE_BF16; break;
+ case GGML_FTYPE_MOSTLY_Q4_0: wtype = GGML_TYPE_Q4_0; break;
+ case GGML_FTYPE_MOSTLY_Q4_1: wtype = GGML_TYPE_Q4_1; break;
+ case GGML_FTYPE_MOSTLY_Q5_0: wtype = GGML_TYPE_Q5_0; break;
+ case GGML_FTYPE_MOSTLY_Q5_1: wtype = GGML_TYPE_Q5_1; break;
+ case GGML_FTYPE_MOSTLY_Q8_0: wtype = GGML_TYPE_Q8_0; break;
+ case GGML_FTYPE_MOSTLY_Q2_K: wtype = GGML_TYPE_Q2_K; break;
+ case GGML_FTYPE_MOSTLY_Q3_K: wtype = GGML_TYPE_Q3_K; break;
+ case GGML_FTYPE_MOSTLY_Q4_K: wtype = GGML_TYPE_Q4_K; break;
+ case GGML_FTYPE_MOSTLY_Q5_K: wtype = GGML_TYPE_Q5_K; break;
+ case GGML_FTYPE_MOSTLY_Q6_K: wtype = GGML_TYPE_Q6_K; break;
+ case GGML_FTYPE_MOSTLY_IQ2_XXS: wtype = GGML_TYPE_IQ2_XXS; break;
+ case GGML_FTYPE_MOSTLY_IQ2_XS: wtype = GGML_TYPE_IQ2_XS; break;
+ case GGML_FTYPE_MOSTLY_IQ3_XXS: wtype = GGML_TYPE_IQ3_XXS; break;
+ case GGML_FTYPE_MOSTLY_IQ1_S: wtype = GGML_TYPE_IQ1_S; break;
+ case GGML_FTYPE_MOSTLY_IQ1_M: wtype = GGML_TYPE_IQ1_M; break;
+ case GGML_FTYPE_MOSTLY_IQ4_NL: wtype = GGML_TYPE_IQ4_NL; break;
+ case GGML_FTYPE_MOSTLY_IQ4_XS: wtype = GGML_TYPE_IQ4_XS; break;
+ case GGML_FTYPE_MOSTLY_IQ3_S: wtype = GGML_TYPE_IQ3_S; break;
+ case GGML_FTYPE_MOSTLY_IQ2_S: wtype = GGML_TYPE_IQ2_S; break;
+ case GGML_FTYPE_MOSTLY_Q4_0_4_4: wtype = GGML_TYPE_Q4_0_4_4; break;
+ case GGML_FTYPE_MOSTLY_Q4_0_4_8: wtype = GGML_TYPE_Q4_0_4_8; break;
+ case GGML_FTYPE_MOSTLY_Q4_0_8_8: wtype = GGML_TYPE_Q4_0_8_8; break;
+ case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break;
+ case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break;
+ }
+
+ GGML_ASSERT(wtype != GGML_TYPE_COUNT);
+
+ return wtype;
+}
+
+size_t ggml_tensor_overhead(void) {
+ return GGML_OBJECT_SIZE + GGML_TENSOR_SIZE;
+}
+
+GGML_CALL bool ggml_is_transposed(const struct ggml_tensor * tensor) {
+ return tensor->nb[0] > tensor->nb[1];
+}
+
+static bool ggml_is_contiguous_n(const struct ggml_tensor * tensor, int n) {
+ size_t next_nb = ggml_type_size(tensor->type);
+ if (tensor->ne[0] != ggml_blck_size(tensor->type) && tensor->nb[0] != next_nb) {
+ return false;
+ }
+ next_nb *= tensor->ne[0]/ggml_blck_size(tensor->type);
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
+ if (tensor->ne[i] != 1) {
+ if (i > n) {
+ if (tensor->nb[i] != next_nb) {
+ return false;
+ }
+ next_nb *= tensor->ne[i];
+ } else {
+ // this dimension does not need to be contiguous
+ next_nb = tensor->ne[i]*tensor->nb[i];
+ }
+ }
+ }
+ return true;
+}
+
+GGML_CALL bool ggml_is_contiguous(const struct ggml_tensor * tensor) {
+ return ggml_is_contiguous_0(tensor);
+}
+
+GGML_CALL bool ggml_is_contiguous_0(const struct ggml_tensor * tensor) {
+ return ggml_is_contiguous_n(tensor, 0);
+}
+
+GGML_CALL bool ggml_is_contiguous_1(const struct ggml_tensor * tensor) {
+ return ggml_is_contiguous_n(tensor, 1);
+}
+
+GGML_CALL bool ggml_is_contiguous_2(const struct ggml_tensor * tensor) {
+ return ggml_is_contiguous_n(tensor, 2);
+}
+
+GGML_CALL bool ggml_is_permuted(const struct ggml_tensor * tensor) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return tensor->nb[0] > tensor->nb[1] || tensor->nb[1] > tensor->nb[2] || tensor->nb[2] > tensor->nb[3];
+}
+
+static inline bool ggml_is_padded_1d(const struct ggml_tensor * tensor) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return
+ tensor->nb[0] == ggml_type_size(tensor->type) &&
+ tensor->nb[2] == tensor->nb[1]*tensor->ne[1] &&
+ tensor->nb[3] == tensor->nb[2]*tensor->ne[2];
+}
+
+GGML_CALL bool ggml_is_empty(const struct ggml_tensor * tensor) {
+ for (int i = 0; i < GGML_MAX_DIMS; ++i) {
+ if (tensor->ne[i] == 0) {
+ // empty if any dimension has no elements
+ return true;
+ }
+ }
+ return false;
+}
+
+bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return
+ (t0->ne[0] == t1->ne[0]) &&
+ (t0->ne[1] == t1->ne[1]) &&
+ (t0->ne[2] == t1->ne[2]) &&
+ (t0->ne[3] == t1->ne[3]);
+}
+
+bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return
+ (t0->nb[0] == t1->nb[0]) &&
+ (t0->nb[1] == t1->nb[1]) &&
+ (t0->nb[2] == t1->nb[2]) &&
+ (t0->nb[3] == t1->nb[3]);
+}
+
+// check if t1 can be represented as a repeatition of t0
+bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return ggml_is_empty(t0) ? ggml_is_empty(t1) :
+ (t1->ne[0]%t0->ne[0] == 0) &&
+ (t1->ne[1]%t0->ne[1] == 0) &&
+ (t1->ne[2]%t0->ne[2] == 0) &&
+ (t1->ne[3]%t0->ne[3] == 0);
+}
+
+static inline bool ggml_can_repeat_rows(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
+
+ return (t0->ne[0] == t1->ne[0]) && ggml_can_repeat(t0, t1);
+}
+
+static inline int ggml_up32(int n) {
+ return (n + 31) & ~31;
+}
+
+//static inline int ggml_up64(int n) {
+// return (n + 63) & ~63;
+//}
+
+static inline int ggml_up(int n, int m) {
+ // assert m is a power of 2
+ GGML_ASSERT((m & (m - 1)) == 0);
+ return (n + m - 1) & ~(m - 1);
+}
+
+// assert that pointer is aligned to GGML_MEM_ALIGN
+#define GGML_ASSERT_ALIGNED(ptr) \
+ GGML_ASSERT(((uintptr_t) (ptr))%GGML_MEM_ALIGN == 0)
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct ggml_context * ggml_init(struct ggml_init_params params) {
+ // make this function thread safe
+ ggml_critical_section_start();
+
+ static bool is_first_call = true;
+
+ if (is_first_call) {
+ // initialize time system (required on Windows)
+ ggml_time_init();
+
+ // initialize GELU, Quick GELU, SILU and EXP F32 tables
+ {
+ const uint64_t t_start = ggml_time_us(); UNUSED(t_start);
+
+ for (int i = 0; i < (1 << 16); ++i) {
+ union {
+ uint16_t u16;
+ ggml_fp16_t fp16;
+ } u = {i};
+ float f = ggml_table_f32_f16[i] = GGML_COMPUTE_FP16_TO_FP32(u.fp16);
+ ggml_table_gelu_f16[i] = GGML_FP32_TO_FP16(ggml_gelu_f32(f));
+ ggml_table_gelu_quick_f16[i] = GGML_FP32_TO_FP16(ggml_gelu_quick_f32(f));
+ }
+
+ const uint64_t t_end = ggml_time_us(); UNUSED(t_end);
+
+ GGML_PRINT_DEBUG("%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\n", __func__, (t_end - t_start)/1000.0f);
+ }
+
+ // initialize g_state
+ {
+ const uint64_t t_start = ggml_time_us(); UNUSED(t_start);
+
+ g_state = (struct ggml_state) {
+ /*.contexts =*/ { { 0 } },
+ /*.numa =*/ {
+ .n_nodes = 0,
+ .total_cpus = 0,
+ },
+ };
+
+ for (int i = 0; i < GGML_MAX_CONTEXTS; ++i) {
+ g_state.contexts[i].used = false;
+ }
+
+ const uint64_t t_end = ggml_time_us(); UNUSED(t_end);
+
+ GGML_PRINT_DEBUG("%s: g_state initialized in %f ms\n", __func__, (t_end - t_start)/1000.0f);
+ }
+
+ is_first_call = false;
+ }
+
+ // find non-used context in g_state
+ struct ggml_context * ctx = NULL;
+
+ for (int i = 0; i < GGML_MAX_CONTEXTS; i++) {
+ if (!g_state.contexts[i].used) {
+ g_state.contexts[i].used = true;
+ ctx = &g_state.contexts[i].context;
+
+ GGML_PRINT_DEBUG("%s: found unused context %d\n", __func__, i);
+ break;
+ }
+ }
+
+ if (ctx == NULL) {
+ GGML_PRINT_DEBUG("%s: no unused context found\n", __func__);
+
+ ggml_critical_section_end();
+
+ return NULL;
+ }
+
+ // allow to call ggml_init with 0 size
+ if (params.mem_size == 0) {
+ params.mem_size = GGML_MEM_ALIGN;
+ }
+
+ const size_t mem_size = params.mem_buffer ? params.mem_size : GGML_PAD(params.mem_size, GGML_MEM_ALIGN);
+
+ *ctx = (struct ggml_context) {
+ /*.mem_size =*/ mem_size,
+ /*.mem_buffer =*/ params.mem_buffer ? params.mem_buffer : GGML_ALIGNED_MALLOC(mem_size),
+ /*.mem_buffer_owned =*/ params.mem_buffer ? false : true,
+ /*.no_alloc =*/ params.no_alloc,
+ /*.no_alloc_save =*/ params.no_alloc,
+ /*.n_objects =*/ 0,
+ /*.objects_begin =*/ NULL,
+ /*.objects_end =*/ NULL,
+ /*.scratch =*/ { 0, 0, NULL, },
+ /*.scratch_save =*/ { 0, 0, NULL, },
+ };
+
+ GGML_ASSERT(ctx->mem_buffer != NULL);
+
+ GGML_ASSERT_ALIGNED(ctx->mem_buffer);
+
+#if defined(__ARM_FEATURE_SVE)
+ if (!ggml_sve_cnt_b) {
+ ggml_sve_cnt_b = PR_SVE_VL_LEN_MASK & prctl(PR_SVE_GET_VL);
+ }
+#endif
+
+ GGML_PRINT_DEBUG("%s: context initialized\n", __func__);
+
+ ggml_critical_section_end();
+
+ return ctx;
+}
+
+void ggml_free(struct ggml_context * ctx) {
+ if (ctx == NULL) {
+ return;
+ }
+
+ // make this function thread safe
+ ggml_critical_section_start();
+
+ bool found = false;
+
+ for (int i = 0; i < GGML_MAX_CONTEXTS; i++) {
+ if (&g_state.contexts[i].context == ctx) {
+ g_state.contexts[i].used = false;
+
+ GGML_PRINT_DEBUG("%s: context %d has been freed. memory used = %zu\n",
+ __func__, i, ggml_used_mem(ctx));
+
+ if (ctx->mem_buffer_owned) {
+ GGML_ALIGNED_FREE(ctx->mem_buffer);
+ }
+
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ GGML_PRINT_DEBUG("%s: context not found\n", __func__);
+ }
+
+ ggml_critical_section_end();
+}
+
+size_t ggml_used_mem(const struct ggml_context * ctx) {
+ return ctx->objects_end == NULL ? 0 : ctx->objects_end->offs + ctx->objects_end->size;
+}
+
+size_t ggml_set_scratch(struct ggml_context * ctx, struct ggml_scratch scratch) {
+ const size_t result = ctx->scratch.data ? ctx->scratch.offs : 0;
+
+ ctx->scratch = scratch;
+
+ return result;
+}
+
+bool ggml_get_no_alloc(struct ggml_context * ctx) {
+ return ctx->no_alloc;
+}
+
+void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc) {
+ ctx->no_alloc = no_alloc;
+}
+
+void * ggml_get_mem_buffer(const struct ggml_context * ctx) {
+ return ctx->mem_buffer;
+}
+
+size_t ggml_get_mem_size(const struct ggml_context * ctx) {
+ return ctx->mem_size;
+}
+
+size_t ggml_get_max_tensor_size(const struct ggml_context * ctx) {
+ size_t max_size = 0;
+
+ for (struct ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor != NULL; tensor = ggml_get_next_tensor(ctx, tensor)) {
+ size_t bytes = ggml_nbytes(tensor);
+ max_size = MAX(max_size, bytes);
+ }
+
+ return max_size;
+}
+
+// IMPORTANT:
+// when creating "opt" tensors, always save and load the scratch buffer
+// this is an error prone process, but it is necessary to support inplace
+// operators when using scratch buffers
+// TODO: implement a better way
+static void ggml_scratch_save(struct ggml_context * ctx) {
+ // this is needed to allow opt tensors to store their data
+ // TODO: again, need to find a better way
+ ctx->no_alloc_save = ctx->no_alloc;
+ ctx->no_alloc = false;
+
+ ctx->scratch_save = ctx->scratch;
+ ctx->scratch.data = NULL;
+}
+
+static void ggml_scratch_load(struct ggml_context * ctx) {
+ ctx->no_alloc = ctx->no_alloc_save;
+
+ ctx->scratch = ctx->scratch_save;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+static struct ggml_object * ggml_new_object(struct ggml_context * ctx, enum ggml_object_type type, size_t size) {
+ // always insert objects at the end of the context's memory pool
+ struct ggml_object * obj_cur = ctx->objects_end;
+
+ const size_t cur_offs = obj_cur == NULL ? 0 : obj_cur->offs;
+ const size_t cur_size = obj_cur == NULL ? 0 : obj_cur->size;
+ const size_t cur_end = cur_offs + cur_size;
+
+ // align to GGML_MEM_ALIGN
+ size_t size_needed = GGML_PAD(size, GGML_MEM_ALIGN);
+
+ char * const mem_buffer = ctx->mem_buffer;
+ struct ggml_object * const obj_new = (struct ggml_object *)(mem_buffer + cur_end);
+
+ if (cur_end + size_needed + GGML_OBJECT_SIZE > ctx->mem_size) {
+ GGML_PRINT("%s: not enough space in the context's memory pool (needed %zu, available %zu)\n",
+ __func__, cur_end + size_needed, ctx->mem_size);
+ assert(false);
+ return NULL;
+ }
+
+ *obj_new = (struct ggml_object) {
+ .offs = cur_end + GGML_OBJECT_SIZE,
+ .size = size_needed,
+ .next = NULL,
+ .type = type,
+ };
+
+ GGML_ASSERT_ALIGNED(mem_buffer + obj_new->offs);
+
+ if (obj_cur != NULL) {
+ obj_cur->next = obj_new;
+ } else {
+ // this is the first object in this context
+ ctx->objects_begin = obj_new;
+ }
+
+ ctx->objects_end = obj_new;
+
+ //printf("%s: inserted new object at %zu, size = %zu\n", __func__, cur_end, obj_new->size);
+
+ return obj_new;
+}
+
+static struct ggml_tensor * ggml_new_tensor_impl(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int n_dims,
+ const int64_t * ne,
+ struct ggml_tensor * view_src,
+ size_t view_offs) {
+
+ GGML_ASSERT(type >= 0 && type < GGML_TYPE_COUNT);
+ GGML_ASSERT(n_dims >= 1 && n_dims <= GGML_MAX_DIMS);
+
+ // find the base tensor and absolute offset
+ if (view_src != NULL && view_src->view_src != NULL) {
+ view_offs += view_src->view_offs;
+ view_src = view_src->view_src;
+ }
+
+ size_t data_size = ggml_row_size(type, ne[0]);
+ for (int i = 1; i < n_dims; i++) {
+ data_size *= ne[i];
+ }
+
+ GGML_ASSERT(view_src == NULL || data_size == 0 || data_size + view_offs <= ggml_nbytes(view_src));
+
+ void * data = view_src != NULL ? view_src->data : NULL;
+ if (data != NULL) {
+ data = (char *) data + view_offs;
+ }
+
+ size_t obj_alloc_size = 0;
+
+ if (view_src == NULL && !ctx->no_alloc) {
+ if (ctx->scratch.data != NULL) {
+ // allocate tensor data in the scratch buffer
+ if (ctx->scratch.offs + data_size > ctx->scratch.size) {
+ GGML_PRINT("%s: not enough space in the scratch memory pool (needed %zu, available %zu)\n",
+ __func__, ctx->scratch.offs + data_size, ctx->scratch.size);
+ assert(false);
+ return NULL;
+ }
+
+ data = (char * const) ctx->scratch.data + ctx->scratch.offs;
+
+ ctx->scratch.offs += data_size;
+ } else {
+ // allocate tensor data in the context's memory pool
+ obj_alloc_size = data_size;
+ }
+ }
+
+ struct ggml_object * const obj_new = ggml_new_object(ctx, GGML_OBJECT_TYPE_TENSOR, GGML_TENSOR_SIZE + obj_alloc_size);
+ GGML_ASSERT(obj_new);
+
+ // TODO: for recoverable errors, we would need to free the data allocated from the scratch buffer here
+
+ struct ggml_tensor * const result = (struct ggml_tensor *)((char *)ctx->mem_buffer + obj_new->offs);
+
+#ifdef __clang__
+ // temporary until ggml_tensor::backend is removed
+ #pragma clang diagnostic push
+ #pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
+ *result = (struct ggml_tensor) {
+ /*.type =*/ type,
+ /*.backend =*/ GGML_BACKEND_TYPE_CPU,
+ /*.buffer =*/ NULL,
+ /*.ne =*/ { 1, 1, 1, 1 },
+ /*.nb =*/ { 0, 0, 0, 0 },
+ /*.op =*/ GGML_OP_NONE,
+ /*.op_params =*/ { 0 },
+ /*.flags =*/ 0,
+ /*.grad =*/ NULL,
+ /*.src =*/ { NULL },
+ /*.view_src =*/ view_src,
+ /*.view_offs =*/ view_offs,
+ /*.data =*/ obj_alloc_size > 0 ? (void *)(result + 1) : data,
+ /*.name =*/ { 0 },
+ /*.extra =*/ NULL,
+ ///*.padding =*/ { 0 },
+ };
+
+#ifdef __clang__
+ #pragma clang diagnostic pop
+#endif
+
+ // TODO: this should not be needed as long as we don't rely on aligned SIMD loads
+ //GGML_ASSERT_ALIGNED(result->data);
+
+ for (int i = 0; i < n_dims; i++) {
+ result->ne[i] = ne[i];
+ }
+
+ result->nb[0] = ggml_type_size(type);
+ result->nb[1] = result->nb[0]*(result->ne[0]/ggml_blck_size(type));
+ for (int i = 2; i < GGML_MAX_DIMS; i++) {
+ result->nb[i] = result->nb[i - 1]*result->ne[i - 1];
+ }
+
+ ctx->n_objects++;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_new_tensor(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int n_dims,
+ const int64_t * ne) {
+ return ggml_new_tensor_impl(ctx, type, n_dims, ne, NULL, 0);
+}
+
+struct ggml_tensor * ggml_new_tensor_1d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0) {
+ return ggml_new_tensor(ctx, type, 1, &ne0);
+}
+
+struct ggml_tensor * ggml_new_tensor_2d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0,
+ int64_t ne1) {
+ const int64_t ne[2] = { ne0, ne1 };
+ return ggml_new_tensor(ctx, type, 2, ne);
+}
+
+struct ggml_tensor * ggml_new_tensor_3d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2) {
+ const int64_t ne[3] = { ne0, ne1, ne2 };
+ return ggml_new_tensor(ctx, type, 3, ne);
+}
+
+struct ggml_tensor * ggml_new_tensor_4d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3) {
+ const int64_t ne[4] = { ne0, ne1, ne2, ne3 };
+ return ggml_new_tensor(ctx, type, 4, ne);
+}
+
+struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value) {
+ ggml_scratch_save(ctx);
+
+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);
+
+ ggml_scratch_load(ctx);
+
+ ggml_set_i32(result, value);
+
+ return result;
+}
+
+struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value) {
+ ggml_scratch_save(ctx);
+
+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+
+ ggml_scratch_load(ctx);
+
+ ggml_set_f32(result, value);
+
+ return result;
+}
+
+struct ggml_tensor * ggml_dup_tensor(struct ggml_context * ctx, const struct ggml_tensor * src) {
+ return ggml_new_tensor(ctx, src->type, GGML_MAX_DIMS, src->ne);
+}
+
+static void ggml_set_op_params(struct ggml_tensor * tensor, const void * params, size_t params_size) {
+ GGML_ASSERT(tensor != NULL); // silence -Warray-bounds warnings
+ assert(params_size <= GGML_MAX_OP_PARAMS);
+ memcpy(tensor->op_params, params, params_size);
+}
+
+static int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_t i) {
+ assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));
+ return ((const int32_t *)(tensor->op_params))[i];
+}
+
+static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t i) {
+ assert(i < GGML_MAX_OP_PARAMS / sizeof(float));
+ return ((const float *)(tensor->op_params))[i];
+}
+
+static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) {
+ assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));
+ ((int32_t *)(tensor->op_params))[i] = value;
+}
+
+static void ggml_set_op_params_f32(struct ggml_tensor * tensor, uint32_t i, float value) {
+ assert(i < GGML_MAX_OP_PARAMS / sizeof(float));
+ ((float *)(tensor->op_params))[i] = value;
+}
+
+struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor) {
+ memset(tensor->data, 0, ggml_nbytes(tensor));
+ return tensor;
+}
+
+struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value) {
+ const int n = ggml_nrows(tensor);
+ const int nc = tensor->ne[0];
+ const size_t n1 = tensor->nb[1];
+
+ char * const data = tensor->data;
+
+ switch (tensor->type) {
+ case GGML_TYPE_I8:
+ {
+ assert(tensor->nb[0] == sizeof(int8_t));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);
+ }
+ } break;
+ case GGML_TYPE_I16:
+ {
+ assert(tensor->nb[0] == sizeof(int16_t));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);
+ }
+ } break;
+ case GGML_TYPE_I32:
+ {
+ assert(tensor->nb[0] == sizeof(int32_t));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);
+ }
+ } break;
+ case GGML_TYPE_F16:
+ {
+ assert(tensor->nb[0] == sizeof(ggml_fp16_t));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_FP32_TO_FP16(value));
+ }
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ assert(tensor->nb[0] == sizeof(ggml_fp16_t));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));
+ }
+ } break;
+ case GGML_TYPE_F32:
+ {
+ assert(tensor->nb[0] == sizeof(float));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_f32(nc, (float *)(data + i*n1), value);
+ }
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+
+ return tensor;
+}
+
+struct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) {
+ const int n = ggml_nrows(tensor);
+ const int nc = tensor->ne[0];
+ const size_t n1 = tensor->nb[1];
+
+ char * const data = tensor->data;
+
+ switch (tensor->type) {
+ case GGML_TYPE_I8:
+ {
+ assert(tensor->nb[0] == sizeof(int8_t));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);
+ }
+ } break;
+ case GGML_TYPE_I16:
+ {
+ assert(tensor->nb[0] == sizeof(int16_t));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);
+ }
+ } break;
+ case GGML_TYPE_I32:
+ {
+ assert(tensor->nb[0] == sizeof(int32_t));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);
+ }
+ } break;
+ case GGML_TYPE_F16:
+ {
+ assert(tensor->nb[0] == sizeof(ggml_fp16_t));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_FP32_TO_FP16(value));
+ }
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ assert(tensor->nb[0] == sizeof(ggml_bf16_t));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));
+ }
+ } break;
+ case GGML_TYPE_F32:
+ {
+ assert(tensor->nb[0] == sizeof(float));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_f32(nc, (float *)(data + i*n1), value);
+ }
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+
+ return tensor;
+}
+
+void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3) {
+ const int64_t ne2 = tensor->ne[2];
+ const int64_t ne1 = tensor->ne[1];
+ const int64_t ne0 = tensor->ne[0];
+
+ const int64_t i3_ = (i/(ne2*ne1*ne0));
+ const int64_t i2_ = (i - i3_*ne2*ne1*ne0)/(ne1*ne0);
+ const int64_t i1_ = (i - i3_*ne2*ne1*ne0 - i2_*ne1*ne0)/ne0;
+ const int64_t i0_ = (i - i3_*ne2*ne1*ne0 - i2_*ne1*ne0 - i1_*ne0);
+
+ if (i0) {
+ * i0 = i0_;
+ }
+ if (i1) {
+ * i1 = i1_;
+ }
+ if (i2) {
+ * i2 = i2_;
+ }
+ if (i3) {
+ * i3 = i3_;
+ }
+}
+
+int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i) {
+ if (!ggml_is_contiguous(tensor)) {
+ int64_t id[4] = { 0, 0, 0, 0 };
+ ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
+ return ggml_get_i32_nd(tensor, id[0], id[1], id[2], id[3]);
+ }
+ switch (tensor->type) {
+ case GGML_TYPE_I8:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));
+ return ((int8_t *)(tensor->data))[i];
+ }
+ case GGML_TYPE_I16:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));
+ return ((int16_t *)(tensor->data))[i];
+ }
+ case GGML_TYPE_I32:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));
+ return ((int32_t *)(tensor->data))[i];
+ }
+ case GGML_TYPE_F16:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
+ return GGML_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);
+ }
+ case GGML_TYPE_BF16:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);
+ }
+ case GGML_TYPE_F32:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(float));
+ return ((float *)(tensor->data))[i];
+ }
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value) {
+ if (!ggml_is_contiguous(tensor)) {
+ int64_t id[4] = { 0, 0, 0, 0 };
+ ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
+ ggml_set_i32_nd(tensor, id[0], id[1], id[2], id[3], value);
+ return;
+ }
+ switch (tensor->type) {
+ case GGML_TYPE_I8:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));
+ ((int8_t *)(tensor->data))[i] = value;
+ } break;
+ case GGML_TYPE_I16:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));
+ ((int16_t *)(tensor->data))[i] = value;
+ } break;
+ case GGML_TYPE_I32:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));
+ ((int32_t *)(tensor->data))[i] = value;
+ } break;
+ case GGML_TYPE_F16:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
+ ((ggml_fp16_t *)(tensor->data))[i] = GGML_FP32_TO_FP16(value);
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));
+ ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);
+ } break;
+ case GGML_TYPE_F32:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(float));
+ ((float *)(tensor->data))[i] = value;
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {
+ void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
+ switch (tensor->type) {
+ case GGML_TYPE_I8:
+ return ((int8_t *) data)[0];
+ case GGML_TYPE_I16:
+ return ((int16_t *) data)[0];
+ case GGML_TYPE_I32:
+ return ((int32_t *) data)[0];
+ case GGML_TYPE_F16:
+ return GGML_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);
+ case GGML_TYPE_BF16:
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);
+ case GGML_TYPE_F32:
+ return ((float *) data)[0];
+ default:
+ GGML_ABORT("fatal error");
+ }
+}
+
+void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value) {
+ void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
+ switch (tensor->type) {
+ case GGML_TYPE_I8:
+ {
+ ((int8_t *)(data))[0] = value;
+ } break;
+ case GGML_TYPE_I16:
+ {
+ ((int16_t *)(data))[0] = value;
+ } break;
+ case GGML_TYPE_I32:
+ {
+ ((int32_t *)(data))[0] = value;
+ } break;
+ case GGML_TYPE_F16:
+ {
+ ((ggml_fp16_t *)(data))[0] = GGML_FP32_TO_FP16(value);
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);
+ } break;
+ case GGML_TYPE_F32:
+ {
+ ((float *)(data))[0] = value;
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i) {
+ if (!ggml_is_contiguous(tensor)) {
+ int64_t id[4] = { 0, 0, 0, 0 };
+ ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
+ return ggml_get_f32_nd(tensor, id[0], id[1], id[2], id[3]);
+ }
+ switch (tensor->type) {
+ case GGML_TYPE_I8:
+ {
+ return ((int8_t *)(tensor->data))[i];
+ }
+ case GGML_TYPE_I16:
+ {
+ return ((int16_t *)(tensor->data))[i];
+ }
+ case GGML_TYPE_I32:
+ {
+ return ((int32_t *)(tensor->data))[i];
+ }
+ case GGML_TYPE_F16:
+ {
+ return GGML_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);
+ }
+ case GGML_TYPE_BF16:
+ {
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);
+ }
+ case GGML_TYPE_F32:
+ {
+ return ((float *)(tensor->data))[i];
+ }
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value) {
+ if (!ggml_is_contiguous(tensor)) {
+ int64_t id[4] = { 0, 0, 0, 0 };
+ ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
+ ggml_set_f32_nd(tensor, id[0], id[1], id[2], id[3], value);
+ return;
+ }
+ switch (tensor->type) {
+ case GGML_TYPE_I8:
+ {
+ ((int8_t *)(tensor->data))[i] = value;
+ } break;
+ case GGML_TYPE_I16:
+ {
+ ((int16_t *)(tensor->data))[i] = value;
+ } break;
+ case GGML_TYPE_I32:
+ {
+ ((int32_t *)(tensor->data))[i] = value;
+ } break;
+ case GGML_TYPE_F16:
+ {
+ ((ggml_fp16_t *)(tensor->data))[i] = GGML_FP32_TO_FP16(value);
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);
+ } break;
+ case GGML_TYPE_F32:
+ {
+ ((float *)(tensor->data))[i] = value;
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {
+ void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
+ switch (tensor->type) {
+ case GGML_TYPE_I8:
+ return ((int8_t *) data)[0];
+ case GGML_TYPE_I16:
+ return ((int16_t *) data)[0];
+ case GGML_TYPE_I32:
+ return ((int32_t *) data)[0];
+ case GGML_TYPE_F16:
+ return GGML_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);
+ case GGML_TYPE_BF16:
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);
+ case GGML_TYPE_F32:
+ return ((float *) data)[0];
+ default:
+ GGML_ABORT("fatal error");
+ }
+}
+
+void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value) {
+ void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
+ switch (tensor->type) {
+ case GGML_TYPE_I8:
+ {
+ ((int8_t *)(data))[0] = value;
+ } break;
+ case GGML_TYPE_I16:
+ {
+ ((int16_t *)(data))[0] = value;
+ } break;
+ case GGML_TYPE_I32:
+ {
+ ((int32_t *)(data))[0] = value;
+ } break;
+ case GGML_TYPE_F16:
+ {
+ ((ggml_fp16_t *)(data))[0] = GGML_FP32_TO_FP16(value);
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);
+ } break;
+ case GGML_TYPE_F32:
+ {
+ ((float *)(data))[0] = value;
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+void * ggml_get_data(const struct ggml_tensor * tensor) {
+ return tensor->data;
+}
+
+float * ggml_get_data_f32(const struct ggml_tensor * tensor) {
+ assert(tensor->type == GGML_TYPE_F32);
+ return (float *)(tensor->data);
+}
+
+GGML_CALL enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor) {
+ GGML_ASSERT(tensor->op == GGML_OP_UNARY);
+ return (enum ggml_unary_op) ggml_get_op_params_i32(tensor, 0);
+}
+
+const char * ggml_get_name(const struct ggml_tensor * tensor) {
+ return tensor->name;
+}
+
+struct ggml_tensor * ggml_set_name(struct ggml_tensor * tensor, const char * name) {
+ size_t i;
+ for (i = 0; i < sizeof(tensor->name) - 1 && name[i] != '\0'; i++) {
+ tensor->name[i] = name[i];
+ }
+ tensor->name[i] = '\0';
+ return tensor;
+}
+
+struct ggml_tensor * ggml_format_name(struct ggml_tensor * tensor, const char * fmt, ...) {
+ va_list args;
+ va_start(args, fmt);
+ vsnprintf(tensor->name, sizeof(tensor->name), fmt, args);
+ va_end(args);
+ return tensor;
+}
+
+struct ggml_tensor * ggml_view_tensor(
+ struct ggml_context * ctx,
+ struct ggml_tensor * src) {
+ struct ggml_tensor * result = ggml_new_tensor_impl(ctx, src->type, GGML_MAX_DIMS, src->ne, src, 0);
+ ggml_format_name(result, "%s (view)", src->name);
+
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
+ result->nb[i] = src->nb[i];
+ }
+
+ return result;
+}
+
+struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx) {
+ struct ggml_object * obj = ctx->objects_begin;
+
+ char * const mem_buffer = ctx->mem_buffer;
+
+ while (obj != NULL) {
+ if (obj->type == GGML_OBJECT_TYPE_TENSOR) {
+ return (struct ggml_tensor *)(mem_buffer + obj->offs);
+ }
+
+ obj = obj->next;
+ }
+
+ return NULL;
+}
+
+struct ggml_tensor * ggml_get_next_tensor(const struct ggml_context * ctx, struct ggml_tensor * tensor) {
+ struct ggml_object * obj = (struct ggml_object *) ((char *)tensor - GGML_OBJECT_SIZE);
+ obj = obj->next;
+
+ char * const mem_buffer = ctx->mem_buffer;
+
+ while (obj != NULL) {
+ if (obj->type == GGML_OBJECT_TYPE_TENSOR) {
+ return (struct ggml_tensor *)(mem_buffer + obj->offs);
+ }
+
+ obj = obj->next;
+ }
+
+ return NULL;
+}
+
+struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name) {
+ struct ggml_object * obj = ctx->objects_begin;
+
+ char * const mem_buffer = ctx->mem_buffer;
+
+ while (obj != NULL) {
+ if (obj->type == GGML_OBJECT_TYPE_TENSOR) {
+ struct ggml_tensor * cur = (struct ggml_tensor *)(mem_buffer + obj->offs);
+ if (strcmp(cur->name, name) == 0) {
+ return cur;
+ }
+ }
+
+ obj = obj->next;
+ }
+
+ return NULL;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// ggml_dup
+
+static struct ggml_tensor * ggml_dup_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && (a->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_DUP;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_dup(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_dup_impl(ctx, a, false);
+}
+
+struct ggml_tensor * ggml_dup_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_dup_impl(ctx, a, true);
+}
+
+// ggml_add
+
+static struct ggml_tensor * ggml_add_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ bool inplace) {
+ GGML_ASSERT(ggml_can_repeat(b, a));
+
+ bool is_node = false;
+
+ if (!inplace && (a->grad || b->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_ADD;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_add(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_add_impl(ctx, a, b, false);
+}
+
+struct ggml_tensor * ggml_add_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_add_impl(ctx, a, b, true);
+}
+
+// ggml_add_cast
+
+static struct ggml_tensor * ggml_add_cast_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ enum ggml_type type) {
+ // TODO: support less-strict constraint
+ // GGML_ASSERT(ggml_can_repeat(b, a));
+ GGML_ASSERT(ggml_can_repeat_rows(b, a));
+
+ // currently only supported for quantized input and f16
+ GGML_ASSERT(ggml_is_quantized(a->type) ||
+ a->type == GGML_TYPE_F16 ||
+ a->type == GGML_TYPE_BF16);
+
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ // TODO: support backward pass for broadcasting
+ GGML_ASSERT(ggml_are_same_shape(a, b));
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor(ctx, type, GGML_MAX_DIMS, a->ne);
+
+ result->op = GGML_OP_ADD;
+ result->grad = is_node ? ggml_new_tensor(ctx, GGML_TYPE_F32, GGML_MAX_DIMS, a->ne) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_add_cast(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ enum ggml_type type) {
+ return ggml_add_cast_impl(ctx, a, b, type);
+}
+
+// ggml_add1
+
+static struct ggml_tensor * ggml_add1_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ bool inplace) {
+ GGML_ASSERT(ggml_is_scalar(b));
+ GGML_ASSERT(ggml_is_padded_1d(a));
+
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_ADD1;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_add1(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_add1_impl(ctx, a, b, false);
+}
+
+struct ggml_tensor * ggml_add1_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_add1_impl(ctx, a, b, true);
+}
+
+// ggml_acc
+
+static struct ggml_tensor * ggml_acc_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset,
+ bool inplace) {
+ GGML_ASSERT(ggml_nelements(b) <= ggml_nelements(a));
+ GGML_ASSERT(ggml_is_contiguous(a));
+ GGML_ASSERT(a->type == GGML_TYPE_F32);
+ GGML_ASSERT(b->type == GGML_TYPE_F32);
+
+ bool is_node = false;
+
+ if (!inplace && (a->grad || b->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ int32_t params[] = { nb1, nb2, nb3, offset, inplace ? 1 : 0 };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_ACC;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_acc(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset) {
+ return ggml_acc_impl(ctx, a, b, nb1, nb2, nb3, offset, false);
+}
+
+struct ggml_tensor * ggml_acc_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset) {
+ return ggml_acc_impl(ctx, a, b, nb1, nb2, nb3, offset, true);
+}
+
+// ggml_sub
+
+static struct ggml_tensor * ggml_sub_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ bool inplace) {
+ GGML_ASSERT(ggml_can_repeat(b, a));
+
+ bool is_node = false;
+
+ if (!inplace && (a->grad || b->grad)) {
+ // TODO: support backward pass for broadcasting
+ GGML_ASSERT(ggml_are_same_shape(a, b));
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_SUB;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_sub(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_sub_impl(ctx, a, b, false);
+}
+
+struct ggml_tensor * ggml_sub_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_sub_impl(ctx, a, b, true);
+}
+
+// ggml_mul
+
+static struct ggml_tensor * ggml_mul_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ bool inplace) {
+ GGML_ASSERT(ggml_can_repeat(b, a));
+
+ bool is_node = false;
+
+ if (!inplace && (a->grad || b->grad)) {
+ // TODO: support backward pass for broadcasting
+ GGML_ASSERT(ggml_are_same_shape(a, b));
+ is_node = true;
+ }
+
+ if (inplace) {
+ GGML_ASSERT(!is_node);
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_MUL;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_mul(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_mul_impl(ctx, a, b, false);
+}
+
+struct ggml_tensor * ggml_mul_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_mul_impl(ctx, a, b, true);
+}
+
+// ggml_div
+
+static struct ggml_tensor * ggml_div_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ bool inplace) {
+ GGML_ASSERT(ggml_can_repeat(b, a));
+
+ bool is_node = false;
+
+ if (!inplace && (a->grad || b->grad)) {
+ is_node = true;
+ }
+
+ if (inplace) {
+ GGML_ASSERT(!is_node);
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_DIV;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_div(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_div_impl(ctx, a, b, false);
+}
+
+struct ggml_tensor * ggml_div_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_div_impl(ctx, a, b, true);
+}
+
+// ggml_sqr
+
+static struct ggml_tensor * ggml_sqr_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && (a->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_SQR;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_sqr(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_sqr_impl(ctx, a, false);
+}
+
+struct ggml_tensor * ggml_sqr_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_sqr_impl(ctx, a, true);
+}
+
+// ggml_sqrt
+
+static struct ggml_tensor * ggml_sqrt_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && (a->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_SQRT;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_sqrt(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_sqrt_impl(ctx, a, false);
+}
+
+struct ggml_tensor * ggml_sqrt_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_sqrt_impl(ctx, a, true);
+}
+
+// ggml_log
+
+static struct ggml_tensor * ggml_log_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && (a->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_LOG;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_log(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_log_impl(ctx, a, false);
+}
+
+struct ggml_tensor * ggml_log_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_log_impl(ctx, a, true);
+}
+
+// ggml_sin
+
+static struct ggml_tensor * ggml_sin_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && (a->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_SIN;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_sin(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_sin_impl(ctx, a, false);
+}
+
+struct ggml_tensor * ggml_sin_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_sin_impl(ctx, a, true);
+}
+
+// ggml_cos
+
+static struct ggml_tensor * ggml_cos_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && (a->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_COS;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_cos(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_cos_impl(ctx, a, false);
+}
+
+struct ggml_tensor * ggml_cos_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_cos_impl(ctx, a, true);
+}
+
+// ggml_sum
+
+struct ggml_tensor * ggml_sum(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, a->type, 1);
+
+ result->op = GGML_OP_SUM;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_sum_rows
+
+struct ggml_tensor * ggml_sum_rows(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ int64_t ne[GGML_MAX_DIMS] = { 1 };
+ for (int i = 1; i < GGML_MAX_DIMS; ++i) {
+ ne[i] = a->ne[i];
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS, ne);
+
+ result->op = GGML_OP_SUM_ROWS;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_mean
+
+struct ggml_tensor * ggml_mean(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ bool is_node = false;
+
+ if (a->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement
+ is_node = true;
+ }
+
+ int64_t ne[4] = { 1, a->ne[1], a->ne[2], a->ne[3] };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ result->op = GGML_OP_MEAN;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_argmax
+
+struct ggml_tensor * ggml_argmax(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ GGML_ASSERT(ggml_is_matrix(a));
+ bool is_node = false;
+
+ if (a->grad) {
+ GGML_ABORT("fatal error");
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, a->ne[1]);
+
+ result->op = GGML_OP_ARGMAX;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_repeat
+
+struct ggml_tensor * ggml_repeat(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ GGML_ASSERT(ggml_can_repeat(a, b));
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS, b->ne);
+
+ result->op = GGML_OP_REPEAT;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_repeat_back
+
+struct ggml_tensor * ggml_repeat_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ GGML_ASSERT(ggml_can_repeat(b, a));
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ if (ggml_are_same_shape(a, b) && !is_node) {
+ return a;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS, b->ne);
+
+ result->op = GGML_OP_REPEAT_BACK;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_concat
+
+struct ggml_tensor * ggml_concat(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int dim) {
+ GGML_ASSERT(dim >= 0 && dim < GGML_MAX_DIMS);
+
+ int64_t ne[GGML_MAX_DIMS];
+ for (int d = 0; d < GGML_MAX_DIMS; ++d) {
+ if (d == dim) {
+ ne[d] = a->ne[d] + b->ne[d];
+ continue;
+ }
+ GGML_ASSERT(a->ne[d] == b->ne[d]);
+ ne[d] = a->ne[d];
+ }
+
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS, ne);
+
+ ggml_set_op_params_i32(result, 0, dim);
+
+ result->op = GGML_OP_CONCAT;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+// ggml_abs
+
+struct ggml_tensor * ggml_abs(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_ABS);
+}
+
+struct ggml_tensor * ggml_abs_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_ABS);
+}
+
+// ggml_sgn
+
+struct ggml_tensor * ggml_sgn(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_SGN);
+}
+
+struct ggml_tensor * ggml_sgn_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_SGN);
+}
+
+// ggml_neg
+
+struct ggml_tensor * ggml_neg(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_NEG);
+}
+
+struct ggml_tensor * ggml_neg_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_NEG);
+}
+
+// ggml_step
+
+struct ggml_tensor * ggml_step(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_STEP);
+}
+
+struct ggml_tensor * ggml_step_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_STEP);
+}
+
+// ggml_tanh
+
+struct ggml_tensor * ggml_tanh(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_TANH);
+}
+
+struct ggml_tensor * ggml_tanh_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_TANH);
+}
+
+// ggml_elu
+
+struct ggml_tensor * ggml_elu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_ELU);
+}
+
+struct ggml_tensor * ggml_elu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_ELU);
+}
+
+// ggml_relu
+
+struct ggml_tensor * ggml_relu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_RELU);
+}
+
+struct ggml_tensor * ggml_relu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_RELU);
+}
+
+// ggml_leaky_relu
+
+struct ggml_tensor * ggml_leaky_relu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, float negative_slope, bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && (a->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params(result, &negative_slope, sizeof(negative_slope));
+
+ result->op = GGML_OP_LEAKY_RELU;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_sigmoid
+
+struct ggml_tensor * ggml_sigmoid(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_SIGMOID);
+}
+
+struct ggml_tensor * ggml_sigmoid_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_SIGMOID);
+}
+
+// ggml_gelu
+
+struct ggml_tensor * ggml_gelu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_GELU);
+}
+
+struct ggml_tensor * ggml_gelu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_GELU);
+}
+
+// ggml_gelu_quick
+
+struct ggml_tensor * ggml_gelu_quick(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_GELU_QUICK);
+}
+
+struct ggml_tensor * ggml_gelu_quick_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_GELU_QUICK);
+}
+
+// ggml_silu
+
+struct ggml_tensor * ggml_silu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_SILU);
+}
+
+struct ggml_tensor * ggml_silu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_SILU);
+}
+
+// ggml_silu_back
+
+struct ggml_tensor * ggml_silu_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ // TODO: implement backward
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_SILU_BACK;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+// ggml hardswish
+struct ggml_tensor * ggml_hardswish(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_HARDSWISH);
+}
+
+// ggml hardsigmoid
+struct ggml_tensor * ggml_hardsigmoid(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_HARDSIGMOID);
+}
+
+// ggml exp
+struct ggml_tensor * ggml_exp(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary(ctx, a, GGML_UNARY_OP_EXP);
+}
+
+struct ggml_tensor * ggml_exp_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_EXP);
+}
+
+// ggml_norm
+
+static struct ggml_tensor * ggml_norm_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && (a->grad)) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params(result, &eps, sizeof(eps));
+
+ result->op = GGML_OP_NORM;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_norm(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps) {
+ return ggml_norm_impl(ctx, a, eps, false);
+}
+
+struct ggml_tensor * ggml_norm_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps) {
+ return ggml_norm_impl(ctx, a, eps, true);
+}
+
+// ggml_rms_norm
+
+static struct ggml_tensor * ggml_rms_norm_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && (a->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params(result, &eps, sizeof(eps));
+
+ result->op = GGML_OP_RMS_NORM;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_rms_norm(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps) {
+ return ggml_rms_norm_impl(ctx, a, eps, false);
+}
+
+struct ggml_tensor * ggml_rms_norm_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps) {
+ return ggml_rms_norm_impl(ctx, a, eps, true);
+}
+
+// ggml_rms_norm_back
+
+struct ggml_tensor * ggml_rms_norm_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ float eps) {
+ bool is_node = false;
+
+ if (a->grad) {
+ // TODO: implement backward
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params(result, &eps, sizeof(eps));
+
+ result->op = GGML_OP_RMS_NORM_BACK;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+// ggml_group_norm
+
+static struct ggml_tensor * ggml_group_norm_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_groups,
+ float eps,
+ bool inplace) {
+
+ bool is_node = false;
+ if (!inplace && (a->grad)) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params_i32(result, 0, n_groups);
+ ggml_set_op_params_f32(result, 1, eps);
+
+ result->op = GGML_OP_GROUP_NORM;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_group_norm(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_groups,
+ float eps) {
+ return ggml_group_norm_impl(ctx, a, n_groups, eps, false);
+}
+
+struct ggml_tensor * ggml_group_norm_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_groups,
+ float eps) {
+ return ggml_group_norm_impl(ctx, a, n_groups, eps, true);
+}
+
+// ggml_mul_mat
+
+struct ggml_tensor * ggml_mul_mat(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ GGML_ASSERT(ggml_can_mul_mat(a, b));
+ GGML_ASSERT(!ggml_is_transposed(a));
+
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ is_node = true;
+ }
+
+ const int64_t ne[4] = { a->ne[1], b->ne[1], b->ne[2], b->ne[3] };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ result->op = GGML_OP_MUL_MAT;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+void ggml_mul_mat_set_prec(
+ struct ggml_tensor * a,
+ enum ggml_prec prec) {
+ GGML_ASSERT(a->op == GGML_OP_MUL_MAT);
+
+ const int32_t prec_i32 = (int32_t) prec;
+
+ ggml_set_op_params_i32(a, 0, prec_i32);
+}
+
+// ggml_mul_mat_id
+
+/*
+ c = ggml_mul_mat_id(ctx, as, b, ids);
+
+ as -> [cols, rows, n_expert]
+ ids -> [n_experts_used, n_tokens] (i32)
+ b -> [cols, n_expert_used, n_tokens]
+ c -> [rows, n_expert_used, n_tokens]
+
+ in b, n_experts_used can be broadcasted to match the n_expert_used of ids
+
+ c ~= as[:,:,i] @ b[:,i%r,t], i = ids[e,t] for all e,t in ids
+*/
+struct ggml_tensor * ggml_mul_mat_id(
+ struct ggml_context * ctx,
+ struct ggml_tensor * as,
+ struct ggml_tensor * b,
+ struct ggml_tensor * ids) {
+ GGML_ASSERT(!ggml_is_transposed(as));
+ GGML_ASSERT(ids->type == GGML_TYPE_I32);
+
+ GGML_ASSERT(as->ne[3] == 1); // as is 3d (one matrix per expert)
+ GGML_ASSERT(b->ne[3] == 1); // b is 3d
+ GGML_ASSERT(ids->ne[2] == 1 && ids->ne[3] == 1); // ids is 2d
+ GGML_ASSERT(ids->ne[1] == b->ne[2]); // must have an expert list per b row
+ GGML_ASSERT(as->ne[0] == b->ne[0]); // can_mul_mat
+ GGML_ASSERT(ids->ne[0] % b->ne[1] == 0); // can broadcast
+
+ bool is_node = false;
+
+ if (as->grad || b->grad) {
+ is_node = true;
+ }
+
+ const int64_t ne[4] = { as->ne[1], ids->ne[0], b->ne[2], 1 };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ result->op = GGML_OP_MUL_MAT_ID;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = as;
+ result->src[1] = b;
+ result->src[2] = ids;
+
+ return result;
+}
+
+// ggml_out_prod
+
+struct ggml_tensor * ggml_out_prod(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ GGML_ASSERT(ggml_can_out_prod(a, b));
+ GGML_ASSERT(!ggml_is_transposed(a));
+
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ is_node = true;
+ }
+
+ // a is broadcastable to b for ne[2] and ne[3] -> use b->ne[2] and b->ne[3]
+ const int64_t ne[4] = { a->ne[0], b->ne[0], b->ne[2], b->ne[3] };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ result->op = GGML_OP_OUT_PROD;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+// ggml_scale
+
+static struct ggml_tensor * ggml_scale_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float s,
+ bool inplace) {
+ GGML_ASSERT(ggml_is_padded_1d(a));
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params(result, &s, sizeof(s));
+
+ result->op = GGML_OP_SCALE;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_scale(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float s) {
+ return ggml_scale_impl(ctx, a, s, false);
+}
+
+struct ggml_tensor * ggml_scale_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float s) {
+ return ggml_scale_impl(ctx, a, s, true);
+}
+
+// ggml_set
+
+static struct ggml_tensor * ggml_set_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset,
+ bool inplace) {
+ GGML_ASSERT(ggml_nelements(a) >= ggml_nelements(b));
+
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ is_node = true;
+ }
+
+ // make a view of the destination
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ int32_t params[] = { nb1, nb2, nb3, offset, inplace ? 1 : 0 };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_SET;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_set(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset) {
+ return ggml_set_impl(ctx, a, b, nb1, nb2, nb3, offset, false);
+}
+
+struct ggml_tensor * ggml_set_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset) {
+ return ggml_set_impl(ctx, a, b, nb1, nb2, nb3, offset, true);
+}
+
+struct ggml_tensor * ggml_set_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t offset) {
+ return ggml_set_impl(ctx, a, b, a->nb[1], a->nb[2], a->nb[3], offset, false);
+}
+
+struct ggml_tensor * ggml_set_1d_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t offset) {
+ return ggml_set_impl(ctx, a, b, a->nb[1], a->nb[2], a->nb[3], offset, true);
+}
+
+struct ggml_tensor * ggml_set_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t offset) {
+ return ggml_set_impl(ctx, a, b, nb1, a->nb[2], a->nb[3], offset, false);
+}
+
+struct ggml_tensor * ggml_set_2d_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t offset) {
+ return ggml_set_impl(ctx, a, b, nb1, a->nb[2], a->nb[3], offset, true);
+}
+
+// ggml_cpy
+
+static struct ggml_tensor * ggml_cpy_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ GGML_ASSERT(ggml_nelements(a) == ggml_nelements(b));
+
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ // inplace is false and either one have a grad
+ is_node = true;
+ }
+
+ // make a view of the destination
+ struct ggml_tensor * result = ggml_view_tensor(ctx, b);
+ if (strlen(b->name) > 0) {
+ ggml_format_name(result, "%s (copy of %s)", b->name, a->name);
+ } else {
+ ggml_format_name(result, "%s (copy)", a->name);
+ }
+
+ result->op = GGML_OP_CPY;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_cpy(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_cpy_impl(ctx, a, b);
+}
+
+struct ggml_tensor * ggml_cast(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_type type) {
+ bool is_node = false;
+
+ struct ggml_tensor * result = ggml_new_tensor(ctx, type, GGML_MAX_DIMS, a->ne);
+ ggml_format_name(result, "%s (copy)", a->name);
+
+ result->op = GGML_OP_CPY;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = result;
+
+ return result;
+}
+
+// ggml_cont
+
+static struct ggml_tensor * ggml_cont_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
+ ggml_format_name(result, "%s (cont)", a->name);
+
+ result->op = GGML_OP_CONT;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_cont(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_cont_impl(ctx, a);
+}
+
+// make contiguous, with new shape
+GGML_API struct ggml_tensor * ggml_cont_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0) {
+ return ggml_cont_4d(ctx, a, ne0, 1, 1, 1);
+}
+
+GGML_API struct ggml_tensor * ggml_cont_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1) {
+ return ggml_cont_4d(ctx, a, ne0, ne1, 1, 1);
+}
+
+GGML_API struct ggml_tensor * ggml_cont_3d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2) {
+ return ggml_cont_4d(ctx, a, ne0, ne1, ne2, 1);
+}
+
+struct ggml_tensor * ggml_cont_4d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3) {
+ GGML_ASSERT(ggml_nelements(a) == (ne0*ne1*ne2*ne3));
+
+ bool is_node = false;
+
+ struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type, ne0, ne1, ne2, ne3);
+ ggml_format_name(result, "%s (cont)", a->name);
+
+ result->op = GGML_OP_CONT;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_reshape
+
+struct ggml_tensor * ggml_reshape(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ GGML_ASSERT(ggml_is_contiguous(a));
+ // as only the shape of b is relevant, and not its memory layout, b is allowed to be non contiguous.
+ GGML_ASSERT(ggml_nelements(a) == ggml_nelements(b));
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ if (b->grad) {
+ // gradient propagation is not supported
+ //GGML_ABORT("fatal error");
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b->ne, a, 0);
+ ggml_format_name(result, "%s (reshaped)", a->name);
+
+ result->op = GGML_OP_RESHAPE;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_reshape_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0) {
+ GGML_ASSERT(ggml_is_contiguous(a));
+ GGML_ASSERT(ggml_nelements(a) == ne0);
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ const int64_t ne[1] = { ne0 };
+ struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 1, ne, a, 0);
+ ggml_format_name(result, "%s (reshaped)", a->name);
+
+ result->op = GGML_OP_RESHAPE;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_reshape_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1) {
+ GGML_ASSERT(ggml_is_contiguous(a));
+ GGML_ASSERT(ggml_nelements(a) == ne0*ne1);
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ const int64_t ne[2] = { ne0, ne1 };
+ struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 2, ne, a, 0);
+ ggml_format_name(result, "%s (reshaped)", a->name);
+
+ result->op = GGML_OP_RESHAPE;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_reshape_3d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2) {
+ GGML_ASSERT(ggml_is_contiguous(a));
+ GGML_ASSERT(ggml_nelements(a) == ne0*ne1*ne2);
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ const int64_t ne[3] = { ne0, ne1, ne2 };
+ struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 3, ne, a, 0);
+ ggml_format_name(result, "%s (reshaped)", a->name);
+
+ result->op = GGML_OP_RESHAPE;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_reshape_4d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3) {
+ GGML_ASSERT(ggml_is_contiguous(a));
+ GGML_ASSERT(ggml_nelements(a) == ne0*ne1*ne2*ne3);
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ const int64_t ne[4] = { ne0, ne1, ne2, ne3 };
+ struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 4, ne, a, 0);
+ ggml_format_name(result, "%s (reshaped)", a->name);
+
+ result->op = GGML_OP_RESHAPE;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+static struct ggml_tensor * ggml_view_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_dims,
+ const int64_t * ne,
+ size_t offset) {
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, n_dims, ne, a, offset);
+ ggml_format_name(result, "%s (view)", a->name);
+
+ ggml_set_op_params(result, &offset, sizeof(offset));
+
+ result->op = GGML_OP_VIEW;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_view_1d
+
+struct ggml_tensor * ggml_view_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ size_t offset) {
+
+ struct ggml_tensor * result = ggml_view_impl(ctx, a, 1, &ne0, offset);
+
+ return result;
+}
+
+// ggml_view_2d
+
+struct ggml_tensor * ggml_view_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ size_t nb1,
+ size_t offset) {
+
+ const int64_t ne[2] = { ne0, ne1 };
+
+ struct ggml_tensor * result = ggml_view_impl(ctx, a, 2, ne, offset);
+
+ result->nb[1] = nb1;
+ result->nb[2] = result->nb[1]*ne1;
+ result->nb[3] = result->nb[2];
+
+ return result;
+}
+
+// ggml_view_3d
+
+struct ggml_tensor * ggml_view_3d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ size_t nb1,
+ size_t nb2,
+ size_t offset) {
+
+ const int64_t ne[3] = { ne0, ne1, ne2 };
+
+ struct ggml_tensor * result = ggml_view_impl(ctx, a, 3, ne, offset);
+
+ result->nb[1] = nb1;
+ result->nb[2] = nb2;
+ result->nb[3] = result->nb[2]*ne2;
+
+ return result;
+}
+
+// ggml_view_4d
+
+struct ggml_tensor * ggml_view_4d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset) {
+
+ const int64_t ne[4] = { ne0, ne1, ne2, ne3 };
+
+ struct ggml_tensor * result = ggml_view_impl(ctx, a, 4, ne, offset);
+
+ result->nb[1] = nb1;
+ result->nb[2] = nb2;
+ result->nb[3] = nb3;
+
+ return result;
+}
+
+// ggml_permute
+
+struct ggml_tensor * ggml_permute(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int axis0,
+ int axis1,
+ int axis2,
+ int axis3) {
+ GGML_ASSERT(axis0 >= 0 && axis0 < GGML_MAX_DIMS);
+ GGML_ASSERT(axis1 >= 0 && axis1 < GGML_MAX_DIMS);
+ GGML_ASSERT(axis2 >= 0 && axis2 < GGML_MAX_DIMS);
+ GGML_ASSERT(axis3 >= 0 && axis3 < GGML_MAX_DIMS);
+
+ GGML_ASSERT(axis0 != axis1);
+ GGML_ASSERT(axis0 != axis2);
+ GGML_ASSERT(axis0 != axis3);
+ GGML_ASSERT(axis1 != axis2);
+ GGML_ASSERT(axis1 != axis3);
+ GGML_ASSERT(axis2 != axis3);
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_view_tensor(ctx, a);
+ ggml_format_name(result, "%s (permuted)", a->name);
+
+ int ne[GGML_MAX_DIMS];
+ int nb[GGML_MAX_DIMS];
+
+ ne[axis0] = a->ne[0];
+ ne[axis1] = a->ne[1];
+ ne[axis2] = a->ne[2];
+ ne[axis3] = a->ne[3];
+
+ nb[axis0] = a->nb[0];
+ nb[axis1] = a->nb[1];
+ nb[axis2] = a->nb[2];
+ nb[axis3] = a->nb[3];
+
+ result->ne[0] = ne[0];
+ result->ne[1] = ne[1];
+ result->ne[2] = ne[2];
+ result->ne[3] = ne[3];
+
+ result->nb[0] = nb[0];
+ result->nb[1] = nb[1];
+ result->nb[2] = nb[2];
+ result->nb[3] = nb[3];
+
+ result->op = GGML_OP_PERMUTE;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ int32_t params[] = { axis0, axis1, axis2, axis3 };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ return result;
+}
+
+// ggml_transpose
+
+struct ggml_tensor * ggml_transpose(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_view_tensor(ctx, a);
+ ggml_format_name(result, "%s (transposed)", a->name);
+
+ result->ne[0] = a->ne[1];
+ result->ne[1] = a->ne[0];
+
+ result->nb[0] = a->nb[1];
+ result->nb[1] = a->nb[0];
+
+ result->op = GGML_OP_TRANSPOSE;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_get_rows
+
+struct ggml_tensor * ggml_get_rows(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ GGML_ASSERT(a->ne[2] == b->ne[1]);
+ GGML_ASSERT(b->ne[3] == 1);
+ GGML_ASSERT(b->type == GGML_TYPE_I32);
+
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ is_node = true;
+ }
+
+ // TODO: implement non F32 return
+ enum ggml_type type = GGML_TYPE_F32;
+ if (a->type == GGML_TYPE_I32) {
+ type = a->type;
+ }
+ struct ggml_tensor * result = ggml_new_tensor_4d(ctx, type, a->ne[0], b->ne[0], b->ne[1], b->ne[2]);
+
+ result->op = GGML_OP_GET_ROWS;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+// ggml_get_rows_back
+
+struct ggml_tensor * ggml_get_rows_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c) {
+ GGML_ASSERT(ggml_is_matrix(a) && ggml_is_vector(b) && b->type == GGML_TYPE_I32);
+ GGML_ASSERT(ggml_is_matrix(c) && (a->ne[0] == c->ne[0]));
+
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ is_node = true;
+ }
+
+ // TODO: implement non F32 return
+ //struct ggml_tensor * result = ggml_new_tensor_2d(ctx, a->type, a->ne[0], b->ne[0]);
+ struct ggml_tensor * result = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, c->ne[0], c->ne[1]);
+
+ result->op = GGML_OP_GET_ROWS_BACK;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+// ggml_diag
+
+struct ggml_tensor * ggml_diag(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ GGML_ASSERT(a->ne[1] == 1);
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ const int64_t ne[4] = { a->ne[0], a->ne[0], a->ne[2], a->ne[3] };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, 4, ne);
+
+ result->op = GGML_OP_DIAG;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_diag_mask_inf
+
+static struct ggml_tensor * ggml_diag_mask_inf_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past,
+ bool inplace) {
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ int32_t params[] = { n_past };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_DIAG_MASK_INF;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_diag_mask_inf(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past) {
+ return ggml_diag_mask_inf_impl(ctx, a, n_past, false);
+}
+
+struct ggml_tensor * ggml_diag_mask_inf_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past) {
+ return ggml_diag_mask_inf_impl(ctx, a, n_past, true);
+}
+
+// ggml_diag_mask_zero
+
+static struct ggml_tensor * ggml_diag_mask_zero_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past,
+ bool inplace) {
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ int32_t params[] = { n_past };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_DIAG_MASK_ZERO;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_diag_mask_zero(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past) {
+ return ggml_diag_mask_zero_impl(ctx, a, n_past, false);
+}
+
+struct ggml_tensor * ggml_diag_mask_zero_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past) {
+ return ggml_diag_mask_zero_impl(ctx, a, n_past, true);
+}
+
+// ggml_soft_max
+
+static struct ggml_tensor * ggml_soft_max_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * mask,
+ float scale,
+ float max_bias,
+ bool inplace) {
+ GGML_ASSERT(ggml_is_contiguous(a));
+
+ if (mask) {
+ GGML_ASSERT(mask->type == GGML_TYPE_F16 || mask->type == GGML_TYPE_F32);
+ GGML_ASSERT(ggml_is_contiguous(mask));
+ GGML_ASSERT(ggml_is_matrix(mask));
+ GGML_ASSERT(mask->ne[0] == a->ne[0]);
+ GGML_ASSERT(mask->ne[1] >= a->ne[1]);
+ }
+
+ if (max_bias > 0.0f) {
+ GGML_ASSERT(mask);
+ }
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ float params[] = { scale, max_bias };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_SOFT_MAX;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = mask;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_soft_max(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_soft_max_impl(ctx, a, NULL, 1.0f, 0.0f, false);
+}
+
+struct ggml_tensor * ggml_soft_max_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a) {
+ return ggml_soft_max_impl(ctx, a, NULL, 1.0f, 0.0f, true);
+}
+
+struct ggml_tensor * ggml_soft_max_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * mask,
+ float scale,
+ float max_bias) {
+ return ggml_soft_max_impl(ctx, a, mask, scale, max_bias, false);
+}
+
+// ggml_soft_max_back
+
+static struct ggml_tensor * ggml_soft_max_back_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ bool inplace) {
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ is_node = true; // TODO : implement backward pass
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_SOFT_MAX_BACK;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_soft_max_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_soft_max_back_impl(ctx, a, b, false);
+}
+
+struct ggml_tensor * ggml_soft_max_back_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_soft_max_back_impl(ctx, a, b, true);
+}
+
+// ggml_rope
+
+static struct ggml_tensor * ggml_rope_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow,
+ bool inplace) {
+ GGML_ASSERT((mode & 1) == 0 && "mode & 1 == 1 is no longer supported");
+
+ GGML_ASSERT(ggml_is_vector(b));
+ GGML_ASSERT(b->type == GGML_TYPE_I32);
+ GGML_ASSERT(a->ne[2] == b->ne[0]);
+
+ if (c) {
+ GGML_ASSERT(c->type == GGML_TYPE_F32);
+ GGML_ASSERT(c->ne[0] >= n_dims / 2);
+ }
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ int32_t params[11] = { /*n_past*/ 0, n_dims, mode, /*n_ctx*/ 0, n_ctx_orig };
+ memcpy(params + 5, &freq_base, sizeof(float));
+ memcpy(params + 6, &freq_scale, sizeof(float));
+ memcpy(params + 7, &ext_factor, sizeof(float));
+ memcpy(params + 8, &attn_factor, sizeof(float));
+ memcpy(params + 9, &beta_fast, sizeof(float));
+ memcpy(params + 10, &beta_slow, sizeof(float));
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_ROPE;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+ result->src[2] = c;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_rope(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode) {
+ return ggml_rope_impl(
+ ctx, a, b, NULL, n_dims, mode, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, false
+ );
+}
+
+struct ggml_tensor * ggml_rope_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode) {
+ return ggml_rope_impl(
+ ctx, a, b, NULL, n_dims, mode, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, true
+ );
+}
+
+struct ggml_tensor * ggml_rope_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow) {
+ return ggml_rope_impl(
+ ctx, a, b, c, n_dims, mode, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow, false
+ );
+}
+
+struct ggml_tensor * ggml_rope_ext_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow) {
+ return ggml_rope_impl(
+ ctx, a, b, c, n_dims, mode, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow, true
+ );
+}
+
+struct ggml_tensor * ggml_rope_custom(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow) {
+ return ggml_rope_impl(
+ ctx, a, b, NULL, n_dims, mode, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow, false
+ );
+}
+
+struct ggml_tensor * ggml_rope_custom_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow) {
+ return ggml_rope_impl(
+ ctx, a, b, NULL, n_dims, mode, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow, true
+ );
+}
+
+// ggml_rope_back
+
+struct ggml_tensor * ggml_rope_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow) {
+ GGML_ASSERT(ggml_is_vector(b));
+ GGML_ASSERT(b->type == GGML_TYPE_I32);
+ GGML_ASSERT(a->ne[2] == b->ne[0]);
+ GGML_ASSERT(c == NULL && "freq factors not implemented yet");
+
+ GGML_ASSERT((mode & 4) == 0 && "ggml_rope_back() for ChatGLM not implemented yet");
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = false; // TODO: implement backward
+ }
+
+ struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
+
+ int32_t params[11] = { /*n_past*/ 0, n_dims, mode, /*n_ctx*/ 0, n_ctx_orig };
+ memcpy(params + 5, &freq_base, sizeof(float));
+ memcpy(params + 6, &freq_scale, sizeof(float));
+ memcpy(params + 7, &ext_factor, sizeof(float));
+ memcpy(params + 8, &attn_factor, sizeof(float));
+ memcpy(params + 9, &beta_fast, sizeof(float));
+ memcpy(params + 10, &beta_slow, sizeof(float));
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_ROPE_BACK;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+// ggml_clamp
+
+struct ggml_tensor * ggml_clamp(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float min,
+ float max) {
+ bool is_node = false;
+
+ if (a->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ // TODO: when implement backward, fix this:
+ struct ggml_tensor * result = ggml_view_tensor(ctx, a);
+
+ float params[] = { min, max };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_CLAMP;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_conv_1d
+
+static int64_t ggml_calc_conv_output_size(int64_t ins, int64_t ks, int s, int p, int d) {
+ return (ins + 2 * p - d * (ks - 1) - 1) / s + 1;
+}
+
+GGML_API struct ggml_tensor * ggml_conv_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int s0,
+ int p0,
+ int d0) {
+ struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F16); // [N, OL, IC * K]
+
+ struct ggml_tensor * result =
+ ggml_mul_mat(ctx,
+ ggml_reshape_2d(ctx, im2col, im2col->ne[0], (im2col->ne[2] * im2col->ne[1])), // [N, OL, IC * K] => [N*OL, IC * K]
+ ggml_reshape_2d(ctx, a, (a->ne[0] * a->ne[1]), a->ne[2])); // [OC，IC, K] => [OC, IC * K]
+
+ result = ggml_reshape_3d(ctx, result, im2col->ne[1], a->ne[2], im2col->ne[2]); // [N, OC, OL]
+
+ return result;
+}
+
+// ggml_conv_1d_ph
+
+struct ggml_tensor* ggml_conv_1d_ph(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int s,
+ int d) {
+ return ggml_conv_1d(ctx, a, b, s, a->ne[0] / 2, d);
+}
+
+// ggml_conv_transpose_1d
+
+static int64_t ggml_calc_conv_transpose_1d_output_size(int64_t ins, int64_t ks, int s, int p, int d) {
+ return (ins - 1) * s - 2 * p + d * (ks - 1) + 1;
+}
+
+GGML_API struct ggml_tensor * ggml_conv_transpose_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int s0,
+ int p0,
+ int d0) {
+ GGML_ASSERT(ggml_is_matrix(b));
+ GGML_ASSERT(a->ne[2] == b->ne[1]);
+ GGML_ASSERT(a->ne[3] == 1);
+
+ GGML_ASSERT(p0 == 0);
+ GGML_ASSERT(d0 == 1);
+
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ const int64_t ne[4] = {
+ ggml_calc_conv_transpose_1d_output_size(b->ne[0], a->ne[0], s0, 0 /*p0*/, 1 /*d0*/),
+ a->ne[1], b->ne[2], 1,
+ };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ int32_t params[] = { s0, p0, d0 };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_CONV_TRANSPOSE_1D;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+// ggml_conv_depthwise
+struct ggml_tensor * ggml_conv_depthwise_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int s0,
+ int s1,
+ int p0,
+ int p1,
+ int d0,
+ int d1) {
+
+ struct ggml_tensor * new_a = ggml_reshape_4d(ctx, a, a->ne[0], a->ne[1], 1, a->ne[2] * a->ne[3]);
+ struct ggml_tensor * im2col = ggml_im2col(ctx, new_a,
+ ggml_reshape_4d(ctx, b, b->ne[0], b->ne[1], 1, b->ne[2] * b->ne[3]),
+ s0, s1, p0, p1, d0, d1, true, GGML_TYPE_F16); // [N * IC, OH, OW, KH * KW]
+ struct ggml_tensor * new_b = ggml_reshape_4d(ctx, im2col, im2col->ne[0], im2col->ne[2] * im2col->ne[1], b->ne[2], b->ne[3]); // [N * IC, OH, OW, KH * KW] => [N, IC, OH * OW, KH * KW]
+
+ new_a = ggml_reshape_4d(ctx, new_a, (new_a->ne[0] * new_a->ne[1]), new_a->ne[2], new_a->ne[3], 1); // [OC，1, KH, KW] => [1, OC, 1, KH * KW]
+ struct ggml_tensor * result = ggml_mul_mat(ctx, new_a, new_b);
+ result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], b->ne[2], b->ne[3]); // [N, OC, OH, OW]
+
+ return result;
+}
+// ggml_conv_2d
+
+// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
+// a: [OC，IC, KH, KW]
+// b: [N, IC, IH, IW]
+// result: [N, OH, OW, IC*KH*KW]
+struct ggml_tensor * ggml_im2col(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int s0,
+ int s1,
+ int p0,
+ int p1,
+ int d0,
+ int d1,
+ bool is_2D,
+ enum ggml_type dst_type) {
+
+ if(is_2D) {
+ GGML_ASSERT(a->ne[2] == b->ne[2]);
+ } else {
+ GGML_ASSERT(a->ne[1] == b->ne[1]);
+ GGML_ASSERT(b->ne[3] == 1);
+ }
+ bool is_node = false;
+
+ if (/*a->grad ||*/ b->grad) { // a is only used for its shape, not its data
+ is_node = true;
+ }
+
+ const int64_t OH = is_2D ? ggml_calc_conv_output_size(b->ne[1], a->ne[1], s1, p1, d1) : 0;
+ const int64_t OW = ggml_calc_conv_output_size(b->ne[0], a->ne[0], s0, p0, d0);
+
+ GGML_ASSERT((!is_2D || OH > 0) && "b too small compared to a");
+ GGML_ASSERT((OW > 0) && "b too small compared to a");
+
+ const int64_t ne[4] = {
+ is_2D ? (a->ne[2] * a->ne[1] * a->ne[0]) : a->ne[1] * a->ne[0],
+ OW,
+ is_2D ? OH : b->ne[2],
+ is_2D ? b->ne[3] : 1,
+ };
+
+ struct ggml_tensor * result = ggml_new_tensor(ctx, dst_type, 4, ne);
+ int32_t params[] = { s0, s1, p0, p1, d0, d1, (is_2D ? 1 : 0) };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_IM2COL;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_im2col_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int64_t * ne,
+ int s0,
+ int s1,
+ int p0,
+ int p1,
+ int d0,
+ int d1,
+ bool is_2D) {
+
+ bool is_node = false;
+
+ if (/*a->grad ||*/ b->grad) { // a is only used for its shape, not its data
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+ int32_t params[] = { s0, s1, p0, p1, d0, d1, (is_2D ? 1 : 0) };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_IM2COL_BACK;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+// a: [OC，IC, KH, KW]
+// b: [N, IC, IH, IW]
+// result: [N, OC, OH, OW]
+struct ggml_tensor * ggml_conv_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int s0,
+ int s1,
+ int p0,
+ int p1,
+ int d0,
+ int d1) {
+ struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, true, a->type); // [N, OH, OW, IC * KH * KW]
+
+ struct ggml_tensor * result =
+ ggml_mul_mat(ctx,
+ ggml_reshape_2d(ctx, im2col, im2col->ne[0], im2col->ne[3] * im2col->ne[2] * im2col->ne[1]), // [N, OH, OW, IC * KH * KW] => [N*OH*OW, IC * KH * KW]
+ ggml_reshape_2d(ctx, a, (a->ne[0] * a->ne[1] * a->ne[2]), a->ne[3])); // [OC，IC, KH, KW] => [OC, IC * KH * KW]
+
+ result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], im2col->ne[3], a->ne[3]); // [OC, N, OH, OW]
+ result = ggml_cont(ctx, ggml_permute(ctx, result, 0, 1, 3, 2)); // [N, OC, OH, OW]
+
+
+ return result;
+}
+
+// ggml_conv_2d_sk_p0
+struct ggml_tensor * ggml_conv_2d_sk_p0(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_conv_2d(ctx, a, b, a->ne[0], a->ne[1], 0, 0, 1, 1);
+}
+
+// ggml_conv_2d_s1_ph
+
+struct ggml_tensor * ggml_conv_2d_s1_ph(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ return ggml_conv_2d(ctx, a, b, 1, 1, a->ne[0] / 2, a->ne[1] / 2, 1, 1);
+}
+
+// ggml_conv_transpose_2d_p0
+
+static int64_t ggml_calc_conv_transpose_output_size(int64_t ins, int64_t ks, int s, int p) {
+ return (ins - 1) * s - 2 * p + ks;
+}
+
+struct ggml_tensor * ggml_conv_transpose_2d_p0(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int stride) {
+ GGML_ASSERT(a->ne[3] == b->ne[2]);
+
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ const int64_t ne[4] = {
+ ggml_calc_conv_transpose_output_size(b->ne[0], a->ne[0], stride, 0 /*p0*/),
+ ggml_calc_conv_transpose_output_size(b->ne[1], a->ne[1], stride, 0 /*p1*/),
+ a->ne[2], b->ne[3],
+ };
+
+ struct ggml_tensor* result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ ggml_set_op_params_i32(result, 0, stride);
+
+ result->op = GGML_OP_CONV_TRANSPOSE_2D;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+// ggml_pool_*
+
+static int64_t ggml_calc_pool_output_size(int64_t ins, int ks, int s, float p) {
+ return (ins + 2 * p - ks) / s + 1;
+}
+
+// ggml_pool_1d
+
+struct ggml_tensor * ggml_pool_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_op_pool op,
+ int k0,
+ int s0,
+ int p0) {
+
+ bool is_node = false;
+
+ if (a->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ const int64_t ne[4] = {
+ ggml_calc_pool_output_size(a->ne[0], k0, s0, p0),
+ a->ne[1],
+ a->ne[2],
+ a->ne[3],
+ };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ int32_t params[] = { op, k0, s0, p0 };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_POOL_1D;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_pool_2d
+
+struct ggml_tensor * ggml_pool_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_op_pool op,
+ int k0,
+ int k1,
+ int s0,
+ int s1,
+ float p0,
+ float p1) {
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result;
+ const int64_t ne[4] = {
+ ggml_calc_pool_output_size(a->ne[0], k0, s0, p0),
+ ggml_calc_pool_output_size(a->ne[1], k1, s1, p1),
+ a->ne[2],
+ a->ne[3],
+ };
+ result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ int32_t params[] = { op, k0, k1, s0, s1, p0, p1 };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_POOL_2D;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ return result;
+}
+
+struct ggml_tensor * ggml_pool_2d_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * af,
+ enum ggml_op_pool op,
+ int k0,
+ int k1,
+ int s0,
+ int s1,
+ float p0,
+ float p1) {
+
+ bool is_node = false;
+
+ if (a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result;
+ result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, af->ne);
+
+ int32_t params[] = { op, k0, k1, s0, s1, p0, p1 };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_POOL_2D_BACK;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = af;
+ return result;
+}
+
+// ggml_upscale
+
+static struct ggml_tensor * ggml_upscale_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int ne0,
+ int ne1,
+ int ne2,
+ int ne3) {
+ bool is_node = false;
+
+ if (a->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ GGML_ASSERT(a->ne[0] <= ne0);
+ GGML_ASSERT(a->ne[1] <= ne1);
+ GGML_ASSERT(a->ne[2] <= ne2);
+ GGML_ASSERT(a->ne[3] <= ne3);
+
+ struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type,
+ ne0,
+ ne1,
+ ne2,
+ ne3
+ );
+
+ result->op = GGML_OP_UPSCALE;
+
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_upscale(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int scale_factor) {
+ return ggml_upscale_impl(ctx, a, a->ne[0] * scale_factor, a->ne[1] * scale_factor, a->ne[2], a->ne[3]);
+}
+
+struct ggml_tensor * ggml_upscale_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int ne0,
+ int ne1,
+ int ne2,
+ int ne3) {
+ return ggml_upscale_impl(ctx, a, ne0, ne1, ne2, ne3);
+}
+
+// ggml_pad
+
+struct ggml_tensor * ggml_pad(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int p0, int p1, int p2, int p3) {
+ bool is_node = false;
+
+ if (a->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type,
+ a->ne[0] + p0,
+ a->ne[1] + p1,
+ a->ne[2] + p2,
+ a->ne[3] + p3);
+
+ result->op = GGML_OP_PAD;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_arange
+
+struct ggml_tensor * ggml_arange(
+ struct ggml_context * ctx,
+ float start,
+ float stop,
+ float step) {
+
+ GGML_ASSERT(stop > start);
+
+ const int64_t steps = (int64_t) ceilf((stop - start) / step);
+
+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, steps);
+
+ result->op = GGML_OP_ARANGE;
+ ggml_set_op_params_f32(result, 0, start);
+ ggml_set_op_params_f32(result, 1, stop);
+ ggml_set_op_params_f32(result, 2, step);
+
+ return result;
+}
+
+// ggml_timestep_embedding
+
+struct ggml_tensor * ggml_timestep_embedding(
+ struct ggml_context * ctx,
+ struct ggml_tensor * timesteps,
+ int dim,
+ int max_period) {
+ bool is_node = false;
+
+ if (timesteps->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ int actual_dim = dim;
+ if (dim % 2 != 0) {
+ actual_dim = dim + 1;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, actual_dim, timesteps->ne[0]);
+
+ result->op = GGML_OP_TIMESTEP_EMBEDDING;
+ ggml_set_op_params_i32(result, 0, dim);
+ ggml_set_op_params_i32(result, 1, max_period);
+
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = timesteps;
+
+ return result;
+}
+
+// ggml_argsort
+
+struct ggml_tensor * ggml_argsort(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_sort_order order) {
+ bool is_node = false;
+
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_I32, GGML_MAX_DIMS, a->ne);
+
+ ggml_set_op_params_i32(result, 0, (int32_t) order);
+
+ result->op = GGML_OP_ARGSORT;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_top_k
+
+struct ggml_tensor * ggml_top_k(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int k) {
+ GGML_ASSERT(a->ne[0] >= k);
+
+ struct ggml_tensor * result = ggml_argsort(ctx, a, GGML_SORT_ORDER_DESC);
+
+ result = ggml_view_4d(ctx, result,
+ k, result->ne[1], result->ne[2], result->ne[3],
+ result->nb[1], result->nb[2], result->nb[3],
+ 0);
+
+ return result;
+}
+
+// ggml_flash_attn_ext
+
+struct ggml_tensor * ggml_flash_attn_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * mask,
+ float scale,
+ float max_bias,
+ float logit_softcap) {
+ GGML_ASSERT(ggml_can_mul_mat(k, q));
+ // TODO: check if vT can be multiplied by (k*qT)
+
+ if (mask) {
+ GGML_ASSERT(ggml_is_contiguous(mask));
+ GGML_ASSERT(mask->ne[2] == 1);
+ GGML_ASSERT(mask->ne[3] == 1);
+ GGML_ASSERT(mask->ne[1] >= GGML_PAD(q->ne[1], GGML_KQ_MASK_PAD) &&
+ "the Flash-Attention kernel requires the mask to be padded to GGML_KQ_MASK_PAD and at least n_queries big");
+ //GGML_ASSERT(ggml_can_repeat_rows(mask, qk));
+ }
+
+ if (max_bias > 0.0f) {
+ GGML_ASSERT(mask);
+ }
+
+ bool is_node = false;
+
+ if (q->grad || k->grad || v->grad) {
+ is_node = true;
+ }
+
+ // permute(0, 2, 1, 3)
+ int64_t ne[4] = { q->ne[0], q->ne[2], q->ne[1], q->ne[3] };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ float params[] = { scale, max_bias, logit_softcap };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_FLASH_ATTN_EXT;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = q;
+ result->src[1] = k;
+ result->src[2] = v;
+ result->src[3] = mask;
+
+ return result;
+}
+
+void ggml_flash_attn_ext_set_prec(
+ struct ggml_tensor * a,
+ enum ggml_prec prec) {
+ GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT);
+
+ const int32_t prec_i32 = (int32_t) prec;
+
+ ggml_set_op_params_i32(a, 3, prec_i32); // scale is on first pos, max_bias on second
+}
+
+// ggml_flash_attn_back
+
+struct ggml_tensor * ggml_flash_attn_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * d,
+ bool masked) {
+ GGML_ABORT("TODO: adapt to ggml_flash_attn_ext() changes");
+
+ GGML_ASSERT(ggml_can_mul_mat(k, q));
+ // TODO: check if vT can be multiplied by (k*qT)
+
+ // d shape [D,N,ne2,ne3]
+ // q shape [D,N,ne2,ne3]
+ // k shape [D,M,kvne2,ne3]
+ // v shape [M,D,kvne2,ne3]
+
+ const int64_t D = q->ne[0];
+ const int64_t N = q->ne[1];
+ const int64_t M = k->ne[1];
+ const int64_t ne2 = q->ne[2];
+ const int64_t ne3 = q->ne[3];
+ const int64_t kvne2 = k->ne[2];
+
+ GGML_ASSERT(k->ne[0] == D);
+ GGML_ASSERT(v->ne[0] == M);
+ GGML_ASSERT(v->ne[1] == D);
+ GGML_ASSERT(d->ne[0] == D);
+ GGML_ASSERT(d->ne[1] == N);
+ GGML_ASSERT(k->ne[2] == kvne2);
+ GGML_ASSERT(k->ne[3] == ne3);
+ GGML_ASSERT(v->ne[2] == kvne2);
+ GGML_ASSERT(v->ne[3] == ne3);
+ GGML_ASSERT(d->ne[2] == ne2);
+ GGML_ASSERT(d->ne[3] == ne3);
+
+ GGML_ASSERT(ne2 % kvne2 == 0);
+
+ bool is_node = false;
+
+ if (q->grad || k->grad || v->grad) {
+ // when using this operation (in backwards pass) these grads are set.
+ // we don't want to create (big) grad of our result, so is_node is false.
+ is_node = false;
+ }
+
+ // store gradients of q, k and v as continuous tensors concatenated in result.
+ // note: v and gradv are actually transposed, i.e. v->ne[0] != D.
+ const int64_t elem_q = ggml_nelements(q);
+ const int64_t elem_k = ggml_nelements(k);
+ const int64_t elem_v = ggml_nelements(v);
+
+ enum ggml_type result_type = GGML_TYPE_F32;
+ GGML_ASSERT(ggml_blck_size(result_type) == 1);
+ const size_t tsize = ggml_type_size(result_type);
+
+ const size_t offs_q = 0;
+ const size_t offs_k = offs_q + GGML_PAD(elem_q * tsize, GGML_MEM_ALIGN);
+ const size_t offs_v = offs_k + GGML_PAD(elem_k * tsize, GGML_MEM_ALIGN);
+ const size_t end = offs_v + GGML_PAD(elem_v * tsize, GGML_MEM_ALIGN);
+
+ const size_t nelements = (end + tsize - 1)/tsize;
+
+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nelements);
+
+ int32_t masked_i = masked ? 1 : 0;
+ ggml_set_op_params(result, &masked_i, sizeof(masked_i));
+
+ result->op = GGML_OP_FLASH_ATTN_BACK;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = q;
+ result->src[1] = k;
+ result->src[2] = v;
+ result->src[3] = d;
+
+ return result;
+}
+
+// ggml_ssm_conv
+
+struct ggml_tensor * ggml_ssm_conv(
+ struct ggml_context * ctx,
+ struct ggml_tensor * sx,
+ struct ggml_tensor * c) {
+ GGML_ASSERT(ggml_is_3d(sx));
+ GGML_ASSERT(ggml_is_matrix(c));
+
+ const int64_t d_conv = c->ne[0];
+ const int64_t d_inner = c->ne[1];
+ const int64_t n_t = sx->ne[0] - d_conv + 1; // tokens per sequence
+ const int64_t n_s = sx->ne[2];
+
+ // TODO: maybe support other strides than 1?
+ GGML_ASSERT(sx->ne[0] == d_conv - 1 + n_t);
+ GGML_ASSERT(sx->ne[1] == d_inner);
+ GGML_ASSERT(n_t >= 0);
+
+ bool is_node = false;
+
+ if (sx->grad || c->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, d_inner, n_t, n_s);
+
+ result->op = GGML_OP_SSM_CONV;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = sx;
+ result->src[1] = c;
+
+ return result;
+}
+
+// ggml_ssm_scan
+
+struct ggml_tensor * ggml_ssm_scan(
+ struct ggml_context * ctx,
+ struct ggml_tensor * s,
+ struct ggml_tensor * x,
+ struct ggml_tensor * dt,
+ struct ggml_tensor * A,
+ struct ggml_tensor * B,
+ struct ggml_tensor * C) {
+ GGML_ASSERT(ggml_is_contiguous(s));
+ GGML_ASSERT(ggml_is_contiguous(x));
+ GGML_ASSERT(ggml_is_contiguous(dt));
+ GGML_ASSERT(ggml_is_contiguous(A));
+ GGML_ASSERT(ggml_is_matrix(A));
+ GGML_ASSERT(ggml_is_3d(B));
+ GGML_ASSERT(ggml_is_3d(s));
+ GGML_ASSERT(B->nb[0] == ggml_type_size(B->type));
+ GGML_ASSERT(C->nb[0] == ggml_type_size(C->type));
+ GGML_ASSERT(ggml_are_same_shape(x, dt));
+ GGML_ASSERT(ggml_are_same_shape(B, C));
+
+ {
+ const int64_t d_state = s->ne[0];
+ const int64_t d_inner = s->ne[1];
+ const int64_t n_seq_tokens = x->ne[1];
+ const int64_t n_seqs = x->ne[2];
+
+ GGML_ASSERT(s->ne[2] == n_seqs);
+ GGML_ASSERT(x->ne[0] == d_inner);
+ GGML_ASSERT(A->ne[0] == d_state);
+ GGML_ASSERT(A->ne[1] == d_inner);
+ GGML_ASSERT(B->ne[0] == d_state);
+ GGML_ASSERT(B->ne[1] == n_seq_tokens);
+ GGML_ASSERT(B->ne[2] == n_seqs);
+ }
+
+ bool is_node = false;
+
+ if (s->grad || x->grad || dt->grad || A->grad || B->grad || C->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement
+ is_node = true;
+ }
+
+ // concatenated y + ssm_states
+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ggml_nelements(x) + ggml_nelements(s));
+
+ result->op = GGML_OP_SSM_SCAN;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = s;
+ result->src[1] = x;
+ result->src[2] = dt;
+ result->src[3] = A;
+ result->src[4] = B;
+ result->src[5] = C;
+
+ return result;
+}
+
+// ggml_win_part
+
+struct ggml_tensor * ggml_win_part(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int w) {
+ GGML_ASSERT(a->ne[3] == 1);
+ GGML_ASSERT(a->type == GGML_TYPE_F32);
+
+ bool is_node = false;
+
+ if (a->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ // padding
+ const int px = (w - a->ne[1]%w)%w;
+ const int py = (w - a->ne[2]%w)%w;
+
+ const int npx = (px + a->ne[1])/w;
+ const int npy = (py + a->ne[2])/w;
+ const int np = npx*npy;
+
+ const int64_t ne[4] = { a->ne[0], w, w, np, };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ int32_t params[] = { npx, npy, w };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_WIN_PART;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_win_unpart
+
+struct ggml_tensor * ggml_win_unpart(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int w0,
+ int h0,
+ int w) {
+ GGML_ASSERT(a->type == GGML_TYPE_F32);
+
+ bool is_node = false;
+
+ if (a->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ const int64_t ne[4] = { a->ne[0], w0, h0, 1, };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 3, ne);
+
+ int32_t params[] = { w };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_WIN_UNPART;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_get_rel_pos
+
+struct ggml_tensor * ggml_get_rel_pos(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int qh,
+ int kh) {
+ GGML_ASSERT(qh == kh);
+ GGML_ASSERT(2*MAX(qh, kh) - 1 == a->ne[1]);
+
+ bool is_node = false;
+
+ if (a->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ const int64_t ne[4] = { a->ne[0], kh, qh, 1, };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F16, 3, ne);
+
+ result->op = GGML_OP_GET_REL_POS;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+// ggml_add_rel_pos
+
+static struct ggml_tensor * ggml_add_rel_pos_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * pw,
+ struct ggml_tensor * ph,
+ bool inplace) {
+ GGML_ASSERT(ggml_are_same_shape(pw, ph));
+ GGML_ASSERT(ggml_is_contiguous(a));
+ GGML_ASSERT(ggml_is_contiguous(pw));
+ GGML_ASSERT(ggml_is_contiguous(ph));
+ GGML_ASSERT(ph->type == GGML_TYPE_F32);
+ GGML_ASSERT(pw->type == GGML_TYPE_F32);
+ GGML_ASSERT(pw->ne[3] == a->ne[2]);
+ GGML_ASSERT(pw->ne[0]*pw->ne[0] == a->ne[0]);
+ GGML_ASSERT(pw->ne[1]*pw->ne[2] == a->ne[1]);
+
+ bool is_node = false;
+
+ if (!inplace && (a->grad || pw->grad || ph->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+ ggml_set_op_params_i32(result, 0, inplace ? 1 : 0);
+
+ result->op = GGML_OP_ADD_REL_POS;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = pw;
+ result->src[2] = ph;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_add_rel_pos(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * pw,
+ struct ggml_tensor * ph) {
+ return ggml_add_rel_pos_impl(ctx, a, pw, ph, false);
+}
+
+struct ggml_tensor * ggml_add_rel_pos_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * pw,
+ struct ggml_tensor * ph) {
+ return ggml_add_rel_pos_impl(ctx, a, pw, ph, true);
+}
+
+// ggml_rwkv_wkv
+
+struct ggml_tensor * ggml_rwkv_wkv(
+ struct ggml_context * ctx,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * r,
+ struct ggml_tensor * tf,
+ struct ggml_tensor * td,
+ struct ggml_tensor * state) {
+ GGML_ASSERT(ggml_is_contiguous(k));
+ GGML_ASSERT(ggml_is_contiguous(v));
+ GGML_ASSERT(ggml_is_contiguous(r));
+ GGML_ASSERT(ggml_is_contiguous(tf));
+ GGML_ASSERT(ggml_is_contiguous(td));
+ GGML_ASSERT(ggml_is_contiguous(state));
+
+ const int64_t S = k->ne[0];
+ const int64_t H = k->ne[2];
+ const int64_t n_tokens = k->ne[3];
+ const int64_t n_seqs = state->ne[1];
+ {
+ GGML_ASSERT(k->ne[1] == 1);
+ GGML_ASSERT(v->ne[0] == 1 && v->ne[1] == S && v->ne[2] == H && v->ne[3] == n_tokens);
+ GGML_ASSERT(r->ne[0] == 1 && r->ne[1] == S && r->ne[2] == H && r->ne[3] == n_tokens);
+ // TODO: RWKV v4 and v5
+ GGML_ASSERT(td->ne[0] == 1 && td->ne[1] == S && td->ne[2] == H && td->ne[3] == n_tokens);
+ GGML_ASSERT(ggml_nelements(state) == S * S * H * n_seqs);
+ }
+
+ bool is_node = false;
+
+ if (k->grad || v->grad || r->grad || tf->grad || td->grad || state->grad) {
+ GGML_ABORT("fatal error"); // TODO: implement backward
+ is_node = true;
+ }
+
+ // concat output and new_state
+ const int64_t ne[4] = { S * H, n_tokens + S * n_seqs, 1, 1 };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ result->op = GGML_OP_RWKV_WKV;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = k;
+ result->src[1] = v;
+ result->src[2] = r;
+ result->src[3] = tf;
+ result->src[4] = td;
+ result->src[5] = state;
+
+ return result;
+}
+
+// ggml_unary
+
+static struct ggml_tensor * ggml_unary_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_unary_op op,
+ bool inplace) {
+ GGML_ASSERT(ggml_is_contiguous_1(a));
+
+ bool is_node = false;
+
+ if (!inplace && (a->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params_i32(result, 0, (int32_t) op);
+
+ result->op = GGML_OP_UNARY;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_unary(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_unary_op op) {
+ return ggml_unary_impl(ctx, a, op, false);
+}
+
+struct ggml_tensor * ggml_unary_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_unary_op op) {
+ return ggml_unary_impl(ctx, a, op, true);
+}
+
+// ggml_map_unary
+
+static struct ggml_tensor * ggml_map_unary_impl_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ const ggml_unary_op_f32_t fun,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params(result, (const void *) &fun, sizeof(fun));
+
+ result->op = GGML_OP_MAP_UNARY;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_map_unary_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ const ggml_unary_op_f32_t fun) {
+ return ggml_map_unary_impl_f32(ctx, a, fun, false);
+}
+
+struct ggml_tensor * ggml_map_unary_inplace_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ const ggml_unary_op_f32_t fun) {
+ return ggml_map_unary_impl_f32(ctx, a, fun, true);
+}
+
+// ggml_map_binary
+
+static struct ggml_tensor * ggml_map_binary_impl_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ const ggml_binary_op_f32_t fun,
+ bool inplace) {
+ GGML_ASSERT(ggml_are_same_shape(a, b));
+
+ bool is_node = false;
+
+ if (!inplace && (a->grad || b->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params(result, (const void *) &fun, sizeof(fun));
+
+ result->op = GGML_OP_MAP_BINARY;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_map_binary_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ const ggml_binary_op_f32_t fun) {
+ return ggml_map_binary_impl_f32(ctx, a, b, fun, false);
+}
+
+struct ggml_tensor * ggml_map_binary_inplace_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ const ggml_binary_op_f32_t fun) {
+ return ggml_map_binary_impl_f32(ctx, a, b, fun, true);
+}
+
+// ggml_map_custom1_f32
+
+static struct ggml_tensor * ggml_map_custom1_impl_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ const ggml_custom1_op_f32_t fun,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params(result, (const void *) &fun, sizeof(fun));
+
+ result->op = GGML_OP_MAP_CUSTOM1_F32;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_map_custom1_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ const ggml_custom1_op_f32_t fun) {
+ return ggml_map_custom1_impl_f32(ctx, a, fun, false);
+}
+
+struct ggml_tensor * ggml_map_custom1_inplace_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ const ggml_custom1_op_f32_t fun) {
+ return ggml_map_custom1_impl_f32(ctx, a, fun, true);
+}
+
+// ggml_map_custom2_f32
+
+static struct ggml_tensor * ggml_map_custom2_impl_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ const ggml_custom2_op_f32_t fun,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && (a->grad || b->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params(result, (const void *) &fun, sizeof(fun));
+
+ result->op = GGML_OP_MAP_CUSTOM2_F32;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_map_custom2_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ const ggml_custom2_op_f32_t fun) {
+ return ggml_map_custom2_impl_f32(ctx, a, b, fun, false);
+}
+
+struct ggml_tensor * ggml_map_custom2_inplace_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ const ggml_custom2_op_f32_t fun) {
+ return ggml_map_custom2_impl_f32(ctx, a, b, fun, true);
+}
+
+// ggml_map_custom3_f32
+
+static struct ggml_tensor * ggml_map_custom3_impl_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ const ggml_custom3_op_f32_t fun,
+ bool inplace) {
+ bool is_node = false;
+
+ if (!inplace && (a->grad || b->grad || c->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ ggml_set_op_params(result, (const void *) &fun, sizeof(fun));
+
+ result->op = GGML_OP_MAP_CUSTOM3_F32;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+ result->src[2] = c;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_map_custom3_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ const ggml_custom3_op_f32_t fun) {
+ return ggml_map_custom3_impl_f32(ctx, a, b, c, fun, false);
+}
+
+struct ggml_tensor * ggml_map_custom3_inplace_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ const ggml_custom3_op_f32_t fun) {
+ return ggml_map_custom3_impl_f32(ctx, a, b, c, fun, true);
+}
+
+// ggml_map_custom1
+struct ggml_map_custom1_op_params {
+ ggml_custom1_op_t fun;
+ int n_tasks;
+ void * userdata;
+};
+
+static struct ggml_tensor * ggml_map_custom1_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ const ggml_custom1_op_t fun,
+ int n_tasks,
+ void * userdata,
+ bool inplace) {
+ GGML_ASSERT(n_tasks == GGML_N_TASKS_MAX || n_tasks > 0);
+
+ bool is_node = false;
+
+ if (!inplace && a->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ struct ggml_map_custom1_op_params params = {
+ /*.fun =*/ fun,
+ /*.n_tasks =*/ n_tasks,
+ /*.userdata =*/ userdata
+ };
+ ggml_set_op_params(result, (const void *) ¶ms, sizeof(params));
+
+ result->op = GGML_OP_MAP_CUSTOM1;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_map_custom1(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ const ggml_custom1_op_t fun,
+ int n_tasks,
+ void * userdata) {
+ return ggml_map_custom1_impl(ctx, a, fun, n_tasks, userdata, false);
+}
+
+struct ggml_tensor * ggml_map_custom1_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ const ggml_custom1_op_t fun,
+ int n_tasks,
+ void * userdata) {
+ return ggml_map_custom1_impl(ctx, a, fun, n_tasks, userdata, true);
+}
+
+// ggml_map_custom2
+
+struct ggml_map_custom2_op_params {
+ ggml_custom2_op_t fun;
+ int n_tasks;
+ void * userdata;
+};
+
+static struct ggml_tensor * ggml_map_custom2_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ const ggml_custom2_op_t fun,
+ int n_tasks,
+ void * userdata,
+ bool inplace) {
+ GGML_ASSERT(n_tasks == GGML_N_TASKS_MAX || n_tasks > 0);
+
+ bool is_node = false;
+
+ if (!inplace && (a->grad || b->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ struct ggml_map_custom2_op_params params = {
+ /*.fun =*/ fun,
+ /*.n_tasks =*/ n_tasks,
+ /*.userdata =*/ userdata
+ };
+ ggml_set_op_params(result, (const void *) ¶ms, sizeof(params));
+
+ result->op = GGML_OP_MAP_CUSTOM2;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_map_custom2(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ const ggml_custom2_op_t fun,
+ int n_tasks,
+ void * userdata) {
+ return ggml_map_custom2_impl(ctx, a, b, fun, n_tasks, userdata, false);
+}
+
+struct ggml_tensor * ggml_map_custom2_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ const ggml_custom2_op_t fun,
+ int n_tasks,
+ void * userdata) {
+ return ggml_map_custom2_impl(ctx, a, b, fun, n_tasks, userdata, true);
+}
+
+// ggml_map_custom3
+
+struct ggml_map_custom3_op_params {
+ ggml_custom3_op_t fun;
+ int n_tasks;
+ void * userdata;
+};
+
+static struct ggml_tensor * ggml_map_custom3_impl(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ const ggml_custom3_op_t fun,
+ int n_tasks,
+ void * userdata,
+ bool inplace) {
+ GGML_ASSERT(n_tasks == GGML_N_TASKS_MAX || n_tasks > 0);
+
+ bool is_node = false;
+
+ if (!inplace && (a->grad || b->grad || c->grad)) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
+
+ struct ggml_map_custom3_op_params params = {
+ /*.fun =*/ fun,
+ /*.n_tasks =*/ n_tasks,
+ /*.userdata =*/ userdata
+ };
+ ggml_set_op_params(result, (const void *) ¶ms, sizeof(params));
+
+ result->op = GGML_OP_MAP_CUSTOM3;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+ result->src[2] = c;
+
+ return result;
+}
+
+struct ggml_tensor * ggml_map_custom3(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ const ggml_custom3_op_t fun,
+ int n_tasks,
+ void * userdata) {
+ return ggml_map_custom3_impl(ctx, a, b, c, fun, n_tasks, userdata, false);
+}
+
+struct ggml_tensor * ggml_map_custom3_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ const ggml_custom3_op_t fun,
+ int n_tasks,
+ void * userdata) {
+ return ggml_map_custom3_impl(ctx, a, b, c, fun, n_tasks, userdata, true);
+}
+
+// ggml_cross_entropy_loss
+
+struct ggml_tensor * ggml_cross_entropy_loss(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ GGML_ASSERT(ggml_are_same_shape(a, b));
+ bool is_node = false;
+
+ if (a->grad || b->grad) {
+ is_node = true;
+ }
+
+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, a->type, 1);
+
+ result->op = GGML_OP_CROSS_ENTROPY_LOSS;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
+// ggml_cross_entropy_loss_back
+
+struct ggml_tensor * ggml_cross_entropy_loss_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c) {
+ GGML_ASSERT(ggml_are_same_shape(a, b));
+ GGML_ASSERT(ggml_is_scalar(c));
+
+ struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_CROSS_ENTROPY_LOSS_BACK;
+ result->grad = NULL;
+ result->src[0] = a;
+ result->src[1] = b;
+ result->src[2] = c;
+
+ return result;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void ggml_set_param(
+ struct ggml_context * ctx,
+ struct ggml_tensor * tensor) {
+ tensor->flags |= GGML_TENSOR_FLAG_PARAM;
+
+ GGML_ASSERT(tensor->grad == NULL);
+ tensor->grad = ggml_dup_tensor(ctx, tensor);
+ ggml_format_name(tensor->grad, "%s (grad)", tensor->name);
+}
+
+// ggml_compute_forward_dup
+
+static void ggml_compute_forward_dup_same_cont(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
+ GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));
+ GGML_ASSERT(src0->type == dst->type);
+
+ const size_t nb00 = src0->nb[0];
+ const size_t nb0 = dst->nb[0];
+
+ const int ith = params->ith; // thread index
+ const int nth = params->nth; // number of threads
+
+ // parallelize by elements
+ const int ne = ggml_nelements(dst);
+ const int dr = (ne + nth - 1) / nth;
+ const int ie0 = dr * ith;
+ const int ie1 = MIN(ie0 + dr, ne);
+
+ if (ie0 < ie1) {
+ memcpy(
+ ((char *) dst->data + ie0*nb0),
+ ((char *) src0->data + ie0*nb00),
+ (ie1 - ie0) * ggml_type_size(src0->type));
+ }
+}
+
+static void ggml_compute_forward_dup_f16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ const int ith = params->ith; // thread index
+ const int nth = params->nth; // number of threads
+
+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst) && src0->type == dst->type) {
+ ggml_compute_forward_dup_same_cont(params, dst);
+ return;
+ }
+
+ // parallelize by rows
+ const int nr = ne01;
+ // number of rows per thread
+ const int dr = (nr + nth - 1) / nth;
+ // row range for this thread
+ const int ir0 = dr * ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (src0->type == dst->type &&
+ ne00 == ne0 &&
+ nb00 == ggml_type_size(src0->type) && nb0 == ggml_type_size(dst->type)) {
+ // copy by rows
+ const size_t rs = ne00*nb00;
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ memcpy(
+ ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),
+ ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),
+ rs);
+ }
+ }
+ }
+ return;
+ }
+
+ // TODO: add more special-case implementations for tensor shapes/strides that can benefit from memcpy
+
+ if (ggml_is_contiguous(dst)) {
+ if (nb00 == sizeof(ggml_fp16_t)) {
+ if (dst->type == GGML_TYPE_F16) {
+ size_t id = 0;
+ const size_t rs = ne00 * nb00;
+ char * dst_ptr = (char *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += rs * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
+ memcpy(dst_ptr + id, src0_ptr, rs);
+ id += rs;
+ }
+ id += rs * (ne01 - ir1);
+ }
+ }
+ } else if (dst->type == GGML_TYPE_F32) {
+ size_t id = 0;
+ float * dst_ptr = (float *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ const ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
+ for (int i00 = 0; i00 < ne00; i00++) {
+ dst_ptr[id] = GGML_FP16_TO_FP32(src0_ptr[i00]);
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
+ } else if (type_traits[dst->type].from_float) {
+ ggml_from_float_t const quantize_row_q = type_traits[dst->type].from_float;
+ float * src0_f32 = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;
+
+ size_t id = 0;
+ size_t rs = nb0 * (ne00 / ggml_blck_size(dst->type));
+ char * dst_ptr = (char *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += rs * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ const ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
+
+ for (int i00 = 0; i00 < ne00; i00++) {
+ src0_f32[i00] = GGML_FP16_TO_FP32(src0_ptr[i00]);
+ }
+
+ quantize_row_q(src0_f32, dst_ptr + id, ne00);
+ id += rs;
+ }
+ id += rs * (ne01 - ir1);
+ }
+ }
+ } else {
+ GGML_ABORT("fatal error"); // TODO: implement
+ }
+ } else {
+ //printf("%s: this is not optimal - fix me\n", __func__);
+
+ if (dst->type == GGML_TYPE_F32) {
+ size_t id = 0;
+ float * dst_ptr = (float *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ const ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+
+ dst_ptr[id] = GGML_FP16_TO_FP32(*src0_ptr);
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
+ } else if (dst->type == GGML_TYPE_F16) {
+ size_t id = 0;
+ ggml_fp16_t * dst_ptr = (ggml_fp16_t *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ const ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+
+ dst_ptr[id] = *src0_ptr;
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
+ } else {
+ GGML_ABORT("fatal error"); // TODO: implement
+ }
+ }
+ return;
+ }
+
+ // dst counters
+ int64_t i10 = 0;
+ int64_t i11 = 0;
+ int64_t i12 = 0;
+ int64_t i13 = 0;
+
+ if (dst->type == GGML_TYPE_F16) {
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ i10 += ne00 * ir0;
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
+
+ memcpy(dst_ptr, src0_ptr, sizeof(ggml_fp16_t));
+
+ if (++i10 == ne00) {
+ i10 = 0;
+ if (++i11 == ne01) {
+ i11 = 0;
+ if (++i12 == ne02) {
+ i12 = 0;
+ if (++i13 == ne03) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ i10 += ne00 * (ne01 - ir1);
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ } else if (dst->type == GGML_TYPE_F32) {
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ i10 += ne00 * ir0;
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
+
+ *(float *) dst_ptr = GGML_FP16_TO_FP32(*(const ggml_fp16_t *) src0_ptr);
+
+ if (++i10 == ne0) {
+ i10 = 0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ i10 += ne00 * (ne01 - ir1);
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ } else {
+ GGML_ABORT("fatal error"); // TODO: implement
+ }
+}
+
+static void ggml_compute_forward_dup_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ const int ith = params->ith; // thread index
+ const int nth = params->nth; // number of threads
+
+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst) && src0->type == dst->type) {
+ ggml_compute_forward_dup_same_cont(params, dst);
+ return;
+ }
+
+ // parallelize by rows
+ const int nr = ne01;
+ // number of rows per thread
+ const int dr = (nr + nth - 1) / nth;
+ // row range for this thread
+ const int ir0 = dr * ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (src0->type == dst->type &&
+ ne00 == ne0 &&
+ nb00 == ggml_type_size(src0->type) && nb0 == ggml_type_size(dst->type)) {
+ // copy by rows
+ const size_t rs = ne00*nb00;
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ memcpy(
+ ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),
+ ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),
+ rs);
+ }
+ }
+ }
+ return;
+ }
+
+ // TODO: add more special-case implementations for tensor shapes/strides that can benefit from memcpy
+
+ if (ggml_is_contiguous(dst)) {
+ if (nb00 == sizeof(ggml_bf16_t)) {
+ if (dst->type == GGML_TYPE_BF16) {
+ size_t id = 0;
+ const size_t rs = ne00 * nb00;
+ char * dst_ptr = (char *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += rs * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
+ memcpy(dst_ptr + id, src0_ptr, rs);
+ id += rs;
+ }
+ id += rs * (ne01 - ir1);
+ }
+ }
+ } else if (dst->type == GGML_TYPE_F16) {
+ size_t id = 0;
+ ggml_fp16_t * dst_ptr = (ggml_fp16_t *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
+ for (int i00 = 0; i00 < ne00; i00++) {
+ dst_ptr[id] = GGML_FP32_TO_FP16(GGML_BF16_TO_FP32(src0_ptr[i00]));
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
+ } else if (dst->type == GGML_TYPE_F32) {
+ size_t id = 0;
+ float * dst_ptr = (float *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
+ for (int i00 = 0; i00 < ne00; i00++) {
+ dst_ptr[id] = GGML_BF16_TO_FP32(src0_ptr[i00]);
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
+ } else if (type_traits[dst->type].from_float) {
+ ggml_from_float_t const quantize_row_q = type_traits[dst->type].from_float;
+ float * src0_f32 = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;
+
+ size_t id = 0;
+ size_t rs = nb0 * (ne00 / ggml_blck_size(dst->type));
+ char * dst_ptr = (char *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += rs * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
+
+ for (int i00 = 0; i00 < ne00; i00++) {
+ src0_f32[i00] = GGML_BF16_TO_FP32(src0_ptr[i00]);
+ }
+
+ quantize_row_q(src0_f32, dst_ptr + id, ne00);
+ id += rs;
+ }
+ id += rs * (ne01 - ir1);
+ }
+ }
+ } else {
+ GGML_ABORT("fatal error"); // TODO: implement
+ }
+ } else {
+ //printf("%s: this is not optimal - fix me\n", __func__);
+
+ if (dst->type == GGML_TYPE_F32) {
+ size_t id = 0;
+ float * dst_ptr = (float *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+
+ dst_ptr[id] = GGML_BF16_TO_FP32(*src0_ptr);
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
+ } else if (dst->type == GGML_TYPE_BF16) {
+ size_t id = 0;
+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+
+ dst_ptr[id] = *src0_ptr;
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
+ } else if (dst->type == GGML_TYPE_F16) {
+ size_t id = 0;
+ ggml_fp16_t * dst_ptr = (ggml_fp16_t *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+
+ dst_ptr[id] = GGML_FP32_TO_FP16(GGML_BF16_TO_FP32(*src0_ptr));
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
+ } else {
+ GGML_ABORT("fatal error"); // TODO: implement
+ }
+ }
+ return;
+ }
+
+ // dst counters
+ int64_t i10 = 0;
+ int64_t i11 = 0;
+ int64_t i12 = 0;
+ int64_t i13 = 0;
+
+ if (dst->type == GGML_TYPE_BF16) {
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ i10 += ne00 * ir0;
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
+
+ memcpy(dst_ptr, src0_ptr, sizeof(ggml_bf16_t));
+
+ if (++i10 == ne00) {
+ i10 = 0;
+ if (++i11 == ne01) {
+ i11 = 0;
+ if (++i12 == ne02) {
+ i12 = 0;
+ if (++i13 == ne03) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ i10 += ne00 * (ne01 - ir1);
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ } else if (dst->type == GGML_TYPE_F16) {
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ i10 += ne00 * ir0;
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
+
+ *(ggml_fp16_t *) dst_ptr = GGML_FP32_TO_FP16(GGML_BF16_TO_FP32(*(const ggml_bf16_t *) src0_ptr));
+
+ if (++i10 == ne0) {
+ i10 = 0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ i10 += ne00 * (ne01 - ir1);
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ } else if (dst->type == GGML_TYPE_F32) {
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ i10 += ne00 * ir0;
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
+
+ *(float *) dst_ptr = GGML_BF16_TO_FP32(*(const ggml_bf16_t *) src0_ptr);
+
+ if (++i10 == ne0) {
+ i10 = 0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ i10 += ne00 * (ne01 - ir1);
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ } else {
+ GGML_ABORT("fatal error"); // TODO: implement
+ }
+}
+
+static void ggml_compute_forward_dup_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ const int ith = params->ith; // thread index
+ const int nth = params->nth; // number of threads
+
+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst) && src0->type == dst->type) {
+ ggml_compute_forward_dup_same_cont(params, dst);
+ return;
+ }
+
+ // parallelize by rows
+ const int nr = ne01;
+ // number of rows per thread
+ const int dr = (nr + nth - 1) / nth;
+ // row range for this thread
+ const int ir0 = dr * ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (src0->type == dst->type &&
+ ne00 == ne0 &&
+ nb00 == ggml_type_size(src0->type) && nb0 == ggml_type_size(dst->type)) {
+ // copy by rows
+ const size_t rs = ne00*nb00;
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ memcpy(
+ ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),
+ ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),
+ rs);
+ }
+ }
+ }
+ return;
+ }
+
+ if (ggml_is_contiguous(dst)) {
+ // TODO: simplify
+ if (nb00 == sizeof(float)) {
+ if (dst->type == GGML_TYPE_F32) {
+ size_t id = 0;
+ const size_t rs = ne00 * nb00;
+ char * dst_ptr = (char *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += rs * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
+ memcpy(dst_ptr + id, src0_ptr, rs);
+ id += rs;
+ }
+ id += rs * (ne01 - ir1);
+ }
+ }
+ } else if (type_traits[dst->type].from_float) {
+ ggml_from_float_t const quantize_row_q = type_traits[dst->type].from_float;
+
+ size_t id = 0;
+ size_t rs = nb0 * (ne00 / ggml_blck_size(dst->type));
+ char * dst_ptr = (char *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += rs * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ const float * src0_ptr = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
+ quantize_row_q(src0_ptr, dst_ptr + id, ne00);
+ id += rs;
+ }
+ id += rs * (ne01 - ir1);
+ }
+ }
+ } else {
+ GGML_ABORT("fatal error"); // TODO: implement
+ }
+ } else {
+ //printf("%s: this is not optimal - fix me\n", __func__);
+
+ if (dst->type == GGML_TYPE_F32) {
+ size_t id = 0;
+ float * dst_ptr = (float *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+
+ dst_ptr[id] = *src0_ptr;
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
+ } else if (dst->type == GGML_TYPE_F16) {
+ size_t id = 0;
+ ggml_fp16_t * dst_ptr = (ggml_fp16_t *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+
+ dst_ptr[id] = GGML_FP32_TO_FP16(*src0_ptr);
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
+ } else if (dst->type == GGML_TYPE_BF16) {
+ size_t id = 0;
+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+
+ dst_ptr[id] = GGML_FP32_TO_BF16(*src0_ptr);
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
+ } else {
+ GGML_ABORT("fatal error"); // TODO: implement
+ }
+ }
+
+ return;
+ }
+
+ // dst counters
+
+ int64_t i10 = 0;
+ int64_t i11 = 0;
+ int64_t i12 = 0;
+ int64_t i13 = 0;
+
+ if (dst->type == GGML_TYPE_F32) {
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ i10 += ne00 * ir0;
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
+
+ memcpy(dst_ptr, src0_ptr, sizeof(float));
+
+ if (++i10 == ne0) {
+ i10 = 0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ i10 += ne00 * (ne01 - ir1);
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ } else if (dst->type == GGML_TYPE_F16) {
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ i10 += ne00 * ir0;
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
+
+ *(ggml_fp16_t *) dst_ptr = GGML_FP32_TO_FP16(*(const float *) src0_ptr);
+
+ if (++i10 == ne0) {
+ i10 = 0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ i10 += ne00 * (ne01 - ir1);
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ } else if (dst->type == GGML_TYPE_BF16) {
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ i10 += ne00 * ir0;
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
+
+ *(ggml_bf16_t *) dst_ptr = GGML_FP32_TO_BF16(*(const float *) src0_ptr);
+
+ if (++i10 == ne0) {
+ i10 = 0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ i10 += ne00 * (ne01 - ir1);
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ } else {
+ GGML_ABORT("fatal error"); // TODO: implement
+ }
+}
+
+// A simplified version of ggml_compute_forward_dup that doesn't do float upcasting, and just plain old memcpy.
+static void ggml_compute_forward_dup_bytes(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
+ GGML_ASSERT(src0->type == dst->type);
+
+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {
+ ggml_compute_forward_dup_same_cont(params, dst);
+ return;
+ }
+
+ GGML_TENSOR_UNARY_OP_LOCALS;
+
+ const size_t type_size = ggml_type_size(src0->type);
+ const int ith = params->ith; // thread index
+ const int nth = params->nth; // number of threads
+
+
+ // parallelize by rows
+ const int nr = ne01;
+ // number of rows per thread
+ const int dr = (nr + nth - 1) / nth;
+ // row range for this thread
+ const int ir0 = dr * ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (src0->type == dst->type &&
+ ne00 == ne0 &&
+ nb00 == type_size && nb0 == type_size) {
+ // copy by rows
+ const size_t rs = ne00 * type_size;
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ memcpy(
+ ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),
+ ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),
+ rs);
+ }
+ }
+ }
+ return;
+ }
+
+ if (ggml_is_contiguous(dst)) {
+ size_t id = 0;
+ char * dst_ptr = (char *) dst->data;
+ const size_t rs = ne00 * type_size;
+
+ if (nb00 == type_size) {
+ // src0 is contigous on first dimension, copy by rows
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ id += rs * ir0;
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
+ memcpy(dst_ptr + id, src0_ptr, rs);
+ id += rs;
+ }
+ id += rs * (ne01 - ir1);
+ }
+ }
+ } else {
+ //printf("%s: this is not optimal - fix me\n", __func__);
+
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ id += rs * ir0;
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const char * src0_ptr = (char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03;
+ memcpy(dst_ptr + id, src0_ptr, type_size);
+
+ id += type_size;
+ }
+ }
+ id += rs * (ne01 - ir1);
+ }
+ }
+ }
+
+ return;
+ }
+
+ // dst counters
+
+ int64_t i10 = 0;
+ int64_t i11 = 0;
+ int64_t i12 = 0;
+ int64_t i13 = 0;
+
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ i10 += ne00 * ir0;
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
+
+ memcpy(dst_ptr, src0_ptr, type_size);
+
+ if (++i10 == ne0) {
+ i10 = 0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ i10 += ne00 * (ne01 - ir1);
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_dup(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (src0->type == dst->type) {
+ ggml_compute_forward_dup_bytes(params, dst);
+ return;
+ }
+
+ switch (src0->type) {
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_dup_f16(params, dst);
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_dup_bf16(params, dst);
+ } break;
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_dup_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_add
+
+static void ggml_compute_forward_add_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT( nb0 == sizeof(float));
+ GGML_ASSERT(nb00 == sizeof(float));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (nb10 == sizeof(float)) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src1 is broadcastable across src0 and dst in i1, i2, i3
+ const int64_t i03 = ir/(ne02*ne01);
+ const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
+ const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+ const int64_t nr0 = ne00 / ne10;
+
+ float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );
+ float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
+
+ for (int64_t r = 0; r < nr0; ++r) {
+#ifdef GGML_USE_ACCELERATE
+ vDSP_vadd(src0_ptr + r*ne10, 1, src1_ptr, 1, dst_ptr + r*ne10, 1, ne10);
+#else
+ ggml_vec_add_f32(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr);
+#endif
+ }
+ }
+ } else {
+ // src1 is not contiguous
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src1 is broadcastable across src0 and dst in i1, i2, i3
+ const int64_t i03 = ir/(ne02*ne01);
+ const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
+ const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+
+ float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );
+ float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
+
+ for (int64_t i0 = 0; i0 < ne0; ++i0) {
+ const int64_t i10 = i0 % ne10;
+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i10*nb10);
+
+ dst_ptr[i0] = src0_ptr[i0] + *src1_ptr;
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_add_f16_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ if (dst->type == GGML_TYPE_F32) {
+ GGML_ASSERT( nb0 == sizeof(float));
+ }
+ else {
+ GGML_ASSERT(dst->type == GGML_TYPE_F16);
+ GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));
+ }
+
+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (nb10 == sizeof(float)) {
+ if (dst->type == GGML_TYPE_F16) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0, src1 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);
+
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(src0_ptr[i]) + src1_ptr[i]);
+ }
+ }
+ } else {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0, src1 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ float * dst_ptr = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);
+
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP16_TO_FP32(src0_ptr[i]) + src1_ptr[i];
+ }
+ }
+ }
+ }
+ else {
+ // src1 is not contiguous
+ GGML_ABORT("fatal error");
+ }
+}
+
+static void ggml_compute_forward_add_bf16_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ if (dst->type == GGML_TYPE_F32) {
+ GGML_ASSERT( nb0 == sizeof(float));
+ }
+ else {
+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);
+ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));
+ }
+
+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (nb10 == sizeof(float)) {
+ if (dst->type == GGML_TYPE_BF16) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0, src1 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);
+
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + src1_ptr[i]);
+ }
+ }
+ } else {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0, src1 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ float * dst_ptr = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);
+
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_BF16_TO_FP32(src0_ptr[i]) + src1_ptr[i];
+ }
+ }
+ }
+ }
+ else {
+ // src1 is not contiguous
+ GGML_ABORT("fatal error");
+ }
+}
+
+static void ggml_compute_forward_add_f16_f16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F16);
+ GGML_ASSERT(dst->type == GGML_TYPE_F16);
+
+ GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (nb10 == sizeof(ggml_fp16_t)) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0, src1 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ ggml_fp16_t * src1_ptr = (ggml_fp16_t *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);
+
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(src0_ptr[i]) + GGML_FP16_TO_FP32(src1_ptr[i]));
+ }
+ }
+ }
+ else {
+ // src1 is not contiguous
+ GGML_ABORT("fatal error");
+ }
+}
+
+static void ggml_compute_forward_add_bf16_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);
+ GGML_ASSERT(src1->type == GGML_TYPE_BF16);
+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);
+
+ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (nb10 == sizeof(ggml_bf16_t)) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0, src1 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ ggml_bf16_t * src1_ptr = (ggml_bf16_t *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);
+
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + GGML_BF16_TO_FP32(src1_ptr[i]));
+ }
+ }
+ }
+ else {
+ // src1 is not contiguous
+ GGML_ABORT("fatal error");
+ }
+}
+
+static void ggml_compute_forward_add_q_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const enum ggml_type type = src0->type;
+ const enum ggml_type dtype = dst->type;
+ ggml_to_float_t const dequantize_row_q = type_traits[type].to_float;
+ ggml_from_float_t const quantize_row_q = type_traits[dtype].from_float;
+
+ // we don't support permuted src0 or src1
+ GGML_ASSERT(nb00 == ggml_type_size(type));
+ GGML_ASSERT(nb10 == sizeof(float));
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 <= nb1);
+ GGML_ASSERT(nb1 <= nb2);
+ GGML_ASSERT(nb2 <= nb3);
+
+ GGML_ASSERT(ggml_is_quantized(src0->type));
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ float * wdata = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0 indices
+ const int i03 = ir/(ne02*ne01);
+ const int i02 = (ir - i03*ne02*ne01)/ne01;
+ const int i01 = (ir - i03*ne02*ne01 - i02*ne01);
+
+ // src1 and dst are same shape as src0 => same indices
+ const int i13 = i03;
+ const int i12 = i02;
+ const int i11 = i01;
+
+ const int i3 = i03;
+ const int i2 = i02;
+ const int i1 = i01;
+
+ void * src0_row = (void *) ((char *) src0->data + (i01*nb01 + i02*nb02 + i03*nb03));
+ float * src1_row = (float *)((char *) src1->data + (i11*nb11 + i12*nb12 + i13*nb13));
+ void * dst_row = (void *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
+
+ assert(ne00 % 32 == 0);
+
+ // unquantize row from src0 to temp buffer
+ dequantize_row_q(src0_row, wdata, ne00);
+ // add src1
+ ggml_vec_acc_f32(ne00, wdata, src1_row);
+ // quantize row to dst
+ if (quantize_row_q != NULL) {
+ quantize_row_q(wdata, dst_row, ne00);
+ } else {
+ memcpy(dst_row, wdata, ne0*nb0);
+ }
+ }
+}
+
+static void ggml_compute_forward_add(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ if (src1->type == GGML_TYPE_F32) {
+ ggml_compute_forward_add_f32(params, dst);
+ }
+ else {
+ GGML_ABORT("fatal error");
+ }
+ } break;
+ case GGML_TYPE_F16:
+ {
+ if (src1->type == GGML_TYPE_F16) {
+ ggml_compute_forward_add_f16_f16(params, dst);
+ }
+ else if (src1->type == GGML_TYPE_F32) {
+ ggml_compute_forward_add_f16_f32(params, dst);
+ }
+ else {
+ GGML_ABORT("fatal error");
+ }
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ if (src1->type == GGML_TYPE_BF16) {
+ ggml_compute_forward_add_bf16_bf16(params, dst);
+ }
+ else if (src1->type == GGML_TYPE_F32) {
+ ggml_compute_forward_add_bf16_f32(params, dst);
+ }
+ else {
+ GGML_ABORT("fatal error");
+ }
+ } break;
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_Q4_0_4_4:
+ case GGML_TYPE_Q4_0_4_8:
+ case GGML_TYPE_Q4_0_8_8:
+ {
+ ggml_compute_forward_add_q_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_add1
+
+static void ggml_compute_forward_add1_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_is_scalar(src1));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ GGML_ASSERT( nb0 == sizeof(float));
+ GGML_ASSERT(nb00 == sizeof(float));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+#ifdef GGML_USE_ACCELERATE
+ UNUSED(ggml_vec_add1_f32);
+
+ vDSP_vadd(
+ (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01), 1,
+ (float *) ((char *) src1->data), 0,
+ (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ), 1,
+ ne0);
+#else
+ ggml_vec_add1_f32(ne0,
+ (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ),
+ (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01),
+ *(float *) src1->data);
+#endif
+ }
+}
+
+static void ggml_compute_forward_add1_f16_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_is_scalar(src1));
+
+ // scalar to add
+ const float v = *(float *) src1->data;
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT(dst->type == GGML_TYPE_F16);
+
+ GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
+ ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(src0_ptr[i]) + v);
+ }
+ }
+}
+
+static void ggml_compute_forward_add1_f16_f16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_is_scalar(src1));
+
+ // scalar to add
+ const float v = GGML_FP16_TO_FP32(*(ggml_fp16_t *) src1->data);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F16);
+ GGML_ASSERT(dst->type == GGML_TYPE_F16);
+
+ GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
+ ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(src0_ptr[i]) + v);
+ }
+ }
+}
+
+static void ggml_compute_forward_add1_q_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_is_scalar(src1));
+
+ // scalar to add
+ const float v = *(float *) src1->data;
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ const enum ggml_type type = src0->type;
+ ggml_to_float_t const dequantize_row_q = type_traits[type].to_float;
+ ggml_from_float_t const quantize_row_q = type_traits[type].from_float;
+
+ // we don't support permuted src0
+ GGML_ASSERT(nb00 == ggml_type_size(type));
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 <= nb1);
+ GGML_ASSERT(nb1 <= nb2);
+ GGML_ASSERT(nb2 <= nb3);
+
+ GGML_ASSERT(ggml_is_quantized(src0->type));
+ GGML_ASSERT(dst->type == src0->type);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ float * wdata = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32) * ith;
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ void * src0_row = (void *) ((char *) src0->data + (i1*nb01 + i2*nb02 + i3*nb03));
+ void * dst_row = (void *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb0 ));
+
+ assert(ne0 % 32 == 0);
+
+ // unquantize row from src0 to temp buffer
+ dequantize_row_q(src0_row, wdata, ne0);
+ // add src1
+ ggml_vec_acc1_f32(ne0, wdata, v);
+ // quantize row to dst
+ quantize_row_q(wdata, dst_row, ne0);
+ }
+}
+
+static void ggml_compute_forward_add1_bf16_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_is_scalar(src1));
+
+ // scalar to add
+ const float v = *(float *) src1->data;
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);
+
+ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);
+ }
+ }
+}
+
+static void ggml_compute_forward_add1_bf16_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_is_scalar(src1));
+
+ // scalar to add
+ const float v = GGML_BF16_TO_FP32(*(ggml_bf16_t *) src1->data);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);
+ GGML_ASSERT(src1->type == GGML_TYPE_BF16);
+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);
+
+ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);
+ }
+ }
+}
+
+static void ggml_compute_forward_add1(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_add1_f32(params, dst);
+ } break;
+ case GGML_TYPE_F16:
+ {
+ if (src1->type == GGML_TYPE_F16) {
+ ggml_compute_forward_add1_f16_f16(params, dst);
+ }
+ else if (src1->type == GGML_TYPE_F32) {
+ ggml_compute_forward_add1_f16_f32(params, dst);
+ }
+ else {
+ GGML_ABORT("fatal error");
+ }
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ if (src1->type == GGML_TYPE_BF16) {
+ ggml_compute_forward_add1_bf16_bf16(params, dst);
+ }
+ else if (src1->type == GGML_TYPE_F32) {
+ ggml_compute_forward_add1_bf16_f32(params, dst);
+ }
+ else {
+ GGML_ABORT("fatal error");
+ }
+ } break;
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_Q4_0_4_4:
+ case GGML_TYPE_Q4_0_4_8:
+ case GGML_TYPE_Q4_0_8_8:
+ {
+ ggml_compute_forward_add1_q_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_acc
+
+static void ggml_compute_forward_acc_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));
+
+ // view src0 and dst with these strides and data offset inbytes during acc
+ // nb0 is implicitly element_size because src0 and dst are contiguous
+ size_t nb1 = ((int32_t *) dst->op_params)[0];
+ size_t nb2 = ((int32_t *) dst->op_params)[1];
+ size_t nb3 = ((int32_t *) dst->op_params)[2];
+ size_t offset = ((int32_t *) dst->op_params)[3];
+ bool inplace = (bool) ((int32_t *) dst->op_params)[4];
+
+ if (!inplace) {
+ if (params->ith == 0) {
+ // memcpy needs to be synchronized across threads to avoid race conditions.
+ // => do it in INIT phase
+ memcpy(
+ ((char *) dst->data),
+ ((char *) src0->data),
+ ggml_nbytes(dst));
+ }
+ ggml_barrier(params->threadpool);
+ }
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src1);
+ const int nc = src1->ne[0];
+
+ GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne)
+ GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)
+
+ // src0 and dst as viewed during acc
+ const size_t nb0 = ggml_element_size(src0);
+
+ const size_t nb00 = nb0;
+ const size_t nb01 = nb1;
+ const size_t nb02 = nb2;
+ const size_t nb03 = nb3;
+
+ GGML_ASSERT(offset + (ne10 == 0 ? 0 : ne10-1)*nb0 + (ne11 == 0 ? 0 : ne11-1)*nb1 + (ne12 == 0 ? 0 : ne12-1)*nb2 + (ne13 == 0 ? 0 : ne13-1)*nb3 < ggml_nbytes(dst));
+ GGML_ASSERT(offset + (ne10 == 0 ? 0 : ne10-1)*nb00 + (ne11 == 0 ? 0 : ne11-1)*nb01 + (ne12 == 0 ? 0 : ne12-1)*nb02 + (ne13 == 0 ? 0 : ne13-1)*nb03 < ggml_nbytes(src0));
+
+ GGML_ASSERT(nb10 == sizeof(float));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0 and dst are viewed with shape of src1 and offset
+ // => same indices
+ const int i3 = ir/(ne12*ne11);
+ const int i2 = (ir - i3*ne12*ne11)/ne11;
+ const int i1 = (ir - i3*ne12*ne11 - i2*ne11);
+
+#ifdef GGML_USE_ACCELERATE
+ vDSP_vadd(
+ (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + offset), 1,
+ (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11), 1,
+ (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + offset), 1, nc);
+#else
+ ggml_vec_add_f32(nc,
+ (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + offset),
+ (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + offset),
+ (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));
+#endif
+ }
+}
+
+static void ggml_compute_forward_acc(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_acc_f32(params, dst);
+ } break;
+ case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_Q4_0_4_4:
+ case GGML_TYPE_Q4_0_4_8:
+ case GGML_TYPE_Q4_0_8_8:
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_sub
+
+static void ggml_compute_forward_sub_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ assert(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT( nb0 == sizeof(float));
+ GGML_ASSERT(nb00 == sizeof(float));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (nb10 == sizeof(float)) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src1 is broadcastable across src0 and dst in i1, i2, i3
+ const int64_t i03 = ir/(ne02*ne01);
+ const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
+ const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+ const int64_t nr0 = ne00 / ne10;
+
+ float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );
+ float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
+
+ for (int64_t r = 0; r < nr0; ++r) {
+#ifdef GGML_USE_ACCELERATE
+ vDSP_vsub(src1_ptr, 1, src0_ptr + r*ne10, 1, dst_ptr + r*ne10, 1, ne10);
+#else
+ ggml_vec_sub_f32(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr);
+#endif
+ }
+ }
+ } else {
+ // src1 is not contiguous
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src1 is broadcastable across src0 and dst in i1, i2, i3
+ const int64_t i03 = ir/(ne02*ne01);
+ const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
+ const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+
+ float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );
+ float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
+
+ for (int64_t i0 = 0; i0 < ne0; ++i0) {
+ const int64_t i10 = i0 % ne10;
+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i10*nb10);
+
+ dst_ptr[i0] = src0_ptr[i0] - *src1_ptr;
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_sub(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_sub_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_mul
+
+static void ggml_compute_forward_mul_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t nr = ggml_nrows(src0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT( nb0 == sizeof(float));
+ GGML_ASSERT(nb00 == sizeof(float));
+
+ if (nb10 == sizeof(float)) {
+ for (int64_t ir = ith; ir < nr; ir += nth) {
+ // src0 and dst are same shape => same indices
+ const int64_t i03 = ir/(ne02*ne01);
+ const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
+ const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+ const int64_t nr0 = ne00 / ne10;
+
+ float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );
+ float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
+
+ for (int64_t r = 0 ; r < nr0; ++r) {
+#ifdef GGML_USE_ACCELERATE
+ UNUSED(ggml_vec_mul_f32);
+
+ vDSP_vmul(src0_ptr + r*ne10, 1, src1_ptr, 1, dst_ptr + r*ne10, 1, ne10);
+#else
+ ggml_vec_mul_f32(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr);
+#endif
+ }
+ }
+ } else {
+ // src1 is not contiguous
+ for (int64_t ir = ith; ir < nr; ir += nth) {
+ // src0 and dst are same shape => same indices
+ // src1 is broadcastable across src0 and dst in i1, i2, i3
+ const int64_t i03 = ir/(ne02*ne01);
+ const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
+ const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+
+ float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );
+ float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
+
+ for (int64_t i0 = 0; i0 < ne00; ++i0) {
+ const int64_t i10 = i0 % ne10;
+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i10*nb10);
+
+ dst_ptr[i0] = src0_ptr[i0] * (*src1_ptr);
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_mul(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(src1->type == GGML_TYPE_F32 && "only f32 src1 supported for now");
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_mul_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_div
+
+static void ggml_compute_forward_div_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t nr = ggml_nrows(src0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT( nb0 == sizeof(float));
+ GGML_ASSERT(nb00 == sizeof(float));
+
+ if (nb10 == sizeof(float)) {
+ for (int64_t ir = ith; ir < nr; ir += nth) {
+ // src0 and dst are same shape => same indices
+ const int64_t i03 = ir/(ne02*ne01);
+ const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
+ const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+ const int64_t nr0 = ne00 / ne10;
+
+ float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );
+ float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
+
+ for (int64_t r = 0; r < nr0; ++r) {
+#ifdef GGML_USE_ACCELERATE
+ UNUSED(ggml_vec_div_f32);
+
+ vDSP_vdiv(src1_ptr, 1, src0_ptr + r*ne10, 1, dst_ptr + r*ne10, 1, ne10);
+#else
+ ggml_vec_div_f32(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr);
+#endif
+ }
+ }
+ } else {
+ // src1 is not contiguous
+ for (int64_t ir = ith; ir < nr; ir += nth) {
+ // src0 and dst are same shape => same indices
+ // src1 is broadcastable across src0 and dst in i1, i2, i3
+ const int64_t i03 = ir/(ne02*ne01);
+ const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
+ const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
+
+ const int64_t i13 = i03 % ne13;
+ const int64_t i12 = i02 % ne12;
+ const int64_t i11 = i01 % ne11;
+
+ float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );
+ float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
+
+ for (int64_t i0 = 0; i0 < ne00; ++i0) {
+ const int64_t i10 = i0 % ne10;
+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i10*nb10);
+
+ dst_ptr[i0] = src0_ptr[i0] / (*src1_ptr);
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_div(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_div_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_sqr
+
+static void ggml_compute_forward_sqr_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ assert( dst->nb[0] == sizeof(float));
+ assert(src0->nb[0] == sizeof(float));
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_sqr_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_sqr(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_sqr_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_sqrt
+
+static void ggml_compute_forward_sqrt_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ assert( dst->nb[0] == sizeof(float));
+ assert(src0->nb[0] == sizeof(float));
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_sqrt_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_sqrt(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_sqrt_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_log
+
+static void ggml_compute_forward_log_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ GGML_ASSERT( dst->nb[0] == sizeof(float));
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_log_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_log(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_log_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_sin
+
+static void ggml_compute_forward_sin_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ GGML_ASSERT( dst->nb[0] == sizeof(float));
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_sin_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_sin(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_sin_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_cos
+
+static void ggml_compute_forward_cos_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ GGML_ASSERT( dst->nb[0] == sizeof(float));
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_cos_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_cos(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_cos_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_sum
+
+static void ggml_compute_forward_sum_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_scalar(dst));
+
+
+ assert(ggml_is_scalar(dst));
+ assert(src0->nb[0] == sizeof(float));
+
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+
+ ggml_float sum = 0;
+ ggml_float row_sum = 0;
+
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ ggml_vec_sum_f32_ggf(ne00,
+ &row_sum,
+ (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
+ sum += row_sum;
+ }
+ }
+ }
+ ((float *) dst->data)[0] = sum;
+}
+
+static void ggml_compute_forward_sum_f16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_scalar(dst));
+
+ assert(src0->nb[0] == sizeof(ggml_fp16_t));
+
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+
+ float sum = 0;
+ float row_sum = 0;
+
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ ggml_vec_sum_f16_ggf(ne00,
+ &row_sum,
+ (ggml_fp16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));
+ sum += row_sum;
+ }
+ }
+ }
+ ((ggml_fp16_t *) dst->data)[0] = GGML_FP32_TO_FP16(sum);
+}
+
+static void ggml_compute_forward_sum_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_scalar(dst));
+
+ assert(src0->nb[0] == sizeof(ggml_bf16_t));
+
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+
+ float sum = 0;
+ float row_sum = 0;
+
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ ggml_vec_sum_bf16_ggf(ne00,
+ &row_sum,
+ (ggml_bf16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));
+ sum += row_sum;
+ }
+ }
+ }
+ ((ggml_bf16_t *) dst->data)[0] = GGML_FP32_TO_BF16(sum);
+}
+
+static void ggml_compute_forward_sum(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_sum_f32(params, dst);
+ } break;
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_sum_f16(params, dst);
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_sum_bf16(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_sum_rows
+
+static void ggml_compute_forward_sum_rows_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+ GGML_ASSERT(dst->nb[0] == sizeof(float));
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ GGML_ASSERT(ne0 == 1);
+ GGML_ASSERT(ne1 == ne01);
+ GGML_ASSERT(ne2 == ne02);
+ GGML_ASSERT(ne3 == ne03);
+
+ for (int64_t i3 = 0; i3 < ne03; i3++) {
+ for (int64_t i2 = 0; i2 < ne02; i2++) {
+ for (int64_t i1 = 0; i1 < ne01; i1++) {
+ float * src_row = (float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
+ float * dst_row = (float *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
+ float row_sum = 0;
+ ggml_vec_sum_f32(ne00, &row_sum, src_row);
+ dst_row[0] = row_sum;
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_sum_rows(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_sum_rows_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_mean
+
+static void ggml_compute_forward_mean_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(src0->nb[0] == sizeof(float));
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ assert(ne0 == 1);
+ assert(ne1 == ne01);
+ assert(ne2 == ne02);
+ assert(ne3 == ne03);
+
+ UNUSED(ne0);
+ UNUSED(ne1);
+ UNUSED(ne2);
+ UNUSED(ne3);
+
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ ggml_vec_sum_f32(ne00,
+ (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),
+ (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
+
+ *(float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3) /= (float) ne00;
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_mean(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_mean_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_argmax
+
+static void ggml_compute_forward_argmax_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(src0->nb[0] == sizeof(float));
+ assert(dst->nb[0] == sizeof(float));
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t ne01 = src0->ne[1];
+
+ const size_t nb01 = src0->nb[1];
+ const size_t nb0 = dst->nb[0];
+
+ for (int64_t i1 = 0; i1 < ne01; i1++) {
+ float * src = (float *) ((char *) src0->data + i1*nb01);
+ int32_t * dst_ = (int32_t *) ((char *) dst->data + i1*nb0);
+ int v = 0;
+ ggml_vec_argmax_f32(ne00, &v, src);
+ dst_[0] = v;
+ }
+}
+
+static void ggml_compute_forward_argmax(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_argmax_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_repeat
+
+static void ggml_compute_forward_repeat_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ GGML_ASSERT(ggml_can_repeat(src0, dst));
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ // guaranteed to be an integer due to the check in ggml_can_repeat
+ const int nr0 = (int)(ne0/ne00);
+ const int nr1 = (int)(ne1/ne01);
+ const int nr2 = (int)(ne2/ne02);
+ const int nr3 = (int)(ne3/ne03);
+
+ // TODO: support for transposed / permuted tensors
+ GGML_ASSERT(nb0 == sizeof(float));
+ GGML_ASSERT(nb00 == sizeof(float));
+
+ // TODO: maybe this is not optimal?
+ for (int i3 = 0; i3 < nr3; i3++) {
+ for (int k3 = 0; k3 < ne03; k3++) {
+ for (int i2 = 0; i2 < nr2; i2++) {
+ for (int k2 = 0; k2 < ne02; k2++) {
+ for (int i1 = 0; i1 < nr1; i1++) {
+ for (int k1 = 0; k1 < ne01; k1++) {
+ for (int i0 = 0; i0 < nr0; i0++) {
+ ggml_vec_cpy_f32(ne00,
+ (float *) ((char *) dst->data + (i3*ne03 + k3)*nb3 + (i2*ne02 + k2)*nb2 + (i1*ne01 + k1)*nb1 + (i0*ne00)*nb0),
+ (float *) ((char *) src0->data + ( k3)*nb03 + ( k2)*nb02 + ( k1)*nb01));
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_repeat_f16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ GGML_ASSERT(ggml_can_repeat(src0, dst));
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ // guaranteed to be an integer due to the check in ggml_can_repeat
+ const int nr0 = (int)(ne0/ne00);
+ const int nr1 = (int)(ne1/ne01);
+ const int nr2 = (int)(ne2/ne02);
+ const int nr3 = (int)(ne3/ne03);
+
+ // TODO: support for transposed / permuted tensors
+ GGML_ASSERT(nb0 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+
+ // TODO: maybe this is not optimal?
+ for (int i3 = 0; i3 < nr3; i3++) {
+ for (int k3 = 0; k3 < ne03; k3++) {
+ for (int i2 = 0; i2 < nr2; i2++) {
+ for (int k2 = 0; k2 < ne02; k2++) {
+ for (int i1 = 0; i1 < nr1; i1++) {
+ for (int k1 = 0; k1 < ne01; k1++) {
+ for (int i0 = 0; i0 < nr0; i0++) {
+ ggml_fp16_t * y = (ggml_fp16_t *) ((char *) dst->data + (i3*ne03 + k3)*nb3 + (i2*ne02 + k2)*nb2 + (i1*ne01 + k1)*nb1 + (i0*ne00)*nb0);
+ ggml_fp16_t * x = (ggml_fp16_t *) ((char *) src0->data + ( k3)*nb03 + ( k2)*nb02 + ( k1)*nb01);
+ // ggml_vec_cpy_f16(ne00, y, x)
+ for (int i = 0; i < ne00; ++i) {
+ y[i] = x[i];
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_repeat(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
+ case GGML_TYPE_I16:
+ {
+ ggml_compute_forward_repeat_f16(params, dst);
+ } break;
+ case GGML_TYPE_F32:
+ case GGML_TYPE_I32:
+ {
+ ggml_compute_forward_repeat_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_repeat_back
+
+static void ggml_compute_forward_repeat_back_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ GGML_ASSERT(ggml_can_repeat(dst, src0));
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ // guaranteed to be an integer due to the check in ggml_can_repeat
+ const int nr0 = (int)(ne00/ne0);
+ const int nr1 = (int)(ne01/ne1);
+ const int nr2 = (int)(ne02/ne2);
+ const int nr3 = (int)(ne03/ne3);
+
+ // TODO: support for transposed / permuted tensors
+ GGML_ASSERT(nb0 == sizeof(float));
+ GGML_ASSERT(nb00 == sizeof(float));
+
+ if (ggml_is_contiguous(dst)) {
+ ggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);
+ } else {
+ for (int k3 = 0; k3 < ne3; k3++) {
+ for (int k2 = 0; k2 < ne2; k2++) {
+ for (int k1 = 0; k1 < ne1; k1++) {
+ ggml_vec_set_f32(ne0,
+ (float *) ((char *) dst->data + k1*nb1 + k2*nb2 + k3*nb3),
+ 0);
+ }
+ }
+ }
+ }
+
+ // TODO: maybe this is not optimal?
+ for (int i3 = 0; i3 < nr3; i3++) {
+ for (int k3 = 0; k3 < ne3; k3++) {
+ for (int i2 = 0; i2 < nr2; i2++) {
+ for (int k2 = 0; k2 < ne2; k2++) {
+ for (int i1 = 0; i1 < nr1; i1++) {
+ for (int k1 = 0; k1 < ne1; k1++) {
+ for (int i0 = 0; i0 < nr0; i0++) {
+ ggml_vec_acc_f32(ne0,
+ (float *) ((char *) dst->data + ( k3)*nb3 + ( k2)*nb2 + ( k1)*nb1),
+ (float *) ((char *) src0->data + (i3*ne3 + k3)*nb03 + (i2*ne2 + k2)*nb02 + (i1*ne1 + k1)*nb01 + (i0*ne0)*nb00));
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_repeat_back(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_repeat_back_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_concat
+
+static void ggml_compute_forward_concat_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int32_t dim = ggml_get_op_params_i32(dst, 0);
+
+ GGML_ASSERT(dim >= 0 && dim < 4);
+
+ int64_t o[4] = {0, 0, 0, 0};
+ o[dim] = src0->ne[dim];
+
+ const float * x;
+
+ // TODO: smarter multi-theading
+ for (int i3 = 0; i3 < ne3; i3++) {
+ for (int i2 = ith; i2 < ne2; i2 += nth) {
+ for (int i1 = 0; i1 < ne1; i1++) {
+ for (int i0 = 0; i0 < ne0; i0++) {
+ if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
+ x = (const float *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
+ } else {
+ x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
+ }
+
+ float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
+
+ *y = *x;
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_concat(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ case GGML_TYPE_I32:
+ {
+ ggml_compute_forward_concat_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_abs
+
+static void ggml_compute_forward_abs_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_abs_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_abs(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_abs_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_sgn
+
+static void ggml_compute_forward_sgn_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_sgn_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_sgn(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_sgn_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_neg
+
+static void ggml_compute_forward_neg_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_neg_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_neg(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_neg_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_step
+
+static void ggml_compute_forward_step_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_step_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_step(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_step_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_tanh
+
+static void ggml_compute_forward_tanh_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_tanh_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_tanh(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_tanh_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_elu
+
+static void ggml_compute_forward_elu_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_elu_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_elu(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_elu_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_relu
+
+static void ggml_compute_forward_relu_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_relu_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_relu(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_relu_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_sigmoid
+
+static void ggml_compute_forward_sigmoid_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_sigmoid_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_sigmoid(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_sigmoid_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_gelu
+
+static void ggml_compute_forward_gelu_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src0->ne[0];
+ const int nr = ggml_nrows(src0);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ ggml_vec_gelu_f32(nc,
+ (float *) ((char *) dst->data + i1*( dst->nb[1])),
+ (float *) ((char *) src0->data + i1*(src0->nb[1])));
+
+#ifndef NDEBUG
+ for (int k = 0; k < nc; k++) {
+ const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
+ UNUSED(x);
+ assert(!isnan(x));
+ assert(!isinf(x));
+ }
+#endif
+ }
+}
+
+static void ggml_compute_forward_gelu(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_gelu_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_gelu_quick
+
+static void ggml_compute_forward_gelu_quick_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src0->ne[0];
+ const int nr = ggml_nrows(src0);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ ggml_vec_gelu_quick_f32(nc,
+ (float *) ((char *) dst->data + i1*( dst->nb[1])),
+ (float *) ((char *) src0->data + i1*(src0->nb[1])));
+
+#ifndef NDEBUG
+ for (int k = 0; k < nc; k++) {
+ const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
+ UNUSED(x);
+ assert(!isnan(x));
+ assert(!isinf(x));
+ }
+#endif
+ }
+}
+
+static void ggml_compute_forward_gelu_quick(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_gelu_quick_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_silu
+
+static void ggml_compute_forward_silu_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src0->ne[0];
+ const int nr = ggml_nrows(src0);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ ggml_vec_silu_f32(nc,
+ (float *) ((char *) dst->data + i1*( dst->nb[1])),
+ (float *) ((char *) src0->data + i1*(src0->nb[1])));
+
+#ifndef NDEBUG
+ for (int k = 0; k < nc; k++) {
+ const float x = ((float *) ((char *) dst->data + i1*(dst->nb[1])))[k];
+ UNUSED(x);
+ assert(!isnan(x));
+ assert(!isinf(x));
+ }
+#endif
+ }
+}
+
+static void ggml_compute_forward_silu(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_silu_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+// ggml_compute_forward_leaky_relu
+
+static void ggml_compute_forward_leaky_relu_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ float negative_slope;
+ memcpy(&negative_slope, dst->op_params, sizeof(float));
+
+ assert(dst->nb[0] == sizeof(float));
+ assert(src0->nb[0] == sizeof(float));
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_leaky_relu_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])), negative_slope);
+ }
+}
+
+static void ggml_compute_forward_leaky_relu(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_leaky_relu_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_silu_back
+
+static void ggml_compute_forward_silu_back_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * grad = dst->src[1];
+
+ assert(ggml_is_contiguous_1(grad));
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+ assert(ggml_are_same_shape(src0, grad));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src0->ne[0];
+ const int nr = ggml_nrows(src0);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ ggml_vec_silu_backward_f32(nc,
+ (float *) ((char *) dst->data + i1*( dst->nb[1])),
+ (float *) ((char *) src0->data + i1*(src0->nb[1])),
+ (float *) ((char *) grad->data + i1*(grad->nb[1])));
+
+#ifndef NDEBUG
+ for (int k = 0; k < nc; k++) {
+ const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
+ UNUSED(x);
+ assert(!isnan(x));
+ assert(!isinf(x));
+ }
+#endif
+ }
+}
+
+static void ggml_compute_forward_silu_back(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_silu_back_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+
+static void ggml_compute_forward_hardswish_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_hardswish_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+static void ggml_compute_forward_hardswish(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_hardswish_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+static void ggml_compute_forward_hardsigmoid_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_hardsigmoid_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_hardsigmoid(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_hardsigmoid_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+static void ggml_compute_forward_exp_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ ggml_vec_exp_f32(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_exp(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_exp_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+
+// ggml_compute_forward_norm
+
+static void ggml_compute_forward_norm_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ float eps;
+ memcpy(&eps, dst->op_params, sizeof(float));
+
+ GGML_ASSERT(eps > 0.0f);
+
+ // TODO: optimize
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
+ const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
+
+ ggml_float sum = 0.0;
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ sum += (ggml_float)x[i00];
+ }
+
+ float mean = sum/ne00;
+
+ float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
+
+ ggml_float sum2 = 0.0;
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ float v = x[i00] - mean;
+ y[i00] = v;
+ sum2 += (ggml_float)(v*v);
+ }
+
+ float variance = sum2/ne00;
+ const float scale = 1.0f/sqrtf(variance + eps);
+
+ ggml_vec_scale_f32(ne00, y, scale);
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_norm(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_norm_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_group_rms_norm
+
+static void ggml_compute_forward_rms_norm_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ float eps;
+ memcpy(&eps, dst->op_params, sizeof(float));
+
+ GGML_ASSERT(eps > 0.0f);
+
+ // TODO: optimize
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
+ const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
+
+ ggml_float sum = 0.0;
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ sum += (ggml_float)(x[i00] * x[i00]);
+ }
+
+ const float mean = sum/ne00;
+
+ float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
+
+ memcpy(y, x, ne00 * sizeof(float));
+ // for (int i00 = 0; i00 < ne00; i00++) {
+ // y[i00] = x[i00];
+ // }
+
+ const float scale = 1.0f/sqrtf(mean + eps);
+
+ ggml_vec_scale_f32(ne00, y, scale);
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_rms_norm(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_rms_norm_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+static void ggml_compute_forward_rms_norm_back_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_are_same_shape(src0, src1));
+
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ float eps;
+ memcpy(&eps, dst->op_params, sizeof(float));
+
+ // TODO: optimize
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
+ // src1 is same shape as src0 => same indices
+ const int64_t i11 = i01;
+ const int64_t i12 = i02;
+ const int64_t i13 = i03;
+
+ const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
+ const float * dz = (float *) ((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13);
+
+ ggml_float sum_xx = 0.0;
+ ggml_float sum_xdz = 0.0;
+
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ sum_xx += (ggml_float)(x[i00] * x[i00]);
+ sum_xdz += (ggml_float)(x[i00] * dz[i00]);
+ }
+
+ //const float mean = (float)(sum_xx)/ne00;
+ const float mean_eps = (float)(sum_xx)/ne00 + eps;
+ const float sum_eps = (float)(sum_xx) + eps*ne00;
+ //const float mean_xdz = (float)(sum_xdz)/ne00;
+ // we could cache rms from forward pass to improve performance.
+ // to do this implement ggml_rms and compose ggml_rms_norm using ggml_rms.
+ //const float rms = sqrtf(mean_eps);
+ const float rrms = 1.0f / sqrtf(mean_eps);
+ //const float scale = -rrms/(ne00 * mean_eps); // -1/(n*rms**3)
+
+ {
+ // z = rms_norm(x)
+ //
+ // rms_norm(src0) =
+ // scale(
+ // src0,
+ // div(
+ // 1,
+ // sqrt(
+ // add(
+ // scale(
+ // sum(
+ // sqr(
+ // src0)),
+ // (1.0/N)),
+ // eps))));
+
+ // postorder:
+ // ## op args grad
+ // 00 param src0 grad[#00]
+ // 01 const 1
+ // 02 sqr (#00) grad[#02]
+ // 03 sum (#02) grad[#03]
+ // 04 const 1/N
+ // 05 scale (#03, #04) grad[#05]
+ // 06 const eps
+ // 07 add (#05, #06) grad[#07]
+ // 08 sqrt (#07) grad[#08]
+ // 09 div (#01,#08) grad[#09]
+ // 10 scale (#00,#09) grad[#10]
+ //
+ // backward pass, given grad[#10]
+ // #10: scale
+ // grad[#00] += scale(grad[#10],#09)
+ // grad[#09] += sum(mul(grad[#10],#00))
+ // #09: div
+ // grad[#08] += neg(mul(grad[#09], div(#09,#08)))
+ // #08: sqrt
+ // grad[#07] += mul(grad[#08], div(0.5, #08))
+ // #07: add
+ // grad[#05] += grad[#07]
+ // #05: scale
+ // grad[#03] += scale(grad[#05],#04)
+ // #03: sum
+ // grad[#02] += repeat(grad[#03], #02)
+ // #02:
+ // grad[#00] += scale(mul(#00, grad[#02]), 2.0)
+ //
+ // substitute and simplify:
+ // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, grad[#02]), 2.0)
+ // grad[#02] = repeat(grad[#03], #02)
+ // grad[#02] = repeat(scale(grad[#05],#04), #02)
+ // grad[#02] = repeat(scale(grad[#07],#04), #02)
+ // grad[#02] = repeat(scale(mul(grad[#08], div(0.5, #08)),#04), #02)
+ // grad[#02] = repeat(scale(mul(neg(mul(grad[#09], div(#09,#08))), div(0.5, #08)),#04), #02)
+ // grad[#02] = repeat(scale(mul(neg(mul(sum(mul(grad[#10],#00)), div(#09,#08))), div(0.5, #08)),#04), #02)
+ // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(#09,#08) * div(0.5, #08) * (1/N)), #02)
+ // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(div(#01,#08),#08) * div(0.5, #08) * (1/N)), #02)
+ // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(1,#08*#08) * div(0.5, #08) * (1/N)), #02)
+ // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N)), #02)
+ // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, grad[#02]), 2.0)
+ // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, repeat(-(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N)), #02)), 2.0)
+ // grad[#00] = scale(grad(#10), #09) + scale(scale(#00, -(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N))), 2.0)
+ // grad[#00] = scale(grad(#10), #09) + scale(#00, -(sum(mul(grad[#10],#00)) * div(1,#07) * div(1,#08) * (1/N)))
+ // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,#07*#08) * (-1/N))
+ // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,#07*#08) * (-1/N))
+ // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,mean_eps*rms) * (-1/N))
+ // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*mean_eps))
+ // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*(sum_xx/N+eps)))
+ // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*sum_xx+rms*N*eps))
+ // grad[#00] = scale(dz, rrms) + scale(x, sum(mul(dz,x)) * div(-1,rms*N*mean_eps))
+ // grad[#00] = scale(dz, rrms) + scale(x, sum_xdz * div(-1,rms*N*mean_eps))
+ // a = b*c + d*e
+ // a = b*c*f/f + d*e*f/f
+ // a = (b*c*f + d*e*f)*(1/f)
+ // a = (b*c*(1/c) + d*e*(1/c))*(1/(1/c))
+ // a = (b + d*e/c)*c
+ // b = dz, c = rrms, d = x, e = sum_xdz * div(-1,rms*N*mean_eps)
+ // a = (dz + x*sum_xdz * div(-1,rms*N*mean_eps)/rrms)*rrms
+ // a = (dz + x*sum_xdz * div(-1,rms*N*mean_eps)*rms)*rrms
+ // a = (dz + x*sum_xdz * div(-rms,rms*N*mean_eps))*rrms
+ // a = (dz + x*sum_xdz * div(-1,N*mean_eps))*rrms
+ // a = (dz + x*div(-sum_xdz,N*mean_eps))*rrms
+ // a = (dz + x*div(-mean_xdz,mean_eps))*rrms
+ // grad[#00] = scale(dz + scale(x, div(-mean_xdz,mean_eps)),rrms)
+ // grad[#00] = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)
+ // dx = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)
+ }
+ // dx = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)
+ // post-order:
+ // dx := x
+ // dx := scale(dx,-mean_xdz/mean_eps)
+ // dx := add(dx, dz)
+ // dx := scale(dx, rrms)
+ float * dx = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
+
+ ggml_vec_cpy_f32 (ne00, dx, x);
+ // ggml_vec_scale_f32(ne00, dx, -mean_xdz/mean_eps);
+ ggml_vec_scale_f32(ne00, dx, (float)(-sum_xdz)/sum_eps);
+ ggml_vec_acc_f32 (ne00, dx, dz);
+ ggml_vec_scale_f32(ne00, dx, rrms);
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_rms_norm_back(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_rms_norm_back_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_group_norm
+
+static void ggml_compute_forward_group_norm_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ // TODO: optimize
+
+ float eps;
+ memcpy(&eps, dst->op_params + 1, sizeof(float));
+
+ int n_channels = src0->ne[2];
+ int n_groups = dst->op_params[0];
+ int n_channels_per_group = (n_channels + n_groups - 1) / n_groups;
+ for (int i = ith; i < n_groups; i += nth) {
+ int start = i * n_channels_per_group;
+ int end = start + n_channels_per_group;
+ if (end > n_channels) {
+ end = n_channels;
+ }
+ int step = end - start;
+
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ ggml_float sum = 0.0;
+ for (int64_t i02 = start; i02 < end; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03);
+
+ ggml_float sumr = 0.0;
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ sumr += (ggml_float)x[i00];
+ }
+ sum += sumr;
+ }
+ }
+ const float mean = sum / (ne00 * ne01 * step);
+
+ ggml_float sum2 = 0.0;
+ for (int64_t i02 = start; i02 < end; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03);
+
+ float * y = (float *)((char *) dst->data + i01 * nb1 + i02 * nb2 + i03 * nb3);
+
+ ggml_float sumr = 0.0;
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ float v = x[i00] - mean;
+ y[i00] = v;
+ sumr += (ggml_float)(v * v);
+ }
+ sum2 += sumr;
+ }
+ }
+ const float variance = sum2 / (ne00 * ne01 * step);
+ const float scale = 1.0f / sqrtf(variance + eps);
+
+ for (int64_t i02 = start; i02 < end; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ float * y = (float *)((char *) dst->data + i01 * nb1 + i02 * nb2 + i03 * nb3);
+ ggml_vec_scale_f32(ne00, y, scale);
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_group_norm(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_group_norm_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_mul_mat
+
+static void ggml_compute_forward_mul_mat_one_chunk(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const int64_t num_rows_per_vec_dot,
+ const int64_t ir0_start,
+ const int64_t ir0_end,
+ const int64_t ir1_start,
+ const int64_t ir1_end) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const enum ggml_type type = src0->type;
+
+ const bool src1_cont = ggml_is_contiguous(src1);
+
+ ggml_vec_dot_t const vec_dot = type_traits[type].vec_dot;
+ enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type;
+
+ // broadcast factors
+ const int64_t r2 = ne12 / ne02;
+ const int64_t r3 = ne13 / ne03;
+
+ //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end);
+
+ // threads with no work simply yield (not sure if it helps)
+ if (ir0_start >= ir0_end || ir1_start >= ir1_end) {
+ return;
+ }
+
+ const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);
+
+ assert(ne12 % ne02 == 0);
+ assert(ne13 % ne03 == 0);
+
+ // block-tiling attempt
+ const int64_t blck_0 = 16;
+ const int64_t blck_1 = 16;
+
+ const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11;
+
+ // attempt to reduce false-sharing (does not seem to make a difference)
+ // 16 * 2, accounting for mmla kernels
+ float tmp[32];
+
+ for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {
+ for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {
+ for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) {
+ const int64_t i13 = (ir1 / (ne12 * ne1));
+ const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1;
+ const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1);
+
+ // broadcast src0 into src1
+ const int64_t i03 = i13 / r3;
+ const int64_t i02 = i12 / r2;
+
+ const int64_t i1 = i11;
+ const int64_t i2 = i12;
+ const int64_t i3 = i13;
+
+ const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03);
+
+ // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides
+ // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using
+ // the original src1 data pointer, so we should index using the indices directly
+ // TODO: this is a bit of a hack, we should probably have a better way to handle this
+ const char * src1_col = (const char*)wdata +
+ (src1_cont || src1->type != vec_dot_type
+ ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size
+ : (i11 * nb11 + i12 * nb12 + i13 * nb13));
+ float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3));
+
+ //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) {
+ // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col);
+ //}
+
+ for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) {
+ vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot);
+ }
+
+ for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) {
+ memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float));
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_mul_mat(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const enum ggml_type type = src0->type;
+
+ enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type;
+ ggml_from_float_t const from_float = type_traits[vec_dot_type].from_float;
+ ggml_from_float_to_mat_t const from_float_to_mat = type_traits[vec_dot_type].from_float_to_mat;
+ int64_t const vec_dot_num_rows = type_traits[type].nrows;
+ int64_t const matmul_num_cols = type_traits[type].ncols;
+ int64_t const blck_size_interleave = type_traits[type].blck_size_interleave;
+ ggml_gemv_t const gemv = type_traits[type].gemv;
+ ggml_gemm_t const gemm = type_traits[type].gemm;
+
+ GGML_ASSERT(ne0 == ne01);
+ GGML_ASSERT(ne1 == ne11);
+ GGML_ASSERT(ne2 == ne12);
+ GGML_ASSERT(ne3 == ne13);
+
+ // we don't support permuted src0 or src1
+ GGML_ASSERT(nb00 == ggml_type_size(type));
+ GGML_ASSERT(nb10 == ggml_type_size(src1->type));
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ GGML_ASSERT(nb0 <= nb1);
+ GGML_ASSERT(nb1 <= nb2);
+ GGML_ASSERT(nb2 <= nb3);
+
+ // nb01 >= nb00 - src0 is not transposed
+ // compute by src0 rows
+
+#if GGML_USE_LLAMAFILE
+ // broadcast factors
+ const int64_t r2 = ne12 / ne02;
+ const int64_t r3 = ne13 / ne03;
+
+ const bool src1_cont = ggml_is_contiguous(src1);
+
+ if (src1_cont) {
+ for (int64_t i13 = 0; i13 < ne13; i13++)
+ for (int64_t i12 = 0; i12 < ne12; i12++)
+ if (!llamafile_sgemm(ne01, ne11, ne00/ggml_blck_size(src0->type),
+ (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,
+ nb01/ggml_type_size(src0->type),
+ (const char *)src1->data + i12*nb12 + i13*nb13,
+ nb11/ggml_type_size(src1->type),
+ (char *)dst->data + i12*nb2 + i13*nb3,
+ nb1/ggml_type_size(dst->type),
+ ith, nth,
+ src0->type,
+ src1->type,
+ dst->type))
+ goto UseGgmlGemm1;
+ return;
+ }
+UseGgmlGemm1:;
+#endif
+
+ if (src1->type != vec_dot_type) {
+ char * wdata = params->wdata;
+
+ const size_t nbw1 = ggml_row_size(vec_dot_type, ne10);
+ const size_t nbw2 = nbw1*ne11;
+ const size_t nbw3 = nbw2*ne12;
+
+ assert(params->wsize >= ne13*nbw3);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ for (int64_t i13 = 0; i13 < ne13; ++i13) {
+ for (int64_t i12 = 0; i12 < ne12; ++i12) {
+ int64_t i11_processed = 0;
+ if ((ggml_n_dims(src1) == 2) && from_float_to_mat && gemm) {
+ for (int64_t i11 = ith * 4; i11 < ne11 - ne11 % 4; i11 += nth * 4) {
+ from_float_to_mat((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),
+ (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),
+ 4, ne10, blck_size_interleave);
+ }
+ i11_processed = ne11 - ne11 % 4;
+ }
+ for (int64_t i11 = i11_processed + ith; i11 < ne11; i11 += nth) {
+ from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),
+ (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),
+ ne10);
+ }
+ }
+ }
+ }
+
+ if (ith == 0) {
+ // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.
+ atomic_store_explicit(¶ms->threadpool->current_chunk, nth, memory_order_relaxed);
+ }
+
+ ggml_barrier(params->threadpool);
+
+#if GGML_USE_LLAMAFILE
+ if (src1->type != vec_dot_type) {
+ const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);
+
+ for (int64_t i13 = 0; i13 < ne13; i13++)
+ for (int64_t i12 = 0; i12 < ne12; i12++)
+ if (!llamafile_sgemm(ne01, ne11, ne00/ggml_blck_size(src0->type),
+ (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,
+ nb01/ggml_type_size(src0->type),
+ (const char *)wdata + (i12*ne11 + i13*ne12*ne11)*row_size,
+ row_size/ggml_type_size(vec_dot_type),
+ (char *)dst->data + i12*nb2 + i13*nb3,
+ nb1/ggml_type_size(dst->type),
+ ith, nth,
+ src0->type,
+ vec_dot_type,
+ dst->type))
+ goto UseGgmlGemm2;
+ return;
+ }
+UseGgmlGemm2:;
+#endif
+
+ // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers)
+ const int64_t nr0 = ne0;
+
+ // This is the size of the rest of the dimensions of the result
+ const int64_t nr1 = ne1 * ne2 * ne3;
+
+ // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols
+ int64_t num_rows_per_vec_dot = vec_dot_num_rows;
+ // TODO: currently the mmla kernels support only even numbered rows/cols.
+ // this check can be removed once they are extended to support odd numbered rows/cols too
+ if ((nr0 % 2 != 0) || (ne11 % 2 != 0)) {
+ num_rows_per_vec_dot = 1;
+ }
+
+ // Now select a reasonable chunk size.
+ int chunk_size = 16;
+
+ // We need to step up the size if it's small
+ if (nr0 == 1 || nr1 == 1) {
+ chunk_size = 64;
+ }
+
+ // distribute the work across the inner or outer loop based on which one is larger
+ // The number of chunks in the 0/1 dim.
+ // CEIL(nr0/chunk_size)
+ int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;
+ int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;
+
+ // If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread.
+ // Also, chunking by thread was measured to have perform better on NUMA systems. See https://github.com/ggerganov/llama.cpp/pull/6915
+ // In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that.
+ if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) {
+ // distribute the thread work across the inner or outer loop based on which one is larger
+ nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows
+ nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows
+ }
+
+ // The number of elements in each chunk
+ const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;
+ const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;
+
+ if ((ggml_n_dims(src0) == 2) && gemv) {
+ const void * src1_wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
+ const size_t src1_col_stride = ggml_is_contiguous(src1) || src1->type != vec_dot_type ? ggml_row_size(vec_dot_type, ne10) : nb11;
+ int64_t src0_start = (ith * ne01) / nth;
+ int64_t src0_end = ((ith + 1) * ne01) / nth;
+ src0_start = (src0_start % matmul_num_cols) ? src0_start + matmul_num_cols - (src0_start % matmul_num_cols): src0_start;
+ src0_end = (src0_end % matmul_num_cols) ? src0_end + matmul_num_cols - (src0_end % matmul_num_cols): src0_end;
+ if (src0_start >= src0_end) return;
+
+ // If there are more than three rows in src1, use gemm; otherwise, use gemv.
+ if (gemm && (ne11 > 3)) {
+ gemm(ne00, (float *)((char *) dst->data) + src0_start, ne01, (const char *) src0->data + src0_start * nb01,
+ (const char *) src1_wdata, ne11 - ne11 % 4, src0_end - src0_start);
+ }
+ for (int iter = gemm ? ne11 - ne11 % 4 : 0; iter < ne11; iter++) {
+ gemv(ne00, (float *)((char *) dst->data + (iter * nb1)) + src0_start, ne01,
+ (const char *) src0->data + src0_start * nb01, (const char *) src1_wdata + (src1_col_stride * iter), 1,
+ src0_end - src0_start);
+ }
+ return;
+ }
+
+ // The first chunk comes from our thread_id, the rest will get auto-assigned.
+ int current_chunk = ith;
+
+ while (current_chunk < nchunk0 * nchunk1) {
+ const int64_t ith0 = current_chunk % nchunk0;
+ const int64_t ith1 = current_chunk / nchunk0;
+
+ const int64_t ir0_start = dr0 * ith0;
+ const int64_t ir0_end = MIN(ir0_start + dr0, nr0);
+
+ const int64_t ir1_start = dr1 * ith1;
+ const int64_t ir1_end = MIN(ir1_start + dr1, nr1);
+
+ ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end);
+
+ if (nth >= nchunk0 * nchunk1) {
+ break;
+ }
+
+ current_chunk = atomic_fetch_add_explicit(¶ms->threadpool->current_chunk, 1, memory_order_relaxed);
+ }
+}
+
+// ggml_compute_forward_mul_mat_id
+
+static void ggml_compute_forward_mul_mat_id(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+ const struct ggml_tensor * ids = dst->src[2];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const enum ggml_type type = src0->type;
+
+ const bool src1_cont = ggml_is_contiguous(src1);
+
+ ggml_vec_dot_t const vec_dot = type_traits[type].vec_dot;
+ enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type;
+ ggml_from_float_t const from_float = type_traits[vec_dot_type].from_float;
+ int64_t const matmul_num_cols = type_traits[type].ncols;
+ ggml_gemv_t const gemv = type_traits[type].gemv;
+
+ // we don't support permuted src0 or src1
+ GGML_ASSERT(nb00 == ggml_type_size(type));
+ GGML_ASSERT(nb10 == ggml_type_size(src1->type));
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ GGML_ASSERT(nb0 <= nb1);
+ GGML_ASSERT(nb1 <= nb2);
+ GGML_ASSERT(nb2 <= nb3);
+
+ // row groups
+ const int n_ids = ids->ne[0]; // n_expert_used
+ const int n_as = ne02; // n_expert
+
+ char * wdata_src1_end = (src1->type == vec_dot_type) ?
+ (char *) params->wdata :
+ (char *) params->wdata + GGML_PAD(ggml_row_size(vec_dot_type, ggml_nelements(src1)), sizeof(int64_t));
+
+ struct mmid_row_mapping {
+ int32_t i1;
+ int32_t i2;
+ };
+
+ int64_t * matrix_row_counts = (int64_t *) (wdata_src1_end); // [n_as]
+ struct mmid_row_mapping * matrix_rows = (struct mmid_row_mapping *)(matrix_row_counts + n_as); // [n_as][ne11]
+
+ if (src1->type != vec_dot_type) {
+ char * wdata = params->wdata;
+
+ const size_t nbw1 = ggml_row_size(vec_dot_type, ne10);
+ const size_t nbw2 = nbw1*ne11;
+ const size_t nbw3 = nbw2*ne12;
+
+ assert(params->wsize >= ne13*nbw3);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ for (int64_t i13 = 0; i13 < ne13; ++i13) {
+ for (int64_t i12 = 0; i12 < ne12; ++i12) {
+ for (int64_t i11 = ith; i11 < ne11; i11 += nth) {
+ from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),
+ (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),
+ ne10);
+ }
+ }
+ }
+ }
+
+#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id)*ne12 + (i1)]
+
+ if (ith == 0) {
+ // initialize matrix_row_counts
+ memset(matrix_row_counts, 0, n_as*sizeof(int64_t));
+
+ // group rows by src0 matrix
+ for (int64_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
+ for (int id = 0; id < n_ids; ++id) {
+ const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1*ids->nb[1] + id*ids->nb[0]);
+
+ assert(i02 >= 0 && i02 < n_as);
+
+ MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = (struct mmid_row_mapping) {id, iid1};
+ matrix_row_counts[i02] += 1;
+ }
+ }
+ }
+
+ ggml_barrier(params->threadpool);
+
+ // compute each matrix multiplication in sequence
+ for (int cur_a = 0; cur_a < n_as; ++cur_a) {
+ const int64_t cne1 = matrix_row_counts[cur_a];
+
+ if (cne1 == 0) {
+ continue;
+ }
+
+ const char * src0_cur = (const char *) src0->data + cur_a*nb02;
+
+ const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);
+
+ const int64_t nr0 = ne01; // src0 rows
+ const int64_t nr1 = cne1; // src1 rows
+
+ if (((ggml_n_dims(src0) - 1) == 2) && gemv) {
+ int64_t src0_cur_start = (ith * ne01) / nth;
+ int64_t src0_cur_end = ((ith + 1) * ne01) / nth;
+ src0_cur_start = (src0_cur_start % matmul_num_cols) ? src0_cur_start + matmul_num_cols - (src0_cur_start % matmul_num_cols): src0_cur_start;
+ src0_cur_end = (src0_cur_end % matmul_num_cols) ? src0_cur_end + matmul_num_cols - (src0_cur_end % matmul_num_cols): src0_cur_end;
+ if (src0_cur_start >= src0_cur_end) return;
+
+ for (int ir1 = 0; ir1 < nr1; ir1++) {
+ struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, ir1);
+ const int id = row_mapping.i1; // selected expert index
+
+ const int64_t i11 = id % ne11;
+ const int64_t i12 = row_mapping.i2; // row index in src1
+
+ const int64_t i1 = id; // selected expert index
+ const int64_t i2 = i12; // row
+
+ const char * src1_col = (const char *) wdata +
+ (src1_cont || src1->type != vec_dot_type
+ ? (i11 + i12 * ne11) * row_size
+ : (i11 * nb11 + i12 * nb12));
+
+ gemv(ne00, (float *)((char *) dst->data + (i1 * nb1 + i2 * nb2)) + src0_cur_start, ne01,
+ (const char *) src0_cur + src0_cur_start * nb01, src1_col, 1, src0_cur_end - src0_cur_start);
+ }
+ continue;
+ }
+
+ // distribute the thread work across the inner or outer loop based on which one is larger
+
+ const int64_t nth0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows
+ const int64_t nth1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows
+
+ const int64_t ith0 = ith % nth0;
+ const int64_t ith1 = ith / nth0;
+
+ const int64_t dr0 = (nr0 + nth0 - 1)/nth0;
+ const int64_t dr1 = (nr1 + nth1 - 1)/nth1;
+
+ const int64_t ir010 = dr0*ith0;
+ const int64_t ir011 = MIN(ir010 + dr0, nr0);
+
+ const int64_t ir110 = dr1*ith1;
+ const int64_t ir111 = MIN(ir110 + dr1, nr1);
+
+ // threads with no work simply yield (not sure if it helps)
+ //if (ir010 >= ir011 || ir110 >= ir111) {
+ // sched_yield();
+ // continue;
+ //}
+
+ // block-tiling attempt
+ const int64_t blck_0 = 16;
+ const int64_t blck_1 = 16;
+
+ // attempt to reduce false-sharing (does not seem to make a difference)
+ float tmp[16];
+
+ for (int64_t iir1 = ir110; iir1 < ir111; iir1 += blck_1) {
+ for (int64_t iir0 = ir010; iir0 < ir011; iir0 += blck_0) {
+ for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir111; ++ir1) {
+ const int64_t _i12 = ir1; // logical row index for this expert
+
+ struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, _i12);
+ const int id = row_mapping.i1; // selected expert index
+
+ const int64_t i11 = id % ne11;
+ const int64_t i12 = row_mapping.i2; // row index in src1
+
+ const int64_t i1 = id; // selected expert index
+ const int64_t i2 = i12; // row
+
+ // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides
+ // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using
+ // the original src1 data pointer, so we should index using the indices directly
+ // TODO: this is a bit of a hack, we should probably have a better way to handle this
+ const char * src1_col = (const char *) wdata +
+ (src1_cont || src1->type != vec_dot_type
+ ? (i11 + i12*ne11)*row_size
+ : (i11*nb11 + i12*nb12));
+
+ float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2));
+
+ //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) {
+ // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col);
+ //}
+
+ for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) {
+ vec_dot(ne00, &tmp[ir0 - iir0], 0, src0_cur + ir0*nb01, 0, src1_col, 0, 1);
+ }
+
+ memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir011) - iir0)*sizeof(float));
+ }
+ }
+ }
+ }
+
+#undef MMID_MATRIX_ROW
+}
+
+// ggml_compute_forward_out_prod
+
+static void ggml_compute_forward_out_prod_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ GGML_ASSERT(ne0 == ne00);
+ GGML_ASSERT(ne1 == ne10);
+ GGML_ASSERT(ne2 == ne02);
+ GGML_ASSERT(ne02 == ne12);
+ GGML_ASSERT(ne3 == ne13);
+ GGML_ASSERT(ne03 == ne13);
+
+ // we don't support permuted src0 or src1
+ GGML_ASSERT(nb00 == sizeof(float));
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ // GGML_ASSERT(nb0 <= nb1);
+ // GGML_ASSERT(nb1 <= nb2);
+ // GGML_ASSERT(nb2 <= nb3);
+
+ // nb01 >= nb00 - src0 is not transposed
+ // compute by src0 rows
+
+ if (ith == 0) {
+ ggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);
+ }
+ ggml_barrier(params->threadpool);
+
+ // dst[:,:,:,:] = 0
+ // for i2,i3:
+ // for i1:
+ // for i01:
+ // for i0:
+ // dst[i0,i1,i2,i3] += src0[i0,i01,i2,i3] * src1[i1,i01,i2,i3]
+
+ // parallelize by last three dimensions
+
+ // total rows in dst
+ const int64_t nr = ne1*ne2*ne3;
+
+ // rows per thread
+ const int64_t dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int64_t ir0 = dr*ith;
+ const int64_t ir1 = MIN(ir0 + dr, nr);
+
+ // block-tiling attempt
+ const int64_t blck_0 = MAX(GGML_VEC_MAD_UNROLL, 32);
+ const int64_t blck_1 = 16;
+
+ for (int64_t bir = ir0; bir < ir1; bir += blck_1) {
+ const int64_t bir1 = MIN(bir + blck_1, ir1);
+ for (int64_t bi01 = 0; bi01 < ne01; bi01 += blck_0) {
+ const int64_t bne01 = MIN(bi01 + blck_0, ne01);
+ for (int64_t ir = bir; ir < bir1; ++ir) {
+ // dst indices
+ const int64_t i3 = ir/(ne2*ne1);
+ const int64_t i2 = (ir - i3*ne2*ne1)/ne1;
+ const int64_t i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ const int64_t i02 = i2;
+ const int64_t i03 = i3;
+
+ //const int64_t i10 = i1;
+ const int64_t i12 = i2;
+ const int64_t i13 = i3;
+
+#if GGML_VEC_MAD_UNROLL > 2
+ const int64_t bne01_unroll = bne01 - (bne01 % GGML_VEC_MAD_UNROLL);
+ for (int64_t i01 = bi01; i01 < bne01_unroll; i01 += GGML_VEC_MAD_UNROLL) {
+ const int64_t i11 = i01;
+
+ float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));
+ float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));
+ float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
+
+ ggml_vec_mad_f32_unroll(ne0, nb01, nb11, d, s0, s1);
+ }
+ for (int64_t i01 = bne01_unroll; i01 < bne01; ++i01) {
+ const int64_t i11 = i01;
+
+ float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));
+ float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));
+ float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
+
+ ggml_vec_mad_f32(ne0, d, s0, *s1);
+ }
+#else
+ for (int64_t i01 = bi01; i01 < bne01; ++i01) {
+ const int64_t i11 = i01;
+
+ float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));
+ float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));
+ float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
+
+ ggml_vec_mad_f32(ne0, d, s0, *s1);
+ }
+#endif
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_out_prod_q_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_TENSOR_BINARY_OP_LOCALS;
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const enum ggml_type type = src0->type;
+ ggml_to_float_t const dequantize_row_q = type_traits[type].to_float;
+
+ GGML_ASSERT(ne02 == ne12);
+ GGML_ASSERT(ne03 == ne13);
+ GGML_ASSERT(ne2 == ne12);
+ GGML_ASSERT(ne3 == ne13);
+
+ // we don't support permuted src0 dim0
+ GGML_ASSERT(nb00 == ggml_type_size(type));
+
+ // dst dim0 cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ // GGML_ASSERT(nb0 <= nb1);
+ // GGML_ASSERT(nb1 <= nb2);
+ // GGML_ASSERT(nb2 <= nb3);
+
+ GGML_ASSERT(ne0 == ne00);
+ GGML_ASSERT(ne1 == ne10);
+ GGML_ASSERT(ne2 == ne02);
+ GGML_ASSERT(ne3 == ne03);
+
+ // nb01 >= nb00 - src0 is not transposed
+ // compute by src0 rows
+
+ if (ith == 0) {
+ ggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);
+ }
+ ggml_barrier(params->threadpool);
+
+ // parallelize by last three dimensions
+
+ // total rows in dst
+ const int64_t nr = ne1*ne2*ne3;
+
+ // rows per thread
+ const int64_t dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int64_t ir0 = dr*ith;
+ const int64_t ir1 = MIN(ir0 + dr, nr);
+
+ // dst[:,:,:,:] = 0
+ // for i2,i3:
+ // for i1:
+ // for i01:
+ // for i0:
+ // dst[i0,i1,i2,i3] += src0[i0,i01,i2,i3] * src1[i1,i01,i2,i3]
+
+ float * wdata = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32) * ith;
+
+ for (int64_t ir = ir0; ir < ir1; ++ir) {
+ // dst indices
+ const int64_t i3 = ir/(ne2*ne1);
+ const int64_t i2 = (ir - i3*ne2*ne1)/ne1;
+ const int64_t i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ const int64_t i02 = i2;
+ const int64_t i03 = i3;
+
+ //const int64_t i10 = i1;
+ const int64_t i12 = i2;
+ const int64_t i13 = i3;
+
+ for (int64_t i01 = 0; i01 < ne01; ++i01) {
+ const int64_t i11 = i01;
+
+ float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));
+ float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));
+ float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
+
+ dequantize_row_q(s0, wdata, ne0);
+ ggml_vec_mad_f32(ne0, d, wdata, *s1);
+ }
+ }
+}
+
+static void ggml_compute_forward_out_prod(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_Q4_0_4_4:
+ case GGML_TYPE_Q4_0_4_8:
+ case GGML_TYPE_Q4_0_8_8:
+ {
+ ggml_compute_forward_out_prod_q_f32(params, dst);
+ } break;
+ case GGML_TYPE_F16:
+ {
+ GGML_ABORT("fatal error"); // todo
+ // ggml_compute_forward_out_prod_f16_f32(params, dst);
+ }
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_out_prod_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_scale
+
+static void ggml_compute_forward_scale_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+ GGML_ASSERT(ggml_is_contiguous(dst));
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+
+ // scale factor
+ float v;
+ memcpy(&v, dst->op_params, sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src0->ne[0];
+ const int nr = ggml_nrows(src0);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ const size_t nb01 = src0->nb[1];
+
+ const size_t nb1 = dst->nb[1];
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ if (dst->data != src0->data) {
+ // src0 is same shape as dst => same indices
+ memcpy((char *)dst->data + i1*nb1, (char *)src0->data + i1*nb01, nc * sizeof(float));
+ }
+ ggml_vec_scale_f32(nc, (float *) ((char *) dst->data + i1*nb1), v);
+ }
+}
+
+static void ggml_compute_forward_scale(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_scale_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_set
+
+static void ggml_compute_forward_set_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));
+
+ // view src0 and dst with these strides and data offset inbytes during set
+ // nb0 is implicitly element_size because src0 and dst are contiguous
+ size_t nb1 = ((int32_t *) dst->op_params)[0];
+ size_t nb2 = ((int32_t *) dst->op_params)[1];
+ size_t nb3 = ((int32_t *) dst->op_params)[2];
+ size_t offset = ((int32_t *) dst->op_params)[3];
+ bool inplace = (bool) ((int32_t *) dst->op_params)[4];
+
+ if (!inplace) {
+ if (params->ith == 0) {
+ // memcpy needs to be synchronized across threads to avoid race conditions.
+ // => do it in INIT phase
+ memcpy(
+ ((char *) dst->data),
+ ((char *) src0->data),
+ ggml_nbytes(dst));
+ }
+ ggml_barrier(params->threadpool);
+ }
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src1);
+ const int nc = src1->ne[0];
+
+ GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne)
+ GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)
+
+ // src0 and dst as viewed during set
+ const size_t nb0 = ggml_element_size(src0);
+
+ const int im0 = (ne10 == 0 ? 0 : ne10-1);
+ const int im1 = (ne11 == 0 ? 0 : ne11-1);
+ const int im2 = (ne12 == 0 ? 0 : ne12-1);
+ const int im3 = (ne13 == 0 ? 0 : ne13-1);
+
+ GGML_ASSERT(offset + im0*nb0 + im1*nb1 + im2*nb2 + im3*nb3 <= ggml_nbytes(dst));
+
+ GGML_ASSERT(nb10 == sizeof(float));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0 and dst are viewed with shape of src1 and offset
+ // => same indices
+ const int i3 = ir/(ne12*ne11);
+ const int i2 = (ir - i3*ne12*ne11)/ne11;
+ const int i1 = (ir - i3*ne12*ne11 - i2*ne11);
+
+ ggml_vec_cpy_f32(nc,
+ (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + offset),
+ (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));
+ }
+}
+
+static void ggml_compute_forward_set(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_set_f32(params, dst);
+ } break;
+ case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_Q4_0_4_4:
+ case GGML_TYPE_Q4_0_4_8:
+ case GGML_TYPE_Q4_0_8_8:
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_cpy
+
+static void ggml_compute_forward_cpy(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ ggml_compute_forward_dup(params, dst);
+}
+
+// ggml_compute_forward_cont
+
+static void ggml_compute_forward_cont(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ ggml_compute_forward_dup(params, dst);
+}
+
+// ggml_compute_forward_reshape
+
+static void ggml_compute_forward_reshape(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ // NOP
+ UNUSED(params);
+ UNUSED(dst);
+}
+
+// ggml_compute_forward_view
+
+static void ggml_compute_forward_view(
+ const struct ggml_compute_params * params,
+ const struct ggml_tensor * dst) {
+ // NOP
+ UNUSED(params);
+ UNUSED(dst);
+}
+
+// ggml_compute_forward_permute
+
+static void ggml_compute_forward_permute(
+ const struct ggml_compute_params * params,
+ const struct ggml_tensor * dst) {
+ // NOP
+ UNUSED(params);
+ UNUSED(dst);
+}
+
+// ggml_compute_forward_transpose
+
+static void ggml_compute_forward_transpose(
+ const struct ggml_compute_params * params,
+ const struct ggml_tensor * dst) {
+ // NOP
+ UNUSED(params);
+ UNUSED(dst);
+}
+
+// ggml_compute_forward_get_rows
+
+static void ggml_compute_forward_get_rows_q(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int64_t nc = ne00;
+ const int64_t nr = ggml_nelements(src1);
+
+ const enum ggml_type type = src0->type;
+ ggml_to_float_t const dequantize_row_q = type_traits[type].to_float;
+
+ assert(ne0 == nc);
+ assert(ne02 == ne11);
+ assert(nb00 == ggml_type_size(type));
+ assert(ggml_nrows(dst) == nr);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int64_t i = ir0; i < ir1; ++i) {
+ const int64_t i12 = i/(ne11*ne10);
+ const int64_t i11 = (i - i12*ne11*ne10)/ne10;
+ const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);
+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);
+
+ assert(i01 >= 0 && i01 < ne01);
+
+ dequantize_row_q(
+ (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),
+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);
+ }
+}
+
+static void ggml_compute_forward_get_rows_f16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int64_t nc = ne00;
+ const int64_t nr = ggml_nelements(src1);
+
+ assert(ne0 == nc);
+ assert(ne02 == ne11);
+ assert(nb00 == sizeof(ggml_fp16_t));
+ assert(ggml_nrows(dst) == nr);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int64_t i = ir0; i < ir1; ++i) {
+ const int64_t i12 = i/(ne11*ne10);
+ const int64_t i11 = (i - i12*ne11*ne10)/ne10;
+ const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);
+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);
+
+ assert(i01 >= 0 && i01 < ne01);
+
+ ggml_fp16_to_fp32_row(
+ (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),
+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);
+ }
+}
+
+static void ggml_compute_forward_get_rows_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int64_t nc = ne00;
+ const int64_t nr = ggml_nelements(src1);
+
+ assert(ne0 == nc);
+ assert(ne02 == ne11);
+ assert(nb00 == sizeof(ggml_bf16_t));
+ assert(ggml_nrows(dst) == nr);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int64_t i = ir0; i < ir1; ++i) {
+ const int64_t i12 = i/(ne11*ne10);
+ const int64_t i11 = (i - i12*ne11*ne10)/ne10;
+ const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);
+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);
+
+ assert(i01 >= 0 && i01 < ne01);
+
+ ggml_bf16_to_fp32_row(
+ (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),
+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);
+ }
+}
+
+static void ggml_compute_forward_get_rows_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int64_t nc = ne00;
+ const int64_t nr = ggml_nelements(src1);
+
+ assert(ne0 == nc);
+ assert(ne02 == ne11);
+ assert(nb00 == sizeof(float));
+ assert(ggml_nrows(dst) == nr);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int64_t i = ir0; i < ir1; ++i) {
+ const int64_t i12 = i/(ne11*ne10);
+ const int64_t i11 = (i - i12*ne11*ne10)/ne10;
+ const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);
+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);
+
+ assert(i01 >= 0 && i01 < ne01);
+
+ ggml_vec_cpy_f32(nc,
+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3),
+ (float *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03));
+ }
+}
+
+static void ggml_compute_forward_get_rows(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_Q4_0_4_4:
+ case GGML_TYPE_Q4_0_4_8:
+ case GGML_TYPE_Q4_0_8_8:
+ {
+ ggml_compute_forward_get_rows_q(params, dst);
+ } break;
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_get_rows_f16(params, dst);
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_get_rows_bf16(params, dst);
+ } break;
+ case GGML_TYPE_F32:
+ case GGML_TYPE_I32:
+ {
+ ggml_compute_forward_get_rows_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+
+ //static bool first = true;
+ //printf("ne0 = %d, ne1 = %d, ne2 = %d\n", dst->ne[0], dst->ne[1], dst->ne[2]);
+ //if (first) {
+ // first = false;
+ //} else {
+ // for (int k = 0; k < dst->ne[1]; ++k) {
+ // for (int j = 0; j < dst->ne[0]/16; ++j) {
+ // for (int i = 0; i < 16; ++i) {
+ // printf("%8.4f ", ((float *) dst->data)[k*dst->ne[0] + j*16 + i]);
+ // }
+ // printf("\n");
+ // }
+ // printf("\n");
+ // }
+ // printf("\n");
+ // exit(0);
+ //}
+}
+
+// ggml_compute_forward_get_rows_back
+
+static void ggml_compute_forward_get_rows_back_f32_f16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ GGML_ASSERT(ggml_is_contiguous(dst));
+
+ // ggml_compute_forward_dup_same_cont(params, opt0, dst);
+
+ memset(dst->data, 0, ggml_nbytes(dst));
+
+ const int nc = src0->ne[0];
+ const int nr = ggml_nelements(src1);
+
+ GGML_ASSERT( dst->ne[0] == nc);
+ GGML_ASSERT(src0->nb[0] == sizeof(ggml_fp16_t));
+
+ for (int i = 0; i < nr; ++i) {
+ const int r = ((int32_t *) src1->data)[i];
+
+ for (int j = 0; j < nc; ++j) {
+ ggml_fp16_t v = ((ggml_fp16_t *) ((char *) src0->data + i*src0->nb[1]))[j];
+ ((float *) ((char *) dst->data + r*dst->nb[1]))[j] += GGML_FP16_TO_FP32(v);
+ }
+ }
+}
+
+static void ggml_compute_forward_get_rows_back_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ GGML_ASSERT(ggml_is_contiguous(dst));
+
+ // ggml_compute_forward_dup_same_cont(params, opt0, dst);
+
+ memset(dst->data, 0, ggml_nbytes(dst));
+
+ const int nc = src0->ne[0];
+ const int nr = ggml_nelements(src1);
+
+ GGML_ASSERT( dst->ne[0] == nc);
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+
+ for (int i = 0; i < nr; ++i) {
+ const int r = ((int32_t *) src1->data)[i];
+
+ ggml_vec_add_f32(nc,
+ (float *) ((char *) dst->data + r*dst->nb[1]),
+ (float *) ((char *) dst->data + r*dst->nb[1]),
+ (float *) ((char *) src0->data + i*src0->nb[1]));
+ }
+}
+
+static void ggml_compute_forward_get_rows_back(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_get_rows_back_f32_f16(params, dst);
+ } break;
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_get_rows_back_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+
+ //static bool first = true;
+ //printf("ne0 = %d, ne1 = %d, ne2 = %d\n", dst->ne[0], dst->ne[1], dst->ne[2]);
+ //if (first) {
+ // first = false;
+ //} else {
+ // for (int k = 0; k < dst->ne[1]; ++k) {
+ // for (int j = 0; j < dst->ne[0]/16; ++j) {
+ // for (int i = 0; i < 16; ++i) {
+ // printf("%8.4f ", ((float *) dst->data)[k*dst->ne[0] + j*16 + i]);
+ // }
+ // printf("\n");
+ // }
+ // printf("\n");
+ // }
+ // printf("\n");
+ // exit(0);
+ //}
+}
+
+// ggml_compute_forward_diag
+
+static void ggml_compute_forward_diag_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ // TODO: handle transposed/permuted matrices
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ GGML_ASSERT(ne00 == ne0);
+ GGML_ASSERT(ne00 == ne1);
+ GGML_ASSERT(ne01 == 1);
+ GGML_ASSERT(ne02 == ne2);
+ GGML_ASSERT(ne03 == ne3);
+
+ GGML_ASSERT(nb00 == sizeof(float));
+ GGML_ASSERT(nb0 == sizeof(float));
+
+ for (int i3 = 0; i3 < ne3; i3++) {
+ for (int i2 = 0; i2 < ne2; i2++) {
+ for (int i1 = 0; i1 < ne1; i1++) {
+ float * d = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ float * s = (float *)((char *) src0->data + i3*nb03 + i2*nb02);
+ for (int i0 = 0; i0 < i1; i0++) {
+ d[i0] = 0;
+ }
+ d[i1] = s[i1];
+ for (int i0 = i1+1; i0 < ne0; i0++) {
+ d[i0] = 0;
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_diag(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_diag_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_diag_mask_inf
+
+static void ggml_compute_forward_diag_mask_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const float value) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int n_past = ((int32_t *) dst->op_params)[0];
+ const bool inplace = src0->data == dst->data;
+
+ GGML_ASSERT(n_past >= 0);
+
+ if (!inplace) {
+ if (ith == 0) {
+ // memcpy needs to be synchronized across threads to avoid race conditions.
+ // => do it in INIT phase
+ GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
+ GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));
+ memcpy(
+ ((char *) dst->data),
+ ((char *) src0->data),
+ ggml_nbytes(dst));
+ }
+ ggml_barrier(params->threadpool);
+ }
+
+ // TODO: handle transposed/permuted matrices
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+ const int nr = src0->ne[1];
+ const int nz = n/nr;
+
+ GGML_ASSERT( dst->nb[0] == sizeof(float));
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+
+ for (int k = 0; k < nz; k++) {
+ for (int j = ith; j < nr; j += nth) {
+ for (int i = n_past; i < nc; i++) {
+ if (i > n_past + j) {
+ *(float *)((char *) dst->data + k*dst->nb[2] + j*dst->nb[1] + i*dst->nb[0]) = value;
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_diag_mask_inf(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_diag_mask_f32(params, dst, -INFINITY);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+static void ggml_compute_forward_diag_mask_zero(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_diag_mask_f32(params, dst, 0);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_soft_max
+
+static void ggml_compute_forward_soft_max_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ assert(ggml_is_contiguous(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ float scale = 1.0f;
+ float max_bias = 0.0f;
+
+ memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));
+ memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));
+
+ // TODO: handle transposed/permuted matrices
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ //const int64_t ne11 = src1 ? src1->ne[1] : 1;
+
+ // TODO: is this supposed to be ceil instead of floor?
+ // https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L370
+ const uint32_t n_head = ne02;
+ const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));
+
+ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
+
+ const int nc = src0->ne[0];
+ const int nr = ggml_nrows(src0);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ float * wp = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith;
+
+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ // ALiBi
+ const uint32_t h = (i1/ne01)%ne02; // head
+ const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;
+
+ float * sp = (float *)((char *) src0->data + i1*src0->nb[1]);
+ float * dp = (float *)((char *) dst->data + i1*dst->nb[1]);
+
+ // broadcast the mask across rows
+ ggml_fp16_t * mp_f16 = src1 ? (ggml_fp16_t *)((char *) src1->data) + (i1%ne01)*ne00 : NULL;
+ float * mp_f32 = src1 ? (float *)((char *) src1->data) + (i1%ne01)*ne00 : NULL;
+
+ ggml_vec_cpy_f32 (nc, wp, sp);
+ ggml_vec_scale_f32(nc, wp, scale);
+ if (mp_f32) {
+ if (use_f16) {
+ for (int i = 0; i < nc; ++i) {
+ wp[i] += slope*GGML_FP16_TO_FP32(mp_f16[i]);
+ }
+ } else {
+ for (int i = 0; i < nc; ++i) {
+ wp[i] += slope*mp_f32[i];
+ }
+ }
+ }
+
+#ifndef NDEBUG
+ for (int i = 0; i < nc; ++i) {
+ //printf("p[%d] = %f\n", i, p[i]);
+ assert(!isnan(wp[i]));
+ }
+#endif
+
+ float max = -INFINITY;
+ ggml_vec_max_f32(nc, &max, wp);
+
+ ggml_float sum = ggml_vec_soft_max_f32(nc, dp, wp, max);
+ assert(sum > 0.0);
+
+ sum = 1.0/sum;
+ ggml_vec_scale_f32(nc, dp, sum);
+
+#ifndef NDEBUG
+ for (int i = 0; i < nc; ++i) {
+ assert(!isnan(dp[i]));
+ assert(!isinf(dp[i]));
+ }
+#endif
+ }
+}
+
+static void ggml_compute_forward_soft_max(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_soft_max_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+
+// ggml_compute_forward_soft_max_back
+
+static void ggml_compute_forward_soft_max_back_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+ GGML_ASSERT(ggml_is_contiguous(src1));
+ GGML_ASSERT(ggml_is_contiguous(dst));
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_are_same_shape(src1, dst));
+
+ // TODO: handle transposed/permuted matrices
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src0->ne[0];
+ const int nr = ggml_nrows(src0);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ float *dy = (float *)((char *) src0->data + i1*src0->nb[1]);
+ float *y = (float *)((char *) src1->data + i1*src1->nb[1]);
+ float *dx = (float *)((char *) dst->data + i1*dst->nb[1]);
+
+#ifndef NDEBUG
+ for (int i = 0; i < nc; ++i) {
+ //printf("p[%d] = %f\n", i, p[i]);
+ assert(!isnan(dy[i]));
+ assert(!isnan(y[i]));
+ }
+#endif
+ // Jii = yi - yi*yi
+ // Jij = -yi*yj
+ // J = diag(y)-y.T*y
+ // dx = J * dy
+ // dxk = sum_i(Jki * dyi)
+ // dxk = sum_i(-yk*yi * dyi) - (-yk*yk)*dyk + (yk - yk*yk)*dyk
+ // dxk = sum_i(-yk*yi * dyi) + yk*yk*dyk + yk*dyk - yk*yk*dyk
+ // dxk = sum_i(-yk*yi * dyi) + yk*dyk
+ // dxk = -yk * sum_i(yi * dyi) + yk*dyk
+ // dxk = -yk * dot(y, dy) + yk*dyk
+ // dxk = yk * (- dot(y, dy) + dyk)
+ // dxk = yk * (dyk - dot(y, dy))
+ //
+ // post-order:
+ // dot_y_dy := dot(y, dy)
+ // dx := dy
+ // dx := dx - dot_y_dy
+ // dx := dx * y
+
+ // linear runtime, no additional memory
+ float dot_y_dy = 0;
+ ggml_vec_dot_f32 (nc, &dot_y_dy, 0, y, 0, dy, 0, 1);
+ ggml_vec_cpy_f32 (nc, dx, dy);
+ ggml_vec_acc1_f32(nc, dx, -dot_y_dy);
+ ggml_vec_mul_f32 (nc, dx, dx, y);
+
+#ifndef NDEBUG
+ for (int i = 0; i < nc; ++i) {
+ assert(!isnan(dx[i]));
+ assert(!isinf(dx[i]));
+ }
+#endif
+ }
+}
+
+static void ggml_compute_forward_soft_max_back(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_soft_max_back_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_clamp
+
+static void ggml_compute_forward_clamp_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ float min;
+ float max;
+ memcpy(&min, (float *) dst->op_params + 0, sizeof(float));
+ memcpy(&max, (float *) dst->op_params + 1, sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ const size_t nb00 = src0->nb[0];
+ const size_t nb01 = src0->nb[1];
+
+ const size_t nb0 = dst->nb[0];
+ const size_t nb1 = dst->nb[1];
+
+ GGML_ASSERT( nb0 == sizeof(float));
+ GGML_ASSERT(nb00 == sizeof(float));
+
+ for (int j = ith; j < n; j += nth) {
+ float * dst_ptr = (float *) ((char *) dst->data + j*nb1);
+ float * src0_ptr = (float *) ((char *) src0->data + j*nb01);
+
+ for (int i = 0; i < nc; i++) {
+ dst_ptr[i] = MAX(MIN(src0_ptr[i], max), min);
+ }
+ }
+}
+
+static void ggml_compute_forward_clamp(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_clamp_f32(params, dst);
+ } break;
+ case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_Q8_K:
+ case GGML_TYPE_Q4_0_4_4:
+ case GGML_TYPE_Q4_0_4_8:
+ case GGML_TYPE_Q4_0_8_8:
+ case GGML_TYPE_I8:
+ case GGML_TYPE_I16:
+ case GGML_TYPE_I32:
+ case GGML_TYPE_I64:
+ case GGML_TYPE_F64:
+ case GGML_TYPE_COUNT:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_rope
+
+static float rope_yarn_ramp(const float low, const float high, const int i0) {
+ const float y = (i0 / 2 - low) / MAX(0.001f, high - low);
+ return 1 - MIN(1, MAX(0, y));
+}
+
+// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn
+// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.
+static void rope_yarn(
+ float theta_extrap, float freq_scale, float corr_dims[2], int64_t i0, float ext_factor, float mscale,
+ float * cos_theta, float * sin_theta) {
+ // Get n-d rotational scaling corrected for extrapolation
+ float theta_interp = freq_scale * theta_extrap;
+ float theta = theta_interp;
+ if (ext_factor != 0.0f) {
+ float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor;
+ theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;
+
+ // Get n-d magnitude scaling corrected for interpolation
+ mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);
+ }
+ *cos_theta = cosf(theta) * mscale;
+ *sin_theta = sinf(theta) * mscale;
+}
+
+// Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get
+// `corr_dim(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`
+static float ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, float base) {
+ return n_dims * logf(n_ctx_orig / (n_rot * 2 * (float)M_PI)) / (2 * logf(base));
+}
+
+static void ggml_rope_cache_init(
+ float theta_base, float freq_scale, const float * freq_factors, float corr_dims[2], int64_t ne0, float ext_factor, float mscale,
+ float * cache, float sin_sign, float theta_scale) {
+ // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py
+ float theta = theta_base;
+ for (int64_t i0 = 0; i0 < ne0; i0 += 2) {
+ const float ff = freq_factors ? freq_factors[i0/2] : 1.0f;
+ rope_yarn(
+ theta/ff, freq_scale, corr_dims, i0, ext_factor, mscale, &cache[i0 + 0], &cache[i0 + 1]
+ );
+ cache[i0 + 1] *= sin_sign;
+
+ theta *= theta_scale;
+ }
+}
+
+GGML_CALL void ggml_rope_yarn_corr_dims(
+ int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]
+) {
+ // start and end correction dims
+ float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base));
+ float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base));
+ dims[0] = MAX(0, start);
+ dims[1] = MIN(n_dims - 1, end);
+}
+
+static void ggml_compute_forward_rope_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const bool forward) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+ const struct ggml_tensor * src2 = dst->src[2];
+
+ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
+
+ //const int n_past = ((int32_t *) dst->op_params)[0];
+ const int n_dims = ((int32_t *) dst->op_params)[1];
+ const int mode = ((int32_t *) dst->op_params)[2];
+ //const int n_ctx = ((int32_t *) dst->op_params)[3];
+ const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
+
+ memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
+ memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
+ memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
+ memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
+ memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
+ memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ //printf("ne0: %d, ne1: %d, ne2: %d, ne3: %d\n", ne0, ne1, ne2, ne3);
+ //printf("n_past = %d, ne2 = %d\n", n_past, ne2);
+
+ GGML_ASSERT(nb00 == sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(dst);
+
+ GGML_ASSERT(n_dims <= ne0);
+ GGML_ASSERT(n_dims % 2 == 0);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ // row index used to determine which thread to use
+ int ir = 0;
+
+ const float theta_scale = powf(freq_base, -2.0f/n_dims);
+
+ float corr_dims[2];
+ ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
+
+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
+
+ const float * freq_factors = NULL;
+ if (src2 != NULL) {
+ GGML_ASSERT(src2->type == GGML_TYPE_F32);
+ GGML_ASSERT(src2->ne[0] >= n_dims / 2);
+ freq_factors = (const float *) src2->data;
+ }
+
+ // backward process uses inverse rotation by cos and sin.
+ // cos and sin build a rotation matrix, where the inverse is the transpose.
+ // this essentially just switches the sign of sin.
+ const float sin_sign = forward ? 1.0f : -1.0f;
+
+ const int32_t * pos = (const int32_t *) src1->data;
+
+ for (int64_t i3 = 0; i3 < ne3; i3++) {
+ for (int64_t i2 = 0; i2 < ne2; i2++) {
+ const int64_t p = pos[i2];
+
+ float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith;
+ ggml_rope_cache_init(p, freq_scale, freq_factors, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);
+
+ for (int64_t i1 = 0; i1 < ne1; i1++) {
+ if (ir++ < ir0) continue;
+ if (ir > ir1) break;
+
+ if (!is_neox) {
+ for (int64_t i0 = 0; i0 < n_dims; i0 += 2) {
+ const float cos_theta = cache[i0 + 0];
+ const float sin_theta = cache[i0 + 1];
+
+ const float * const src = (float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+ float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ const float x0 = src[0];
+ const float x1 = src[1];
+
+ dst_data[0] = x0*cos_theta - x1*sin_theta;
+ dst_data[1] = x0*sin_theta + x1*cos_theta;
+ }
+ } else {
+ for (int64_t i0 = 0; i0 < n_dims; i0 += 2) {
+ const int64_t ic = i0/2;
+
+ const float cos_theta = cache[i0 + 0];
+ const float sin_theta = cache[i0 + 1];
+
+ const float * const src = (float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
+ float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);
+
+ const float x0 = src[0];
+ const float x1 = src[n_dims/2];
+
+ dst_data[0] = x0*cos_theta - x1*sin_theta;
+ dst_data[n_dims/2] = x0*sin_theta + x1*cos_theta;
+ }
+ }
+
+ for (int64_t i0 = n_dims; i0 < ne0; i0 += 2) {
+ const float * const src = (float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+ float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ dst_data[0] = src[0];
+ dst_data[1] = src[1];
+ }
+ }
+ }
+ }
+}
+
+// TODO: deduplicate f16/f32 code
+static void ggml_compute_forward_rope_f16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const bool forward) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+ const struct ggml_tensor * src2 = dst->src[2];
+
+ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
+
+ //const int n_past = ((int32_t *) dst->op_params)[0];
+ const int n_dims = ((int32_t *) dst->op_params)[1];
+ const int mode = ((int32_t *) dst->op_params)[2];
+ //const int n_ctx = ((int32_t *) dst->op_params)[3];
+ const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
+ memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
+ memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
+ memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
+ memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
+ memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
+ memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ //printf("ne0: %d, ne1: %d, ne2: %d, ne3: %d\n", ne0, ne1, ne2, ne3);
+ //printf("n_past = %d, ne2 = %d\n", n_past, ne2);
+
+ GGML_ASSERT(nb0 == sizeof(ggml_fp16_t));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(dst);
+
+ GGML_ASSERT(n_dims <= ne0);
+ GGML_ASSERT(n_dims % 2 == 0);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ // row index used to determine which thread to use
+ int ir = 0;
+
+ const float theta_scale = powf(freq_base, -2.0f/n_dims);
+
+ float corr_dims[2];
+ ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
+
+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
+
+ const float * freq_factors = NULL;
+ if (src2 != NULL) {
+ GGML_ASSERT(src2->type == GGML_TYPE_F32);
+ GGML_ASSERT(src2->ne[0] >= n_dims / 2);
+ freq_factors = (const float *) src2->data;
+ }
+
+ // backward process uses inverse rotation by cos and sin.
+ // cos and sin build a rotation matrix, where the inverse is the transpose.
+ // this essentially just switches the sign of sin.
+ const float sin_sign = forward ? 1.0f : -1.0f;
+
+ const int32_t * pos = (const int32_t *) src1->data;
+
+ for (int64_t i3 = 0; i3 < ne3; i3++) {
+ for (int64_t i2 = 0; i2 < ne2; i2++) {
+ const int64_t p = pos[i2];
+
+ float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith;
+ ggml_rope_cache_init(p, freq_scale, freq_factors, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);
+
+ for (int64_t i1 = 0; i1 < ne1; i1++) {
+ if (ir++ < ir0) continue;
+ if (ir > ir1) break;
+
+ if (!is_neox) {
+ for (int64_t i0 = 0; i0 < n_dims; i0 += 2) {
+ const float cos_theta = cache[i0 + 0];
+ const float sin_theta = cache[i0 + 1];
+
+ const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+ ggml_fp16_t * dst_data = (ggml_fp16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ const float x0 = GGML_FP16_TO_FP32(src[0]);
+ const float x1 = GGML_FP16_TO_FP32(src[1]);
+
+ dst_data[0] = GGML_FP32_TO_FP16(x0*cos_theta - x1*sin_theta);
+ dst_data[1] = GGML_FP32_TO_FP16(x0*sin_theta + x1*cos_theta);
+ }
+ } else {
+ for (int64_t i0 = 0; i0 < n_dims; i0 += 2) {
+ const int64_t ic = i0/2;
+
+ const float cos_theta = cache[i0 + 0];
+ const float sin_theta = cache[i0 + 1];
+
+ const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
+ ggml_fp16_t * dst_data = (ggml_fp16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);
+
+ const float x0 = GGML_FP16_TO_FP32(src[0]);
+ const float x1 = GGML_FP16_TO_FP32(src[n_dims/2]);
+
+ dst_data[0] = GGML_FP32_TO_FP16(x0*cos_theta - x1*sin_theta);
+ dst_data[n_dims/2] = GGML_FP32_TO_FP16(x0*sin_theta + x1*cos_theta);
+ }
+ }
+
+ for (int64_t i0 = n_dims; i0 < ne0; i0 += 2) {
+ const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+ ggml_fp16_t * dst_data = (ggml_fp16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ dst_data[0] = src[0];
+ dst_data[1] = src[1];
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_rope(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_rope_f16(params, dst, true);
+ } break;
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_rope_f32(params, dst, true);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_rope_back
+
+static void ggml_compute_forward_rope_back(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_rope_f16(params, dst, false);
+ } break;
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_rope_f32(params, dst, false);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_conv_transpose_1d
+
+static void ggml_compute_forward_conv_transpose_1d_f16_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nk = ne00*ne01*ne02;
+
+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nb10 == sizeof(float));
+
+ if (ith == 0) {
+ memset(params->wdata, 0, params->wsize);
+
+ // permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)
+ {
+ ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;
+
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i02*nb02 + i01*nb01);
+ ggml_fp16_t * dst_data = wdata + i01*ne00*ne02;
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ dst_data[i00*ne02 + i02] = src[i00];
+ }
+ }
+ }
+ }
+
+ // permute source data (src1) from (L x Cin) to (Cin x L)
+ {
+ ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk;
+ ggml_fp16_t * dst_data = wdata;
+
+ for (int64_t i11 = 0; i11 < ne11; i11++) {
+ const float * const src = (float *)((char *) src1->data + i11*nb11);
+ for (int64_t i10 = 0; i10 < ne10; i10++) {
+ dst_data[i10*ne11 + i11] = GGML_FP32_TO_FP16(src[i10]);
+ }
+ }
+ }
+
+ // need to zero dst since we are accumulating into it
+ memset(dst->data, 0, ggml_nbytes(dst));
+ }
+ ggml_barrier(params->threadpool);
+
+ const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
+
+ // total rows in dst
+ const int nr = ne1;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;
+ ggml_fp16_t * const wdata_src = wdata + nk;
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ float * dst_data = (float *)((char *) dst->data + i1*nb1);
+ ggml_fp16_t * wdata_kernel = wdata + i1*ne02*ne00;
+ for (int i10 = 0; i10 < ne10; i10++) {
+ const int i1n = i10*ne11;
+ for (int i00 = 0; i00 < ne00; i00++) {
+ float v = 0;
+ ggml_vec_dot_f16(ne02, &v, 0,
+ (ggml_fp16_t *) wdata_src + i1n, 0,
+ (ggml_fp16_t *) wdata_kernel + i00*ne02, 0, 1);
+ dst_data[i10*s0 + i00] += v;
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_conv_transpose_1d_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nk = ne00*ne01*ne02;
+
+ GGML_ASSERT(nb00 == sizeof(float));
+ GGML_ASSERT(nb10 == sizeof(float));
+
+ if (ith == 0) {
+ memset(params->wdata, 0, params->wsize);
+
+ // prepare kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)
+ {
+ float * const wdata = (float *) params->wdata + 0;
+
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ const float * const src = (float *)((char *) src0->data + i02*nb02 + i01*nb01);
+ float * dst_data = wdata + i01*ne00*ne02;
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ dst_data[i00*ne02 + i02] = src[i00];
+ }
+ }
+ }
+ }
+
+ // prepare source data (src1)
+ {
+ float * const wdata = (float *) params->wdata + nk;
+ float * dst_data = wdata;
+
+ for (int64_t i11 = 0; i11 < ne11; i11++) {
+ const float * const src = (float *)((char *) src1->data + i11*nb11);
+ for (int64_t i10 = 0; i10 < ne10; i10++) {
+ dst_data[i10*ne11 + i11] = src[i10];
+ }
+ }
+ }
+
+ // need to zero dst since we are accumulating into it
+ memset(dst->data, 0, ggml_nbytes(dst));
+ }
+ ggml_barrier(params->threadpool);
+
+ const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
+
+ // total rows in dst
+ const int nr = ne1;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ float * const wdata = (float *) params->wdata + 0;
+ float * const wdata_src = wdata + nk;
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ float * dst_data = (float *)((char *) dst->data + i1*nb1);
+ float * wdata_kernel = wdata + i1*ne02*ne00;
+ for (int i10 = 0; i10 < ne10; i10++) {
+ const int i1n = i10*ne11;
+ for (int i00 = 0; i00 < ne00; i00++) {
+ float v = 0;
+ ggml_vec_dot_f32(ne02, &v, 0,
+ wdata_src + i1n, 0,
+ wdata_kernel + i00*ne02, 0, 1);
+ dst_data[i10*s0 + i00] += v;
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_conv_transpose_1d(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_conv_transpose_1d_f16_f32(params, dst);
+ } break;
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_conv_transpose_1d_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_im2col_f32
+// src0: kernel [OC, IC, KH, KW]
+// src1: image [N, IC, IH, IW]
+// dst: result [N, OH, OW, IC*KH*KW]
+static void ggml_compute_forward_im2col_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ GGML_TENSOR_BINARY_OP_LOCALS;
+
+ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
+ const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
+ const int32_t p1 = ((const int32_t *)(dst->op_params))[3];
+ const int32_t d0 = ((const int32_t *)(dst->op_params))[4];
+ const int32_t d1 = ((const int32_t *)(dst->op_params))[5];
+ const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t N = is_2D ? ne13 : ne12;
+ const int64_t IC = is_2D ? ne12 : ne11;
+ const int64_t IH = is_2D ? ne11 : 1;
+ const int64_t IW = ne10;
+
+ const int64_t KH = is_2D ? ne01 : 1;
+ const int64_t KW = ne00;
+
+ const int64_t OH = is_2D ? ne2 : 1;
+ const int64_t OW = ne1;
+
+ int ofs0 = is_2D ? nb13 : nb12;
+ int ofs1 = is_2D ? nb12 : nb11;
+
+ GGML_ASSERT(nb10 == sizeof(float));
+
+ // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
+ {
+ float * const wdata = (float *) dst->data;
+
+ for (int64_t in = 0; in < N; in++) {
+ for (int64_t ioh = 0; ioh < OH; ioh++) { // 1
+ for (int64_t iow = 0; iow < OW; iow++) {
+ for (int64_t iic = ith; iic < IC; iic += nth) {
+
+ // micro kernel
+ float * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
+ const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]
+
+ for (int64_t ikh = 0; ikh < KH; ikh++) { // 1
+ for (int64_t ikw = 0; ikw < KW; ikw++) {
+ const int64_t iiw = iow*s0 + ikw*d0 - p0;
+ const int64_t iih = ioh*s1 + ikh*d1 - p1;
+
+ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;
+ } else {
+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = (src_data[iih*IW + iiw]);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+
+// ggml_compute_forward_im2col_f16
+// src0: kernel [OC, IC, KH, KW]
+// src1: image [N, IC, IH, IW]
+// dst: result [N, OH, OW, IC*KH*KW]
+static void ggml_compute_forward_im2col_f16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F16);
+
+ GGML_TENSOR_BINARY_OP_LOCALS;
+
+ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
+ const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
+ const int32_t p1 = ((const int32_t *)(dst->op_params))[3];
+ const int32_t d0 = ((const int32_t *)(dst->op_params))[4];
+ const int32_t d1 = ((const int32_t *)(dst->op_params))[5];
+ const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t N = is_2D ? ne13 : ne12;
+ const int64_t IC = is_2D ? ne12 : ne11;
+ const int64_t IH = is_2D ? ne11 : 1;
+ const int64_t IW = ne10;
+
+ const int64_t KH = is_2D ? ne01 : 1;
+ const int64_t KW = ne00;
+
+ const int64_t OH = is_2D ? ne2 : 1;
+ const int64_t OW = ne1;
+
+ int ofs0 = is_2D ? nb13 : nb12;
+ int ofs1 = is_2D ? nb12 : nb11;
+
+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nb10 == sizeof(float));
+
+ // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
+ {
+ ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;
+
+ for (int64_t in = 0; in < N; in++) {
+ for (int64_t ioh = 0; ioh < OH; ioh++) { // 1
+ for (int64_t iow = 0; iow < OW; iow++) {
+ for (int64_t iic = ith; iic < IC; iic += nth) {
+
+ // micro kernel
+ ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
+ const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]
+
+ for (int64_t ikh = 0; ikh < KH; ikh++) { // 1
+ for (int64_t ikw = 0; ikw < KW; ikw++) {
+ const int64_t iiw = iow*s0 + ikw*d0 - p0;
+ const int64_t iih = ioh*s1 + ikh*d1 - p1;
+
+ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;
+ } else {
+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_FP16(src_data[iih*IW + iiw]);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_im2col(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ switch (dst->type) {
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_im2col_f16(params, dst);
+ } break;
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_im2col_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_im2col_back_f32
+
+static void ggml_compute_forward_im2col_back_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ GGML_TENSOR_BINARY_OP_LOCALS;
+
+ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
+ const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
+ const int32_t p1 = ((const int32_t *)(dst->op_params))[3];
+ const int32_t d0 = ((const int32_t *)(dst->op_params))[4];
+ const int32_t d1 = ((const int32_t *)(dst->op_params))[5];
+ const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t N = is_2D ? ne3 : ne2;
+ const int64_t IC = is_2D ? ne2 : ne1;
+ const int64_t IH = is_2D ? ne1 : 1;
+ const int64_t IW = ne0;
+
+ const int64_t KH = is_2D ? ne01 : 1;
+ const int64_t KW = ne00;
+
+ const int64_t OH = is_2D ? ne12 : 1;
+ const int64_t OW = ne11;
+
+ int ofs0 = is_2D ? nb3 : nb2;
+ int ofs1 = is_2D ? nb2 : nb1;
+
+ GGML_ASSERT(nb0 == sizeof(float));
+
+ // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
+ {
+ float * const wdata = (float *) dst->data;
+
+ for (int64_t in = 0; in < N; in++) {
+ for (int64_t iic = ith; iic < IC; iic += nth) {
+ for (int64_t iih = 0; iih < IH; iih++) {
+ for (int64_t iiw = 0; iiw < IW; iiw++) {
+
+ // micro kernel
+ float grad = 0.0f;
+ for (int64_t ikh = 0; ikh < KH; ikh++) {
+ for (int64_t ikw = 0; ikw < KW; ikw++) {
+ // For s0 > 1 some values were skipped over in the forward pass.
+ // These values have tmpw % s0 != 0 and need to be skipped in the backwards pass as well.
+ const int64_t tmpw = (iiw + p0 - ikw*d0);
+ if (tmpw % s0 != 0) {
+ continue;
+ }
+ const int64_t iow = tmpw / s0;
+
+ // Equivalent logic as above except for s1.
+ int64_t ioh;
+ if (is_2D) {
+ const int64_t tmph = iih + p1 - ikh*d1;
+
+ if (tmph % s1 != 0) {
+ continue;
+ }
+
+ ioh = tmph / s1;
+ } else {
+ ioh = 0;
+ }
+
+ if (iow < 0 || iow >= OW || ioh < 0 || ioh >= OH) {
+ continue;
+ }
+
+ const float * const src_data = (const float *) src1->data
+ + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
+ grad += src_data[iic*(KH*KW) + ikh*KW + ikw];
+ }
+ }
+ float * dst_data = (float *)((char *) wdata + (in*ofs0 + iic*ofs1)); // [IH, IW]
+ dst_data[iih*IW + iiw] = grad;
+ }
+ }
+ }
+ }
+ }
+}
+
+// ggml_compute_forward_conv_transpose_2d
+
+static void ggml_compute_forward_conv_transpose_2d(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nk = ne00*ne01*ne02*ne03;
+
+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nb10 == sizeof(float));
+
+ if (ith == 0) {
+ memset(params->wdata, 0, params->wsize);
+
+ // permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout)
+ {
+ ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;
+
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i03*nb03 + i02*nb02);
+ ggml_fp16_t * dst_data = wdata + i02*ne01*ne00*ne03;
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ dst_data[i01*ne00*ne03 + i00*ne03 + i03] = src[i01 * ne00 + i00];
+ }
+ }
+ }
+ }
+ }
+
+ // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh)
+ {
+ ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk;
+ for (int i12 = 0; i12 < ne12; i12++) {
+ for (int i11 = 0; i11 < ne11; i11++) {
+ const float * const src = (float *)((char *) src1->data + i12*nb12 + i11*nb11);
+ ggml_fp16_t * dst_data = wdata + i11*ne10*ne12;
+ for (int i10 = 0; i10 < ne10; i10++) {
+ dst_data[i10*ne12 + i12] = GGML_FP32_TO_FP16(src[i10]);
+ }
+ }
+ }
+ }
+
+ memset(dst->data, 0, ggml_nbytes(dst));
+ }
+ ggml_barrier(params->threadpool);
+
+ const int32_t stride = ggml_get_op_params_i32(dst, 0);
+
+ // total patches in dst
+ const int np = ne2;
+
+ // patches per thread
+ const int dp = (np + nth - 1)/nth;
+
+ // patch range for this thread
+ const int ip0 = dp*ith;
+ const int ip1 = MIN(ip0 + dp, np);
+
+ ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;
+ ggml_fp16_t * const wdata_src = wdata + nk;
+
+ for (int i2 = ip0; i2 < ip1; i2++) { // Cout
+ float * dst_data = (float *)((char *) dst->data + i2*nb2);
+ ggml_fp16_t * wdata_kernel = wdata + i2*ne01*ne00*ne03;
+ for (int i11 = 0; i11 < ne11; i11++) {
+ for (int i10 = 0; i10 < ne10; i10++) {
+ const int i1n = i11*ne10*ne12 + i10*ne12;
+ for (int i01 = 0; i01 < ne01; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ float v = 0;
+ ggml_vec_dot_f16(ne03, &v, 0,
+ wdata_src + i1n, 0,
+ wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
+ dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;
+ }
+ }
+ }
+ }
+ }
+}
+
+// ggml_compute_forward_pool_1d_sk_p0
+
+static void ggml_compute_forward_pool_1d_sk_p0(
+ const struct ggml_compute_params * params,
+ const enum ggml_op_pool op,
+ const int k,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src = dst->src[0];
+
+ assert(src->type == GGML_TYPE_F32 || src->type == GGML_TYPE_F16);
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ const char * cdata = (const char *)src->data;
+ const char * const data_end = cdata + ggml_nbytes(src);
+ float * drow = (float *)dst->data;
+
+ const int64_t rs = dst->ne[0];
+
+ while (cdata < data_end) {
+ const void * srow = (const void *)cdata;
+ int j = 0;
+ for (int64_t i = 0; i < rs; ++i) {
+ switch (op) {
+ case GGML_OP_POOL_AVG: drow[i] = 0; break;
+ case GGML_OP_POOL_MAX: drow[i] = -FLT_MAX; break;
+ case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error");
+ }
+ for (int ki = 0; ki < k; ++ki) {
+ const float srow_j = (src->type == GGML_TYPE_F32) ? ((const float*)srow)[j] : GGML_FP16_TO_FP32(((const ggml_fp16_t*)srow)[j]);
+ switch (op) {
+ case GGML_OP_POOL_AVG: drow[i] += srow_j; break;
+ case GGML_OP_POOL_MAX: if (srow_j > drow[i]) drow[i] = srow_j; break;
+ case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error");
+ }
+ ++j;
+ }
+ switch (op) {
+ case GGML_OP_POOL_AVG: drow[i] /= k; break;
+ case GGML_OP_POOL_MAX: break;
+ case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error");
+ }
+ }
+
+ cdata += src->nb[1];
+ drow += rs;
+ }
+}
+
+// ggml_compute_forward_pool_1d
+
+static void ggml_compute_forward_pool_1d(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const int32_t * opts = (const int32_t *)dst->op_params;
+ enum ggml_op_pool op = opts[0];
+ const int k0 = opts[1];
+ const int s0 = opts[2];
+ const int p0 = opts[3];
+ GGML_ASSERT(p0 == 0); // padding not supported
+ GGML_ASSERT(k0 == s0); // only s = k supported
+
+ ggml_compute_forward_pool_1d_sk_p0(params, op, k0, dst);
+}
+
+// ggml_compute_forward_pool_2d
+
+static void ggml_compute_forward_pool_2d(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src = dst->src[0];
+
+ assert(src->type == GGML_TYPE_F32 || src->type == GGML_TYPE_F16);
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ const int32_t * opts = (const int32_t *)dst->op_params;
+ enum ggml_op_pool op = opts[0];
+ const int k0 = opts[1];
+ const int k1 = opts[2];
+ const int s0 = opts[3];
+ const int s1 = opts[4];
+ const int p0 = opts[5];
+ const int p1 = opts[6];
+ const char * cdata = (const char*)src->data;
+ const char * const data_end = cdata + ggml_nbytes(src);
+
+ const int64_t px = dst->ne[0];
+ const int64_t py = dst->ne[1];
+ const int64_t pa = px * py;
+
+ float * dplane = (float *)dst->data;
+
+ const int ka = k0 * k1;
+ const int offset0 = -p0;
+ const int offset1 = -p1;
+
+ while (cdata < data_end) {
+ for (int oy = 0; oy < py; ++oy) {
+ float * const drow = dplane + oy * px;
+ for (int ox = 0; ox < px; ++ox) {
+ float * const out = drow + ox;
+ switch (op) {
+ case GGML_OP_POOL_AVG: *out = 0; break;
+ case GGML_OP_POOL_MAX: *out = -FLT_MAX; break;
+ case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error");
+ }
+
+ const int ix = offset0 + ox * s0;
+ const int iy = offset1 + oy * s1;
+
+ for (int ky = 0; ky < k1; ++ky) {
+ if (iy + ky < 0 || iy + ky >= src->ne[1]) continue;
+ const void * srow = (const void *)(cdata + src->nb[1] * (iy + ky));
+ for (int kx = 0; kx < k0; ++kx) {
+ int j = ix + kx;
+ if (j < 0 || j >= src->ne[0]) continue;
+ const float srow_j = (src->type == GGML_TYPE_F32) ? ((const float*)srow)[j] : GGML_FP16_TO_FP32(((const ggml_fp16_t*)srow)[j]);
+ switch (op) {
+ case GGML_OP_POOL_AVG: *out += srow_j; break;
+ case GGML_OP_POOL_MAX: if (srow_j > *out) *out = srow_j; break;
+ case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error");
+ }
+ }
+ }
+ switch (op) {
+ case GGML_OP_POOL_AVG: *out /= ka; break;
+ case GGML_OP_POOL_MAX: break;
+ case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error");
+ }
+ }
+ }
+
+ cdata += src->nb[2];
+ dplane += pa;
+ }
+}
+
+// ggml_compute_forward_pool_2d_back
+
+static void ggml_compute_forward_pool_2d_back(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src = dst->src[0];
+ const struct ggml_tensor * dstf = dst->src[1]; // forward tensor of dst
+
+ assert(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ const int32_t * opts = (const int32_t *)dst->op_params;
+ enum ggml_op_pool op = opts[0];
+ const int k0 = opts[1];
+ const int k1 = opts[2];
+ const int s0 = opts[3];
+ const int s1 = opts[4];
+ const int p0 = opts[5];
+ const int p1 = opts[6];
+
+ char * cdata = (char *) dst->data;
+ const char * cdataf = (const char *) dstf->data;
+ const char * const data_end = cdata + ggml_nbytes(dst);
+
+ GGML_ASSERT(params->ith == 0);
+ memset(cdata, 0, ggml_nbytes(dst));
+
+ const int64_t px = src->ne[0];
+ const int64_t py = src->ne[1];
+ const int64_t pa = px * py;
+
+ const float * splane = (const float *) src->data;
+
+ const int ka = k0 * k1;
+ const int offset0 = -p0;
+ const int offset1 = -p1;
+
+ while (cdata < data_end) {
+ for (int oy = 0; oy < py; ++oy) {
+ const float * const srow = splane + oy * px;
+ for (int ox = 0; ox < px; ++ox) {
+ const float grad0 = srow[ox];
+
+ const int ix = offset0 + ox * s0;
+ const int iy = offset1 + oy * s1;
+
+ if (op == GGML_OP_POOL_MAX) {
+ float maxval = -FLT_MAX;
+ int kxmax = -1;
+ int kymax = -1;
+
+ for (int ky = 0; ky < k1; ++ky) {
+ if (iy + ky < 0 || iy + ky >= dst->ne[1]) {
+ continue;
+ }
+ const void * drowf = (const void *)(cdataf + dst->nb[1] * (iy + ky));
+ for (int kx = 0; kx < k0; ++kx) {
+ int j = ix + kx;
+ if (j < 0 || j >= dst->ne[0]) {
+ continue;
+ }
+
+ const float val = dst->type == GGML_TYPE_F32 ?
+ ((const float *) drowf)[j] : GGML_FP16_TO_FP32(((const ggml_fp16_t *) drowf)[j]);
+ if (val <= maxval) {
+ continue;
+ }
+
+ maxval = val;
+ kxmax = kx;
+ kymax = ky;
+ }
+ }
+
+ if (kxmax == -1 || kymax == -1) {
+ continue;
+ }
+
+ void * drow = (void *)(cdata + dst->nb[1] * (iy + kymax));
+ const int j = ix + kxmax;
+ if (dst->type == GGML_TYPE_F32) {
+ ((float *) drow)[j] += grad0;
+ } else {
+ ((ggml_fp16_t *) drow)[j] = GGML_FP32_TO_FP16(grad0 + GGML_FP16_TO_FP32(((const ggml_fp16_t *) drow)[j]));
+ }
+ } else if (op == GGML_OP_POOL_AVG) {
+ const float grad = grad0 / ka;
+
+ for (int ky = 0; ky < k1; ++ky) {
+ if (iy + ky < 0 || iy + ky >= dst->ne[1]) {
+ continue;
+ }
+ void * drow = (void *)(cdata + dst->nb[1] * (iy + ky));
+ for (int kx = 0; kx < k0; ++kx) {
+ int j = ix + kx;
+ if (j < 0 || j >= dst->ne[0]) {
+ continue;
+ }
+
+ if (dst->type == GGML_TYPE_F32) {
+ ((float *) drow)[j] += grad;
+ } else {
+ ((ggml_fp16_t *) drow)[j] += GGML_FP32_TO_FP16(grad);
+ }
+ }
+ }
+ } else {
+ GGML_ASSERT(false);
+ }
+ }
+ }
+
+ cdata += dst->nb[2];
+ cdataf += dst->nb[2];
+ splane += pa;
+ }
+}
+
+// ggml_compute_forward_upscale
+
+static void ggml_compute_forward_upscale_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ const float sf0 = (float)ne0/src0->ne[0];
+ const float sf1 = (float)ne1/src0->ne[1];
+ const float sf2 = (float)ne2/src0->ne[2];
+ const float sf3 = (float)ne3/src0->ne[3];
+
+ // TODO: optimize
+
+ for (int64_t i3 = 0; i3 < ne3; i3++) {
+ const int64_t i03 = i3 / sf3;
+ for (int64_t i2 = ith; i2 < ne2; i2 += nth) {
+ const int64_t i02 = i2 / sf2;
+ for (int64_t i1 = 0; i1 < ne1; i1++) {
+ const int64_t i01 = i1 / sf1;
+ for (int64_t i0 = 0; i0 < ne0; i0++) {
+ const int64_t i00 = i0 / sf0;
+
+ const float * x = (float *)((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ float * y = (float *)((char *) dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
+
+ *y = *x;
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_upscale(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_upscale_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+
+// ggml_compute_forward_pad
+
+static void ggml_compute_forward_pad_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+ GGML_ASSERT( dst->nb[0] == sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ float * dst_ptr = (float *) dst->data;
+
+ // TODO: optimize
+
+ for (int64_t i2 = 0; i2 < ne2; ++i2) {
+ for (int64_t i1 = ith; i1 < ne1; i1 += nth) {
+ for (int64_t i0 = 0; i0 < ne0; ++i0) {
+ for (int64_t i3 = 0; i3 < ne3; ++i3) {
+ const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0;
+
+ const float * src_ptr = (const float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+
+ if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
+ dst_ptr[dst_idx] = *src_ptr;
+ } else {
+ dst_ptr[dst_idx] = 0;
+ }
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_pad(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_pad_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+
+// ggml_compute_forward_arange
+
+static void ggml_compute_forward_arange_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ GGML_ASSERT(dst->nb[0] == sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const float start = ggml_get_op_params_f32(dst, 0);
+ const float stop = ggml_get_op_params_f32(dst, 1);
+ const float step = ggml_get_op_params_f32(dst, 2);
+
+ const int64_t steps = (int64_t) ceilf((stop - start) / step);
+
+ GGML_ASSERT(ggml_nelements(dst) == steps);
+
+ for (int64_t i = ith; i < steps; i+= nth) {
+ float value = start + step * i;
+ ((float *)dst->data)[i] = value;
+ }
+}
+
+static void ggml_compute_forward_arange(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ switch (dst->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_arange_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+static void ggml_compute_forward_timestep_embedding_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ const int dim = ggml_get_op_params_i32(dst, 0);
+ const int max_period = ggml_get_op_params_i32(dst, 1);
+
+ int half = dim / 2;
+
+ for (int64_t i = 0; i < ne00; i++) {
+ float * embed_data = (float *)((char *) dst->data + i*nb1);
+ for (int64_t j = ith; j < half; j += nth) {
+ float timestep = ((float *)src0->data)[i];
+ float freq = (float)expf(-logf(max_period) * j / half);
+ float arg = timestep * freq;
+ embed_data[j] = cosf(arg);
+ embed_data[j + half] = sinf(arg);
+ }
+ if (dim % 2 != 0 && ith == 0) {
+ embed_data[dim] = 0.f;
+ }
+ }
+}
+
+static void ggml_compute_forward_timestep_embedding(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_timestep_embedding_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_argsort
+
+static void ggml_compute_forward_argsort_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ GGML_ASSERT(nb0 == sizeof(float));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t nr = ggml_nrows(src0);
+
+ enum ggml_sort_order order = (enum ggml_sort_order) ggml_get_op_params_i32(dst, 0);
+
+ for (int64_t i = ith; i < nr; i += nth) {
+ int32_t * dst_data = (int32_t *)((char *) dst->data + i*nb1);
+ const float * src_data = (float *)((char *) src0->data + i*nb01);
+
+ for (int64_t j = 0; j < ne0; j++) {
+ dst_data[j] = j;
+ }
+
+ // C doesn't have a functional sort, so we do a bubble sort instead
+ for (int64_t j = 0; j < ne0; j++) {
+ for (int64_t k = j + 1; k < ne0; k++) {
+ if ((order == GGML_SORT_ORDER_ASC && src_data[dst_data[j]] > src_data[dst_data[k]]) ||
+ (order == GGML_SORT_ORDER_DESC && src_data[dst_data[j]] < src_data[dst_data[k]])) {
+ int32_t tmp = dst_data[j];
+ dst_data[j] = dst_data[k];
+ dst_data[k] = tmp;
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_argsort(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_argsort_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_flash_attn_ext
+
+static void ggml_compute_forward_flash_attn_ext_f16(
+ const struct ggml_compute_params * params,
+ const struct ggml_tensor * q,
+ const struct ggml_tensor * k,
+ const struct ggml_tensor * v,
+ const struct ggml_tensor * mask,
+ struct ggml_tensor * dst) {
+
+ GGML_TENSOR_LOCALS(int64_t, neq, q, ne)
+ GGML_TENSOR_LOCALS(size_t, nbq, q, nb)
+ GGML_TENSOR_LOCALS(int64_t, nek, k, ne)
+ GGML_TENSOR_LOCALS(size_t, nbk, k, nb)
+ GGML_TENSOR_LOCALS(int64_t, nev, v, ne)
+ GGML_TENSOR_LOCALS(size_t, nbv, v, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t D = neq0;
+ const int64_t N = neq1;
+
+ GGML_ASSERT(ne0 == D);
+ GGML_ASSERT(ne2 == N);
+
+ // input tensor rows must be contiguous
+ GGML_ASSERT(nbq0 == ggml_type_size(q->type));
+ GGML_ASSERT(nbk0 == ggml_type_size(k->type));
+ GGML_ASSERT(nbv0 == ggml_type_size(v->type));
+
+ GGML_ASSERT(neq0 == D);
+ GGML_ASSERT(nek0 == D);
+ GGML_ASSERT(nev0 == D);
+
+ GGML_ASSERT(neq1 == N);
+ GGML_ASSERT(nev0 == D);
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ GGML_ASSERT(nb0 <= nb1);
+ GGML_ASSERT(nb1 <= nb2);
+ GGML_ASSERT(nb2 <= nb3);
+
+ // broadcast factors
+ const int64_t rk2 = neq2/nek2;
+ const int64_t rk3 = neq3/nek3;
+
+ const int64_t rv2 = neq2/nev2;
+ const int64_t rv3 = neq3/nev3;
+
+ // parallelize by q rows using ggml_vec_dot_f32
+
+ // total rows in q
+ const int nr = neq1*neq2*neq3;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ float scale = 1.0f;
+ float max_bias = 0.0f;
+ float logit_softcap = 0.0f;
+
+ memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));
+ memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));
+ memcpy(&logit_softcap, (float *) dst->op_params + 2, sizeof(float));
+
+ if (logit_softcap != 0) {
+ scale /= logit_softcap;
+ }
+
+ const uint32_t n_head = neq2;
+ const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));
+
+ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
+
+ enum ggml_type const k_vec_dot_type = type_traits[k->type].vec_dot_type;
+ ggml_from_float_t const q_to_vec_dot = type_traits[k_vec_dot_type].from_float;
+ ggml_vec_dot_t const kq_vec_dot = type_traits[k->type].vec_dot;
+ ggml_to_float_t const v_to_float = type_traits[v->type].to_float;
+
+ // loop over n_batch and n_head
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // q indices
+ const int iq3 = ir/(neq2*neq1);
+ const int iq2 = (ir - iq3*neq2*neq1)/neq1;
+ const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);
+
+ const uint32_t h = iq2; // head index
+ const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;
+
+ float S = 0.0f; // sum
+ float M = -INFINITY; // maximum KQ value
+
+ float * VKQ32 = (float *) params->wdata + ith*(3*D + CACHE_LINE_SIZE_F32); // FP32 VKQ accumulator
+ float * V32 = (VKQ32 + 1*D); // (temporary) FP32 V buffer
+ ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*D); // (temporary) FP16 VKQ accumulator
+ ggml_fp16_t * Q_q = (ggml_fp16_t *) (VKQ32 + 2*D); // (temporary) buffer for Q converted to quantized/FP16
+
+ if (v->type == GGML_TYPE_F16) {
+ memset(VKQ16, 0, D*sizeof(ggml_fp16_t));
+ } else {
+ memset(VKQ32, 0, D*sizeof(float));
+ }
+
+ const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1]) : NULL;
+
+ // k indices
+ const int ik3 = iq3 / rk3;
+ const int ik2 = iq2 / rk2;
+
+ // v indices
+ const int iv3 = iq3 / rv3;
+ const int iv2 = iq2 / rv2;
+
+ const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3));
+ q_to_vec_dot(pq, Q_q, D);
+
+ // online softmax / attention
+ // loop over n_kv and n_head_kv
+ // ref: https://arxiv.org/pdf/2112.05682.pdf
+ for (int64_t ic = 0; ic < nek1; ++ic) {
+ const float mv = mp ? slope*GGML_FP16_TO_FP32(mp[ic]) : 0.0f;
+ if (mv == -INFINITY) {
+ continue;
+ }
+
+ float s; // KQ value
+
+ const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);
+ kq_vec_dot(D, &s, 0, k_data, 0, Q_q, 0, 1);
+
+ s = s*scale; // scale KQ value
+
+ if (logit_softcap != 0.0f) {
+ s = logit_softcap*tanhf(s);
+ }
+
+ s += mv; // apply mask
+
+ const float Mold = M;
+
+ float ms = 1.0f; // upon new higher max val, scale VKQ and KQ sum with this value
+ float vs = 1.0f; // post-softmax KQ value, expf(s - M)
+
+ const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));
+
+ if (v->type == GGML_TYPE_F16) {
+ if (s > M) {
+ // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f
+ M = s;
+ ms = expf(Mold - M);
+
+ // V = V*expf(Mold - M)
+ ggml_vec_scale_f16(D, VKQ16, ms);
+ } else {
+ // no new maximum, ms == 1.0f, vs != 1.0f
+ vs = expf(s - M);
+ }
+
+ // V += v*expf(s - M)
+ ggml_vec_mad_f16(D, VKQ16, (const ggml_fp16_t *) v_data, vs);
+ } else {
+ if (s > M) {
+ // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f
+ M = s;
+ ms = expf(Mold - M);
+
+ // V = V*expf(Mold - M)
+ ggml_vec_scale_f32(D, VKQ32, ms);
+ } else {
+ // no new maximum, ms == 1.0f, vs != 1.0f
+ vs = expf(s - M);
+ }
+
+ v_to_float(v_data, V32, D);
+
+ // V += v*expf(s - M)
+ ggml_vec_mad_f32(D, VKQ32, V32, vs);
+ }
+
+ S = S*ms + vs; // scale and increment sum with partial sum
+ }
+
+ if (v->type == GGML_TYPE_F16) {
+ for (int64_t d = 0; d < D; ++d) {
+ VKQ32[d] = GGML_FP16_TO_FP32(VKQ16[d]);
+ }
+ }
+
+ // V /= S
+ const float S_inv = 1.0f/S;
+ ggml_vec_scale_f32(D, VKQ32, S_inv);
+
+ // dst indices
+ const int i1 = iq1;
+ const int i2 = iq2;
+ const int i3 = iq3;
+
+ // original
+ //memcpy((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3), V, nev0*sizeof(float));
+
+ // permute(0, 2, 1, 3)
+ memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, VKQ32, nb1);
+ }
+}
+
+static void ggml_compute_forward_flash_attn_ext(
+ const struct ggml_compute_params * params,
+ const struct ggml_tensor * q,
+ const struct ggml_tensor * k,
+ const struct ggml_tensor * v,
+ const struct ggml_tensor * mask,
+ struct ggml_tensor * dst) {
+ switch (dst->op_params[3]) {
+ case GGML_PREC_DEFAULT:
+ case GGML_PREC_F32:
+ {
+ // uses F32 accumulators
+ ggml_compute_forward_flash_attn_ext_f16(params, q, k, v, mask, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_flash_attn_back
+
+static void ggml_compute_forward_flash_attn_back_f32(
+ const struct ggml_compute_params * params,
+ const bool masked,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * q = dst->src[0];
+ const struct ggml_tensor * k = dst->src[1];
+ const struct ggml_tensor * v = dst->src[2];
+ const struct ggml_tensor * d = dst->src[3];
+
+ GGML_TENSOR_LOCALS(int64_t, neq, q, ne)
+ GGML_TENSOR_LOCALS(size_t, nbq, q, nb)
+ GGML_TENSOR_LOCALS(int64_t, nek, k, ne)
+ GGML_TENSOR_LOCALS(size_t, nbk, k, nb)
+ GGML_TENSOR_LOCALS(int64_t, nev, v, ne)
+ GGML_TENSOR_LOCALS(size_t, nbv, v, nb)
+ GGML_TENSOR_LOCALS(int64_t, ned, d, ne)
+ GGML_TENSOR_LOCALS(size_t, nbd, d, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t D = neq0;
+ const int64_t N = neq1;
+ const int64_t P = nek1 - N;
+ const int64_t M = P + N;
+
+ const int Mup = ggml_up(M, GGML_SOFT_MAX_UNROLL);
+ const int mxDM = MAX(D, Mup);
+
+ // GGML_ASSERT(ne0 == D);
+ // GGML_ASSERT(ne1 == N);
+ GGML_ASSERT(P >= 0);
+
+ GGML_ASSERT(nbq0 == sizeof(float));
+ GGML_ASSERT(nbk0 == sizeof(float));
+ GGML_ASSERT(nbv0 == sizeof(float));
+
+ GGML_ASSERT(neq0 == D);
+ GGML_ASSERT(nek0 == D);
+ GGML_ASSERT(nev1 == D);
+ GGML_ASSERT(ned0 == D);
+
+ GGML_ASSERT(neq1 == N);
+ GGML_ASSERT(nek1 == N + P);
+ GGML_ASSERT(nev1 == D);
+ GGML_ASSERT(ned1 == N);
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ GGML_ASSERT(nb0 <= nb1);
+ GGML_ASSERT(nb1 <= nb2);
+ GGML_ASSERT(nb2 <= nb3);
+
+ if (ith == 0) {
+ memset(dst->data, 0, nb0*ne0*ne1*ne2*ne3);
+ }
+ ggml_barrier(params->threadpool);
+
+ const int64_t elem_q = ggml_nelements(q);
+ const int64_t elem_k = ggml_nelements(k);
+
+ enum ggml_type result_type = dst->type;
+ GGML_ASSERT(ggml_blck_size(result_type) == 1);
+ const size_t tsize = ggml_type_size(result_type);
+
+ const size_t offs_q = 0;
+ const size_t offs_k = offs_q + GGML_PAD(elem_q * tsize, GGML_MEM_ALIGN);
+ const size_t offs_v = offs_k + GGML_PAD(elem_k * tsize, GGML_MEM_ALIGN);
+
+ void * grad_q = (char *) dst->data;
+ void * grad_k = (char *) dst->data + offs_k;
+ void * grad_v = (char *) dst->data + offs_v;
+
+ const size_t nbgq1 = nb0*neq0;
+ const size_t nbgq2 = nb0*neq0*neq1;
+ const size_t nbgq3 = nb0*neq0*neq1*neq2;
+
+ const size_t nbgk1 = nb0*nek0;
+ const size_t nbgk2 = nb0*nek0*nek1;
+ const size_t nbgk3 = nb0*nek0*nek1*neq2;
+
+ const size_t nbgv1 = nb0*nev0;
+ const size_t nbgv2 = nb0*nev0*nev1;
+ const size_t nbgv3 = nb0*nev0*nev1*neq2;
+
+ // parallelize by k rows using ggml_vec_dot_f32
+
+ // total rows in k
+ const int nr = nek2*nek3;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ const float scale = 1.0f/sqrtf(D);
+
+ //printf("P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\n", P, N, D, ir0, ir1, scale);
+
+ // how often k2 (and v2) is repeated in q2
+ int nrep = neq2/nek2;
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // q indices
+ const int ik3 = ir/(nek2);
+ const int ik2 = ir - ik3*nek2;
+
+ const int iq3 = ik3;
+ const int id3 = ik3;
+ const int iv3 = ik3;
+ const int iv2 = ik2;
+
+ for (int irep = 0; irep < nrep; ++irep) {
+ const int iq2 = ik2 + irep*nek2;
+ const int id2 = iq2;
+
+ // (ik2 + irep*nek2) % nek2 == ik2
+ for (int iq1 = 0; iq1 < neq1; ++iq1) {
+ const int id1 = iq1;
+
+ // not sure about CACHE_LINE_SIZE_F32..
+ // - maybe it must not be multiplied by 2 and excluded from .. in SM 1*(..) offset?
+ float * S = (float *) params->wdata + ith*2*(mxDM + CACHE_LINE_SIZE_F32) + 0*(mxDM+CACHE_LINE_SIZE_F32);
+ float * SM = (float *) params->wdata + ith*2*(mxDM + CACHE_LINE_SIZE_F32) + 1*(mxDM+CACHE_LINE_SIZE_F32);
+
+ for (int i = M; i < Mup; ++i) {
+ S[i] = -INFINITY;
+ }
+
+ const int64_t masked_begin = masked ? (P + iq1 + 1) : M;
+ for (int64_t ic = 0; ic < masked_begin; ++ic) {
+ // k indices
+ const int ik1 = ic;
+
+ // S indices
+ const int i1 = ik1;
+
+ ggml_vec_dot_f32(neq0,
+ S + i1, 0,
+ (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,
+ (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);
+ }
+
+ // scale
+ ggml_vec_scale_f32(masked_begin, S, scale);
+
+ for (int64_t i = masked_begin; i < M; i++) {
+ S[i] = -INFINITY;
+ }
+
+ // softmax
+ // exclude known -INF S[..] values from max and loop
+ // dont forget to set their SM values to zero
+ {
+ float max = -INFINITY;
+ ggml_vec_max_f32(masked_begin, &max, S);
+
+ ggml_float sum = 0.0;
+ {
+#ifdef GGML_SOFT_MAX_ACCELERATE
+ max = -max;
+ vDSP_vsadd(SM, 1, &max, SM, 1, Mup);
+ vvexpf(SM, SM, &Mup);
+ ggml_vec_sum_f32(Mup, &sum, SM);
+#else
+ sum = ggml_vec_soft_max_f32(Mup, SM, S, max);
+#endif
+ }
+
+ assert(sum > 0.0);
+
+ sum = 1.0/sum;
+ ggml_vec_scale_f32(masked_begin, SM, sum);
+
+ }
+
+ // step-by-step explanation
+ {
+ // forward-process shape grads from backward process
+ // parallel_for ik2,ik3:
+ // for irep:
+ // iq2 = ik2 + irep*nek2
+ // k[:D,:M,:,:] [D,M,:,:] grad[k][:D,:M,ik2,ik3] += grad[kcur]
+ // q[:D,:N,:,:] [D,N,:,:] grad[q][:D,iq1,iq2,iq3] += grad[qcur]
+ // v[:M,:D,:,:] [M,D,:,:] grad[v][:M,:D,iv2,iv3] += grad[vcur]
+ // for iq1:
+ // kcur = k[:D,:M,ik2,ik3] [D,M,1,1] grad[kcur] = grad[S1].T @ qcur
+ // qcur = q[:D,iq1,iq2,iq3] [D,1,1,1] grad[qcur] = grad[S1] @ kcur
+ // vcur = v[:M,:D,iv2,iv3] [M,D,1,1] grad[vcur] = grad[S5].T @ S4
+ // S0 = -Inf [D,1,1,1]
+ // ~S1[i] = dot(kcur[:D,i], qcur)
+ // S1 = qcur @ kcur.T [M,1,1,1] grad[S1] = grad[S2] * scale
+ // S2 = S1 * scale [M,1,1,1] grad[S2] = diag_mask_zero(grad[S3], P)
+ // S3 = diag_mask_inf(S2, P) [M,1,1,1] grad[S3] = S4 * (grad[S4] - dot(S4, grad[S4]))
+ // S4 = softmax(S3) [M,1,1,1] grad[S4] = grad[S5] @ vcur
+ // ~S5[i] = dot(vcur[:,i], S4)
+ // S5 = S4 @ vcur.T [D,1,1,1] grad[S5] = d[:D,id1,id2,id3]
+ // ~dst[i,iq1,iq2,iq3] = S5[i] ^
+ // dst[:D,iq1,iq2,iq3] = S5 | grad[dst[:D,iq1,iq2,iq3]] = d[:D,id1,id2,id3]
+ // dst backward-/ grad[dst] = d
+ //
+ // output gradients with their dependencies:
+ //
+ // grad[kcur] = grad[S1].T @ qcur
+ // grad[S1] = diag_mask_zero(grad[S3], P) * scale
+ // grad[S3] = S4 * (grad[S4] - dot(S4, grad[S4]))
+ // grad[S4] = grad[S5] @ vcur
+ // grad[S4] = d[:D,id1,id2,id3] @ vcur
+ // grad[qcur] = grad[S1] @ kcur
+ // grad[vcur] = grad[S5].T @ S4
+ // grad[vcur] = d[:D,id1,id2,id3].T @ S4
+ //
+ // in post-order:
+ //
+ // S1 = qcur @ kcur.T
+ // S2 = S1 * scale
+ // S3 = diag_mask_inf(S2, P)
+ // S4 = softmax(S3)
+ // grad[S4] = d[:D,id1,id2,id3] @ vcur
+ // grad[S3] = S4 * (grad[S4] - dot(S4, grad[S4]))
+ // grad[S1] = diag_mask_zero(grad[S3], P) * scale
+ // grad[qcur] = grad[S1] @ kcur
+ // grad[kcur] = grad[S1].T @ qcur
+ // grad[vcur] = d[:D,id1,id2,id3].T @ S4
+ //
+ // using less variables (SM=S4):
+ //
+ // S = diag_mask_inf(qcur @ kcur.T * scale, P)
+ // SM = softmax(S)
+ // S = d[:D,iq1,iq2,iq3] @ vcur
+ // dot_SM_gradSM = dot(SM, S)
+ // S = SM * (S - dot(SM, S))
+ // S = diag_mask_zero(S, P) * scale
+ //
+ // grad[q][:D,iq1,iq2,iq3] += S @ kcur
+ // grad[k][:D,:M,ik2,ik3] += S.T @ qcur
+ // grad[v][:M,:D,iv2,iv3] += d[:D,id1,id2,id3].T @ SM
+ }
+
+ // S = gradSM = d[:D,id1,id2,id3] @ vcur[:,:,iv2,iv3]
+ // S = d[:D,id1,id2,id3] @ vcur[:,:,iv2,iv3]
+ // for ic:
+ // S[:M] += vcur[:M,ic,iv2,iv3] * d[ic,id1,id2,id3]
+ // exclude known future zero S[..] values from operation
+ ggml_vec_set_f32(masked_begin, S, 0);
+ for (int64_t ic = 0; ic < D; ++ic) {
+ ggml_vec_mad_f32(masked_begin,
+ S,
+ (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)),
+ *(float *) ((char *) d->data + (ic*nbd0 + id1*nbd1 + id2*nbd2 + id3*nbd3)));
+ }
+
+ // S = SM * (S - dot(SM, S))
+ float dot_SM_gradSM = 0;
+ ggml_vec_dot_f32 (masked_begin, &dot_SM_gradSM, 0, SM, 0, S, 0, 1);
+ ggml_vec_acc1_f32(M, S, -dot_SM_gradSM);
+ ggml_vec_mul_f32 (masked_begin, S, S, SM);
+
+ // S = diag_mask_zero(S, P) * scale
+ // already done by above ggml_vec_set_f32
+
+ // exclude known zero S[..] values from operation
+ ggml_vec_scale_f32(masked_begin, S, scale);
+
+ // S shape [M,1]
+ // SM shape [M,1]
+ // kcur shape [D,M]
+ // qcur shape [D,1]
+ // vcur shape [M,D]
+
+ // grad[q][:D,iq1,iq2,iq3] += S @ kcur
+ // grad[q][:D,iq1,iq2,iq3] += shape[M,1] @ shape[D,M]
+ // for ic:
+ // grad[q][:D,iq1,iq2,iq3] += S[ic] * kcur[:D,ic,ik2,ik3]
+ // exclude known zero S[..] values from loop
+ for (int64_t ic = 0; ic < masked_begin; ++ic) {
+ ggml_vec_mad_f32(D,
+ (float *) ((char *) grad_q + (iq1*nbgq1 + iq2*nbgq2 + iq3*nbgq3)),
+ (float *) ((char *) k->data + (ic*nbk1 + ik2*nbk2 + ik3*nbk3)),
+ S[ic]);
+ }
+
+ // grad[k][:D,:M,iq2,iq3] += S.T @ qcur
+ // for ic:
+ // grad[k][:D,ic,iq2,iq3] += S.T[0,ic] * qcur[:D,0]
+ // grad[k][:D,ic,iq2,iq3] += S[ic] * qcur[:D,0]
+ // exclude known zero S[..] values from loop
+ for (int64_t ic = 0; ic < masked_begin; ++ic) {
+ ggml_vec_mad_f32(D,
+ (float *) ((char *) grad_k + (ic*nbgk1 + ik2*nbgk2 + ik3*nbgk3)),
+ (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)),
+ S[ic]);
+ }
+
+ // grad[v][:M,:D,iv2,iv3] += d[:D,id1,id2,id3].T @ SM
+ // for ic:
+ // grad[v][:M,ic,iv2,iv3] += d[:D,id1,id2,id3].T[0,ic] * SM[:M]
+ // grad[v][:M,ic,iv2,iv3] += d[ic,id1,id2,id3] * SM[:M]
+ // exclude known zero SM[..] values from mad
+ for (int64_t ic = 0; ic < D; ++ic) {
+ ggml_vec_mad_f32(masked_begin,
+ (float *) ((char *) grad_v + ( ic*nbgv1 + iv2*nbgv2 + iv3*nbgv3)),
+ SM,
+ *(float *) ((char *) d->data + (ic*nbd0 + id1*nbd1 + id2*nbd2 + id3*nbd3)));
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_flash_attn_back(
+ const struct ggml_compute_params * params,
+ const bool masked,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * q = dst->src[0];
+
+ switch (q->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_flash_attn_back_f32(params, masked, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_ssm_conv
+
+static void ggml_compute_forward_ssm_conv_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ const struct ggml_tensor * src0 = dst->src[0]; // conv_x
+ const struct ggml_tensor * src1 = dst->src[1]; // conv1d.weight
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src1->ne[0]; // d_conv
+ const int ncs = src0->ne[0]; // d_conv - 1 + n_t
+ const int nr = src0->ne[1]; // d_inner
+ const int n_t = dst->ne[1]; // tokens per sequence
+ const int n_s = dst->ne[2]; // number of sequences in the batch
+
+ GGML_ASSERT( dst->ne[0] == nr);
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+ GGML_ASSERT(src1->nb[0] == sizeof(float));
+ GGML_ASSERT(src0->nb[1] == src0->ne[0]*sizeof(float));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+ const int ir = ir1 - ir0;
+
+ for (int i3 = 0; i3 < n_s; ++i3) {
+ for (int i2 = 0; i2 < n_t; ++i2) {
+ // {d_conv - 1 + n_t, d_inner, n_seqs}
+ // sliding window
+ const float * s = (const float *) ((const char *) src0->data + ir0*(src0->nb[1]) + i2*(src0->nb[0]) + i3*(src0->nb[2])); // {d_conv, d_inner, n_s}
+ const float * c = (const float *) ((const char *) src1->data + ir0*(src1->nb[1])); // {d_conv, d_inner}
+ float * x = (float *) ((char *) dst->data + ir0*(dst->nb[0]) + i2*(dst->nb[1]) + i3*(dst->nb[2])); // {d_inner, n_t, n_s}
+
+ // TODO: transpose the output for smaller strides for big batches?
+ // d_inner
+ for (int i1 = 0; i1 < ir; ++i1) {
+ // rowwise dot product
+ // NOTE: not using ggml_vec_dot_f32, because its sum is in double precision
+ float sumf = 0.0f;
+
+ // d_conv
+ for (int i0 = 0; i0 < nc; ++i0) {
+ sumf += s[i0 + i1*ncs] * c[i0 + i1*nc];
+ }
+ x[i1] = sumf;
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_ssm_conv(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ switch (dst->src[0]->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_ssm_conv_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_ssm_scan
+
+static void ggml_compute_forward_ssm_scan_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ const struct ggml_tensor * src0 = dst->src[0]; // s
+ const struct ggml_tensor * src1 = dst->src[1]; // x
+ const struct ggml_tensor * src2 = dst->src[2]; // dt
+ const struct ggml_tensor * src3 = dst->src[3]; // A
+ const struct ggml_tensor * src4 = dst->src[4]; // B
+ const struct ggml_tensor * src5 = dst->src[5]; // C
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t nc = src0->ne[0]; // d_state
+ const int64_t nr = src0->ne[1]; // d_inner
+ const int64_t n_t = src1->ne[1]; // number of tokens per sequence
+ const int64_t n_s = src0->ne[2]; // number of sequences in the batch
+
+ GGML_ASSERT(ggml_nelements(src1) + ggml_nelements(src0) == ggml_nelements(dst));
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
+ GGML_ASSERT(src1->nb[0] == sizeof(float));
+ GGML_ASSERT(src2->nb[0] == sizeof(float));
+ GGML_ASSERT(src3->nb[0] == sizeof(float));
+ GGML_ASSERT(src4->nb[0] == sizeof(float));
+ GGML_ASSERT(src5->nb[0] == sizeof(float));
+ // required for the dot product between s and C
+ GGML_ASSERT(src0->nb[1] == src0->ne[0]*sizeof(float));
+ // required for per-sequence offsets for states
+ GGML_ASSERT(src0->nb[2] == src0->ne[0]*src0->ne[1]*sizeof(float));
+ // required to get correct offset for state destination (i.e. src1->nb[3])
+ GGML_ASSERT(src1->nb[3] == src1->ne[0]*src1->ne[1]*src1->ne[2]*sizeof(float));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+ const int ir = ir1 - ir0;
+
+ for (int i3 = 0; i3 < n_s; ++i3) {
+ for (int i2 = 0; i2 < n_t; ++i2) {
+ const float * s0 = (const float *) ((const char *) src0->data + ir0*(src0->nb[1]) + i3*(src0->nb[2])); // {d_state, d_inner, n_s}
+ const float * x = (const float *) ((const char *) src1->data + ir0*(src1->nb[0]) + i2*(src1->nb[1]) + i3*(src1->nb[2])); // {d_inner, n_t, n_s}
+ const float * dt = (const float *) ((const char *) src2->data + ir0*(src2->nb[0]) + i2*(src2->nb[1]) + i3*(src2->nb[2])); // {d_inner, n_t, n_s}
+ const float * A = (const float *) ((const char *) src3->data + ir0*(src3->nb[1])); // {d_state, d_inner}
+ const float * B = (const float *) ((const char *) src4->data + i2*(src4->nb[1]) + i3*(src4->nb[2])); // {d_state, n_t, n_s}
+ const float * C = (const float *) ((const char *) src5->data + i2*(src5->nb[1]) + i3*(src5->nb[2])); // {d_state, n_t, n_s}
+ float * y = ( float *) (( char *) dst->data + ir0*(src1->nb[0]) + i2*(src1->nb[1]) + i3*(src1->nb[2])); // {d_inner, n_t, n_s}
+ float * s = ( float *) (( char *) dst->data + ir0*(src0->nb[1]) + i3*(src0->nb[2]) + src1->nb[3]); // {d_state, d_inner, n_s}
+
+ // use the output as the source for the next token-wise iterations
+ if (i2 > 0) { s0 = s; }
+
+ // d_inner
+ for (int i1 = 0; i1 < ir; ++i1) {
+ // ref: https://github.com/state-spaces/mamba/blob/34076d664838588a3c97727b263478ab9f621a07/mamba_ssm/ops/triton/selective_state_update.py#L78
+ float dt_soft_plus = dt[i1] <= 20.0f ? log1pf(expf(dt[i1])) : dt[i1];
+ float x_dt = x[i1] * dt_soft_plus;
+ float sumf = 0.0f;
+ // d_state
+ for (int i0 = 0; i0 < nc; ++i0) {
+ int i = i0 + i1*nc;
+ // state = prev_state * dA + dB * x
+ float state = (s0[i] * expf(dt_soft_plus * A[i])) + (B[i0] * x_dt);
+ // y = rowwise_dotprod(state, C)
+ sumf += state * C[i0];
+ s[i] = state;
+ }
+ y[i1] = sumf;
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_ssm_scan(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ switch (dst->src[0]->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_ssm_scan_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_win_part
+
+static void ggml_compute_forward_win_part_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ UNUSED(params);
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+
+ const int32_t nep0 = ((const int32_t *)(dst->op_params))[0];
+ const int32_t nep1 = ((const int32_t *)(dst->op_params))[1];
+ const int32_t w = ((const int32_t *)(dst->op_params))[2];
+
+ assert(ne00 == ne0);
+ assert(ne3 == nep0*nep1);
+
+ // TODO: optimize / multi-thread
+ for (int py = 0; py < nep1; ++py) {
+ for (int px = 0; px < nep0; ++px) {
+ const int64_t i3 = py*nep0 + px;
+ for (int64_t i2 = 0; i2 < ne2; ++i2) {
+ for (int64_t i1 = 0; i1 < ne1; ++i1) {
+ for (int64_t i0 = 0; i0 < ne0; ++i0) {
+ const int64_t i02 = py*w + i2;
+ const int64_t i01 = px*w + i1;
+ const int64_t i00 = i0;
+
+ const int64_t i = i3*ne2*ne1*ne0 + i2*ne1*ne0 + i1*ne0 + i0;
+ const int64_t j = i02*ne01*ne00 + i01*ne00 + i00;
+
+ if (py*w + i2 >= ne02 || px*w + i1 >= ne01) {
+ ((float *) dst->data)[i] = 0.0f;
+ } else {
+ ((float *) dst->data)[i] = ((float *) src0->data)[j];
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_win_part(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_win_part_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_win_unpart
+
+static void ggml_compute_forward_win_unpart_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ UNUSED(params);
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+
+ const int32_t w = ((const int32_t *)(dst->op_params))[0];
+
+ // padding
+ const int px = (w - ne1%w)%w;
+ //const int py = (w - ne2%w)%w;
+
+ const int npx = (px + ne1)/w;
+ //const int npy = (py + ne2)/w;
+
+ assert(ne0 == ne00);
+
+ // TODO: optimize / multi-thread
+ for (int64_t i2 = 0; i2 < ne2; ++i2) {
+ for (int64_t i1 = 0; i1 < ne1; ++i1) {
+ for (int64_t i0 = 0; i0 < ne0; ++i0) {
+ const int ip2 = i2/w;
+ const int ip1 = i1/w;
+
+ const int64_t i02 = i2%w;
+ const int64_t i01 = i1%w;
+ const int64_t i00 = i0;
+
+ const int64_t i = (ip2*npx + ip1)*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00 + i00;
+ const int64_t j = i2*ne1*ne0 + i1*ne0 + i0;
+
+ ((float *) dst->data)[j] = ((float *) src0->data)[i];
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_win_unpart(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_win_unpart_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+//gmml_compute_forward_unary
+
+static void ggml_compute_forward_unary(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const enum ggml_unary_op op = ggml_get_unary_op(dst);
+
+ switch (op) {
+ case GGML_UNARY_OP_ABS:
+ {
+ ggml_compute_forward_abs(params, dst);
+ } break;
+ case GGML_UNARY_OP_SGN:
+ {
+ ggml_compute_forward_sgn(params, dst);
+ } break;
+ case GGML_UNARY_OP_NEG:
+ {
+ ggml_compute_forward_neg(params, dst);
+ } break;
+ case GGML_UNARY_OP_STEP:
+ {
+ ggml_compute_forward_step(params, dst);
+ } break;
+ case GGML_UNARY_OP_TANH:
+ {
+ ggml_compute_forward_tanh(params, dst);
+ } break;
+ case GGML_UNARY_OP_ELU:
+ {
+ ggml_compute_forward_elu(params, dst);
+ } break;
+ case GGML_UNARY_OP_RELU:
+ {
+ ggml_compute_forward_relu(params, dst);
+ } break;
+ case GGML_UNARY_OP_SIGMOID:
+ {
+ ggml_compute_forward_sigmoid(params, dst);
+ } break;
+ case GGML_UNARY_OP_GELU:
+ {
+ ggml_compute_forward_gelu(params, dst);
+ } break;
+ case GGML_UNARY_OP_GELU_QUICK:
+ {
+ ggml_compute_forward_gelu_quick(params, dst);
+ } break;
+ case GGML_UNARY_OP_SILU:
+ {
+ ggml_compute_forward_silu(params, dst);
+ } break;
+ case GGML_UNARY_OP_HARDSWISH:
+ {
+ ggml_compute_forward_hardswish(params, dst);
+ } break;
+ case GGML_UNARY_OP_HARDSIGMOID:
+ {
+ ggml_compute_forward_hardsigmoid(params, dst);
+ } break;
+ case GGML_UNARY_OP_EXP:
+ {
+ ggml_compute_forward_exp(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_get_rel_pos
+
+static void ggml_compute_forward_get_rel_pos_f16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ UNUSED(params);
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L292-L322
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ const int64_t w = ne1;
+
+ ggml_fp16_t * src0_data = (ggml_fp16_t *) src0->data;
+ ggml_fp16_t * dst_data = (ggml_fp16_t *) dst->data;
+
+ for (int64_t i2 = 0; i2 < ne2; ++i2) {
+ for (int64_t i1 = 0; i1 < ne1; ++i1) {
+ const int64_t pos = (w - i1 - 1) + i2;
+ for (int64_t i0 = 0; i0 < ne0; ++i0) {
+ dst_data[i2*ne1*ne0 + i1*ne0 + i0] = src0_data[pos*ne00 + i0];
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_get_rel_pos(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_get_rel_pos_f16(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_add_rel_pos
+
+static void ggml_compute_forward_add_rel_pos_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+ const struct ggml_tensor * src2 = dst->src[2];
+
+ const bool inplace = (bool) ((int32_t *) dst->op_params)[0];
+ if (!inplace) {
+ if (params->ith == 0) {
+ memcpy((char *) dst->data, (char *) src0->data, ggml_nbytes(dst));
+ }
+ ggml_barrier(params->threadpool);
+ }
+ // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L357-L359
+
+ float * src1_data = (float *) src1->data;
+ float * src2_data = (float *) src2->data;
+ float * dst_data = (float *) dst->data;
+
+ const int64_t ne10 = src1->ne[0];
+ const int64_t ne11 = src1->ne[1];
+ const int64_t ne12 = src1->ne[2];
+ const int64_t ne13 = src1->ne[3];
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ // total patches in dst
+ const int np = ne13;
+
+ // patches per thread
+ const int dp = (np + nth - 1)/nth;
+
+ // patch range for this thread
+ const int ip0 = dp*ith;
+ const int ip1 = MIN(ip0 + dp, np);
+
+ for (int64_t i13 = ip0; i13 < ip1; ++i13) {
+ for (int64_t i12 = 0; i12 < ne12; ++i12) {
+ for (int64_t i11 = 0; i11 < ne11; ++i11) {
+ const int64_t jp1 = i13*ne12*ne11*ne10 + i12*ne11*ne10 + i11*ne10;
+ for (int64_t i10 = 0; i10 < ne10; ++i10) {
+ const int64_t jp0 = jp1 + i10;
+ const float src1_e = src1_data[jp0];
+ const float src2_e = src2_data[jp0];
+
+ const int64_t jdh = jp0 * ne10;
+ const int64_t jdw = jdh - (ne10 - 1) * i10;
+
+ for (int64_t j = 0; j < ne10; ++j) {
+ dst_data[jdh + j ] += src2_e;
+ dst_data[jdw + j*ne10] += src1_e;
+ }
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_add_rel_pos(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_add_rel_pos_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_rwkv_wkv
+
+static void ggml_compute_forward_rwkv_wkv_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+ const size_t T = dst->src[1]->ne[3];
+ const size_t C = dst->ne[0];
+ const size_t H = dst->src[1]->ne[2];
+ const size_t n_seqs = dst->src[5]->ne[1];
+
+ float * dst_data = (float *) dst->data;
+ float * state = ((float *) dst->data) + C * T;
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ memset(dst_data, 0, T * C * sizeof(float));
+
+ float * k = (float *) dst->src[0]->data;
+ float * v = (float *) dst->src[1]->data;
+ float * r = (float *) dst->src[2]->data;
+ float * time_faaaa = (float *) dst->src[3]->data;
+ float * time_decay = (float *) dst->src[4]->data;
+
+ size_t t_stride = H * (C / H);
+
+ size_t h_stride = C / H;
+ size_t h_stride_2d = (C / H) * (C / H);
+
+ // basically fused operations:
+ // dst = r @ (time_faaaa * (k @ v) + state),
+ // state = time_decay * state + (k @ v),
+ // recursive through each token
+ for (size_t t = 0; t < T; t++) {
+ size_t t_offset = t * t_stride;
+ size_t state_offset = (C / H) * C * (t / (T / n_seqs));
+ float * state_cur = state + state_offset;
+ float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[5]->data + state_offset;
+
+ for (size_t h = 0; h < H; h++) {
+ size_t h_offset = h * h_stride;
+ size_t t_h_offset = t_offset + h_offset;
+ size_t h_2d_offset = h * h_stride_2d;
+
+ for (size_t i = 0; i < C / H; i++) {
+ size_t t_h_i_offset = t_h_offset + i;
+ size_t h_i_offset = h_offset + i;
+ size_t h_2d_i_offset = h_2d_offset + i * h_stride;
+
+ float k_val = k[t_h_i_offset];
+ float r_val = r[t_h_i_offset];
+ float time_faaaa_val = time_faaaa[h_i_offset];
+ // RWKV v6: different time_decay for each token.
+ float time_decay_val = time_decay[t_h_i_offset];
+
+ for (size_t j = 0; j < C / H; j ++) {
+ size_t t_h_j_offset = t_h_offset + j;
+ size_t h_2d_i_j_offset = h_2d_i_offset + j;
+
+ float v_val = v[t_h_j_offset];
+ float kv_val = v_val * k_val;
+ float prev_state_val = state_prev[h_2d_i_j_offset];
+ float temp_val = kv_val * time_faaaa_val + prev_state_val;
+ dst_data[t_h_j_offset] += temp_val * r_val;
+ state_cur[h_2d_i_j_offset] = prev_state_val * time_decay_val + kv_val;
+ }
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_rwkv_wkv(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_rwkv_wkv_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_map_unary
+
+static void ggml_compute_forward_map_unary_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const ggml_unary_op_f32_t fun) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ fun(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_map_unary(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const ggml_unary_op_f32_t fun) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_map_unary_f32(params, dst, fun);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_map_binary
+
+static void ggml_compute_forward_map_binary_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const ggml_binary_op_f32_t fun) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ assert(ggml_is_contiguous_1(src0));
+ assert(ggml_is_contiguous_1(src1));
+ assert(ggml_is_contiguous_1(dst));
+ assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
+
+ const int n = ggml_nrows(src0);
+ const int nc = src0->ne[0];
+
+ for (int i = 0; i < n; i++) {
+ fun(nc,
+ (float *) ((char *) dst->data + i*( dst->nb[1])),
+ (float *) ((char *) src0->data + i*(src0->nb[1])),
+ (float *) ((char *) src1->data + i*(src1->nb[1])));
+ }
+}
+
+static void ggml_compute_forward_map_binary(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const ggml_binary_op_f32_t fun) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_map_binary_f32(params, dst, fun);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_map_custom1
+
+static void ggml_compute_forward_map_custom1_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const ggml_custom1_op_f32_t fun) {
+
+ const struct ggml_tensor * a = dst->src[0];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ fun(dst, a);
+}
+
+// ggml_compute_forward_map_custom2
+
+static void ggml_compute_forward_map_custom2_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const ggml_custom2_op_f32_t fun) {
+
+ const struct ggml_tensor * a = dst->src[0];
+ const struct ggml_tensor * b = dst->src[1];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ fun(dst, a, b);
+}
+
+// ggml_compute_forward_map_custom3
+
+static void ggml_compute_forward_map_custom3_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const ggml_custom3_op_f32_t fun) {
+
+ const struct ggml_tensor * a = dst->src[0];
+ const struct ggml_tensor * b = dst->src[1];
+ const struct ggml_tensor * c = dst->src[1];
+
+ if (params->ith != 0) {
+ return;
+ }
+
+ fun(dst, a, b, c);
+}
+
+// ggml_compute_forward_map_custom1
+
+static void ggml_compute_forward_map_custom1(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * a = dst->src[0];
+
+ struct ggml_map_custom1_op_params p;
+ memcpy(&p, dst->op_params, sizeof(p));
+
+ p.fun(dst, a, params->ith, params->nth, p.userdata);
+}
+
+// ggml_compute_forward_map_custom2
+
+static void ggml_compute_forward_map_custom2(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * a = dst->src[0];
+ const struct ggml_tensor * b = dst->src[1];
+
+ struct ggml_map_custom2_op_params p;
+ memcpy(&p, dst->op_params, sizeof(p));
+
+ p.fun(dst, a, b, params->ith, params->nth, p.userdata);
+}
+
+// ggml_compute_forward_map_custom3
+
+static void ggml_compute_forward_map_custom3(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * a = dst->src[0];
+ const struct ggml_tensor * b = dst->src[1];
+ const struct ggml_tensor * c = dst->src[2];
+
+ struct ggml_map_custom3_op_params p;
+ memcpy(&p, dst->op_params, sizeof(p));
+
+ p.fun(dst, a, b, c, params->ith, params->nth, p.userdata);
+}
+
+// ggml_compute_forward_cross_entropy_loss
+
+static void ggml_compute_forward_cross_entropy_loss_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+ GGML_ASSERT(ggml_is_contiguous(src1));
+ GGML_ASSERT(ggml_is_scalar(dst));
+ GGML_ASSERT(ggml_are_same_shape(src0, src1));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ float * sums = (float *) params->wdata;
+
+ // TODO: handle transposed/permuted matrices
+ const int nc = src0->ne[0];
+ const int nr = ggml_nrows(src0);
+
+ GGML_ASSERT(params->wsize >= sizeof(float) * (nth + nth * nc));
+
+ if (ith == 0) {
+ memset(sums, 0, sizeof(float) * (nth + nth * nc));
+ }
+ ggml_barrier(params->threadpool);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ float * s0 = (float *)((char *) src0->data + i1*src0->nb[1]);
+ float * s1 = (float *)((char *) src1->data + i1*src1->nb[1]);
+ float * st = ((float *) params->wdata) + nth + ith*nc;
+
+#ifndef NDEBUG
+ for (int i = 0; i < nc; ++i) {
+ //printf("p[%d] = %f\n", i, p[i]);
+ assert(!isnan(s0[i]));
+ assert(!isnan(s1[i]));
+ }
+#endif
+
+ float max = -INFINITY;
+ ggml_vec_max_f32(nc, &max, s0);
+ ggml_float sum = ggml_vec_log_soft_max_f32(nc, st, s0, max);
+ assert(sum >= 0.0);
+
+ ggml_vec_add1_f32(nc, st, st, -sum);
+ ggml_vec_mul_f32(nc, st, st, s1);
+
+ float st_sum = 0.0f;
+ ggml_vec_sum_f32(nc, &st_sum, st);
+ sums[ith] += st_sum;
+
+#ifndef NDEBUG
+ for (int i = 0; i < nc; ++i) {
+ assert(!isnan(st[i]));
+ assert(!isinf(st[i]));
+ }
+#endif
+ }
+ ggml_barrier(params->threadpool);
+
+ if (ith == 0) {
+ float * dp = (float *) dst->data;
+ ggml_vec_sum_f32(nth, dp, sums);
+ dp[0] *= -1.0f / (float) nr;
+ }
+}
+
+static void ggml_compute_forward_cross_entropy_loss(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_cross_entropy_loss_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+// ggml_compute_forward_cross_entropy_loss_back
+
+static void ggml_compute_forward_cross_entropy_loss_back_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+ const struct ggml_tensor * opt0 = dst->src[2];
+
+ GGML_ASSERT(ggml_is_contiguous(dst));
+ GGML_ASSERT(ggml_is_contiguous(src0));
+ GGML_ASSERT(ggml_is_contiguous(src1));
+ GGML_ASSERT(ggml_is_contiguous(opt0));
+ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
+
+ const int64_t ith = params->ith;
+ const int64_t nth = params->nth;
+
+ // TODO: handle transposed/permuted matrices
+ const int64_t nc = src0->ne[0];
+ const int64_t nr = ggml_nrows(src0);
+
+ // rows per thread
+ const int64_t dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int64_t ir0 = dr*ith;
+ const int64_t ir1 = MIN(ir0 + dr, nr);
+
+ float * d = (float *) opt0->data;
+
+ for (int64_t i1 = ir0; i1 < ir1; i1++) {
+ float * ds0 = (float *)((char *) dst->data + i1*dst->nb[1]);
+ float * s0 = (float *)((char *) src0->data + i1*src0->nb[1]);
+ float * s1 = (float *)((char *) src1->data + i1*src1->nb[1]);
+
+#ifndef NDEBUG
+ for (int i = 0; i < nc; ++i) {
+ //printf("p[%d] = %f\n", i, p[i]);
+ assert(!isnan(s0[i]));
+ assert(!isnan(s1[i]));
+ }
+#endif
+
+ // soft_max
+ float max = -INFINITY;
+ ggml_vec_max_f32(nc, &max, s0);
+ ggml_float sum = ggml_vec_soft_max_f32(nc, ds0, s0, max);
+ assert(sum > 0.0);
+ ggml_vec_scale_f32(nc, ds0, 1.0/sum);
+
+ // grad(src0) = (softmax(src0) - src1) * grad(cross_entropy_loss(src0, src1)) / nr
+ ggml_vec_sub_f32(nc, ds0, ds0, s1);
+ ggml_vec_scale_f32(nc, ds0, d[0] / (float) nr);
+
+#ifndef NDEBUG
+ for (int i = 0; i < nc; ++i) {
+ assert(!isnan(ds0[i]));
+ assert(!isinf(ds0[i]));
+ }
+#endif
+ }
+}
+
+static void ggml_compute_forward_cross_entropy_loss_back(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_cross_entropy_loss_back_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+/////////////////////////////////
+
+static void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) {
+ GGML_ASSERT(params);
+
+ if (tensor->op == GGML_OP_NONE || ggml_is_empty(tensor)) {
+ return;
+ }
+
+ switch (tensor->op) {
+ case GGML_OP_DUP:
+ {
+ ggml_compute_forward_dup(params, tensor);
+ } break;
+ case GGML_OP_ADD:
+ {
+ ggml_compute_forward_add(params, tensor);
+ } break;
+ case GGML_OP_ADD1:
+ {
+ ggml_compute_forward_add1(params, tensor);
+ } break;
+ case GGML_OP_ACC:
+ {
+ ggml_compute_forward_acc(params, tensor);
+ } break;
+ case GGML_OP_SUB:
+ {
+ ggml_compute_forward_sub(params, tensor);
+ } break;
+ case GGML_OP_MUL:
+ {
+ ggml_compute_forward_mul(params, tensor);
+ } break;
+ case GGML_OP_DIV:
+ {
+ ggml_compute_forward_div(params, tensor);
+ } break;
+ case GGML_OP_SQR:
+ {
+ ggml_compute_forward_sqr(params, tensor);
+ } break;
+ case GGML_OP_SQRT:
+ {
+ ggml_compute_forward_sqrt(params, tensor);
+ } break;
+ case GGML_OP_LOG:
+ {
+ ggml_compute_forward_log(params, tensor);
+ } break;
+ case GGML_OP_SIN:
+ {
+ ggml_compute_forward_sin(params, tensor);
+ } break;
+ case GGML_OP_COS:
+ {
+ ggml_compute_forward_cos(params, tensor);
+ } break;
+ case GGML_OP_SUM:
+ {
+ ggml_compute_forward_sum(params, tensor);
+ } break;
+ case GGML_OP_SUM_ROWS:
+ {
+ ggml_compute_forward_sum_rows(params, tensor);
+ } break;
+ case GGML_OP_MEAN:
+ {
+ ggml_compute_forward_mean(params, tensor);
+ } break;
+ case GGML_OP_ARGMAX:
+ {
+ ggml_compute_forward_argmax(params, tensor);
+ } break;
+ case GGML_OP_REPEAT:
+ {
+ ggml_compute_forward_repeat(params, tensor);
+ } break;
+ case GGML_OP_REPEAT_BACK:
+ {
+ ggml_compute_forward_repeat_back(params, tensor);
+ } break;
+ case GGML_OP_CONCAT:
+ {
+ ggml_compute_forward_concat(params, tensor);
+ } break;
+ case GGML_OP_SILU_BACK:
+ {
+ ggml_compute_forward_silu_back(params, tensor);
+ } break;
+ case GGML_OP_NORM:
+ {
+ ggml_compute_forward_norm(params, tensor);
+ } break;
+ case GGML_OP_RMS_NORM:
+ {
+ ggml_compute_forward_rms_norm(params, tensor);
+ } break;
+ case GGML_OP_RMS_NORM_BACK:
+ {
+ ggml_compute_forward_rms_norm_back(params, tensor);
+ } break;
+ case GGML_OP_GROUP_NORM:
+ {
+ ggml_compute_forward_group_norm(params, tensor);
+ } break;
+ case GGML_OP_MUL_MAT:
+ {
+ ggml_compute_forward_mul_mat(params, tensor);
+ } break;
+ case GGML_OP_MUL_MAT_ID:
+ {
+ ggml_compute_forward_mul_mat_id(params, tensor);
+ } break;
+ case GGML_OP_OUT_PROD:
+ {
+ ggml_compute_forward_out_prod(params, tensor);
+ } break;
+ case GGML_OP_SCALE:
+ {
+ ggml_compute_forward_scale(params, tensor);
+ } break;
+ case GGML_OP_SET:
+ {
+ ggml_compute_forward_set(params, tensor);
+ } break;
+ case GGML_OP_CPY:
+ {
+ ggml_compute_forward_cpy(params, tensor);
+ } break;
+ case GGML_OP_CONT:
+ {
+ ggml_compute_forward_cont(params, tensor);
+ } break;
+ case GGML_OP_RESHAPE:
+ {
+ ggml_compute_forward_reshape(params, tensor);
+ } break;
+ case GGML_OP_VIEW:
+ {
+ ggml_compute_forward_view(params, tensor);
+ } break;
+ case GGML_OP_PERMUTE:
+ {
+ ggml_compute_forward_permute(params, tensor);
+ } break;
+ case GGML_OP_TRANSPOSE:
+ {
+ ggml_compute_forward_transpose(params, tensor);
+ } break;
+ case GGML_OP_GET_ROWS:
+ {
+ ggml_compute_forward_get_rows(params, tensor);
+ } break;
+ case GGML_OP_GET_ROWS_BACK:
+ {
+ ggml_compute_forward_get_rows_back(params, tensor);
+ } break;
+ case GGML_OP_DIAG:
+ {
+ ggml_compute_forward_diag(params, tensor);
+ } break;
+ case GGML_OP_DIAG_MASK_INF:
+ {
+ ggml_compute_forward_diag_mask_inf(params, tensor);
+ } break;
+ case GGML_OP_DIAG_MASK_ZERO:
+ {
+ ggml_compute_forward_diag_mask_zero(params, tensor);
+ } break;
+ case GGML_OP_SOFT_MAX:
+ {
+ ggml_compute_forward_soft_max(params, tensor);
+ } break;
+ case GGML_OP_SOFT_MAX_BACK:
+ {
+ ggml_compute_forward_soft_max_back(params, tensor);
+ } break;
+ case GGML_OP_ROPE:
+ {
+ ggml_compute_forward_rope(params, tensor);
+ } break;
+ case GGML_OP_ROPE_BACK:
+ {
+ ggml_compute_forward_rope_back(params, tensor);
+ } break;
+ case GGML_OP_CLAMP:
+ {
+ ggml_compute_forward_clamp(params, tensor);
+ } break;
+ case GGML_OP_CONV_TRANSPOSE_1D:
+ {
+ ggml_compute_forward_conv_transpose_1d(params, tensor);
+ } break;
+ case GGML_OP_IM2COL:
+ {
+ ggml_compute_forward_im2col(params, tensor);
+ } break;
+ case GGML_OP_IM2COL_BACK:
+ {
+ ggml_compute_forward_im2col_back_f32(params, tensor);
+ } break;
+ case GGML_OP_CONV_TRANSPOSE_2D:
+ {
+ ggml_compute_forward_conv_transpose_2d(params, tensor);
+ } break;
+ case GGML_OP_POOL_1D:
+ {
+ ggml_compute_forward_pool_1d(params, tensor);
+ } break;
+ case GGML_OP_POOL_2D:
+ {
+ ggml_compute_forward_pool_2d(params, tensor);
+ } break;
+ case GGML_OP_POOL_2D_BACK:
+ {
+ ggml_compute_forward_pool_2d_back(params, tensor);
+ } break;
+ case GGML_OP_UPSCALE:
+ {
+ ggml_compute_forward_upscale(params, tensor);
+ } break;
+ case GGML_OP_PAD:
+ {
+ ggml_compute_forward_pad(params, tensor);
+ } break;
+ case GGML_OP_ARANGE:
+ {
+ ggml_compute_forward_arange(params, tensor);
+ } break;
+ case GGML_OP_TIMESTEP_EMBEDDING:
+ {
+ ggml_compute_forward_timestep_embedding(params, tensor);
+ } break;
+ case GGML_OP_ARGSORT:
+ {
+ ggml_compute_forward_argsort(params, tensor);
+ } break;
+ case GGML_OP_LEAKY_RELU:
+ {
+ ggml_compute_forward_leaky_relu(params, tensor);
+ } break;
+ case GGML_OP_FLASH_ATTN_EXT:
+ {
+ ggml_compute_forward_flash_attn_ext(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor->src[3], tensor);
+ } break;
+ case GGML_OP_FLASH_ATTN_BACK:
+ {
+ int32_t t = ggml_get_op_params_i32(tensor, 0);
+ GGML_ASSERT(t == 0 || t == 1);
+ bool masked = t != 0;
+ ggml_compute_forward_flash_attn_back(params, masked, tensor);
+ } break;
+ case GGML_OP_SSM_CONV:
+ {
+ ggml_compute_forward_ssm_conv(params, tensor);
+ } break;
+ case GGML_OP_SSM_SCAN:
+ {
+ ggml_compute_forward_ssm_scan(params, tensor);
+ } break;
+ case GGML_OP_WIN_PART:
+ {
+ ggml_compute_forward_win_part(params, tensor);
+ } break;
+ case GGML_OP_WIN_UNPART:
+ {
+ ggml_compute_forward_win_unpart(params, tensor);
+ } break;
+ case GGML_OP_UNARY:
+ {
+ ggml_compute_forward_unary(params, tensor);
+ } break;
+ case GGML_OP_GET_REL_POS:
+ {
+ ggml_compute_forward_get_rel_pos(params, tensor);
+ } break;
+ case GGML_OP_ADD_REL_POS:
+ {
+ ggml_compute_forward_add_rel_pos(params, tensor);
+ } break;
+ case GGML_OP_RWKV_WKV:
+ {
+ ggml_compute_forward_rwkv_wkv(params, tensor);
+ } break;
+ case GGML_OP_MAP_UNARY:
+ {
+ ggml_unary_op_f32_t fun;
+ memcpy(&fun, tensor->op_params, sizeof(fun));
+ ggml_compute_forward_map_unary(params, tensor, fun);
+ }
+ break;
+ case GGML_OP_MAP_BINARY:
+ {
+ ggml_binary_op_f32_t fun;
+ memcpy(&fun, tensor->op_params, sizeof(fun));
+ ggml_compute_forward_map_binary(params, tensor, fun);
+ }
+ break;
+ case GGML_OP_MAP_CUSTOM1_F32:
+ {
+ ggml_custom1_op_f32_t fun;
+ memcpy(&fun, tensor->op_params, sizeof(fun));
+ ggml_compute_forward_map_custom1_f32(params, tensor, fun);
+ }
+ break;
+ case GGML_OP_MAP_CUSTOM2_F32:
+ {
+ ggml_custom2_op_f32_t fun;
+ memcpy(&fun, tensor->op_params, sizeof(fun));
+ ggml_compute_forward_map_custom2_f32(params, tensor, fun);
+ }
+ break;
+ case GGML_OP_MAP_CUSTOM3_F32:
+ {
+ ggml_custom3_op_f32_t fun;
+ memcpy(&fun, tensor->op_params, sizeof(fun));
+ ggml_compute_forward_map_custom3_f32(params, tensor, fun);
+ }
+ break;
+ case GGML_OP_MAP_CUSTOM1:
+ {
+ ggml_compute_forward_map_custom1(params, tensor);
+ }
+ break;
+ case GGML_OP_MAP_CUSTOM2:
+ {
+ ggml_compute_forward_map_custom2(params, tensor);
+ }
+ break;
+ case GGML_OP_MAP_CUSTOM3:
+ {
+ ggml_compute_forward_map_custom3(params, tensor);
+ }
+ break;
+ case GGML_OP_CROSS_ENTROPY_LOSS:
+ {
+ ggml_compute_forward_cross_entropy_loss(params, tensor);
+ }
+ break;
+ case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
+ {
+ ggml_compute_forward_cross_entropy_loss_back(params, tensor);
+ }
+ break;
+ case GGML_OP_NONE:
+ {
+ // nop
+ } break;
+ case GGML_OP_COUNT:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct ggml_hash_set ggml_hash_set_new(size_t size) {
+ size = ggml_hash_size(size);
+ struct ggml_hash_set result;
+ result.size = size;
+ result.keys = GGML_MALLOC(sizeof(struct ggml_tensor *) * size);
+ result.used = GGML_CALLOC(ggml_bitset_size(size), sizeof(ggml_bitset_t));
+ return result;
+}
+
+void ggml_hash_set_reset(struct ggml_hash_set * hash_set) {
+ memset(hash_set->used, 0, sizeof(ggml_bitset_t) * ggml_bitset_size(hash_set->size));
+}
+
+void ggml_hash_set_free(struct ggml_hash_set * hash_set) {
+ GGML_FREE(hash_set->used);
+ GGML_FREE(hash_set->keys);
+}
+
+size_t ggml_hash_size(size_t min_sz) {
+ // next primes after powers of two
+ static const size_t primes[] = {
+ 2, 3, 5, 11, 17, 37, 67, 131, 257, 521, 1031,
+ 2053, 4099, 8209, 16411, 32771, 65537, 131101,
+ 262147, 524309, 1048583, 2097169, 4194319, 8388617,
+ 16777259, 33554467, 67108879, 134217757, 268435459,
+ 536870923, 1073741827, 2147483659
+ };
+ static const size_t n_primes = sizeof(primes)/sizeof(primes[0]);
+
+ // find the smallest prime that is larger or equal than min_sz
+ size_t l = 0;
+ size_t r = n_primes;
+ while (l < r) {
+ size_t m = (l + r)/2;
+ if (primes[m] < min_sz) {
+ l = m + 1;
+ } else {
+ r = m;
+ }
+ }
+ size_t sz = l < n_primes ? primes[l] : min_sz | 1;
+ return sz;
+}
+
+struct hash_map {
+ struct ggml_hash_set set;
+ struct ggml_tensor ** vals;
+};
+
+static struct hash_map * ggml_new_hash_map(size_t size) {
+ struct hash_map * result = GGML_MALLOC(sizeof(struct hash_map));
+ result->set = ggml_hash_set_new(size);
+ result->vals = GGML_CALLOC(result->set.size, sizeof(struct ggml_tensor *));
+ return result;
+}
+
+static void ggml_hash_map_free(struct hash_map * map) {
+ ggml_hash_set_free(&map->set);
+ GGML_FREE(map->vals);
+ GGML_FREE(map);
+}
+
+// gradient checkpointing
+
+static struct ggml_tensor * ggml_recompute_graph_node(
+ struct ggml_context * ctx,
+ struct ggml_cgraph * graph,
+ struct hash_map * replacements,
+ struct ggml_tensor * node) {
+
+ if (node == NULL) {
+ return NULL;
+ }
+
+ if (node->flags & GGML_TENSOR_FLAG_PARAM) {
+ return node;
+ }
+
+ if (!ggml_hash_contains(&graph->visited_hash_set, node)) {
+ return node;
+ }
+
+ int count_children = 0;
+ for (int k = 0; k < GGML_MAX_SRC; ++k) {
+ if (node->src[k]) {
+ ++count_children;
+ }
+ }
+
+ if (count_children == 0) {
+ return node;
+ }
+
+ size_t i = ggml_hash_find(&replacements->set, node);
+ GGML_ASSERT(i != GGML_HASHSET_FULL); // assert that not full
+ if (replacements->set.keys[i] == node) {
+ return replacements->vals[i];
+ }
+
+ struct ggml_tensor * clone = ggml_new_tensor(ctx, node->type, GGML_MAX_DIMS, node->ne);
+
+ // insert clone into replacements
+ GGML_ASSERT(replacements->set.keys[i] == NULL); // assert that we don't overwrite
+ replacements->set.keys[i] = node;
+ replacements->vals[i] = clone;
+
+ clone->op = node->op;
+ clone->grad = node->grad;
+ clone->flags = node->flags;
+ clone->extra = node->extra;
+ for (int k = 0; k < GGML_MAX_DIMS; ++k) {
+ clone->nb[k] = node->nb[k];
+ }
+ for (int k = 0; k < GGML_MAX_SRC; ++k) {
+ clone->src[k] = ggml_recompute_graph_node(ctx, graph, replacements, node->src[k]);
+ }
+ if (node->view_src != NULL) {
+ clone->data = (node->view_src->data == NULL)
+ ? NULL // view_src not yet allocated
+ : (char *) node->view_src->data // view_src already allocated
+ + node->view_offs;
+ clone->view_src = node->view_src;
+ clone->view_offs = node->view_offs;
+ }
+
+ GGML_ASSERT(sizeof(node->op_params) == sizeof(int32_t) * (GGML_MAX_OP_PARAMS / sizeof(int32_t)));
+ GGML_ASSERT(sizeof(node->name) == GGML_MAX_NAME);
+ memcpy(clone->op_params, node->op_params, sizeof(node->op_params));
+ ggml_format_name(clone, "%s (clone)", ggml_get_name(node));
+
+ return clone;
+}
+
+void ggml_build_backward_gradient_checkpointing(
+ struct ggml_context * ctx,
+ struct ggml_cgraph * gf,
+ struct ggml_cgraph * gb,
+ struct ggml_cgraph * gb_tmp,
+ struct ggml_tensor * * checkpoints,
+ int n_checkpoints) {
+ ggml_graph_cpy(gf, gb_tmp);
+ ggml_build_backward_expand(ctx, gf, gb_tmp, true);
+
+ if (n_checkpoints <= 0) {
+ ggml_graph_cpy(gb_tmp, gb);
+ return;
+ }
+
+ struct hash_map * replacements = ggml_new_hash_map(gf->n_nodes + gf->n_leafs + n_checkpoints);
+
+ // insert checkpoints in replacements
+ for (int i = 0; i < n_checkpoints; ++i) {
+ size_t k = ggml_hash_find(&replacements->set, checkpoints[i]);
+ GGML_ASSERT(k != GGML_HASHSET_FULL); // assert that not full
+ GGML_ASSERT(replacements->set.keys[k] == NULL); // assert that we don't overwrite
+ replacements->set.keys[k] = checkpoints[i];
+ replacements->vals[k] = checkpoints[i];
+ }
+
+ ggml_graph_cpy(gf, gb);
+ // rewrite gb_tmp->nodes[gf->n_nodes:gb_tmp->n_nodes],
+ // replacing references to gb_tmp->nodes[0:gf->n_nodes] ( == gf->nodes[0:gf->n_nodes]),
+ // by recomputing them from checkpoints
+ for (int i = gf->n_nodes; i<gb_tmp->n_nodes; ++i) {
+ struct ggml_tensor * node = gb_tmp->nodes[i];
+ for (int k = 0; k < GGML_MAX_SRC; ++k) {
+ // insert new tensors recomputing src, reusing already made replacements,
+ // remember replacements: remember new tensors with mapping from corresponding gf nodes
+ // recurse for input tensors,
+ // unless (i.e. terminating when) input tensors are replacements (like checkpoints)
+ node->src[k] = ggml_recompute_graph_node(ctx, gf, replacements, node->src[k]);
+ }
+ // insert rewritten backward node with replacements made into resulting backward graph gb
+ ggml_build_forward_expand(gb, node);
+ }
+
+ ggml_hash_map_free(replacements);
+}
+
+// functions to change gradients considering the case that input a might be initial gradient with zero value
+
+static struct ggml_tensor * ggml_add_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, struct ggml_hash_set * zero_table) {
+ if (ggml_hash_contains(zero_table, a)) {
+ return b;
+ } else {
+ return ggml_add_impl(ctx, a, b, false);
+ }
+}
+
+static struct ggml_tensor * ggml_acc_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, size_t nb1, size_t nb2, size_t nb3, size_t offset, struct ggml_hash_set * zero_table) {
+ if (ggml_hash_contains(zero_table, a)) {
+ struct ggml_tensor * a_zero = ggml_scale(ctx, a, 0.0f);
+ return ggml_acc_impl(ctx, a_zero, b, nb1, nb2, nb3, offset, false);
+ } else {
+ return ggml_acc_impl(ctx, a, b, nb1, nb2, nb3, offset, false);
+ }
+}
+
+static struct ggml_tensor * ggml_add1_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, struct ggml_hash_set * zero_table) {
+ if (ggml_hash_contains(zero_table, a)) {
+ return ggml_repeat(ctx, b, a);
+ } else {
+ return ggml_add1_impl(ctx, a, b, false);
+ }
+}
+
+static struct ggml_tensor * ggml_sub_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, struct ggml_hash_set * zero_table) {
+ if (ggml_hash_contains(zero_table, a)) {
+ return ggml_neg(ctx, b);
+ } else {
+ return ggml_sub_impl(ctx, a, b, false);
+ }
+}
+
+static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor * tensor, struct ggml_hash_set * zero_table) {
+ struct ggml_tensor * src0 = tensor->src[0];
+ struct ggml_tensor * src1 = tensor->src[1];
+ struct ggml_tensor * src2 = tensor->src[2];
+
+ switch (tensor->op) {
+ case GGML_OP_DUP:
+ {
+ if (src0->grad) {
+ src0->grad = ggml_add_or_set(ctx, src0->grad, tensor->grad, zero_table);
+ }
+ } break;
+ case GGML_OP_ADD:
+ {
+ if (src0->grad) {
+ src0->grad = ggml_add_or_set(ctx, src0->grad, tensor->grad, zero_table);
+ }
+ if (src1->grad) {
+ if (ggml_are_same_shape(src0, src1)) {
+ src1->grad = ggml_add_or_set(ctx, src1->grad, tensor->grad, zero_table);
+ } else {
+ src1->grad = ggml_add_or_set(ctx, src1->grad, ggml_repeat_back(ctx, tensor->grad, src1), zero_table);
+ }
+ }
+ } break;
+ case GGML_OP_ADD1:
+ {
+ if (src0->grad) {
+ src0->grad = ggml_add_or_set(ctx, src0->grad, tensor->grad, zero_table);
+ }
+ if (src1->grad) {
+ src1->grad = ggml_add_or_set(ctx,
+ src1->grad,
+ ggml_mean(ctx, tensor->grad), // TODO: should probably be sum instead of mean
+ zero_table);
+ }
+ } break;
+ case GGML_OP_ACC:
+ {
+ if (src0->grad) {
+ src0->grad = ggml_add_or_set(ctx, src0->grad, tensor->grad, zero_table);
+ }
+ if (src1->grad) {
+ const size_t nb1 = ((int32_t *) tensor->op_params)[0];
+ const size_t nb2 = ((int32_t *) tensor->op_params)[1];
+ const size_t nb3 = ((int32_t *) tensor->op_params)[2];
+ const size_t offset = ((int32_t *) tensor->op_params)[3];
+
+ struct ggml_tensor * tensor_grad_view = ggml_view_4d(ctx,
+ tensor->grad,
+ src1->grad->ne[0],
+ src1->grad->ne[1],
+ src1->grad->ne[2],
+ src1->grad->ne[3],
+ nb1, nb2, nb3, offset);
+
+ src1->grad =
+ ggml_add_or_set(ctx,
+ src1->grad,
+ ggml_reshape(ctx,
+ ggml_cont(ctx, tensor_grad_view),
+ src1->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_SUB:
+ {
+ if (src0->grad) {
+ src0->grad = ggml_add_or_set(ctx, src0->grad, tensor->grad, zero_table);
+ }
+ if (src1->grad) {
+ src1->grad = ggml_sub_or_set(ctx, src1->grad, tensor->grad, zero_table);
+ }
+ } break;
+ case GGML_OP_MUL:
+ {
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_mul(ctx, src1, tensor->grad),
+ zero_table);
+ }
+ if (src1->grad) {
+ src1->grad =
+ ggml_add_or_set(ctx,
+ src1->grad,
+ ggml_mul(ctx, src0, tensor->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_DIV:
+ {
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_div(ctx, tensor->grad, src1),
+ zero_table);
+ }
+ if (src1->grad) {
+ src1->grad =
+ ggml_sub_or_set(ctx,
+ src1->grad,
+ ggml_mul(ctx,
+ tensor->grad,
+ ggml_div(ctx, tensor, src1)),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_SQR:
+ {
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_scale(ctx,
+ ggml_mul(ctx, src0, tensor->grad),
+ 2.0f),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_SQRT:
+ {
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_scale(ctx,
+ ggml_div(ctx,
+ tensor->grad,
+ tensor),
+ 0.5f),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_LOG:
+ {
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_div(ctx,
+ tensor->grad,
+ src0),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_SIN:
+ {
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_mul(ctx,
+ tensor->grad,
+ ggml_cos(ctx, src0)),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_COS:
+ {
+ if (src0->grad) {
+ src0->grad =
+ ggml_sub_or_set(ctx,
+ src0->grad,
+ ggml_mul(ctx,
+ tensor->grad,
+ ggml_sin(ctx, src0)),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_SUM:
+ {
+ if (src0->grad) {
+ src0->grad =
+ ggml_add1_or_set(ctx,
+ src0->grad,
+ tensor->grad,
+ zero_table);
+ }
+ } break;
+ case GGML_OP_SUM_ROWS:
+ {
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_repeat(ctx,
+ tensor->grad,
+ src0->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_MEAN:
+ case GGML_OP_ARGMAX:
+ {
+ GGML_ABORT("fatal error"); // TODO: implement
+ }
+ case GGML_OP_REPEAT:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_repeat_back(ctx, tensor->grad, src0->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_REPEAT_BACK:
+ {
+ if (src0->grad) {
+ // TODO: test this
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_repeat(ctx, tensor->grad, src0->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_CONCAT:
+ {
+ GGML_ABORT("fatal error"); // TODO: implement
+ }
+ case GGML_OP_SILU_BACK:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_NORM:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_RMS_NORM:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ float eps;
+ memcpy(&eps, tensor->op_params, sizeof(float));
+
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_rms_norm_back(ctx, src0, tensor->grad, eps),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_RMS_NORM_BACK:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_GROUP_NORM:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_MUL_MAT:
+ {
+ // https://cs231n.github.io/optimization-2/#staged
+ // # forward pass
+ // s0 = np.random.randn(5, 10)
+ // s1 = np.random.randn(10, 3)
+ // t = s0.dot(s1)
+
+ // # now suppose we had the gradient on t from above in the circuit
+ // dt = np.random.randn(*t.shape) # same shape as t
+ // ds0 = dt.dot(s1.T) #.T gives the transpose of the matrix
+ // ds1 = t.T.dot(dt)
+
+ // tensor.shape [m,p,qq,rr]
+ // src0.shape [n,m,q1,r1]
+ // src1.shape [n,p,qq,rr]
+
+ // necessary for llama
+ if (src0->grad) {
+ struct ggml_tensor * s1_tg =
+ ggml_out_prod(ctx, // [n,m,qq,rr]
+ src1, // [n,p,qq,rr]
+ tensor->grad); // [m,p,qq,rr]
+ const int64_t qq = s1_tg->ne[2];
+ const int64_t rr = s1_tg->ne[3];
+ const int64_t q1 = src0->ne[2];
+ const int64_t r1 = src0->ne[3];
+ const bool ne2_broadcasted = qq > q1;
+ const bool ne3_broadcasted = rr > r1;
+ if (ne2_broadcasted || ne3_broadcasted) {
+ // sum broadcast repetitions of s1_tg into shape of src0
+ s1_tg = ggml_repeat_back(ctx, s1_tg, src0);
+ }
+ src0->grad =
+ ggml_add_or_set(ctx,
+ src0->grad, // [n,m,q1,r1]
+ s1_tg, // [n,m,q1,r1]
+ zero_table);
+ }
+ if (src1->grad) {
+ src1->grad =
+ ggml_add_or_set(ctx,
+ src1->grad, // [n,p,qq,rr]
+ // ggml_mul_mat(ctx, // [n,p,qq,rr]
+ // ggml_cont(ctx, // [m,n,q1,r1]
+ // ggml_transpose(ctx, src0)), // [m,n,q1,r1]
+ // tensor->grad), // [m,p,qq,rr]
+
+ // // when src0 is bigger than tensor->grad (this is mostly the case in llama),
+ // // avoid transpose of src0, rather transpose smaller tensor->grad
+ // // and then use ggml_out_prod
+ ggml_out_prod(ctx, // [n,p,qq,rr]
+ src0, // [n,m,q1,r1]
+ ggml_transpose(ctx, // [p,m,qq,rr]
+ tensor->grad)), // [m,p,qq,rr]
+ zero_table);
+ }
+ } break;
+ case GGML_OP_MUL_MAT_ID:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_OUT_PROD:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_SCALE:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ float s;
+ memcpy(&s, tensor->op_params, sizeof(float));
+
+ src0->grad =
+ ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_scale_impl(ctx, tensor->grad, s, false),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_SET:
+ {
+ const size_t nb1 = ((int32_t *) tensor->op_params)[0];
+ const size_t nb2 = ((int32_t *) tensor->op_params)[1];
+ const size_t nb3 = ((int32_t *) tensor->op_params)[2];
+ const size_t offset = ((int32_t *) tensor->op_params)[3];
+
+ struct ggml_tensor * tensor_grad_view = NULL;
+
+ if (src0->grad || src1->grad) {
+ GGML_ASSERT(src0->type == tensor->type);
+ GGML_ASSERT(tensor->grad->type == tensor->type);
+ GGML_ASSERT(tensor->grad->type == src1->grad->type);
+
+ tensor_grad_view = ggml_view_4d(ctx,
+ tensor->grad,
+ src1->grad->ne[0],
+ src1->grad->ne[1],
+ src1->grad->ne[2],
+ src1->grad->ne[3],
+ nb1, nb2, nb3, offset);
+ }
+
+ if (src0->grad) {
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_acc_impl(ctx,
+ tensor->grad,
+ ggml_neg(ctx, tensor_grad_view),
+ nb1, nb2, nb3, offset, false),
+ zero_table);
+ }
+
+ if (src1->grad) {
+ src1->grad =
+ ggml_add_or_set(ctx,
+ src1->grad,
+ ggml_reshape(ctx,
+ ggml_cont(ctx, tensor_grad_view),
+ src1->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_CPY:
+ {
+ // necessary for llama
+ // cpy overwrites value of src1 by src0 and returns view(src1)
+ // the overwriting is mathematically equivalent to:
+ // tensor = src0 * 1 + src1 * 0
+ if (src0->grad) {
+ // dsrc0 = dtensor * 1
+ src0->grad = ggml_add_or_set(ctx, src0->grad, tensor->grad, zero_table);
+ }
+ if (src1->grad) {
+ // dsrc1 = dtensor * 0 -> noop
+ }
+ } break;
+ case GGML_OP_CONT:
+ {
+ // same as cpy
+ if (src0->grad) {
+ GGML_ASSERT(ggml_is_contiguous(src0->grad));
+ GGML_ASSERT(ggml_is_contiguous(tensor->grad));
+ src0->grad = ggml_add_or_set(ctx, src0->grad, tensor->grad, zero_table);
+ }
+ } break;
+ case GGML_OP_RESHAPE:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx, src0->grad,
+ ggml_reshape(ctx,
+ ggml_is_contiguous(tensor->grad)
+ ? tensor->grad
+ : ggml_cont(ctx, tensor->grad),
+ src0->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_VIEW:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ size_t offset;
+
+ memcpy(&offset, tensor->op_params, sizeof(offset));
+
+ size_t nb1 = tensor->nb[1];
+ size_t nb2 = tensor->nb[2];
+ size_t nb3 = tensor->nb[3];
+
+ if (src0->type != src0->grad->type) {
+ // gradient is typically F32, but src0 could be other type
+ size_t ng = ggml_element_size(src0->grad);
+ size_t n0 = ggml_element_size(src0);
+ GGML_ASSERT(offset % n0 == 0);
+ GGML_ASSERT(nb1 % n0 == 0);
+ GGML_ASSERT(nb2 % n0 == 0);
+ GGML_ASSERT(nb3 % n0 == 0);
+ offset = (offset / n0) * ng;
+ nb1 = (nb1 / n0) * ng;
+ nb2 = (nb2 / n0) * ng;
+ nb3 = (nb3 / n0) * ng;
+ }
+
+ src0->grad = ggml_acc_or_set(ctx, src0->grad, tensor->grad, nb1, nb2, nb3, offset, zero_table);
+ }
+ } break;
+ case GGML_OP_PERMUTE:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ int32_t * axes = (int32_t *) tensor->op_params;
+ int axis0 = axes[0] & 0x3;
+ int axis1 = axes[1] & 0x3;
+ int axis2 = axes[2] & 0x3;
+ int axis3 = axes[3] & 0x3;
+ int axes_backward[4] = {0,0,0,0};
+ axes_backward[axis0] = 0;
+ axes_backward[axis1] = 1;
+ axes_backward[axis2] = 2;
+ axes_backward[axis3] = 3;
+ src0->grad =
+ ggml_add_or_set(ctx, src0->grad,
+ ggml_permute(ctx,
+ tensor->grad,
+ axes_backward[0],
+ axes_backward[1],
+ axes_backward[2],
+ axes_backward[3]),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_TRANSPOSE:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx, src0->grad,
+ ggml_transpose(ctx, tensor->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_GET_ROWS:
+ {
+ // necessary for llama (only for tokenizer)
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx, src0->grad,
+ // last ggml_get_rows_back argument src0->grad is only
+ // necessary to setup correct output shape
+ ggml_get_rows_back(ctx, tensor->grad, src1, src0->grad),
+ zero_table);
+ }
+ if (src1->grad) {
+ // noop
+ }
+ } break;
+ case GGML_OP_GET_ROWS_BACK:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_DIAG:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_DIAG_MASK_INF:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ const int n_past = ((int32_t *) tensor->op_params)[0];
+ src0->grad =
+ ggml_add_or_set(ctx, src0->grad,
+ /* ggml_diag_mask_inf_impl() shouldn't be here */
+ /* ref: https://github.com/ggerganov/llama.cpp/pull/4203#discussion_r1412377992 */
+ ggml_diag_mask_zero_impl(ctx, tensor->grad, n_past, false),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_DIAG_MASK_ZERO:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ const int n_past = ((int32_t *) tensor->op_params)[0];
+ src0->grad =
+ ggml_add_or_set(ctx, src0->grad,
+ ggml_diag_mask_zero_impl(ctx, tensor->grad, n_past, false),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_SOFT_MAX:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx, src0->grad,
+ ggml_soft_max_back(ctx, tensor->grad, tensor),
+ zero_table);
+ }
+
+ } break;
+ case GGML_OP_SOFT_MAX_BACK:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_ROPE:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ //const int n_past = ((int32_t *) tensor->op_params)[0];
+ const int n_dims = ((int32_t *) tensor->op_params)[1];
+ const int mode = ((int32_t *) tensor->op_params)[2];
+ //const int n_ctx = ((int32_t *) tensor->op_params)[3];
+ const int n_ctx_orig = ((int32_t *) tensor->op_params)[4];
+ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
+
+ memcpy(&freq_base, (int32_t *) tensor->op_params + 5, sizeof(float));
+ memcpy(&freq_scale, (int32_t *) tensor->op_params + 6, sizeof(float));
+ memcpy(&ext_factor, (int32_t *) tensor->op_params + 7, sizeof(float));
+ memcpy(&attn_factor, (int32_t *) tensor->op_params + 8, sizeof(float));
+ memcpy(&beta_fast, (int32_t *) tensor->op_params + 9, sizeof(float));
+ memcpy(&beta_slow, (int32_t *) tensor->op_params + 10, sizeof(float));
+
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_rope_back(ctx,
+ tensor->grad,
+ src1,
+ src2,
+ n_dims,
+ mode,
+ n_ctx_orig,
+ freq_base,
+ freq_scale,
+ ext_factor,
+ attn_factor,
+ beta_fast,
+ beta_slow),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_ROPE_BACK:
+ {
+ if (src0->grad) {
+ //const int n_past = ((int32_t *) tensor->op_params)[0];
+ const int n_dims = ((int32_t *) tensor->op_params)[1];
+ const int mode = ((int32_t *) tensor->op_params)[2];
+ //const int n_ctx = ((int32_t *) tensor->op_params)[3];
+ const int n_ctx_orig = ((int32_t *) tensor->op_params)[4];
+ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
+
+ memcpy(&freq_base, (int32_t *) tensor->op_params + 5, sizeof(float));
+ memcpy(&freq_scale, (int32_t *) tensor->op_params + 6, sizeof(float));
+ memcpy(&ext_factor, (int32_t *) tensor->op_params + 7, sizeof(float));
+ memcpy(&attn_factor, (int32_t *) tensor->op_params + 8, sizeof(float));
+ memcpy(&beta_fast, (int32_t *) tensor->op_params + 9, sizeof(float));
+ memcpy(&beta_slow, (int32_t *) tensor->op_params + 10, sizeof(float));
+
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_rope_impl(ctx,
+ tensor->grad,
+ src1,
+ src2,
+ n_dims,
+ mode,
+ n_ctx_orig,
+ freq_base,
+ freq_scale,
+ ext_factor,
+ attn_factor,
+ beta_fast,
+ beta_slow,
+ false),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_CLAMP:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_CONV_TRANSPOSE_1D:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_IM2COL:
+ {
+ if (src1->grad) {
+ const int32_t s0 = ggml_get_op_params_i32(tensor, 0);
+ const int32_t s1 = ggml_get_op_params_i32(tensor, 1);
+ const int32_t p0 = ggml_get_op_params_i32(tensor, 2);
+ const int32_t p1 = ggml_get_op_params_i32(tensor, 3);
+ const int32_t d0 = ggml_get_op_params_i32(tensor, 4);
+ const int32_t d1 = ggml_get_op_params_i32(tensor, 5);
+ const bool is_2D = ggml_get_op_params_i32(tensor, 6) == 1;
+
+ src1->grad = ggml_add_or_set(ctx,
+ src1->grad,
+ ggml_im2col_back(ctx, src0, tensor->grad, src1->ne, s0, s1, p0, p1, d0, d1, is_2D),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_IM2COL_BACK:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_CONV_TRANSPOSE_2D:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_POOL_1D:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_POOL_2D:
+ {
+ if (src0->grad) {
+ const enum ggml_op_pool op = ggml_get_op_params_i32(tensor, 0);
+ const int32_t k0 = ggml_get_op_params_i32(tensor, 1);
+ const int32_t k1 = ggml_get_op_params_i32(tensor, 2);
+ const int32_t s0 = ggml_get_op_params_i32(tensor, 3);
+ const int32_t s1 = ggml_get_op_params_i32(tensor, 4);
+ const int32_t p0 = ggml_get_op_params_i32(tensor, 5);
+ const int32_t p1 = ggml_get_op_params_i32(tensor, 6);
+
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_pool_2d_back(ctx, tensor->grad, src0, op, k0, k1, s0, s1, p0, p1),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_POOL_2D_BACK:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_UPSCALE:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_PAD:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_ARANGE:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_TIMESTEP_EMBEDDING:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_ARGSORT:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_LEAKY_RELU:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_FLASH_ATTN_EXT:
+ {
+ struct ggml_tensor * flash_grad = NULL;
+ if (src0->grad || src1->grad || tensor->src[2]->grad) {
+ int32_t t = ggml_get_op_params_i32(tensor, 0);
+ GGML_ASSERT(t == 0 || t == 1);
+ bool masked = t != 0;
+ flash_grad =
+ ggml_flash_attn_back(ctx,
+ src0,
+ src1,
+ tensor->src[2],
+ tensor->grad,
+ masked);
+ }
+
+ const int64_t elem_q = ggml_nelements(src0);
+ const int64_t elem_k = ggml_nelements(src1);
+ const int64_t elem_v = ggml_nelements(src2);
+
+ enum ggml_type result_type = flash_grad->type;
+ GGML_ASSERT(ggml_blck_size(result_type) == 1);
+ const size_t tsize = ggml_type_size(result_type);
+
+ const size_t offs_q = 0;
+ const size_t offs_k = offs_q + GGML_PAD(elem_q * tsize, GGML_MEM_ALIGN);
+ const size_t offs_v = offs_k + GGML_PAD(elem_k * tsize, GGML_MEM_ALIGN);
+
+ if (src0->grad) {
+ struct ggml_tensor * view_q = ggml_view_1d(ctx, flash_grad, elem_q, offs_q);
+ struct ggml_tensor * grad_q = ggml_reshape(ctx, view_q, src0);
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ grad_q,
+ zero_table);
+ }
+ if (src1->grad) {
+ struct ggml_tensor * view_k = ggml_view_1d(ctx, flash_grad, elem_k, offs_k);
+ struct ggml_tensor * grad_k = ggml_reshape(ctx, view_k, src1);
+ src1->grad = ggml_add_or_set(ctx,
+ src1->grad,
+ grad_k,
+ zero_table);
+ }
+ if (src2->grad) {
+ struct ggml_tensor * view_v = ggml_view_1d(ctx, flash_grad, elem_v, offs_v);
+ struct ggml_tensor * grad_v = ggml_reshape(ctx, view_v, src2);
+ src2->grad = ggml_add_or_set(ctx,
+ src2->grad,
+ grad_v,
+ zero_table);
+ }
+ } break;
+ case GGML_OP_FLASH_ATTN_BACK:
+ {
+ GGML_ABORT("fatal error"); // not supported
+ }
+ case GGML_OP_SSM_CONV:
+ case GGML_OP_SSM_SCAN:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_OP_WIN_PART:
+ case GGML_OP_WIN_UNPART:
+ case GGML_OP_UNARY:
+ {
+ switch (ggml_get_unary_op(tensor)) {
+ case GGML_UNARY_OP_ABS:
+ {
+ if (src0->grad) {
+ src0->grad =
+ ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_mul(ctx,
+ ggml_sgn(ctx, src0),
+ tensor->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_UNARY_OP_SGN:
+ {
+ if (src0->grad) {
+ // noop
+ }
+ } break;
+ case GGML_UNARY_OP_NEG:
+ {
+ if (src0->grad) {
+ src0->grad = ggml_sub_or_set(ctx, src0->grad, tensor->grad, zero_table);
+ }
+ } break;
+ case GGML_UNARY_OP_STEP:
+ {
+ if (src0->grad) {
+ // noop
+ }
+ } break;
+ case GGML_UNARY_OP_TANH:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_UNARY_OP_ELU:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_UNARY_OP_RELU:
+ {
+ if (src0->grad) {
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_mul(ctx,
+ ggml_step(ctx, src0),
+ tensor->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_UNARY_OP_SIGMOID:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_UNARY_OP_GELU:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_UNARY_OP_GELU_QUICK:
+ {
+ GGML_ABORT("fatal error"); // TODO: not implemented
+ }
+ case GGML_UNARY_OP_SILU:
+ {
+ // necessary for llama
+ if (src0->grad) {
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_silu_back(ctx, src0, tensor->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_UNARY_OP_EXP:
+ {
+ if (src0->grad) {
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_mul(ctx, tensor, tensor->grad),
+ zero_table);
+ }
+ } break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+ } break;
+ case GGML_OP_GET_REL_POS:
+ case GGML_OP_ADD_REL_POS:
+ case GGML_OP_RWKV_WKV:
+ case GGML_OP_MAP_UNARY:
+ case GGML_OP_MAP_BINARY:
+ case GGML_OP_MAP_CUSTOM1_F32:
+ case GGML_OP_MAP_CUSTOM2_F32:
+ case GGML_OP_MAP_CUSTOM3_F32:
+ case GGML_OP_MAP_CUSTOM1:
+ case GGML_OP_MAP_CUSTOM2:
+ case GGML_OP_MAP_CUSTOM3:
+ {
+ GGML_ABORT("fatal error"); // not supported
+ }
+ case GGML_OP_CROSS_ENTROPY_LOSS:
+ {
+ if (src0->grad) {
+ src0->grad = ggml_add_or_set(ctx,
+ src0->grad,
+ ggml_cross_entropy_loss_back(ctx,
+ src0,
+ src1,
+ tensor->grad),
+ zero_table);
+ }
+ } break;
+ case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
+ {
+ GGML_ABORT("fatal error"); // not supported
+ }
+ case GGML_OP_NONE:
+ {
+ // nop
+ } break;
+ case GGML_OP_COUNT:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+
+ for (int i = 0; i < GGML_MAX_SRC; ++i) {
+ if (tensor->src[i] && tensor->src[i]->grad) {
+ GGML_ASSERT(ggml_are_same_shape(tensor->src[i], tensor->src[i]->grad));
+ }
+ }
+}
+
+static void ggml_visit_parents(struct ggml_cgraph * cgraph, struct ggml_tensor * node) {
+ if (node->grad == NULL) {
+ // this usually happens when we generate intermediate nodes from constants in the backward pass
+ // it can also happen during forward pass, if the user performs computations with constants
+ if (node->op != GGML_OP_NONE) {
+ //GGML_PRINT_DEBUG("%s: warning: node %p has no grad, but op %d\n", __func__, (void *) node, node->op);
+ }
+ }
+
+ // check if already visited
+ if (ggml_hash_insert(&cgraph->visited_hash_set, node) == GGML_HASHSET_ALREADY_EXISTS) {
+ return;
+ }
+
+ for (int i = 0; i < GGML_MAX_SRC; ++i) {
+ const int k =
+ (cgraph->order == GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT) ? i :
+ (cgraph->order == GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT) ? (GGML_MAX_SRC-1-i) :
+ /* unknown order, just fall back to using i*/ i;
+ if (node->src[k]) {
+ ggml_visit_parents(cgraph, node->src[k]);
+ }
+ }
+
+ if (node->op == GGML_OP_NONE && node->grad == NULL) {
+ // reached a leaf node, not part of the gradient graph (e.g. a constant)
+ GGML_ASSERT(cgraph->n_leafs < cgraph->size);
+
+ if (strlen(node->name) == 0) {
+ ggml_format_name(node, "leaf_%d", cgraph->n_leafs);
+ }
+
+ cgraph->leafs[cgraph->n_leafs] = node;
+ cgraph->n_leafs++;
+ } else {
+ GGML_ASSERT(cgraph->n_nodes < cgraph->size);
+
+ if (strlen(node->name) == 0) {
+ ggml_format_name(node, "node_%d", cgraph->n_nodes);
+ }
+
+ cgraph->nodes[cgraph->n_nodes] = node;
+ if (cgraph->grads) {
+ cgraph->grads[cgraph->n_nodes] = node->grad;
+ }
+ cgraph->n_nodes++;
+ }
+}
+
+static void ggml_build_forward_impl(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor, bool expand) {
+ if (!expand) {
+ // TODO: this branch isn't accessible anymore, maybe move this to ggml_build_forward_expand
+ ggml_graph_clear(cgraph);
+ }
+
+ const int n0 = cgraph->n_nodes;
+
+ ggml_visit_parents(cgraph, tensor);
+
+ const int n_new = cgraph->n_nodes - n0;
+ GGML_PRINT_DEBUG("%s: visited %d new nodes\n", __func__, n_new);
+
+ if (n_new > 0) {
+ // the last added node should always be starting point
+ GGML_ASSERT(cgraph->nodes[cgraph->n_nodes - 1] == tensor);
+ }
+}
+
+void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor) {
+ ggml_build_forward_impl(cgraph, tensor, true);
+}
+
+void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_cgraph * gb, bool keep) {
+ GGML_ASSERT(gf->n_nodes > 0);
+ GGML_ASSERT(gf->grads);
+
+ // if we are keeping the gradient graph, we have to detach the gradient nodes from the original graph
+ if (keep) {
+ for (int i = 0; i < gf->n_nodes; i++) {
+ struct ggml_tensor * node = gf->nodes[i];
+
+ if (node->grad) {
+ node->grad = ggml_dup_tensor(ctx, node);
+ gf->grads[i] = node->grad;
+ }
+ }
+ }
+
+ // remember original gradients which start with zero values
+ struct ggml_hash_set zero_table = ggml_hash_set_new(gf->size);
+ for (int i = 0; i < gf->n_nodes; i++) {
+ if (gf->grads[i]) {
+ ggml_hash_insert(&zero_table, gf->grads[i]);
+ }
+ }
+
+ for (int i = gf->n_nodes - 1; i >= 0; i--) {
+ struct ggml_tensor * node = gf->nodes[i];
+
+ // inplace operations to add gradients are not created by ggml_compute_backward
+ // use allocator to automatically make inplace operations
+ if (node->grad) {
+ ggml_compute_backward(ctx, node, &zero_table);
+ }
+ }
+
+ for (int i = 0; i < gf->n_nodes; i++) {
+ struct ggml_tensor * node = gf->nodes[i];
+
+ if (node->flags & GGML_TENSOR_FLAG_PARAM) {
+ GGML_PRINT_DEBUG("%s: found root node %p\n", __func__, (void *) node);
+ ggml_build_forward_expand(gb, node->grad);
+ }
+ }
+
+ ggml_hash_set_free(&zero_table);
+}
+
+static void * incr_ptr_aligned(void ** p, size_t size, size_t align) {
+ void * ptr = *p;
+ ptr = (void *) GGML_PAD((uintptr_t) ptr, align);
+ *p = (void *) ((char *) ptr + size);
+ return ptr;
+}
+
+static size_t ggml_graph_nbytes(size_t size, bool grads) {
+ size_t hash_size = ggml_hash_size(size * 2);
+ void * p = 0;
+ incr_ptr_aligned(&p, sizeof(struct ggml_cgraph), 1);
+ incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)); // nodes
+ incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)); // leafs
+ incr_ptr_aligned(&p, hash_size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)); // hash keys
+ if (grads) {
+ incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)); // grads
+ }
+ incr_ptr_aligned(&p, ggml_bitset_size(hash_size) * sizeof(ggml_bitset_t), sizeof(ggml_bitset_t));
+
+ size_t nbytes = (size_t) p;
+ return nbytes;
+}
+
+size_t ggml_graph_overhead_custom(size_t size, bool grads) {
+ return GGML_OBJECT_SIZE + GGML_PAD(ggml_graph_nbytes(size, grads), GGML_MEM_ALIGN);
+}
+
+size_t ggml_graph_overhead(void) {
+ return ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, false);
+}
+
+struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads) {
+ const size_t obj_size = ggml_graph_nbytes(size, grads);
+ struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_GRAPH, obj_size);
+ struct ggml_cgraph * cgraph = (struct ggml_cgraph *) ((char *) ctx->mem_buffer + obj->offs);
+
+ // the size of the hash table is doubled since it needs to hold both nodes and leafs
+ size_t hash_size = ggml_hash_size(size * 2);
+
+ void * p = cgraph + 1;
+
+ struct ggml_tensor ** nodes_ptr = incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *));
+ struct ggml_tensor ** leafs_ptr = incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *));
+ struct ggml_tensor ** hash_keys_ptr = incr_ptr_aligned(&p, hash_size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *));
+ struct ggml_tensor ** grads_ptr = grads ? incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)) : NULL;
+ ggml_bitset_t * hash_used = incr_ptr_aligned(&p, ggml_bitset_size(hash_size) * sizeof(ggml_bitset_t), sizeof(ggml_bitset_t));
+
+ // check that we allocated the correct amount of memory
+ assert(obj_size == (size_t)((char *)p - (char *)cgraph));
+
+ *cgraph = (struct ggml_cgraph) {
+ /*.size =*/ size,
+ /*.n_nodes =*/ 0,
+ /*.n_leafs =*/ 0,
+ /*.nodes =*/ nodes_ptr,
+ /*.grads =*/ grads_ptr,
+ /*.leafs =*/ leafs_ptr,
+ /*.hash_table =*/ { hash_size, hash_used, hash_keys_ptr },
+ /*.order =*/ GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT,
+ };
+
+ ggml_hash_set_reset(&cgraph->visited_hash_set);
+
+ return cgraph;
+}
+
+struct ggml_cgraph * ggml_new_graph(struct ggml_context * ctx) {
+ return ggml_new_graph_custom(ctx, GGML_DEFAULT_GRAPH_SIZE, false);
+}
+
+struct ggml_cgraph ggml_graph_view(struct ggml_cgraph * cgraph0, int i0, int i1) {
+ struct ggml_cgraph cgraph = {
+ /*.size =*/ 0,
+ /*.n_nodes =*/ i1 - i0,
+ /*.n_leafs =*/ 0,
+ /*.nodes =*/ cgraph0->nodes + i0,
+ /*.grads =*/ cgraph0->grads ? cgraph0->grads + i0 : NULL,
+ /*.leafs =*/ NULL,
+ /*.hash_table =*/ { 0, NULL, NULL },
+ /*.order =*/ cgraph0->order,
+ };
+
+ return cgraph;
+}
+
+void ggml_graph_cpy(struct ggml_cgraph * src, struct ggml_cgraph * dst) {
+ GGML_ASSERT(dst->size >= src->n_leafs);
+ GGML_ASSERT(dst->size >= src->n_nodes);
+ GGML_ASSERT(dst->visited_hash_set.size >= src->visited_hash_set.size);
+
+ dst->n_leafs = src->n_leafs;
+ dst->n_nodes = src->n_nodes;
+ dst->order = src->order;
+
+ for (int i = 0; i < src->n_leafs; ++i) {
+ dst->leafs[i] = src->leafs[i];
+ }
+
+ for (int i = 0; i < src->n_nodes; ++i) {
+ dst->nodes[i] = src->nodes[i];
+ }
+
+ if (src->grads) {
+ GGML_ASSERT(dst->grads != NULL);
+ for (int i = 0; i < src->n_nodes; ++i) {
+ dst->grads[i] = src->grads[i];
+ }
+ }
+
+ for (size_t i = 0; i < src->visited_hash_set.size; ++i) {
+ if (src->visited_hash_set.keys[i]) {
+ ggml_hash_insert(&dst->visited_hash_set, src->visited_hash_set.keys[i]);
+ }
+ }
+}
+
+struct ggml_cgraph * ggml_graph_dup(struct ggml_context * ctx, struct ggml_cgraph * cgraph) {
+ struct ggml_cgraph * result = ggml_new_graph_custom(ctx, cgraph->size, cgraph->grads != NULL);
+ ggml_graph_cpy(cgraph, result);
+ return result;
+}
+
+void ggml_graph_reset(struct ggml_cgraph * cgraph) {
+ GGML_ASSERT(cgraph->grads != NULL);
+
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ struct ggml_tensor * grad = cgraph->grads[i];
+
+ if (grad) {
+ ggml_set_zero(grad);
+ }
+ }
+}
+
+void ggml_graph_clear(struct ggml_cgraph * cgraph) {
+ cgraph->n_leafs = 0;
+ cgraph->n_nodes = 0;
+ ggml_hash_set_reset(&cgraph->visited_hash_set);
+}
+
+// Android's libc implementation "bionic" does not support setting affinity
+#if defined(__gnu_linux__)
+static void set_numa_thread_affinity(int thread_n) {
+ if (!ggml_is_numa()) {
+ return;
+ }
+
+ int node_num;
+ int rv;
+ size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus);
+
+ switch(g_state.numa.numa_strategy) {
+ case GGML_NUMA_STRATEGY_DISTRIBUTE:
+ // run thread on node_num thread_n / (threads per node)
+ node_num = thread_n % g_state.numa.n_nodes;
+ break;
+ case GGML_NUMA_STRATEGY_ISOLATE:
+ // run thread on current_node
+ node_num = g_state.numa.current_node;
+ break;
+ case GGML_NUMA_STRATEGY_NUMACTL:
+ // use the cpuset that numactl gave us
+ rv = pthread_setaffinity_np(pthread_self(), setsize, &g_state.numa.cpuset);
+ if (rv) {
+ fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n",strerror(rv));
+ }
+ return;
+ default:
+ return;
+ }
+
+ struct ggml_numa_node * node = &g_state.numa.nodes[node_num];
+
+ cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus);
+ CPU_ZERO_S(setsize, cpus);
+ for (size_t i = 0; i < node->n_cpus; ++i) {
+ CPU_SET_S(node->cpus[i], setsize, cpus);
+ }
+
+ rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);
+ if (rv) {
+ fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv));
+ }
+
+ CPU_FREE(cpus);
+}
+
+static void clear_numa_thread_affinity(void) {
+ if (!ggml_is_numa()) {
+ return;
+ }
+
+ size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus);
+
+ cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus);
+ CPU_ZERO_S(setsize, cpus);
+ for (unsigned i = 0; i < g_state.numa.total_cpus; ++i) {
+ CPU_SET_S(i, setsize, cpus);
+ }
+
+ int rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);
+ if (rv) {
+ fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv));
+ }
+
+ CPU_FREE(cpus);
+}
+#else
+// TODO: Windows etc.
+// (the linux implementation may also work on BSD, someone should test)
+static void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n); }
+static void clear_numa_thread_affinity(void) {}
+#endif
+
+static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
+ int n_tasks = 0;
+
+ if (ggml_is_empty(node)) {
+ // no need to multi-thread a no-op
+ n_tasks = 1;
+ return n_tasks;
+ }
+
+ switch (node->op) {
+ case GGML_OP_CPY:
+ case GGML_OP_DUP:
+ case GGML_OP_CONT:
+ case GGML_OP_ADD:
+ case GGML_OP_ADD1:
+ case GGML_OP_ACC:
+ {
+ n_tasks = n_threads;
+ } break;
+ case GGML_OP_SUB:
+ case GGML_OP_SQR:
+ case GGML_OP_SQRT:
+ case GGML_OP_LOG:
+ case GGML_OP_SIN:
+ case GGML_OP_COS:
+ case GGML_OP_SUM:
+ case GGML_OP_SUM_ROWS:
+ case GGML_OP_MEAN:
+ case GGML_OP_ARGMAX:
+ case GGML_OP_REPEAT:
+ case GGML_OP_REPEAT_BACK:
+ case GGML_OP_LEAKY_RELU:
+ {
+ n_tasks = 1;
+ } break;
+ case GGML_OP_UNARY:
+ switch (ggml_get_unary_op(node)) {
+ case GGML_UNARY_OP_ABS:
+ case GGML_UNARY_OP_SGN:
+ case GGML_UNARY_OP_NEG:
+ case GGML_UNARY_OP_STEP:
+ case GGML_UNARY_OP_TANH:
+ case GGML_UNARY_OP_ELU:
+ case GGML_UNARY_OP_RELU:
+ case GGML_UNARY_OP_SIGMOID:
+ case GGML_UNARY_OP_HARDSWISH:
+ case GGML_UNARY_OP_HARDSIGMOID:
+ case GGML_UNARY_OP_EXP:
+ {
+ n_tasks = 1;
+ } break;
+
+ case GGML_UNARY_OP_GELU:
+ case GGML_UNARY_OP_GELU_QUICK:
+ case GGML_UNARY_OP_SILU:
+ {
+ n_tasks = n_threads;
+ } break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+ break;
+ case GGML_OP_SILU_BACK:
+ case GGML_OP_MUL:
+ case GGML_OP_DIV:
+ case GGML_OP_NORM:
+ case GGML_OP_RMS_NORM:
+ case GGML_OP_RMS_NORM_BACK:
+ case GGML_OP_GROUP_NORM:
+ case GGML_OP_CONCAT:
+ case GGML_OP_MUL_MAT:
+ case GGML_OP_MUL_MAT_ID:
+ case GGML_OP_OUT_PROD:
+ {
+ n_tasks = n_threads;
+ } break;
+ case GGML_OP_GET_ROWS:
+ {
+ // FIXME: get_rows can use additional threads, but the cost of launching additional threads
+ // decreases performance with GPU offloading
+ //n_tasks = n_threads;
+ n_tasks = 1;
+ } break;
+ case GGML_OP_SCALE:
+ case GGML_OP_SET:
+ case GGML_OP_RESHAPE:
+ case GGML_OP_VIEW:
+ case GGML_OP_PERMUTE:
+ case GGML_OP_TRANSPOSE:
+ case GGML_OP_GET_ROWS_BACK:
+ case GGML_OP_DIAG:
+ {
+ n_tasks = 1;
+ } break;
+ case GGML_OP_DIAG_MASK_ZERO:
+ case GGML_OP_DIAG_MASK_INF:
+ case GGML_OP_SOFT_MAX_BACK:
+ case GGML_OP_ROPE:
+ case GGML_OP_ROPE_BACK:
+ case GGML_OP_ADD_REL_POS:
+ {
+ n_tasks = n_threads;
+ } break;
+ case GGML_OP_CLAMP:
+ {
+ n_tasks = 1; //TODO
+ } break;
+ case GGML_OP_SOFT_MAX:
+ {
+ n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));
+ } break;
+ case GGML_OP_IM2COL:
+ case GGML_OP_IM2COL_BACK:
+ case GGML_OP_CONV_TRANSPOSE_1D:
+ case GGML_OP_CONV_TRANSPOSE_2D:
+ {
+ n_tasks = n_threads;
+ } break;
+ case GGML_OP_POOL_1D:
+ case GGML_OP_POOL_2D:
+ case GGML_OP_POOL_2D_BACK:
+ {
+ n_tasks = 1;
+ } break;
+ case GGML_OP_UPSCALE:
+ case GGML_OP_PAD:
+ case GGML_OP_ARANGE:
+ case GGML_OP_TIMESTEP_EMBEDDING:
+ case GGML_OP_ARGSORT:
+ case GGML_OP_FLASH_ATTN_EXT:
+ case GGML_OP_FLASH_ATTN_BACK:
+ case GGML_OP_SSM_CONV:
+ case GGML_OP_SSM_SCAN:
+ {
+ n_tasks = n_threads;
+ } break;
+ case GGML_OP_WIN_PART:
+ case GGML_OP_WIN_UNPART:
+ case GGML_OP_GET_REL_POS:
+ case GGML_OP_RWKV_WKV:
+ case GGML_OP_MAP_UNARY:
+ case GGML_OP_MAP_BINARY:
+ case GGML_OP_MAP_CUSTOM1_F32:
+ case GGML_OP_MAP_CUSTOM2_F32:
+ case GGML_OP_MAP_CUSTOM3_F32:
+ {
+ n_tasks = 1;
+ } break;
+ case GGML_OP_MAP_CUSTOM1:
+ {
+ struct ggml_map_custom1_op_params p;
+ memcpy(&p, node->op_params, sizeof(p));
+ if (p.n_tasks == GGML_N_TASKS_MAX) {
+ n_tasks = n_threads;
+ } else {
+ n_tasks = MIN(p.n_tasks, n_threads);
+ }
+ } break;
+ case GGML_OP_MAP_CUSTOM2:
+ {
+ struct ggml_map_custom2_op_params p;
+ memcpy(&p, node->op_params, sizeof(p));
+ if (p.n_tasks == GGML_N_TASKS_MAX) {
+ n_tasks = n_threads;
+ } else {
+ n_tasks = MIN(p.n_tasks, n_threads);
+ }
+ } break;
+ case GGML_OP_MAP_CUSTOM3:
+ {
+ struct ggml_map_custom3_op_params p;
+ memcpy(&p, node->op_params, sizeof(p));
+ if (p.n_tasks == GGML_N_TASKS_MAX) {
+ n_tasks = n_threads;
+ } else {
+ n_tasks = MIN(p.n_tasks, n_threads);
+ }
+ } break;
+ case GGML_OP_CROSS_ENTROPY_LOSS:
+ case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
+ {
+ n_tasks = n_threads;
+ } break;
+ case GGML_OP_NONE:
+ {
+ n_tasks = 1;
+ } break;
+ case GGML_OP_COUNT:
+ {
+ GGML_ABORT("fatal error");
+ }
+ default:
+ {
+ fprintf(stderr, "%s: op not implemented: ", __func__);
+ if (node->op < GGML_OP_COUNT) {
+ fprintf(stderr, "%s\n", ggml_op_name(node->op));
+ } else {
+ fprintf(stderr, "%d\n", node->op);
+ }
+ GGML_ABORT("fatal error");
+ }
+ }
+
+ assert(n_tasks > 0);
+
+ return n_tasks;
+}
+
+static thread_ret_t ggml_graph_compute_secondary_thread(void* data);
+
+#if defined(_WIN32)
+#include "windows.h"
+
+// TODO: support > 64 CPUs
+bool ggml_thread_apply_affinity(bool * mask) {
+ HANDLE h = GetCurrentThread();
+ uint64_t bitmask = 0ULL;
+
+ assert(GGML_MAX_N_THREADS >= 64);
+
+ for (int32_t i = 0; i < 8; i++) {
+ int32_t idx = i * 8;
+ uint8_t val = 0;
+ val |= mask[idx + 0] << 0;
+ val |= mask[idx + 1] << 1;
+ val |= mask[idx + 2] << 2;
+ val |= mask[idx + 3] << 3;
+ val |= mask[idx + 4] << 4;
+ val |= mask[idx + 5] << 5;
+ val |= mask[idx + 6] << 6;
+ val |= mask[idx + 7] << 7;
+ bitmask |= (uint64_t)val << idx;
+ }
+
+ for (int32_t i = 64; i < GGML_MAX_N_THREADS; i++) {
+ if (mask[i]) {
+ fprintf(stderr, "warn: setting thread-affinity for > 64 CPUs isn't supported on windows!\n");
+ break;
+ }
+ }
+
+ DWORD_PTR m = (DWORD_PTR)bitmask;
+
+ m = SetThreadAffinityMask(h, m);
+
+ return m != 0;
+}
+
+static bool ggml_thread_apply_priority(int32_t prio) {
+ // Note that on Windows the Process Priority Class must be updated in order to set Thread priority.
+ // This is up to the applications.
+ DWORD p = THREAD_PRIORITY_NORMAL;
+ switch (prio) {
+ case GGML_SCHED_PRIO_NORMAL: p = THREAD_PRIORITY_NORMAL; break;
+ case GGML_SCHED_PRIO_MEDIUM: p = THREAD_PRIORITY_ABOVE_NORMAL; break;
+ case GGML_SCHED_PRIO_HIGH: p = THREAD_PRIORITY_HIGHEST; break;
+ case GGML_SCHED_PRIO_REALTIME: p = THREAD_PRIORITY_TIME_CRITICAL; break;
+ }
+
+ if (prio == GGML_SCHED_PRIO_NORMAL) {
+ // Keep inherited policy/priority
+ return true;
+ }
+
+ if (!SetThreadPriority(GetCurrentThread(), p)) {
+ fprintf(stderr, "warn: failed to set thread priority %d : (%d)\n", prio, (int) GetLastError());
+ return false;
+ }
+
+ return true;
+}
+
+#elif defined(__APPLE__)
+#include <sys/types.h>
+#include <sys/resource.h>
+
+static bool ggml_thread_apply_affinity(const bool * mask) {
+ // Not supported on Apple platforms
+ UNUSED(mask);
+ return true;
+}
+
+static bool ggml_thread_apply_priority(int32_t prio) {
+ struct sched_param p;
+ int32_t policy = SCHED_OTHER;
+ switch (prio) {
+ case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER; p.sched_priority = 0; break;
+ case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO; p.sched_priority = 40; break;
+ case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO; p.sched_priority = 80; break;
+ case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO; p.sched_priority = 90; break;
+ }
+
+ if (prio == GGML_SCHED_PRIO_NORMAL) {
+ // Keep inherited policy/priority
+ return true;
+ }
+
+ int32_t err = pthread_setschedparam(pthread_self(), policy, &p);
+ if (err != 0) {
+ fprintf(stderr, "warn: failed to set thread priority %d : %s (%d)\n", prio, strerror(err), err);
+ return false;
+ }
+
+ return true;
+}
+
+#else // posix?
+
+static bool ggml_thread_apply_affinity(const bool * mask) {
+ cpu_set_t cpuset;
+ int err;
+
+ CPU_ZERO(&cpuset);
+
+ for (uint32_t i = 0; i < GGML_MAX_N_THREADS; i++) {
+ if (mask[i]) {
+ GGML_PRINT_DEBUG("Thread %lx: adding %d to cpuset\n", pthread_self(), i);
+ CPU_SET(i, &cpuset);
+ }
+ }
+
+#ifdef __ANDROID__
+ err = sched_setaffinity(0, sizeof(cpuset), &cpuset);
+ if (err < 0) {
+ err = errno;
+ }
+#else
+ err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
+#endif
+ if (err != 0) {
+ fprintf(stderr, "warn: failed to set affinity mask 0x%llx : %s (%d)\n", (unsigned long long)mask, strerror(err), err);
+ return false;
+ }
+
+ return true;
+}
+
+static bool ggml_thread_apply_priority(int32_t prio) {
+ struct sched_param p;
+ int32_t policy = SCHED_OTHER;
+ switch (prio) {
+ case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER; p.sched_priority = 0; break;
+ case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO; p.sched_priority = 40; break;
+ case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO; p.sched_priority = 80; break;
+ case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO; p.sched_priority = 90; break;
+ }
+
+ if (prio == GGML_SCHED_PRIO_NORMAL) {
+ // Keep inherited policy/priority
+ return true;
+ }
+
+ int32_t err = pthread_setschedparam(pthread_self(), policy, &p);
+ if (err != 0) {
+ fprintf(stderr, "warn: failed to set thread priority %d : %s (%d)\n", prio, strerror(err), err);
+ return false;
+ }
+
+ return true;
+}
+
+#endif
+
+static bool ggml_thread_cpumask_is_valid(const bool * mask) {
+ for (int i = 0; i < GGML_MAX_N_THREADS; i++) {
+ if (mask[i]) { return true; }
+ }
+ return false;
+}
+
+static void ggml_thread_cpumask_next(const bool * global_mask, bool * local_mask, bool strict, int32_t* iter) {
+ if (!strict) {
+ memcpy(local_mask, global_mask, GGML_MAX_N_THREADS);
+ return;
+ } else {
+ memset(local_mask, 0, GGML_MAX_N_THREADS);
+ int32_t base_idx = *iter;
+ for (int32_t i = 0; i < GGML_MAX_N_THREADS; i++) {
+ int32_t idx = base_idx + i;
+ if (idx >= GGML_MAX_N_THREADS) {
+ // Just a cheaper modulo
+ idx -= GGML_MAX_N_THREADS;
+ }
+ if (global_mask[idx]) {
+ local_mask[idx] = 1;
+ *iter = idx + 1;
+ return;
+ }
+ }
+ }
+}
+
+void ggml_threadpool_free(struct ggml_threadpool* threadpool) {
+ if (!threadpool) return;
+
+#ifndef GGML_USE_OPENMP
+ struct ggml_compute_state* workers = threadpool->workers;
+ const int n_threads = threadpool->n_threads_max;
+
+ ggml_mutex_lock(&threadpool->mutex);
+
+ threadpool->stop = true;
+ threadpool->pause = false;
+
+ ggml_cond_broadcast(&threadpool->cond);
+ ggml_mutex_unlock(&threadpool->mutex);
+
+ for (int j = 1; j < n_threads; j++) {
+ int32_t rc = ggml_thread_join(workers[j].thrd, NULL);
+ GGML_ASSERT(rc == GGML_EXIT_SUCCESS || rc == GGML_EXIT_ABORTED);
+ UNUSED(rc);
+ }
+
+ ggml_mutex_destroy(&threadpool->mutex);
+ ggml_cond_destroy(&threadpool->cond);
+#endif // GGML_USE_OPENMP
+
+ GGML_ALIGNED_FREE(threadpool->workers);
+ GGML_ALIGNED_FREE(threadpool);
+}
+
+#ifndef GGML_USE_OPENMP
+// pause/resume must be called under mutex
+static void ggml_threadpool_pause_locked(struct ggml_threadpool * threadpool) {
+ GGML_PRINT_DEBUG("Pausing threadpool\n");
+ threadpool->pause = true;
+ ggml_cond_broadcast(&threadpool->cond);
+}
+
+static void ggml_threadpool_resume_locked(struct ggml_threadpool * threadpool) {
+ GGML_PRINT_DEBUG("Resuming threadpool\n");
+ threadpool->pause = false;
+ ggml_cond_broadcast(&threadpool->cond);
+}
+#endif
+
+void ggml_threadpool_pause(struct ggml_threadpool * threadpool) {
+#ifndef GGML_USE_OPENMP
+ ggml_mutex_lock(&threadpool->mutex);
+ if (!threadpool->pause) {
+ ggml_threadpool_pause_locked(threadpool);
+ }
+ ggml_mutex_unlock(&threadpool->mutex);
+#else
+ UNUSED(threadpool);
+#endif
+}
+
+void ggml_threadpool_resume(struct ggml_threadpool * threadpool) {
+#ifndef GGML_USE_OPENMP
+ ggml_mutex_lock(&threadpool->mutex);
+ if (threadpool->pause) {
+ ggml_threadpool_resume_locked(threadpool);
+ }
+ ggml_mutex_unlock(&threadpool->mutex);
+#else
+ UNUSED(threadpool);
+#endif
+}
+
+struct ggml_cplan ggml_graph_plan(
+ const struct ggml_cgraph * cgraph,
+ int n_threads,
+ struct ggml_threadpool * threadpool) {
+
+ if (threadpool == NULL) {
+ GGML_PRINT_DEBUG("Threadpool is not specified. Will create a disposable threadpool : n_threads %d\n", n_threads);
+ }
+ if (n_threads <= 0) {
+ n_threads = threadpool ? threadpool->n_threads_max : GGML_DEFAULT_N_THREADS;
+ }
+
+ size_t work_size = 0;
+
+ struct ggml_cplan cplan;
+ memset(&cplan, 0, sizeof(struct ggml_cplan));
+
+ int max_tasks = 1;
+
+ // thread scheduling for the different operations + work buffer size estimation
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ struct ggml_tensor * node = cgraph->nodes[i];
+
+ const int n_tasks = ggml_get_n_tasks(node, n_threads);
+
+ max_tasks = MAX(max_tasks, n_tasks);
+
+ size_t cur = 0;
+
+ switch (node->op) {
+ case GGML_OP_CPY:
+ case GGML_OP_DUP:
+ {
+ if (ggml_is_quantized(node->type) ||
+ // F16 -> BF16 and BF16 -> F16 copies go through intermediate F32
+ (node->src[0]->type == GGML_TYPE_F16 && node->src[1] && node->src[1]->type == GGML_TYPE_BF16) ||
+ (node->src[0]->type == GGML_TYPE_BF16 && node->src[1] && node->src[1]->type == GGML_TYPE_F16)) {
+ cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;
+ }
+ } break;
+ case GGML_OP_ADD:
+ case GGML_OP_ADD1:
+ {
+ if (ggml_is_quantized(node->src[0]->type)) {
+ cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
+ }
+ } break;
+ case GGML_OP_ACC:
+ {
+ if (ggml_is_quantized(node->src[0]->type)) {
+ cur = ggml_type_size(GGML_TYPE_F32) * node->src[1]->ne[0] * n_tasks;
+ }
+ } break;
+ case GGML_OP_MUL_MAT:
+ {
+ const enum ggml_type vec_dot_type = type_traits[node->src[0]->type].vec_dot_type;
+
+ if (node->src[1]->type != vec_dot_type) {
+ cur = ggml_row_size(vec_dot_type, ggml_nelements(node->src[1]));
+ }
+ } break;
+ case GGML_OP_MUL_MAT_ID:
+ {
+ cur = 0;
+ const struct ggml_tensor * src0 = node->src[0];
+ const struct ggml_tensor * src1 = node->src[1];
+ const enum ggml_type vec_dot_type = type_traits[src0->type].vec_dot_type;
+ if (src1->type != vec_dot_type) {
+ cur += ggml_row_size(vec_dot_type, ggml_nelements(src1));
+ }
+ const int n_as = src0->ne[2];
+ cur += GGML_PAD(cur, sizeof(int64_t)); // align
+ cur += n_as * sizeof(int64_t); // matrix_row_counts
+ cur += n_as * src1->ne[2] * sizeof(int64_t); // matrix_rows
+ } break;
+ case GGML_OP_OUT_PROD:
+ {
+ if (ggml_is_quantized(node->src[0]->type)) {
+ cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
+ }
+ } break;
+ case GGML_OP_SOFT_MAX:
+ case GGML_OP_ROPE:
+ {
+ cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;
+ } break;
+ case GGML_OP_CONV_TRANSPOSE_1D:
+ {
+ GGML_ASSERT(node->src[0]->ne[3] == 1);
+ GGML_ASSERT(node->src[1]->ne[2] == 1);
+ GGML_ASSERT(node->src[1]->ne[3] == 1);
+
+ const int64_t ne00 = node->src[0]->ne[0]; // K
+ const int64_t ne01 = node->src[0]->ne[1]; // Cout
+ const int64_t ne02 = node->src[0]->ne[2]; // Cin
+
+ const int64_t ne10 = node->src[1]->ne[0]; // L
+ const int64_t ne11 = node->src[1]->ne[1]; // Cin
+
+ if ((node->src[0]->type == GGML_TYPE_F16 ||
+ node->src[0]->type == GGML_TYPE_BF16) &&
+ node->src[1]->type == GGML_TYPE_F32) {
+ cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02;
+ cur += sizeof(ggml_fp16_t)*ne10*ne11;
+ } else if (node->src[0]->type == GGML_TYPE_F32 &&
+ node->src[1]->type == GGML_TYPE_F32) {
+ cur += sizeof(float)*ne00*ne01*ne02;
+ cur += sizeof(float)*ne10*ne11;
+ } else {
+ GGML_ABORT("fatal error");
+ }
+ } break;
+ case GGML_OP_CONV_TRANSPOSE_2D:
+ {
+ const int64_t ne00 = node->src[0]->ne[0]; // W
+ const int64_t ne01 = node->src[0]->ne[1]; // H
+ const int64_t ne02 = node->src[0]->ne[2]; // Channels Out
+ const int64_t ne03 = node->src[0]->ne[3]; // Channels In
+
+ const int64_t ne10 = node->src[1]->ne[0]; // W
+ const int64_t ne11 = node->src[1]->ne[1]; // H
+ const int64_t ne12 = node->src[1]->ne[2]; // Channels In
+
+ cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02*ne03;
+ cur += sizeof(ggml_fp16_t)*ne10*ne11*ne12;
+ } break;
+ case GGML_OP_FLASH_ATTN_EXT:
+ {
+ const int64_t ne00 = node->src[0]->ne[0]; // D
+
+ cur = 3*sizeof(float)*ne00*n_tasks; // 3x head size/thread
+ } break;
+ case GGML_OP_FLASH_ATTN_BACK:
+ {
+ const int64_t D = node->src[0]->ne[0];
+ const int64_t ne11 = ggml_up(node->src[1]->ne[1], GGML_SOFT_MAX_UNROLL);
+ const int64_t mxDn = MAX(D, ne11) * 2; // *2 because of S and SM in ggml_compute_forward_flash_attn_back
+ if (node->src[1]->type == GGML_TYPE_F32) {
+ cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
+ cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
+ } else if (node->src[1]->type == GGML_TYPE_F16) {
+ cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
+ cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
+ } else if (node->src[1]->type == GGML_TYPE_BF16) {
+ cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
+ cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
+ }
+ } break;
+
+ case GGML_OP_CROSS_ENTROPY_LOSS:
+ {
+ cur = ggml_type_size(node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks);
+ } break;
+ case GGML_OP_COUNT:
+ {
+ GGML_ABORT("fatal error");
+ }
+ default:
+ break;
+ }
+
+ work_size = MAX(work_size, cur);
+ }
+
+ if (work_size > 0) {
+ work_size += CACHE_LINE_SIZE*(n_threads);
+ }
+
+ cplan.threadpool = threadpool;
+ cplan.n_threads = MIN(max_tasks, n_threads);
+ cplan.work_size = work_size;
+ cplan.work_data = NULL;
+
+ return cplan;
+}
+
+static thread_ret_t ggml_graph_compute_thread(void * data) {
+ struct ggml_compute_state * state = (struct ggml_compute_state *) data;
+
+ const struct ggml_cgraph * cgraph = state->threadpool->cgraph;
+ const struct ggml_cplan * cplan = state->threadpool->cplan;
+
+ set_numa_thread_affinity(state->ith);
+
+ struct ggml_compute_params params = {
+ /*.ith =*/ state->ith,
+ /*.nth =*/ state->threadpool->n_threads_cur,
+ /*.wsize =*/ cplan->work_size,
+ /*.wdata =*/ cplan->work_data,
+ /*.threadpool=*/ state->threadpool,
+ };
+
+ for (int node_n = 0; node_n < cgraph->n_nodes; node_n++) {
+ struct ggml_tensor * node = cgraph->nodes[node_n];
+
+ ggml_compute_forward(¶ms, node);
+
+ if (state->ith == 0 && cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) {
+ state->threadpool->ec = GGML_STATUS_ABORTED;
+ }
+
+ ggml_barrier(state->threadpool);
+
+ if (state->threadpool->ec != GGML_STATUS_SUCCESS) {
+ break;
+ }
+ }
+
+ return 0;
+}
+
+#ifndef GGML_USE_OPENMP
+
+static inline bool ggml_graph_compute_ready(struct ggml_compute_state * state) {
+ struct ggml_threadpool * threadpool = state->threadpool;
+
+ if (state->pending || threadpool->stop || threadpool->pause) { return true; }
+
+ // check for new graph/work
+ int new_graph = atomic_load_explicit(&threadpool->n_graph, memory_order_relaxed);
+ if (new_graph != state->last_graph) {
+ state->pending = (state->ith < threadpool->n_threads_cur);
+ state->last_graph = new_graph;
+ }
+
+ return state->pending;
+}
+
+static inline bool ggml_graph_compute_poll_for_work(struct ggml_compute_state * state) {
+ struct ggml_threadpool * threadpool = state->threadpool;
+
+ // This seems to make 0 ... 100 a decent range for polling level across modern processors.
+ // Perhaps, we can adjust it dynamically based on load and things.
+ const uint64_t n_rounds = 1024UL * 128 * threadpool->poll;
+
+ for (uint64_t i=0; !ggml_graph_compute_ready(state) && i<n_rounds; i++) {
+ // No new work. Keep polling.
+ ggml_thread_cpu_relax();
+ }
+
+ return state->pending;
+}
+
+static inline bool ggml_graph_compute_check_for_work(struct ggml_compute_state * state) {
+ struct ggml_threadpool * threadpool = state->threadpool;
+
+ if (ggml_graph_compute_poll_for_work(state)) {
+ return state->pending;
+ }
+
+ ggml_mutex_lock_shared(&threadpool->mutex);
+ while (!ggml_graph_compute_ready(state)) {
+ // No new work. Wait for the signal.
+ GGML_PRINT_DEBUG("thread #%d waiting for work\n", state->ith);
+ ggml_cond_wait(&threadpool->cond, &threadpool->mutex);
+ }
+ ggml_mutex_unlock_shared(&threadpool->mutex);
+
+ return state->pending;
+}
+
+static thread_ret_t ggml_graph_compute_secondary_thread(void* data) {
+ struct ggml_compute_state * state = (struct ggml_compute_state *) data;
+ struct ggml_threadpool * threadpool = state->threadpool;
+
+ ggml_thread_apply_priority(threadpool->prio);
+ if (ggml_thread_cpumask_is_valid(state->cpumask)) {
+ ggml_thread_apply_affinity(state->cpumask);
+ }
+
+ while (true) {
+ // Check if we need to sleep
+ while (threadpool->pause) {
+ GGML_PRINT_DEBUG("thread #%d inside pause loop\n", state->ith);
+ ggml_mutex_lock_shared(&threadpool->mutex);
+ if (threadpool->pause) {
+ ggml_cond_wait(&threadpool->cond, &threadpool->mutex);
+ }
+ GGML_PRINT_DEBUG("thread #%d resuming after wait\n", state->ith);
+ ggml_mutex_unlock_shared(&threadpool->mutex);
+ }
+
+ // This needs to be checked for after the cond_wait
+ if (threadpool->stop) break;
+
+ // Check if there is new work
+ // The main thread is the only one that can dispatch new work
+
+ ggml_graph_compute_check_for_work(state);
+ if (state->pending) {
+ state->pending = false;
+
+ ggml_graph_compute_thread(state);
+ }
+ }
+
+ return (thread_ret_t) 0;
+}
+
+// Start processing new graph
+static void ggml_graph_compute_kickoff(struct ggml_threadpool * threadpool)
+{
+ // always take the mutex here because the worker threads are doing hybrid poll/wait
+
+ ggml_mutex_lock(&threadpool->mutex);
+
+ atomic_fetch_add_explicit(&threadpool->n_graph, 1, memory_order_relaxed);
+
+ if (threadpool->pause) {
+ // Update main thread prio and affinity to match the threadpool settings
+ ggml_thread_apply_priority(threadpool->prio);
+ if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {
+ ggml_thread_apply_affinity(threadpool->workers[0].cpumask);
+ }
+
+ // resume does cond broadcast
+ ggml_threadpool_resume_locked(threadpool);
+ } else {
+ ggml_cond_broadcast(&threadpool->cond);
+ }
+
+ ggml_mutex_unlock(&threadpool->mutex);
+}
+
+#endif // GGML_USE_OPENMP
+
+void ggml_threadpool_params_init(struct ggml_threadpool_params * p, int n_threads) {
+ p->n_threads = n_threads;
+ p->prio = 0; // default priority (usually means normal or inherited)
+ p->poll = 50; // hybrid-polling enabled
+ p->strict_cpu = false; // no strict placement (all threads share same cpumask)
+ p->paused = false; // threads are ready to go
+ memset(p->cpumask, 0, GGML_MAX_N_THREADS); // all-zero means use the default affinity (usually inherited)
+}
+
+struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads) {
+ struct ggml_threadpool_params p;
+ ggml_threadpool_params_init(&p, n_threads);
+ return p;
+}
+
+bool ggml_threadpool_params_match(const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1) {
+ if (p0->n_threads != p1->n_threads ) return false;
+ if (p0->prio != p1->prio ) return false;
+ if (p0->poll != p1->poll ) return false;
+ if (p0->strict_cpu != p1->strict_cpu ) return false;
+ return memcmp(p0->cpumask, p1->cpumask, GGML_MAX_N_THREADS) == 0;
+}
+
+static struct ggml_threadpool * ggml_threadpool_new_impl(
+ struct ggml_threadpool_params * tpp,
+ struct ggml_cgraph * cgraph,
+ struct ggml_cplan * cplan) {
+
+ struct ggml_threadpool * threadpool =
+ GGML_ALIGNED_MALLOC(sizeof(struct ggml_threadpool));
+ {
+ threadpool->cgraph = cgraph;
+ threadpool->cplan = cplan;
+ threadpool->n_graph = 0;
+ threadpool->n_barrier = 0;
+ threadpool->n_barrier_passed = 0;
+ threadpool->current_chunk = 0;
+ threadpool->stop = false;
+ threadpool->pause = tpp->paused;
+ threadpool->workers = NULL;
+ threadpool->n_threads_max = tpp->n_threads;
+ threadpool->n_threads_cur = tpp->n_threads;
+ threadpool->poll = tpp->poll;
+ threadpool->prio = tpp->prio;
+ threadpool->ec = GGML_STATUS_SUCCESS;
+ }
+
+ // Allocate and init workers state
+ const size_t workers_size = sizeof(struct ggml_compute_state) * tpp->n_threads;
+ struct ggml_compute_state * workers = GGML_ALIGNED_MALLOC(workers_size);
+
+ memset(workers, 0, workers_size);
+ for (int j = 0; j < tpp->n_threads; j++) {
+ workers[j].threadpool = threadpool;
+ workers[j].ith = j;
+ }
+
+ threadpool->workers = workers;
+
+#ifndef GGML_USE_OPENMP
+ ggml_mutex_init(&threadpool->mutex);
+ ggml_cond_init(&threadpool->cond);
+
+ // Spin the threads for all workers, and update CPU placements.
+ // Place the main thread last (towards the higher numbered CPU cores).
+
+ int32_t cpumask_iter = 0;
+
+ for (int j = 1; j < tpp->n_threads; j++) {
+ ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter);
+
+ int32_t rc = ggml_thread_create(&workers[j].thrd, NULL, ggml_graph_compute_secondary_thread, &workers[j]);
+ GGML_ASSERT(rc == 0);
+ }
+
+ ggml_thread_cpumask_next(tpp->cpumask, workers[0].cpumask, tpp->strict_cpu, &cpumask_iter);
+
+ if (!threadpool->pause) {
+ // Update main thread prio and affinity at the start, otherwise we'll do it in resume
+ ggml_thread_apply_priority(threadpool->prio);
+ if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {
+ ggml_thread_apply_affinity(threadpool->workers[0].cpumask);
+ }
+ }
+#endif // GGML_USE_OPENMP
+
+ return threadpool;
+}
+
+struct ggml_threadpool * ggml_threadpool_new(struct ggml_threadpool_params * tpp) {
+ return ggml_threadpool_new_impl(tpp, NULL, NULL);
+}
+
+enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {
+ GGML_ASSERT(cplan);
+ GGML_ASSERT(cplan->n_threads > 0);
+ GGML_ASSERT(cplan->work_size == 0 || cplan->work_data != NULL);
+
+ int n_threads = cplan->n_threads;
+ struct ggml_threadpool * threadpool = cplan->threadpool;
+
+ bool disposable_threadpool = false;
+
+ if (threadpool == NULL) {
+ GGML_PRINT_DEBUG("Threadpool is not specified. Will create a disposable threadpool : n_threads %d\n", n_threads);
+ disposable_threadpool = true;
+
+ struct ggml_threadpool_params ttp = ggml_threadpool_params_default(n_threads);
+ threadpool = ggml_threadpool_new_impl(&ttp, cgraph, cplan);
+ } else {
+ // Reset some of the parameters that need resetting
+ // No worker threads should be accessing the parameters below at this stage
+ threadpool->cgraph = cgraph;
+ threadpool->cplan = cplan;
+ threadpool->n_threads_cur = n_threads;
+ threadpool->current_chunk = 0;
+ threadpool->ec = GGML_STATUS_SUCCESS;
+ }
+
+ if (n_threads > threadpool->n_threads_max) {
+ GGML_PRINT("WARNING: cplan is requesting more threads than the threadpool contains. Expect a bad time!\n");
+ }
+
+#ifdef GGML_USE_OPENMP
+ if (n_threads > 1) {
+ #pragma omp parallel num_threads(n_threads)
+ {
+ #pragma omp single
+ {
+ // update the number of threads from the actual number of threads that we got from OpenMP
+ n_threads = omp_get_num_threads();
+ threadpool->n_threads_cur = n_threads;
+ }
+
+ ggml_graph_compute_thread(&threadpool->workers[omp_get_thread_num()]);
+ }
+ } else {
+ ggml_graph_compute_thread(&threadpool->workers[0]);
+ }
+#else
+ // Kick all threads to start the new graph
+ ggml_graph_compute_kickoff(threadpool);
+
+ // This is a work thread too
+ ggml_graph_compute_thread(&threadpool->workers[0]);
+#endif
+
+ // don't leave affinity set on the main thread
+ clear_numa_thread_affinity();
+
+ enum ggml_status ret = threadpool->ec;
+
+ if (disposable_threadpool) {
+ ggml_threadpool_free(threadpool);
+ }
+
+ return ret;
+}
+
+enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) {
+ struct ggml_cplan cplan = ggml_graph_plan(cgraph, n_threads, NULL);
+
+ struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_WORK_BUFFER, cplan.work_size);
+
+ cplan.work_data = (uint8_t *)ctx->mem_buffer + obj->offs;
+
+ return ggml_graph_compute(cgraph, &cplan);
+}
+
+struct ggml_tensor * ggml_graph_get_tensor(struct ggml_cgraph * cgraph, const char * name) {
+ for (int i = 0; i < cgraph->n_leafs; i++) {
+ struct ggml_tensor * leaf = cgraph->leafs[i];
+
+ if (strcmp(leaf->name, name) == 0) {
+ return leaf;
+ }
+ }
+
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ struct ggml_tensor * node = cgraph->nodes[i];
+
+ if (strcmp(node->name, name) == 0) {
+ return node;
+ }
+ }
+
+ return NULL;
+}
+
+static void ggml_graph_export_leaf(const struct ggml_tensor * tensor, FILE * fout) {
+ const int64_t * ne = tensor->ne;
+ const size_t * nb = tensor->nb;
+
+ fprintf(fout, "%-6s %-12s %8d %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %16zu %16zu %16zu %16zu %16p %32s\n",
+ ggml_type_name(tensor->type),
+ ggml_op_name (tensor->op),
+ ggml_n_dims(tensor),
+ ne[0], ne[1], ne[2], ne[3],
+ nb[0], nb[1], nb[2], nb[3],
+ tensor->data,
+ tensor->name);
+}
+
+static void ggml_graph_export_node(const struct ggml_tensor * tensor, const char * arg, FILE * fout) {
+ const int64_t * ne = tensor->ne;
+ const size_t * nb = tensor->nb;
+
+ fprintf(fout, "%-6s %-6s %-12s %8d %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %16zu %16zu %16zu %16zu %16p %32s\n",
+ arg,
+ ggml_type_name(tensor->type),
+ ggml_op_name (tensor->op),
+ ggml_n_dims(tensor),
+ ne[0], ne[1], ne[2], ne[3],
+ nb[0], nb[1], nb[2], nb[3],
+ tensor->data,
+ tensor->name);
+}
+
+void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname) {
+ uint64_t size_eval = 0;
+
+ // compute size of intermediate results
+ // TODO: does not take into account scratch buffers !!!!
+ for (int i = 0; i < cgraph->n_nodes; ++i) {
+ size_eval += ggml_nbytes_pad(cgraph->nodes[i]);
+ }
+
+ // print
+ {
+ FILE * fout = stdout;
+
+ fprintf(fout, "\n");
+ fprintf(fout, "%-16s %8x\n", "magic", GGML_FILE_MAGIC);
+ fprintf(fout, "%-16s %8d\n", "version", GGML_FILE_VERSION);
+ fprintf(fout, "%-16s %8d\n", "leafs", cgraph->n_leafs);
+ fprintf(fout, "%-16s %8d\n", "nodes", cgraph->n_nodes);
+ fprintf(fout, "%-16s %" PRIu64 "\n", "eval", size_eval);
+
+ // header
+ fprintf(fout, "\n");
+ fprintf(fout, "%-6s %-12s %8s %8s %8s %8s %8s %16s %16s %16s %16s %16s %16s\n",
+ "TYPE", "OP", "NDIMS", "NE0", "NE1", "NE2", "NE3", "NB0", "NB1", "NB2", "NB3", "DATA", "NAME");
+
+ for (int i = 0; i < cgraph->n_leafs; ++i) {
+ ggml_graph_export_leaf(cgraph->leafs[i], fout);
+
+ GGML_ASSERT(cgraph->leafs[i]->op == GGML_OP_NONE);
+ GGML_ASSERT(cgraph->leafs[i]->src[0] == NULL);
+ GGML_ASSERT(cgraph->leafs[i]->src[1] == NULL);
+ }
+
+ // header
+ fprintf(fout, "\n");
+ fprintf(fout, "%-6s %-6s %-12s %8s %8s %8s %8s %8s %16s %16s %16s %16s %8s %16s %16s\n",
+ "ARG", "TYPE", "OP", "NDIMS", "NE0", "NE1", "NE2", "NE3", "NB0", "NB1", "NB2", "NB3", "NTASKS", "DATA", "NAME");
+
+ for (int i = 0; i < cgraph->n_nodes; ++i) {
+ ggml_graph_export_node(cgraph->nodes[i], "DST", fout);
+
+ for (int j = 0; j < GGML_MAX_SRC; ++j) {
+ if (cgraph->nodes[i]->src[j]) {
+ ggml_graph_export_node(cgraph->nodes[i]->src[j], "SRC", fout);
+ }
+ }
+
+ fprintf(fout, "\n");
+ }
+
+ fprintf(fout, "\n");
+ }
+
+ // write binary data
+ {
+ FILE * fout = ggml_fopen(fname, "wb");
+
+ if (!fout) {
+ fprintf(stderr, "%s: failed to open %s: %s\n", __func__, fname, strerror(errno));
+ return;
+ }
+
+ // header
+ {
+ const uint32_t magic = GGML_FILE_MAGIC;
+ const uint32_t version = GGML_FILE_VERSION;
+ const uint32_t n_leafs = cgraph->n_leafs;
+ const uint32_t n_nodes = cgraph->n_nodes;
+
+ fwrite(&magic, sizeof(uint32_t), 1, fout);
+ fwrite(&version, sizeof(uint32_t), 1, fout);
+ fwrite(&n_leafs, sizeof(uint32_t), 1, fout);
+ fwrite(&n_nodes, sizeof(uint32_t), 1, fout);
+ fwrite(&size_eval, sizeof(uint64_t), 1, fout);
+ }
+
+ // leafs
+ {
+ for (int i = 0; i < cgraph->n_leafs; ++i) {
+ const struct ggml_tensor * tensor = cgraph->leafs[i];
+
+ const uint32_t type = tensor->type;
+ const uint32_t op = tensor->op;
+ const int32_t flags = tensor->flags;
+
+ fwrite(&type, sizeof(uint32_t), 1, fout);
+ fwrite(&op, sizeof(uint32_t), 1, fout);
+ fwrite(&flags, sizeof(int32_t), 1, fout);
+
+ for (int j = 0; j < GGML_MAX_DIMS; ++j) {
+ const uint64_t ne = tensor->ne[j];
+ const uint64_t nb = tensor->nb[j];
+
+ fwrite(&ne, sizeof(uint64_t), 1, fout);
+ fwrite(&nb, sizeof(uint64_t), 1, fout);
+ }
+
+ fwrite(tensor->name, sizeof(char), GGML_MAX_NAME, fout);
+ fwrite(tensor->op_params, sizeof(char), GGML_MAX_OP_PARAMS, fout);
+
+ // dump the data
+ // TODO: pad this to 32 byte boundary
+ {
+ const size_t size = ggml_nbytes(tensor);
+
+ fwrite(tensor->data, sizeof(char), size, fout);
+ }
+ }
+ }
+
+ // nodes
+ {
+ for (int i = 0; i < cgraph->n_nodes; ++i) {
+ const struct ggml_tensor * tensor = cgraph->nodes[i];
+
+ const uint32_t type = tensor->type;
+ const uint32_t op = tensor->op;
+ const int32_t flags = tensor->flags;
+
+ fwrite(&type, sizeof(uint32_t), 1, fout);
+ fwrite(&op, sizeof(uint32_t), 1, fout);
+ fwrite(&flags, sizeof(int32_t), 1, fout);
+
+ for (int j = 0; j < GGML_MAX_DIMS; ++j) {
+ const uint64_t ne = tensor->ne[j];
+ const uint64_t nb = tensor->nb[j];
+
+ fwrite(&ne, sizeof(uint64_t), 1, fout);
+ fwrite(&nb, sizeof(uint64_t), 1, fout);
+ }
+
+ fwrite(tensor->name, sizeof(char), GGML_MAX_NAME, fout);
+ fwrite(tensor->op_params, sizeof(char), GGML_MAX_OP_PARAMS, fout);
+
+ // output the op arguments
+ {
+ struct ggml_tensor * args[GGML_MAX_SRC] = { NULL };
+
+ for (int j = 0; j < GGML_MAX_SRC; ++j) {
+ args[j] = tensor->src[j];
+ }
+
+ for (int j = 0; j < GGML_MAX_SRC; ++j) {
+ if (args[j]) {
+ int32_t idx = -1;
+
+ // check if leaf
+ {
+ for (int k = 0; k < cgraph->n_leafs; ++k) {
+ if (args[j] == cgraph->leafs[k]) {
+ idx = k;
+ break;
+ }
+ }
+ }
+
+ // check if node
+ if (idx == -1) {
+ for (int k = 0; k < cgraph->n_nodes; ++k) {
+ if (args[j] == cgraph->nodes[k]) {
+ idx = cgraph->n_leafs + k;
+ break;
+ }
+ }
+ }
+
+ if (idx == -1) {
+ fprintf(stderr, "%s: failed to find tensor, arg = %d, node = %d\n", __func__, j, i);
+ fclose(fout);
+ return;
+ }
+
+ fwrite(&idx, sizeof(int32_t), 1, fout);
+ } else {
+ const int32_t nul = -1;
+
+ fwrite(&nul, sizeof(int32_t), 1, fout);
+ }
+ }
+ }
+
+ // dump the data
+ // TODO: pad this to 32 byte boundary
+ if ((flags & GGML_TENSOR_FLAG_PARAM)) {
+ const size_t size = ggml_nbytes(tensor);
+
+ fwrite(tensor->data, sizeof(char), size, fout);
+ }
+ }
+ }
+
+ fclose(fout);
+ }
+}
+
+struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context ** ctx_data, struct ggml_context ** ctx_eval) {
+ assert(*ctx_data == NULL);
+ assert(*ctx_eval == NULL);
+
+ struct ggml_cgraph * result = NULL;
+
+ struct ggml_tensor * data = NULL;
+
+ // read file into data
+ {
+ FILE * fin = ggml_fopen(fname, "rb");
+ if (!fin) {
+ fprintf(stderr, "%s: failed to open %s: %s\n", __func__, fname, strerror(errno));
+ return result;
+ }
+
+ size_t fsize = 0;
+
+ fseek(fin, 0, SEEK_END);
+ fsize = ftell(fin);
+ fseek(fin, 0, SEEK_SET);
+
+ // create the data context
+ {
+ const size_t overhead = 1*ggml_tensor_overhead();
+
+ struct ggml_init_params params = {
+ .mem_size = fsize + overhead,
+ .mem_buffer = NULL,
+ .no_alloc = false,
+ };
+
+ *ctx_data = ggml_init(params);
+
+ if (!*ctx_data) {
+ fprintf(stderr, "%s: failed to create ggml context\n", __func__);
+ fclose(fin);
+ return result;
+ }
+ }
+
+ data = ggml_new_tensor_1d(*ctx_data, GGML_TYPE_I8, fsize);
+
+ {
+ const size_t ret = fread(data->data, sizeof(char), fsize, fin);
+ if (ret != fsize) {
+ fprintf(stderr, "%s: failed to read %s\n", __func__, fname);
+ fclose(fin);
+ return result;
+ }
+ }
+
+ fclose(fin);
+ }
+
+ // populate result
+ {
+ char * ptr = (char *) data->data;
+
+ const uint32_t magic = *(const uint32_t *) ptr; ptr += sizeof(magic);
+
+ if (magic != GGML_FILE_MAGIC) {
+ fprintf(stderr, "%s: invalid magic number, got %08x\n", __func__, magic);
+ return result;
+ }
+
+ const uint32_t version = *(const uint32_t *) ptr; ptr += sizeof(version);
+
+ if (version != GGML_FILE_VERSION) {
+ fprintf(stderr, "%s: invalid version number\n", __func__);
+ return result;
+ }
+
+ const uint32_t n_leafs = *(const uint32_t *) ptr; ptr += sizeof(n_leafs);
+ const uint32_t n_nodes = *(const uint32_t *) ptr; ptr += sizeof(n_nodes);
+ const uint64_t size_eval = *(const uint64_t *) ptr; ptr += sizeof(size_eval);
+ const int graph_size = MAX(n_leafs, n_nodes);
+
+ // create the data context
+ {
+ const size_t overhead = (n_leafs + n_nodes)*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false);
+
+ struct ggml_init_params params = {
+ .mem_size = size_eval + overhead,
+ .mem_buffer = NULL,
+ .no_alloc = true,
+ };
+
+ *ctx_eval = ggml_init(params);
+
+ if (!*ctx_eval) {
+ fprintf(stderr, "%s: failed to create ggml context\n", __func__);
+ return result;
+ }
+ }
+
+ result = ggml_new_graph_custom(*ctx_eval, graph_size, false);
+
+ result->n_leafs = n_leafs;
+ result->n_nodes = n_nodes;
+
+
+ // leafs
+ {
+ uint32_t type;
+ uint32_t op;
+ int32_t flags;
+
+ for (uint32_t i = 0; i < n_leafs; ++i) {
+ type = *(const uint32_t *) ptr; ptr += sizeof(type);
+ op = *(const uint32_t *) ptr; ptr += sizeof(op);
+ flags = *(const int32_t *) ptr; ptr += sizeof(flags);
+
+ int64_t ne[GGML_MAX_DIMS];
+ size_t nb[GGML_MAX_DIMS];
+
+ for (int j = 0; j < GGML_MAX_DIMS; ++j) {
+ uint64_t ne_cur;
+ uint64_t nb_cur;
+
+ ne_cur = *(const uint64_t *) ptr; ptr += sizeof(ne_cur);
+ nb_cur = *(const uint64_t *) ptr; ptr += sizeof(nb_cur);
+
+ ne[j] = ne_cur;
+ nb[j] = nb_cur;
+ }
+
+ struct ggml_tensor * tensor = ggml_new_tensor(*ctx_eval, (enum ggml_type) type, GGML_MAX_DIMS, ne);
+
+ tensor->op = (enum ggml_op) op;
+ tensor->flags = flags;
+
+ memcpy(tensor->name, ptr, GGML_MAX_NAME); ptr += GGML_MAX_NAME;
+ memcpy(tensor->op_params, ptr, GGML_MAX_OP_PARAMS); ptr += GGML_MAX_OP_PARAMS;
+
+ for (int j = 0; j < GGML_MAX_DIMS; ++j) {
+ tensor->nb[j] = nb[j];
+ }
+
+ tensor->data = (void *) ptr; ptr += ggml_nbytes(tensor);
+
+ result->leafs[i] = tensor;
+
+ fprintf(stderr, "%s: loaded leaf %u: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor));
+ }
+ }
+
+ ggml_set_no_alloc(*ctx_eval, false);
+
+ // nodes
+ {
+ uint32_t type;
+ uint32_t op;
+ int32_t flags;
+
+ for (uint32_t i = 0; i < n_nodes; ++i) {
+ type = *(const uint32_t *) ptr; ptr += sizeof(type);
+ op = *(const uint32_t *) ptr; ptr += sizeof(op);
+ flags = *(const int32_t *) ptr; ptr += sizeof(flags);
+
+ enum ggml_op eop = (enum ggml_op) op;
+
+ int64_t ne[GGML_MAX_DIMS];
+ size_t nb[GGML_MAX_DIMS];
+
+ for (int j = 0; j < GGML_MAX_DIMS; ++j) {
+ uint64_t ne_cur;
+ uint64_t nb_cur;
+
+ ne_cur = *(const uint64_t *) ptr; ptr += sizeof(ne_cur);
+ nb_cur = *(const uint64_t *) ptr; ptr += sizeof(nb_cur);
+
+ ne[j] = ne_cur;
+ nb[j] = nb_cur;
+ }
+
+ const char * ptr_name = ptr; ptr += GGML_MAX_NAME;
+ const char * ptr_op_params = ptr; ptr += GGML_MAX_OP_PARAMS;
+
+ const int32_t * ptr_arg_idx = (const int32_t *) ptr; ptr += GGML_MAX_SRC*sizeof(int32_t);
+
+ struct ggml_tensor * args[GGML_MAX_SRC] = { NULL };
+
+ // parse args
+ for (int j = 0; j < GGML_MAX_SRC; ++j) {
+ const int32_t arg_idx = ptr_arg_idx[j];
+
+ if (arg_idx == -1) {
+ continue;
+ }
+
+ if (arg_idx < result->n_leafs) {
+ args[j] = result->leafs[arg_idx];
+ } else {
+ args[j] = result->nodes[arg_idx - result->n_leafs];
+ }
+ }
+
+ // create the tensor
+ // "view" operations are handled differently
+ // TODO: handle inplace ops - currently a copy is always made
+
+ struct ggml_tensor * tensor = NULL;
+
+ switch (eop) {
+ // TODO: implement other view ops
+ case GGML_OP_RESHAPE:
+ {
+ tensor = ggml_reshape_4d(*ctx_eval, args[0], ne[0], ne[1], ne[2], ne[3]);
+ } break;
+ case GGML_OP_VIEW:
+ {
+ tensor = ggml_view_4d(*ctx_eval, args[0], ne[0], ne[1], ne[2], ne[3], 0, 0, 0, 0);
+
+ size_t offs;
+ memcpy(&offs, ptr_op_params, sizeof(offs));
+
+ tensor->data = ((char *) tensor->data) + offs;
+ } break;
+ case GGML_OP_TRANSPOSE:
+ {
+ tensor = ggml_transpose(*ctx_eval, args[0]);
+ } break;
+ case GGML_OP_PERMUTE:
+ {
+ tensor = ggml_view_4d(*ctx_eval, args[0], ne[0], ne[1], ne[2], ne[3], 0, 0, 0, 0);
+ } break;
+ default:
+ {
+ tensor = ggml_new_tensor(*ctx_eval, (enum ggml_type) type, GGML_MAX_DIMS, ne);
+
+ tensor->op = eop;
+ } break;
+ }
+
+ memcpy(tensor->name, ptr_name, GGML_MAX_NAME);
+ memcpy(tensor->op_params, ptr_op_params, GGML_MAX_OP_PARAMS);
+
+ for (int j = 0; j < GGML_MAX_DIMS; ++j) {
+ tensor->nb[j] = nb[j];
+ }
+
+ for (int j = 0; j < GGML_MAX_SRC; ++j) {
+ tensor->src[j] = args[j];
+ }
+
+ result->nodes[i] = tensor;
+
+ // TODO tensor data is be duplicated due to ggml_new_tensor call above
+ if (flags & GGML_TENSOR_FLAG_PARAM) {
+ tensor->data = (void *) ptr; ptr += ggml_nbytes(tensor);
+ }
+
+ fprintf(stderr, "%s: loaded node %u: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor));
+ }
+ }
+ }
+
+ return result;
+}
+
+void ggml_graph_print(const struct ggml_cgraph * cgraph) {
+ GGML_PRINT("=== GRAPH ===\n");
+
+ GGML_PRINT("n_nodes = %d\n", cgraph->n_nodes);
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ struct ggml_tensor * node = cgraph->nodes[i];
+
+ GGML_PRINT(" - %3d: [ %5" PRId64 ", %5" PRId64 ", %5" PRId64 "] %16s %s\n",
+ i,
+ node->ne[0], node->ne[1], node->ne[2],
+ ggml_op_name(node->op), (node->flags & GGML_TENSOR_FLAG_PARAM) ? "x" : node->grad ? "g" : " ");
+ }
+
+ GGML_PRINT("n_leafs = %d\n", cgraph->n_leafs);
+ for (int i = 0; i < cgraph->n_leafs; i++) {
+ struct ggml_tensor * node = cgraph->leafs[i];
+
+ GGML_PRINT(" - %3d: [ %5" PRId64 ", %5" PRId64 "] %8s %16s\n",
+ i,
+ node->ne[0], node->ne[1],
+ ggml_op_name(node->op),
+ ggml_get_name(node));
+ }
+
+ GGML_PRINT("========================================\n");
+}
+
+// check if node is part of the graph
+static bool ggml_graph_find(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node) {
+ if (cgraph == NULL) {
+ return true;
+ }
+
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ if (cgraph->nodes[i] == node) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+static struct ggml_tensor * ggml_graph_get_parent(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node) {
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ struct ggml_tensor * parent = cgraph->nodes[i];
+
+ if (parent->grad == node) {
+ return parent;
+ }
+ }
+
+ return NULL;
+}
+
+static void ggml_graph_dump_dot_node_edge(FILE * fp, const struct ggml_cgraph * gb, struct ggml_tensor * node, struct ggml_tensor * parent, const char * label) {
+ struct ggml_tensor * gparent = ggml_graph_get_parent(gb, node);
+ struct ggml_tensor * gparent0 = ggml_graph_get_parent(gb, parent);
+ fprintf(fp, " \"%p\":%s -> \"%p\":%s [ arrowhead = %s; style = %s; label = \"%s\"; ]\n",
+ gparent0 ? (void *) gparent0 : (void *) parent,
+ gparent0 ? "g" : "x",
+ gparent ? (void *) gparent : (void *) node,
+ gparent ? "g" : "x",
+ gparent ? "empty" : "vee",
+ gparent ? "dashed" : "solid",
+ label);
+}
+
+static void ggml_graph_dump_dot_leaf_edge(FILE * fp, struct ggml_tensor * node, struct ggml_tensor * parent, const char * label) {
+ fprintf(fp, " \"%p\":%s -> \"%p\":%s [ label = \"%s\"; ]\n",
+ (void *) parent, "x",
+ (void *) node, "x",
+ label);
+}
+
+void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename) {
+ char color[16];
+
+ FILE * fp = ggml_fopen(filename, "w");
+ GGML_ASSERT(fp);
+
+ fprintf(fp, "digraph G {\n");
+ fprintf(fp, " newrank = true;\n");
+ fprintf(fp, " rankdir = TB;\n");
+
+ for (int i = 0; i < gb->n_nodes; i++) {
+ struct ggml_tensor * node = gb->nodes[i];
+
+ if (ggml_graph_get_parent(gb, node) != NULL) {
+ continue;
+ }
+
+ if (node->flags & GGML_TENSOR_FLAG_PARAM) {
+ snprintf(color, sizeof(color), "yellow");
+ } else if (node->grad) {
+ if (ggml_graph_find(gf, node)) {
+ snprintf(color, sizeof(color), "green");
+ } else {
+ snprintf(color, sizeof(color), "lightblue");
+ }
+ } else {
+ snprintf(color, sizeof(color), "white");
+ }
+
+ fprintf(fp, " \"%p\" [ "
+ "style = filled; fillcolor = %s; shape = record; "
+ "label=\"",
+ (void *) node, color);
+
+ if (strlen(node->name) > 0) {
+ fprintf(fp, "%s (%s)|", node->name, ggml_type_name(node->type));
+ } else {
+ fprintf(fp, "(%s)|", ggml_type_name(node->type));
+ }
+
+ if (ggml_is_matrix(node)) {
+ fprintf(fp, "%d [%" PRId64 ", %" PRId64 "] | <x>%s", i, node->ne[0], node->ne[1], ggml_op_symbol(node->op));
+ } else {
+ fprintf(fp, "%d [%" PRId64 ", %" PRId64 ", %" PRId64 "] | <x>%s", i, node->ne[0], node->ne[1], node->ne[2], ggml_op_symbol(node->op));
+ }
+
+ if (node->grad) {
+ fprintf(fp, " | <g>%s\"; ]\n", ggml_op_symbol(node->grad->op));
+ } else {
+ fprintf(fp, "\"; ]\n");
+ }
+ }
+
+ for (int i = 0; i < gb->n_leafs; i++) {
+ struct ggml_tensor * node = gb->leafs[i];
+
+ snprintf(color, sizeof(color), "pink");
+
+ fprintf(fp, " \"%p\" [ "
+ "style = filled; fillcolor = %s; shape = record; "
+ "label=\"<x>",
+ (void *) node, color);
+
+ if (strlen(node->name) > 0) {
+ fprintf(fp, "%s (%s)|", node->name, ggml_type_name(node->type));
+ } else {
+ fprintf(fp, "(%s)|", ggml_type_name(node->type));
+ }
+
+ fprintf(fp, "CONST %d [%" PRId64 ", %" PRId64 "]", i, node->ne[0], node->ne[1]);
+ if (ggml_nelements(node) < 5 && node->data != NULL) {
+ fprintf(fp, " | (");
+ for (int j = 0; j < ggml_nelements(node); j++) {
+ if (node->type == GGML_TYPE_I8 || node->type == GGML_TYPE_I16 || node->type == GGML_TYPE_I32) {
+ fprintf(fp, "%d", ggml_get_i32_1d(node, j));
+ }
+ else if (node->type == GGML_TYPE_F32 ||
+ node->type == GGML_TYPE_F16 ||
+ node->type == GGML_TYPE_BF16) {
+ fprintf(fp, "%.1e", (double)ggml_get_f32_1d(node, j));
+ }
+ else {
+ fprintf(fp, "#");
+ }
+ if (j < ggml_nelements(node) - 1) {
+ fprintf(fp, ", ");
+ }
+ }
+ fprintf(fp, ")");
+ }
+ fprintf(fp, "\"; ]\n");
+ }
+
+ for (int i = 0; i < gb->n_nodes; i++) {
+ struct ggml_tensor * node = gb->nodes[i];
+
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ if (node->src[j]) {
+ char label[16];
+ snprintf(label, sizeof(label), "src %d", j);
+ ggml_graph_dump_dot_node_edge(fp, gb, node, node->src[j], label);
+ }
+ }
+ }
+
+ for (int i = 0; i < gb->n_leafs; i++) {
+ struct ggml_tensor * node = gb->leafs[i];
+
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ if (node->src[j]) {
+ char label[16];
+ snprintf(label, sizeof(label), "src %d", j);
+ ggml_graph_dump_dot_leaf_edge(fp, node, node->src[j], label);
+ }
+ }
+ }
+
+ fprintf(fp, "}\n");
+
+ fclose(fp);
+
+ GGML_PRINT("%s: dot -Tpng %s -o %s.png && open %s.png\n", __func__, filename, filename, filename);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+static void ggml_opt_set_params(int np, struct ggml_tensor * const ps[], const float * x) {
+ int i = 0;
+ for (int p = 0; p < np; ++p) {
+ const int64_t ne = ggml_nelements(ps[p]) ;
+ // TODO: add function to set tensor from array
+ for (int64_t j = 0; j < ne; ++j) {
+ ggml_set_f32_1d(ps[p], j, x[i++]);
+ }
+ }
+}
+
+static void ggml_opt_get_params(int np, struct ggml_tensor * const ps[], float * x) {
+ int i = 0;
+ for (int p = 0; p < np; ++p) {
+ const int64_t ne = ggml_nelements(ps[p]) ;
+ // TODO: add function to get all elements at once
+ for (int64_t j = 0; j < ne; ++j) {
+ x[i++] = ggml_get_f32_1d(ps[p], j);
+ }
+ }
+}
+
+static void ggml_opt_get_grad(int np, struct ggml_tensor * const ps[], float * g) {
+ int64_t i = 0;
+ for (int p = 0; p < np; ++p) {
+ const int64_t ne = ggml_nelements(ps[p]) ;
+ // TODO: add function to get all elements at once
+ for (int64_t j = 0; j < ne; ++j) {
+ g[i++] = ggml_get_f32_1d(ps[p]->grad, j);
+ }
+ }
+}
+
+static void ggml_opt_acc_grad(int np, struct ggml_tensor * const ps[], float * g, float scale) {
+ int64_t i = 0;
+ for (int p = 0; p < np; ++p) {
+ const int64_t ne = ggml_nelements(ps[p]) ;
+ // TODO: add function to get all elements at once
+ for (int64_t j = 0; j < ne; ++j) {
+ g[i++] += ggml_get_f32_1d(ps[p]->grad, j) * scale;
+ }
+ }
+}
+
+//
+// Using AdamW - ref: https://arxiv.org/pdf/1711.05101v3.pdf
+//
+// (Original Adam - ref: https://arxiv.org/pdf/1412.6980.pdf)
+//
+
+static enum ggml_opt_result ggml_opt_adam(
+ struct ggml_context * ctx,
+ struct ggml_opt_context * opt,
+ struct ggml_opt_params params,
+ struct ggml_tensor * f,
+ struct ggml_cgraph * gf,
+ struct ggml_cgraph * gb,
+ ggml_opt_callback callback,
+ void * callback_data) {
+ GGML_ASSERT(ggml_is_scalar(f));
+ GGML_ASSERT(f->type == GGML_TYPE_F32);
+
+ // these will store the parameters we want to optimize
+ struct ggml_tensor * ps[GGML_MAX_PARAMS];
+
+ int np = 0;
+ int64_t nx = 0;
+ for (int i = 0; i < gf->n_nodes; ++i) {
+ if (gf->nodes[i]->flags & GGML_TENSOR_FLAG_PARAM) {
+ GGML_PRINT_DEBUG("found param %d: grad->op = %d\n", np, gf->nodes[i]->grad->op);
+
+ GGML_ASSERT(np < GGML_MAX_PARAMS);
+
+ ps[np++] = gf->nodes[i];
+ nx += ggml_nelements(gf->nodes[i]);
+ }
+ }
+
+ if ((opt->params.type != params.type) || (opt->nx != nx) || (opt->params.past != params.past)) {
+ int iter = opt->iter;
+ ggml_opt_init(opt->ctx, opt, params, nx);
+ opt->iter = iter;
+ }
+
+ // constants
+ float sched = params.adam.sched;
+ const float alpha = params.adam.alpha;
+ const float decay = params.adam.decay * alpha;
+ const float beta1 = params.adam.beta1;
+ const float beta2 = params.adam.beta2;
+ const float eps = params.adam.eps;
+ const float gclip = params.adam.gclip;
+ const int decay_min_ndim = params.adam.decay_min_ndim;
+ const int n_accum = MAX(1, params.n_gradient_accumulation);
+ const float accum_norm = 1.0f / (float) n_accum;
+
+ float * g = opt->adam.g->data; // gradients
+ float * m = opt->adam.m->data; // first moment
+ float * v = opt->adam.v->data; // second moment
+
+ float * pf = params.past > 0 ? opt->adam.pf->data : NULL; // past function values
+
+ struct ggml_cplan cplan = ggml_graph_plan(gb, params.n_threads, NULL);
+ struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_WORK_BUFFER, cplan.work_size);
+ cplan.work_data = (uint8_t *)ctx->mem_buffer + obj->offs;
+
+ bool cancel = false;
+
+ // compute the function value
+ float fx = 0;
+ ggml_set_zero(opt->adam.g);
+ for (int accum_step = 0; accum_step < n_accum; ++accum_step) {
+ if (callback) {
+ callback(callback_data, accum_step, &sched, &cancel);
+ if (cancel) {
+ return GGML_OPT_RESULT_CANCEL;
+ }
+ }
+ // ggml_graph_reset (gf);
+ ggml_set_f32 (f->grad, 1.0f);
+ ggml_graph_compute(gb, &cplan);
+ ggml_opt_acc_grad(np, ps, g, accum_norm);
+ fx += ggml_get_f32_1d(f, 0);
+ }
+ fx *= accum_norm;
+
+ opt->adam.fx_prev = fx;
+ opt->adam.fx_best = opt->adam.fx_prev;
+ if (pf) {
+ pf[opt->iter % params.past] = opt->adam.fx_prev;
+ }
+
+ opt->loss_before = opt->adam.fx_prev;
+ opt->loss_after = opt->adam.fx_prev;
+
+ // initialize
+ if (opt->just_initialized) {
+ opt->adam.n_no_improvement = 0;
+ opt->just_initialized = false;
+ }
+
+ float * fx_best = &opt->adam.fx_best;
+ float * fx_prev = &opt->adam.fx_prev;
+ int * n_no_improvement = &opt->adam.n_no_improvement;
+
+ int iter0 = opt->iter;
+
+ // run the optimizer
+ for (int t = 0; t < params.adam.n_iter; ++t) {
+ opt->iter = iter0 + t + 1;
+ GGML_PRINT_DEBUG ("=== iter %d ===\n", t);
+
+ GGML_PRINT_DEBUG ("f = %10.6f\n", ggml_get_f32_1d(f, 0));
+ GGML_PRINT_DEBUG_5("df/dx0 = %10.6f\n", ggml_get_f32_1d(ps[0]->grad, 0));
+ GGML_PRINT_DEBUG_5("df/dx1 = %10.6f\n", ggml_get_f32_1d(ps[1]->grad, 0));
+
+ for (int i = 0; i < np; ++i) {
+ GGML_PRINT_DEBUG("param %d: %10.6f, g = %10.6f\n", i,
+ ggml_get_f32_1d(ps[i], 0), ggml_get_f32_1d(ps[i]->grad, 0));
+ }
+
+ const int64_t t_start_wall = ggml_time_us();
+ const int64_t t_start_cpu = ggml_cycles();
+ UNUSED(t_start_wall);
+ UNUSED(t_start_cpu);
+
+ {
+ float gnorm = 1.0f;
+ if (gclip > 0.0f) {
+ // gradient clipping
+ ggml_float sum = 0.0;
+ for (int64_t i = 0; i < nx; ++i) {
+ sum += (ggml_float)(g[i]*g[i]);
+ }
+ ggml_float norm = sqrt(sum);
+ if (norm > (ggml_float) gclip) {
+ gnorm = (float) ((ggml_float) gclip / norm);
+ }
+ }
+ const float beta1h = alpha*sched/(1.0f - powf(beta1, opt->iter));
+ const float beta2h = 1.0f/(1.0f - powf(beta2, opt->iter));
+ int64_t i = 0;
+ for (int p = 0; p < np; ++p) {
+ const int64_t ne = ggml_nelements(ps[p]);
+ const float p_decay = ((ggml_n_dims(ps[p]) >= decay_min_ndim) ? decay : 0.0f) * sched;
+ for (int64_t j = 0; j < ne; ++j) {
+ float x = ggml_get_f32_1d(ps[p], j);
+ float g_ = g[i]*gnorm;
+ m[i] = m[i]*beta1 + g_*(1.0f - beta1);
+ v[i] = v[i]*beta2 + g_*g_*(1.0f - beta2);
+ float mh = m[i]*beta1h;
+ float vh = v[i]*beta2h;
+ vh = sqrtf(vh) + eps;
+ x = x*(1.0f - p_decay) - mh/vh;
+ ggml_set_f32_1d(ps[p], j, x);
+ ++i;
+ }
+ }
+ }
+
+ fx = 0;
+ ggml_set_zero(opt->adam.g);
+ for (int accum_step = 0; accum_step < n_accum; ++accum_step) {
+ if (callback) {
+ callback(callback_data, accum_step, &sched, &cancel);
+ if (cancel) {
+ return GGML_OPT_RESULT_CANCEL;;
+ }
+ }
+ // ggml_graph_reset (gf);
+ ggml_set_f32 (f->grad, 1.0f);
+ ggml_graph_compute(gb, &cplan);
+ ggml_opt_acc_grad(np, ps, g, accum_norm);
+ fx += ggml_get_f32_1d(f, 0);
+ }
+ fx *= accum_norm;
+
+ opt->loss_after = fx;
+
+ // check convergence
+ if (fabsf(fx - fx_prev[0])/fx < params.adam.eps_f) {
+ GGML_PRINT_DEBUG("converged\n");
+
+ return GGML_OPT_RESULT_OK;
+ }
+
+ // delta-based convergence test
+ if (pf != NULL) {
+ // need at least params.past iterations to start checking for convergence
+ if (params.past <= iter0 + t) {
+ const float rate = (pf[(iter0 + t)%params.past] - fx)/fx;
+
+ if (fabsf(rate) < params.delta) {
+ return GGML_OPT_RESULT_OK;
+ }
+ }
+
+ pf[(iter0 + t)%params.past] = fx;
+ }
+
+ // check for improvement
+ if (params.max_no_improvement > 0) {
+ if (fx_best[0] > fx) {
+ fx_best[0] = fx;
+ n_no_improvement[0] = 0;
+ } else {
+ ++n_no_improvement[0];
+
+ if (n_no_improvement[0] >= params.max_no_improvement) {
+ return GGML_OPT_RESULT_OK;
+ }
+ }
+ }
+
+ fx_prev[0] = fx;
+
+ {
+ const int64_t t_end_cpu = ggml_cycles();
+ GGML_PRINT_DEBUG("time iter: %5.3f s\n", ((float)(t_end_cpu - t_start_cpu))/CLOCKS_PER_SEC);
+ UNUSED(t_end_cpu);
+
+ const int64_t t_end_wall = ggml_time_us();
+ GGML_PRINT_DEBUG("wall time iter: %5.3f s\n", (t_end_wall - t_start_wall)/1e6);
+ UNUSED(t_end_wall);
+ }
+ }
+
+ return GGML_OPT_RESULT_DID_NOT_CONVERGE;
+}
+
+//
+// L-BFGS
+//
+// the L-BFGS implementation below is based on the following implementation:
+//
+// https://github.com/chokkan/liblbfgs
+//
+
+struct ggml_lbfgs_iteration_data {
+ float alpha;
+ float ys;
+ float * s;
+ float * y;
+};
+
+static enum ggml_opt_result linesearch_backtracking(
+ const struct ggml_opt_params * params,
+ int nx,
+ float * x,
+ float * fx,
+ float * g,
+ float * d,
+ float * step,
+ const float * xp,
+ struct ggml_tensor * f,
+ struct ggml_cgraph * gb,
+ struct ggml_cplan * cplan,
+ const int np,
+ struct ggml_tensor * ps[],
+ bool * cancel,
+ ggml_opt_callback callback,
+ void * callback_data) {
+ int count = 0;
+
+ float width = 0.0f;
+ float dg = 0.0f;
+ float finit = 0.0f;
+ float dginit = 0.0f;
+ float dgtest = 0.0f;
+
+ const float dec = 0.5f;
+ const float inc = 2.1f;
+
+ const int n_accum = MAX(1, params->n_gradient_accumulation);
+ const float accum_norm = 1.0f / (float) n_accum;
+
+ if (*step <= 0.f) {
+ return GGML_LINESEARCH_INVALID_PARAMETERS;
+ }
+
+ // compute the initial gradient in the search direction
+ ggml_vec_dot_f32(nx, &dginit, 0, g, 0, d, 0, 1);
+
+ // make sure that d points to a descent direction
+ if (0 < dginit) {
+ return GGML_LINESEARCH_FAIL;
+ }
+
+ // initialize local variables
+ finit = *fx;
+ dgtest = params->lbfgs.ftol*dginit;
+
+ while (true) {
+ ggml_vec_cpy_f32(nx, x, xp);
+ ggml_vec_mad_f32(nx, x, d, *step);
+
+ // evaluate the function and gradient values
+ {
+ ggml_opt_set_params(np, ps, x);
+
+ *fx = 0;
+ memset(g, 0, sizeof(float)*nx);
+ for (int accum_step = 0; accum_step < n_accum; ++accum_step) {
+ if (callback) {
+ // LBFG-S does not support learning rate -> ignore learning schedule
+ float sched = 0;
+ callback(callback_data, accum_step, &sched, cancel);
+ if (*cancel) {
+ return GGML_OPT_RESULT_CANCEL;
+ }
+ }
+ // ggml_graph_reset (gf);
+ ggml_set_f32 (f->grad, 1.0f);
+ ggml_graph_compute(gb, cplan);
+ ggml_opt_acc_grad(np, ps, g, accum_norm);
+ *fx += ggml_get_f32_1d(f, 0);
+ }
+ *fx *= accum_norm;
+
+ }
+
+ ++count;
+
+ if (*fx > finit + (*step)*dgtest) {
+ width = dec;
+ } else {
+ // Armijo condition is satisfied
+ if (params->lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_ARMIJO) {
+ return count;
+ }
+
+ ggml_vec_dot_f32(nx, &dg, 0, g, 0, d, 0, 1);
+
+ // check the Wolfe condition
+ if (dg < params->lbfgs.wolfe * dginit) {
+ width = inc;
+ } else {
+ if(params->lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_WOLFE) {
+ // regular Wolfe conditions
+ return count;
+ }
+
+ if(dg > -params->lbfgs.wolfe*dginit) {
+ width = dec;
+ } else {
+ // strong Wolfe condition (GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE)
+ return count;
+ }
+ }
+ }
+
+ if (*step < params->lbfgs.min_step) {
+ return GGML_LINESEARCH_MINIMUM_STEP;
+ }
+ if (*step > params->lbfgs.max_step) {
+ return GGML_LINESEARCH_MAXIMUM_STEP;
+ }
+ if (params->lbfgs.max_linesearch <= count) {
+ return GGML_LINESEARCH_MAXIMUM_ITERATIONS;
+ }
+
+ (*step) *= width;
+ }
+
+ GGML_ABORT("line search failed");
+
+ //return GGML_LINESEARCH_FAIL;
+}
+
+static enum ggml_opt_result ggml_opt_lbfgs(
+ struct ggml_context * ctx,
+ struct ggml_opt_context * opt,
+ struct ggml_opt_params params,
+ struct ggml_tensor * f,
+ struct ggml_cgraph * gf,
+ struct ggml_cgraph * gb,
+ ggml_opt_callback callback,
+ void * callback_data) {
+ if (params.lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_WOLFE ||
+ params.lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE) {
+ if (params.lbfgs.wolfe <= params.lbfgs.ftol || 1.f <= params.lbfgs.wolfe) {
+ return GGML_OPT_RESULT_INVALID_WOLFE;
+ }
+ }
+
+ const int m = params.lbfgs.m;
+
+ // these will store the parameters we want to optimize
+ struct ggml_tensor * ps[GGML_MAX_PARAMS];
+
+ int np = 0;
+ int nx = 0;
+ for (int i = 0; i < gf->n_nodes; ++i) {
+ if (gf->nodes[i]->flags & GGML_TENSOR_FLAG_PARAM) {
+ GGML_PRINT_DEBUG("found param %d: grad->op = %d\n", np, gf->nodes[i]->grad->op);
+
+ GGML_ASSERT(np < GGML_MAX_PARAMS);
+
+ ps[np++] = gf->nodes[i];
+ nx += ggml_nelements(gf->nodes[i]);
+ }
+ }
+
+ if ((opt->params.type != params.type) || (opt->nx != nx) || (opt->params.past != params.past) || (opt->params.lbfgs.m != params.lbfgs.m)) {
+ int iter = opt->iter;
+ ggml_opt_init(ctx, opt, params, nx);
+ opt->iter = iter;
+ }
+
+ struct ggml_cplan cplan = ggml_graph_plan(gb, params.n_threads, NULL);
+ struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_WORK_BUFFER, cplan.work_size);
+ cplan.work_data = (uint8_t *)ctx->mem_buffer + obj->offs;
+
+ float * x = opt->lbfgs.x->data; // current parameters
+ float * xp = opt->lbfgs.xp->data; // previous parameters
+ float * g = opt->lbfgs.g->data; // current gradient
+ float * gp = opt->lbfgs.gp->data; // previous gradient
+ float * d = opt->lbfgs.d->data; // search direction
+
+ float * pf = params.past > 0 ? opt->lbfgs.pf->data : NULL; // past function values
+
+ const int n_accum = MAX(1, params.n_gradient_accumulation);
+ const float accum_norm = 1.0f / (float) n_accum;
+
+ float fx = 0.0f; // cost function value
+ float xnorm = 0.0f; // ||x||
+ float gnorm = 0.0f; // ||g||
+
+ // initialize x from the graph nodes
+ ggml_opt_get_params(np, ps, x);
+
+ // the L-BFGS memory
+ float * lm_alpha = opt->lbfgs.lmal->data;
+ float * lm_ys = opt->lbfgs.lmys->data;
+ float * lm_s = opt->lbfgs.lms->data;
+ float * lm_y = opt->lbfgs.lmy->data;
+
+ bool cancel = false;
+
+ // evaluate the function value and its gradient
+ {
+ ggml_opt_set_params(np, ps, x);
+
+ fx = 0;
+ memset(g, 0, sizeof(float)*nx);
+ for (int accum_step = 0; accum_step < n_accum; ++accum_step) {
+ if (callback) {
+ // LBFG-S does not support learning rate -> ignore learning schedule
+ float sched = 0;
+ callback(callback_data, accum_step, &sched, &cancel);
+ if (cancel) {
+ return GGML_OPT_RESULT_CANCEL;
+ }
+ }
+ // ggml_graph_reset (gf);
+ ggml_set_f32 (f->grad, 1.0f);
+ ggml_graph_compute(gb, &cplan);
+ ggml_opt_acc_grad(np, ps, g, accum_norm);
+ fx += ggml_get_f32_1d(f, 0);
+ }
+ fx *= accum_norm;
+
+ opt->loss_before = fx;
+ opt->loss_after = fx;
+ }
+
+ // search direction = -gradient
+ ggml_vec_neg_f32(nx, d, g);
+
+ // ||x||, ||g||
+ ggml_vec_norm_f32(nx, &xnorm, x);
+ ggml_vec_norm_f32(nx, &gnorm, g);
+
+ if (xnorm < 1.0f) {
+ xnorm = 1.0f;
+ }
+
+ // already optimized
+ if (gnorm/xnorm <= params.lbfgs.eps) {
+ return GGML_OPT_RESULT_OK;
+ }
+
+ if (opt->just_initialized) {
+ if (pf) {
+ pf[0] = fx;
+ }
+ opt->lbfgs.fx_best = fx;
+
+ // initial step
+ ggml_vec_norm_inv_f32(nx, &opt->lbfgs.step, d);
+ opt->lbfgs.j = 0;
+ opt->lbfgs.k = 1;
+ opt->lbfgs.end = 0;
+ opt->lbfgs.n_no_improvement = 0;
+ opt->just_initialized = false;
+ }
+
+ float * fx_best = &opt->lbfgs.fx_best;
+ float * step = &opt->lbfgs.step;
+ int * j = &opt->lbfgs.j;
+ int * k = &opt->lbfgs.k;
+ int * end = &opt->lbfgs.end;
+ int * n_no_improvement = &opt->lbfgs.n_no_improvement;
+
+ int ls = 0;
+ int bound = 0;
+
+ float ys = 0.0f;
+ float yy = 0.0f;
+ float beta = 0.0f;
+
+ int it = 0;
+
+ while (true) {
+ // store the current position and gradient vectors
+ ggml_vec_cpy_f32(nx, xp, x);
+ ggml_vec_cpy_f32(nx, gp, g);
+
+ // TODO: instead of passing &cancel here, use the return code of the linesearch
+ // to determine if the optimization should be cancelled
+ // this is a simple change, but not doing this atm, since I don't have a nice
+ // way to test and don't want to break something with so many changes lined up
+ ls = linesearch_backtracking(¶ms, nx, x, &fx, g, d, step, xp, f, gb, &cplan, np, ps, &cancel, callback, callback_data);
+ if (cancel) {
+ return GGML_OPT_RESULT_CANCEL;
+ }
+
+ if (ls < 0) {
+ // linesearch failed - go back to the previous point and return
+ ggml_vec_cpy_f32(nx, x, xp);
+ ggml_vec_cpy_f32(nx, g, gp);
+
+ return ls;
+ }
+
+ opt->loss_after = fx;
+
+ ggml_vec_norm_f32(nx, &xnorm, x);
+ ggml_vec_norm_f32(nx, &gnorm, g);
+
+ GGML_PRINT_DEBUG("f = %10.6f\n", ggml_get_f32_1d(f, 0));
+
+ if (xnorm < 1.0f) {
+ xnorm = 1.0f;
+ }
+ if (gnorm/xnorm <= params.lbfgs.eps) {
+ // converged
+ return GGML_OPT_RESULT_OK;
+ }
+
+ // delta-based convergence test
+ if (pf != NULL) {
+ // need at least params.past iterations to start checking for convergence
+ if (params.past <= k[0]) {
+ const float rate = (pf[k[0]%params.past] - fx)/fx;
+
+ if (fabsf(rate) < params.delta) {
+ return GGML_OPT_RESULT_OK;
+ }
+ }
+
+ pf[k[0]%params.past] = fx;
+ }
+
+ // check for improvement
+ if (params.max_no_improvement > 0) {
+ if (fx < fx_best[0]) {
+ fx_best[0] = fx;
+ n_no_improvement[0] = 0;
+ } else {
+ n_no_improvement[0]++;
+
+ if (n_no_improvement[0] >= params.max_no_improvement) {
+ return GGML_OPT_RESULT_OK;
+ }
+ }
+ }
+
+ if (params.lbfgs.n_iter != 0 && params.lbfgs.n_iter < it + 1) {
+ // reached the maximum number of iterations
+ return GGML_OPT_RESULT_DID_NOT_CONVERGE;
+ }
+
+ // update vectors s and y:
+ // s_{k+1} = x_{k+1} - x_{k} = \step * d_{k}.
+ // y_{k+1} = g_{k+1} - g_{k}.
+ //
+ ggml_vec_sub_f32(nx, &lm_s[end[0]*nx], x, xp);
+ ggml_vec_sub_f32(nx, &lm_y[end[0]*nx], g, gp);
+
+ // compute scalars ys and yy:
+ // ys = y^t \cdot s -> 1 / \rho.
+ // yy = y^t \cdot y.
+ //
+ ggml_vec_dot_f32(nx, &ys, 0, &lm_y[end[0]*nx], 0, &lm_s[end[0]*nx], 0, 1);
+ ggml_vec_dot_f32(nx, &yy, 0, &lm_y[end[0]*nx], 0, &lm_y[end[0]*nx], 0, 1);
+
+ lm_ys[end[0]] = ys;
+
+ // find new search direction
+ // ref: https://en.wikipedia.org/wiki/Limited-memory_BFGS
+
+ bound = (m <= k[0]) ? m : k[0];
+ k[0]++;
+ it++;
+ end[0] = (end[0] + 1)%m;
+
+ // initialize search direction with -g
+ ggml_vec_neg_f32(nx, d, g);
+
+ j[0] = end[0];
+ for (int i = 0; i < bound; ++i) {
+ j[0] = (j[0] + m - 1) % m;
+ // \alpha_{j} = \rho_{j} s^{t}_{j} \cdot q_{k+1}
+ ggml_vec_dot_f32(nx, &lm_alpha[j[0]], 0, &lm_s[j[0]*nx], 0, d, 0, 1);
+ lm_alpha[j[0]] /= lm_ys[j[0]];
+ // q_{i} = q_{i+1} - \alpha_{i} y_{i}
+ ggml_vec_mad_f32(nx, d, &lm_y[j[0]*nx], -lm_alpha[j[0]]);
+ }
+
+ ggml_vec_scale_f32(nx, d, ys/yy);
+
+ for (int i = 0; i < bound; ++i) {
+ // \beta_{j} = \rho_{j} y^t_{j} \cdot \gamma_{i}
+ ggml_vec_dot_f32(nx, &beta, 0, &lm_y[j[0]*nx], 0, d, 0, 1);
+ beta /= lm_ys[j[0]];
+ // \gamma_{i+1} = \gamma_{i} + (\alpha_{j} - \beta_{j}) s_{j}
+ ggml_vec_mad_f32(nx, d, &lm_s[j[0]*nx], lm_alpha[j[0]] - beta);
+ j[0] = (j[0] + 1)%m;
+ }
+
+ step[0] = 1.0;
+ }
+
+ GGML_ABORT("lbfgs failed");
+
+ //return GGML_OPT_RESULT_DID_NOT_CONVERGE;
+}
+
+struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type) {
+ struct ggml_opt_params result;
+
+ switch (type) {
+ case GGML_OPT_TYPE_ADAM:
+ {
+ result = (struct ggml_opt_params) {
+ .type = GGML_OPT_TYPE_ADAM,
+ .graph_size = GGML_DEFAULT_GRAPH_SIZE,
+ .n_threads = 1, // FIXME: GGML_DEFAULT_N_THREADS ?
+ .past = 0,
+ .delta = 1e-5f,
+
+ .max_no_improvement = 100,
+
+ .print_forward_graph = true,
+ .print_backward_graph = true,
+
+ .n_gradient_accumulation = 1,
+
+ .adam = {
+ .n_iter = 10000,
+ .sched = 1.000f,
+ .decay = 0.0f,
+ .decay_min_ndim = 2,
+ .alpha = 0.001f,
+ .beta1 = 0.9f,
+ .beta2 = 0.999f,
+ .eps = 1e-8f,
+ .eps_f = 1e-5f,
+ .eps_g = 1e-3f,
+ .gclip = 0.0f,
+ },
+ };
+ } break;
+ case GGML_OPT_TYPE_LBFGS:
+ {
+ result = (struct ggml_opt_params) {
+ .type = GGML_OPT_TYPE_LBFGS,
+ .graph_size = GGML_DEFAULT_GRAPH_SIZE,
+ .n_threads = 1,
+ .past = 0,
+ .delta = 1e-5f,
+
+ .max_no_improvement = 0,
+
+ .print_forward_graph = true,
+ .print_backward_graph = true,
+
+ .n_gradient_accumulation = 1,
+
+ .lbfgs = {
+ .m = 6,
+ .n_iter = 100,
+ .max_linesearch = 20,
+
+ .eps = 1e-5f,
+ .ftol = 1e-4f,
+ .wolfe = 0.9f,
+ .min_step = 1e-20f,
+ .max_step = 1e+20f,
+
+ .linesearch = GGML_LINESEARCH_DEFAULT,
+ },
+ };
+ } break;
+ }
+
+ return result;
+}
+
+GGML_API void ggml_opt_init(
+ struct ggml_context * ctx,
+ struct ggml_opt_context * opt,
+ struct ggml_opt_params params,
+ int64_t nx) {
+ opt->ctx = ctx;
+ opt->params = params;
+ opt->iter = 0;
+ opt->nx = nx;
+ opt->just_initialized = true;
+ if (opt->ctx == NULL) {
+ struct ggml_init_params ctx_opt_params;
+ if (opt->params.type == GGML_OPT_TYPE_ADAM) {
+ ctx_opt_params.mem_size = GGML_MEM_ALIGN*3 + ggml_tensor_overhead()*3 + ggml_type_size(GGML_TYPE_F32)*nx*3;
+ if (opt->params.past > 0) {
+ ctx_opt_params.mem_size += GGML_MEM_ALIGN + ggml_tensor_overhead() + ggml_type_size(GGML_TYPE_F32)*opt->params.past;
+ }
+ } else if (opt->params.type == GGML_OPT_TYPE_LBFGS) {
+ ctx_opt_params.mem_size = GGML_MEM_ALIGN*9 + ggml_tensor_overhead()*9 + ggml_type_size(GGML_TYPE_F32)*(nx*5 + opt->params.lbfgs.m*2 + nx*opt->params.lbfgs.m*2);
+ if (opt->params.past > 0) {
+ ctx_opt_params.mem_size += GGML_MEM_ALIGN + ggml_tensor_overhead() + ggml_type_size(GGML_TYPE_F32)*opt->params.past;
+ }
+ }
+ ctx_opt_params.mem_buffer = NULL;
+ ctx_opt_params.no_alloc = false;
+
+ opt->ctx = ggml_init(ctx_opt_params);
+ }
+ switch (opt->params.type) {
+ case GGML_OPT_TYPE_ADAM:
+ {
+ opt->adam.g = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx);
+ opt->adam.m = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx);
+ opt->adam.v = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx);
+ opt->adam.pf = params.past > 0
+ ? ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, params.past)
+ : NULL;
+ ggml_set_zero(opt->adam.m);
+ ggml_set_zero(opt->adam.v);
+ if (opt->adam.pf) {
+ ggml_set_zero(opt->adam.pf);
+ }
+ } break;
+ case GGML_OPT_TYPE_LBFGS:
+ {
+ opt->lbfgs.x = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx);
+ opt->lbfgs.xp = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx);
+ opt->lbfgs.g = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx);
+ opt->lbfgs.gp = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx);
+ opt->lbfgs.d = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx);
+ opt->lbfgs.pf = params.past > 0
+ ? ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, params.past)
+ : NULL;
+ opt->lbfgs.lmal = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, params.lbfgs.m);
+ opt->lbfgs.lmys = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, params.lbfgs.m);
+ opt->lbfgs.lms = ggml_new_tensor_2d(opt->ctx, GGML_TYPE_F32, nx, params.lbfgs.m);
+ opt->lbfgs.lmy = ggml_new_tensor_2d(opt->ctx, GGML_TYPE_F32, nx, params.lbfgs.m);
+ ggml_set_zero(opt->lbfgs.x);
+ ggml_set_zero(opt->lbfgs.xp);
+ ggml_set_zero(opt->lbfgs.g);
+ ggml_set_zero(opt->lbfgs.gp);
+ ggml_set_zero(opt->lbfgs.d);
+ if (opt->lbfgs.pf) {
+ ggml_set_zero(opt->lbfgs.pf);
+ }
+ ggml_set_zero(opt->lbfgs.lmal);
+ ggml_set_zero(opt->lbfgs.lmys);
+ ggml_set_zero(opt->lbfgs.lms);
+ ggml_set_zero(opt->lbfgs.lmy);
+ } break;
+ }
+}
+
+enum ggml_opt_result ggml_opt(
+ struct ggml_context * ctx,
+ struct ggml_opt_params params,
+ struct ggml_tensor * f) {
+ GGML_ASSERT(f->grad && "ggml_set_param called for at least one parent tensor.");
+
+ bool free_ctx = false;
+ if (ctx == NULL) {
+ struct ggml_init_params params_ctx = {
+ .mem_size = 16*1024*1024,
+ .mem_buffer = NULL,
+ .no_alloc = false,
+ };
+
+ ctx = ggml_init(params_ctx);
+ if (ctx == NULL) {
+ return GGML_OPT_RESULT_NO_CONTEXT;
+ }
+
+ free_ctx = true;
+ }
+
+ enum ggml_opt_result result = GGML_OPT_RESULT_OK;
+
+ struct ggml_opt_context * opt = (struct ggml_opt_context *) alloca(sizeof(struct ggml_opt_context));
+
+ ggml_opt_init(ctx, opt, params, 0);
+ result = ggml_opt_resume(ctx, opt, f);
+
+ if (free_ctx) {
+ ggml_free(ctx);
+ }
+
+ return result;
+}
+
+enum ggml_opt_result ggml_opt_resume(
+ struct ggml_context * ctx,
+ struct ggml_opt_context * opt,
+ struct ggml_tensor * f) {
+
+ // build forward + backward compute graphs
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, opt->params.graph_size, true);
+ ggml_build_forward_expand(gf, f);
+
+ struct ggml_cgraph * gb = ggml_graph_dup(ctx, gf);
+ ggml_build_backward_expand(ctx, gf, gb, true);
+
+ return ggml_opt_resume_g(ctx, opt, f, gf, gb, NULL, NULL);
+}
+
+enum ggml_opt_result ggml_opt_resume_g(
+ struct ggml_context * ctx,
+ struct ggml_opt_context * opt,
+ struct ggml_tensor * f,
+ struct ggml_cgraph * gf,
+ struct ggml_cgraph * gb,
+ ggml_opt_callback callback,
+ void * callback_data) {
+
+ GGML_ASSERT(f->grad && "ggml_set_param must be called for at least one ancestor");
+
+ // build forward + backward compute graphs
+ enum ggml_opt_result result = GGML_OPT_RESULT_OK;
+
+ switch (opt->params.type) {
+ case GGML_OPT_TYPE_ADAM:
+ {
+ result = ggml_opt_adam(ctx, opt, opt->params, f, gf, gb, callback, callback_data);
+ } break;
+ case GGML_OPT_TYPE_LBFGS:
+ {
+ result = ggml_opt_lbfgs(ctx, opt, opt->params, f, gf, gb, callback, callback_data);
+ } break;
+ }
+
+ if (opt->params.print_forward_graph) {
+ ggml_graph_print (gf);
+ ggml_graph_dump_dot(gf, NULL, "opt-forward.dot");
+ }
+
+ if (opt->params.print_backward_graph) {
+ ggml_graph_print (gb);
+ ggml_graph_dump_dot(gb, gf, "opt-backward.dot");
+ }
+
+ return result;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void ggml_set_input(struct ggml_tensor * tensor) {
+ tensor->flags |= GGML_TENSOR_FLAG_INPUT;
+}
+
+void ggml_set_output(struct ggml_tensor * tensor) {
+ tensor->flags |= GGML_TENSOR_FLAG_OUTPUT;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void ggml_quantize_init(enum ggml_type type) {
+ ggml_critical_section_start();
+
+ switch (type) {
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M: iq2xs_init_impl(type); break;
+ case GGML_TYPE_IQ3_XXS: iq3xs_init_impl(256); break;
+ case GGML_TYPE_IQ3_S: iq3xs_init_impl(512); break;
+ default: // nothing
+ break;
+ }
+
+ ggml_critical_section_end();
+}
+
+void ggml_quantize_free(void) {
+ ggml_critical_section_start();
+
+ iq2xs_free_impl(GGML_TYPE_IQ2_XXS);
+ iq2xs_free_impl(GGML_TYPE_IQ2_XS);
+ iq2xs_free_impl(GGML_TYPE_IQ1_S);
+ iq3xs_free_impl(256);
+
+ ggml_critical_section_end();
+}
+
+bool ggml_quantize_requires_imatrix(enum ggml_type type) {
+ return
+ type == GGML_TYPE_IQ2_XXS ||
+ type == GGML_TYPE_IQ2_XS ||
+ type == GGML_TYPE_IQ1_S;// ||
+ //type == GGML_TYPE_IQ1_M;
+}
+
+size_t ggml_quantize_chunk(
+ enum ggml_type type,
+ const float * src,
+ void * dst,
+ int64_t start,
+ int64_t nrows,
+ int64_t n_per_row,
+ const float * imatrix) {
+ const int64_t n = (int64_t) nrows * n_per_row;
+
+ if (ggml_quantize_requires_imatrix(type)) {
+ GGML_ASSERT(imatrix != NULL);
+ }
+
+ GGML_ASSERT(start % type_traits[type].blck_size == 0);
+ GGML_ASSERT(start % n_per_row == 0);
+
+ ggml_quantize_init(type); // this is noop if already initialized
+
+ const size_t start_row = start / n_per_row;
+ const size_t row_size = ggml_row_size(type, n_per_row);
+
+ size_t result = 0;
+
+ switch (type) {
+ case GGML_TYPE_Q4_0: result = quantize_q4_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q4_1: result = quantize_q4_1(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q5_0: result = quantize_q5_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q5_1: result = quantize_q5_1(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q8_0: result = quantize_q8_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q2_K: result = quantize_q2_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q3_K: result = quantize_q3_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q4_K: result = quantize_q4_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q5_K: result = quantize_q5_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q6_K: result = quantize_q6_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_IQ2_XXS: result = quantize_iq2_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_IQ2_XS: result = quantize_iq2_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_IQ3_XXS: result = quantize_iq3_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_IQ3_S: result = quantize_iq3_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_IQ2_S: result = quantize_iq2_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_IQ1_S: result = quantize_iq1_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_IQ1_M: result = quantize_iq1_m (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_IQ4_NL: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_IQ4_XS: result = quantize_iq4_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q4_0_4_4: result = quantize_q4_0_4x4(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q4_0_4_8: result = quantize_q4_0_4x8(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q4_0_8_8: result = quantize_q4_0_8x8(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_F16:
+ {
+ size_t elemsize = sizeof(ggml_fp16_t);
+ ggml_fp32_to_fp16_row(src + start, (ggml_fp16_t *)dst + start, n);
+ result = n * elemsize;
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ size_t elemsize = sizeof(ggml_bf16_t);
+ ggml_fp32_to_bf16_row_ref(src + start, (ggml_bf16_t *)dst + start, n);
+ result = n * elemsize;
+ } break;
+ case GGML_TYPE_F32:
+ {
+ size_t elemsize = sizeof(float);
+ result = n * elemsize;
+ memcpy((uint8_t *)dst + start * elemsize, src + start, result);
+ } break;
+ default:
+ assert(false);
+ }
+
+ GGML_ASSERT(result == nrows * row_size);
+
+ return result;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct gguf_str {
+ uint64_t n; // GGUFv2
+ char * data;
+};
+
+static const size_t GGUF_TYPE_SIZE[GGUF_TYPE_COUNT] = {
+ [GGUF_TYPE_UINT8] = sizeof(uint8_t),
+ [GGUF_TYPE_INT8] = sizeof(int8_t),
+ [GGUF_TYPE_UINT16] = sizeof(uint16_t),
+ [GGUF_TYPE_INT16] = sizeof(int16_t),
+ [GGUF_TYPE_UINT32] = sizeof(uint32_t),
+ [GGUF_TYPE_INT32] = sizeof(int32_t),
+ [GGUF_TYPE_FLOAT32] = sizeof(float),
+ [GGUF_TYPE_BOOL] = sizeof(bool),
+ [GGUF_TYPE_STRING] = sizeof(struct gguf_str),
+ [GGUF_TYPE_UINT64] = sizeof(uint64_t),
+ [GGUF_TYPE_INT64] = sizeof(int64_t),
+ [GGUF_TYPE_FLOAT64] = sizeof(double),
+ [GGUF_TYPE_ARRAY] = 0, // undefined
+};
+static_assert(GGUF_TYPE_COUNT == 13, "GGUF_TYPE_COUNT != 13");
+
+static const char * GGUF_TYPE_NAME[GGUF_TYPE_COUNT] = {
+ [GGUF_TYPE_UINT8] = "u8",
+ [GGUF_TYPE_INT8] = "i8",
+ [GGUF_TYPE_UINT16] = "u16",
+ [GGUF_TYPE_INT16] = "i16",
+ [GGUF_TYPE_UINT32] = "u32",
+ [GGUF_TYPE_INT32] = "i32",
+ [GGUF_TYPE_FLOAT32] = "f32",
+ [GGUF_TYPE_BOOL] = "bool",
+ [GGUF_TYPE_STRING] = "str",
+ [GGUF_TYPE_ARRAY] = "arr",
+ [GGUF_TYPE_UINT64] = "u64",
+ [GGUF_TYPE_INT64] = "i64",
+ [GGUF_TYPE_FLOAT64] = "f64",
+};
+static_assert(GGUF_TYPE_COUNT == 13, "GGUF_TYPE_COUNT != 13");
+
+union gguf_value {
+ uint8_t uint8;
+ int8_t int8;
+ uint16_t uint16;
+ int16_t int16;
+ uint32_t uint32;
+ int32_t int32;
+ float float32;
+ uint64_t uint64;
+ int64_t int64;
+ double float64;
+ bool bool_;
+
+ struct gguf_str str;
+
+ struct {
+ enum gguf_type type;
+
+ uint64_t n; // GGUFv2
+ void * data;
+ } arr;
+};
+
+struct gguf_kv {
+ struct gguf_str key;
+
+ enum gguf_type type;
+ union gguf_value value;
+};
+
+struct gguf_header {
+ char magic[4];
+
+ uint32_t version;
+ uint64_t n_tensors; // GGUFv2
+ uint64_t n_kv; // GGUFv2
+};
+
+struct gguf_tensor_info {
+ struct gguf_str name;
+
+ uint32_t n_dims;
+ uint64_t ne[GGML_MAX_DIMS];
+
+ enum ggml_type type;
+
+ uint64_t offset; // offset from start of `data`, must be a multiple of `ALIGNMENT`
+
+ // for writing API
+ const void * data;
+ size_t size;
+};
+
+struct gguf_context {
+ struct gguf_header header;
+
+ struct gguf_kv * kv;
+ struct gguf_tensor_info * infos;
+
+ size_t alignment;
+ size_t offset; // offset of `data` from beginning of file
+ size_t size; // size of `data` in bytes
+
+ //uint8_t * padding;
+ void * data;
+};
+
+static size_t gguf_type_size(enum gguf_type type) {
+ GGML_ASSERT(0 <= type && type < GGUF_TYPE_COUNT);
+ return GGUF_TYPE_SIZE[type];
+}
+
+static void gguf_tensor_info_sanitize(struct gguf_tensor_info * info) {
+ GGML_ASSERT(info->n_dims <= GGML_MAX_DIMS);
+ GGML_ASSERT(0 <= info->type && info->type < GGML_TYPE_COUNT);
+
+ for (uint32_t i = 0; i < info->n_dims; ++i) {
+ GGML_ASSERT(info->ne[i] > 0);
+ }
+
+ // prevent overflow for total number of elements
+ GGML_ASSERT(INT64_MAX/info->ne[1] > info->ne[0]);
+ GGML_ASSERT(INT64_MAX/info->ne[2] > info->ne[0]*info->ne[1]);
+ GGML_ASSERT(INT64_MAX/info->ne[3] > info->ne[0]*info->ne[1]*info->ne[2]);
+}
+
+static bool gguf_fread_el(FILE * file, void * dst, size_t size, size_t * offset) {
+ const size_t n = fread(dst, 1, size, file);
+ *offset += n;
+ return n == size;
+}
+
+static bool gguf_fread_str(FILE * file, struct gguf_str * p, size_t * offset) {
+ p->n = 0;
+ p->data = NULL;
+
+ bool ok = true;
+
+ ok = ok && gguf_fread_el(file, &p->n, sizeof(p->n), offset);
+
+ // early exit if string length is invalid, prevents from integer overflow
+ if (p->n == SIZE_MAX) {
+ fprintf(stderr, "%s: invalid string length (%" PRIu64 ")\n", __func__, p->n);
+ return false;
+ }
+
+ p->data = GGML_CALLOC(p->n + 1, 1);
+
+ ok = ok && gguf_fread_el(file, p->data, p->n, offset);
+
+ return ok;
+}
+
+static void gguf_free_kv(struct gguf_kv * kv) {
+ if (kv->key.data) {
+ GGML_FREE(kv->key.data);
+ }
+
+ if (kv->type == GGUF_TYPE_STRING) {
+ if (kv->value.str.data) {
+ GGML_FREE(kv->value.str.data);
+ }
+ }
+
+ if (kv->type == GGUF_TYPE_ARRAY) {
+ if (kv->value.arr.data) {
+ if (kv->value.arr.type == GGUF_TYPE_STRING) {
+ for (uint64_t j = 0; j < kv->value.arr.n; ++j) {
+ struct gguf_str * str = &((struct gguf_str *) kv->value.arr.data)[j];
+ if (str->data) {
+ GGML_FREE(str->data);
+ }
+ }
+ }
+ GGML_FREE(kv->value.arr.data);
+ }
+ }
+}
+
+struct gguf_context * gguf_init_empty(void) {
+ struct gguf_context * ctx = GGML_CALLOC(1, sizeof(struct gguf_context));
+
+ memcpy(ctx->header.magic, GGUF_MAGIC, sizeof(ctx->header.magic));
+ ctx->header.version = GGUF_VERSION;
+ ctx->header.n_tensors = 0;
+ ctx->header.n_kv = 0;
+
+ ctx->kv = NULL;
+ ctx->infos = NULL;
+
+ ctx->alignment = GGUF_DEFAULT_ALIGNMENT;
+ ctx->offset = 0;
+ ctx->size = 0;
+
+ ctx->data = NULL;
+
+ return ctx;
+}
+
+struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params) {
+ FILE * file = ggml_fopen(fname, "rb");
+ if (!file) {
+ fprintf(stderr, "%s: failed to open '%s': '%s'\n", __func__, fname, strerror(errno));
+ return NULL;
+ }
+
+ // offset from start of file
+ size_t offset = 0;
+
+ char magic[4];
+
+ // check the magic before making allocations
+ {
+ gguf_fread_el(file, &magic, sizeof(magic), &offset);
+
+ for (uint32_t i = 0; i < sizeof(magic); i++) {
+ if (magic[i] != GGUF_MAGIC[i]) {
+ fprintf(stderr, "%s: invalid magic characters '%c%c%c%c'\n", __func__, magic[0], magic[1], magic[2], magic[3]);
+ fclose(file);
+ return NULL;
+ }
+ }
+ }
+
+ bool ok = true;
+
+ struct gguf_context * ctx = GGML_CALLOC(1, sizeof(struct gguf_context));
+
+ // read the header
+ {
+ strncpy(ctx->header.magic, magic, 4);
+
+ ctx->kv = NULL;
+ ctx->infos = NULL;
+ ctx->data = NULL;
+
+ ok = ok && gguf_fread_el(file, &ctx->header.version, sizeof(ctx->header.version), &offset);
+ ok = ok && gguf_fread_el(file, &ctx->header.n_tensors, sizeof(ctx->header.n_tensors), &offset);
+ ok = ok && gguf_fread_el(file, &ctx->header.n_kv, sizeof(ctx->header.n_kv), &offset);
+
+ if (ctx->header.version == 1) {
+ fprintf(stderr, "%s: GGUFv1 is no longer supported. please use a more up-to-date version\n", __func__);
+ fclose(file);
+ gguf_free(ctx);
+ return NULL;
+ }
+
+ // sanity-checks to prevent from integer/buffer overflows
+
+ ok = ok && (ctx->header.n_tensors < (SIZE_MAX/2)/sizeof(struct gguf_tensor_info));
+ ok = ok && (ctx->header.n_tensors < (SIZE_MAX/2)/ggml_tensor_overhead());
+ ok = ok && (ctx->header.n_kv < (SIZE_MAX/2)/sizeof(struct gguf_kv));
+
+ if (!ok) {
+ fprintf(stderr, "%s: failed to read header\n", __func__);
+ fclose(file);
+ gguf_free(ctx);
+ return NULL;
+ }
+ }
+
+ // read the kv pairs
+ {
+ const uint64_t n_kv = ctx->header.n_kv;
+
+ // header.n_kv will hold the actual value of pairs that were successfully read in the loop below
+ ctx->header.n_kv = 0;
+ ctx->kv = GGML_CALLOC(n_kv, sizeof(struct gguf_kv));
+
+ for (uint64_t i = 0; i < n_kv; ++i) {
+ struct gguf_kv * kv = &ctx->kv[i];
+
+ //fprintf(stderr, "%s: reading kv %d\n", __func__, i);
+
+ ok = ok && gguf_fread_str(file, &kv->key, &offset);
+ ok = ok && gguf_fread_el (file, &kv->type, sizeof(kv->type), &offset);
+
+ //fprintf(stderr, "%s: reading kv with key %s\n", __func__, kv->key.data);
+
+ switch (kv->type) {
+ case GGUF_TYPE_UINT8: ok = ok && gguf_fread_el (file, &kv->value.uint8, sizeof(kv->value.uint8), &offset); break;
+ case GGUF_TYPE_INT8: ok = ok && gguf_fread_el (file, &kv->value.int8, sizeof(kv->value.int8), &offset); break;
+ case GGUF_TYPE_UINT16: ok = ok && gguf_fread_el (file, &kv->value.uint16, sizeof(kv->value.uint16), &offset); break;
+ case GGUF_TYPE_INT16: ok = ok && gguf_fread_el (file, &kv->value.int16, sizeof(kv->value.int16), &offset); break;
+ case GGUF_TYPE_UINT32: ok = ok && gguf_fread_el (file, &kv->value.uint32, sizeof(kv->value.uint32), &offset); break;
+ case GGUF_TYPE_INT32: ok = ok && gguf_fread_el (file, &kv->value.int32, sizeof(kv->value.int32), &offset); break;
+ case GGUF_TYPE_FLOAT32: ok = ok && gguf_fread_el (file, &kv->value.float32, sizeof(kv->value.float32), &offset); break;
+ case GGUF_TYPE_UINT64: ok = ok && gguf_fread_el (file, &kv->value.uint64, sizeof(kv->value.uint64), &offset); break;
+ case GGUF_TYPE_INT64: ok = ok && gguf_fread_el (file, &kv->value.int64, sizeof(kv->value.int64), &offset); break;
+ case GGUF_TYPE_FLOAT64: ok = ok && gguf_fread_el (file, &kv->value.float64, sizeof(kv->value.float64), &offset); break;
+ case GGUF_TYPE_BOOL: ok = ok && gguf_fread_el (file, &kv->value.bool_, sizeof(kv->value.bool_), &offset); break;
+ case GGUF_TYPE_STRING: ok = ok && gguf_fread_str(file, &kv->value.str, &offset); break;
+ case GGUF_TYPE_ARRAY:
+ {
+ ok = ok && gguf_fread_el(file, &kv->value.arr.type, sizeof(kv->value.arr.type), &offset);
+ ok = ok && gguf_fread_el(file, &kv->value.arr.n, sizeof(kv->value.arr.n), &offset);
+
+ switch (kv->value.arr.type) {
+ case GGUF_TYPE_UINT8:
+ case GGUF_TYPE_INT8:
+ case GGUF_TYPE_UINT16:
+ case GGUF_TYPE_INT16:
+ case GGUF_TYPE_UINT32:
+ case GGUF_TYPE_INT32:
+ case GGUF_TYPE_FLOAT32:
+ case GGUF_TYPE_UINT64:
+ case GGUF_TYPE_INT64:
+ case GGUF_TYPE_FLOAT64:
+ case GGUF_TYPE_BOOL:
+ {
+ // prevent from integer overflow in the malloc below
+ if (kv->value.arr.n >= SIZE_MAX/gguf_type_size(kv->value.arr.type)) {
+ fprintf(stderr, "%s: array size is too large (%" PRIu64 ")\n", __func__, kv->value.arr.n);
+ fclose(file);
+ gguf_free(ctx);
+ return NULL;
+ }
+
+ kv->value.arr.data = GGML_CALLOC(kv->value.arr.n, gguf_type_size(kv->value.arr.type));
+
+ ok = ok && gguf_fread_el(file, kv->value.arr.data, kv->value.arr.n * gguf_type_size(kv->value.arr.type), &offset);
+ } break;
+ case GGUF_TYPE_STRING:
+ {
+ // prevent from integer overflow in the malloc below
+ if (kv->value.arr.n >= SIZE_MAX/sizeof(struct gguf_str)) {
+ fprintf(stderr, "%s: array size is too large (%" PRIu64 ")\n", __func__, kv->value.arr.n);
+ fclose(file);
+ gguf_free(ctx);
+ return NULL;
+ }
+
+ kv->value.arr.data = GGML_CALLOC(kv->value.arr.n, sizeof(struct gguf_str));
+
+ for (uint64_t j = 0; j < kv->value.arr.n; ++j) {
+ ok = ok && gguf_fread_str(file, &((struct gguf_str *) kv->value.arr.data)[j], &offset);
+ }
+ } break;
+ case GGUF_TYPE_ARRAY:
+ default: GGML_ABORT("invalid type");
+ }
+ } break;
+ default: GGML_ABORT("invalid type");
+ }
+
+ if (!ok) {
+ break;
+ }
+
+ ctx->header.n_kv++;
+ }
+
+ if (!ok) {
+ fprintf(stderr, "%s: failed to read key-value pairs\n", __func__);
+ fclose(file);
+ gguf_free(ctx);
+ return NULL;
+ }
+ }
+
+ // read the tensor infos
+ if (ctx->header.n_tensors > 0) {
+ ctx->infos = GGML_CALLOC(ctx->header.n_tensors, sizeof(struct gguf_tensor_info));
+
+ for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {
+ struct gguf_tensor_info * info = &ctx->infos[i];
+
+ for (int j = 0; j < GGML_MAX_DIMS; ++j) {
+ info->ne[j] = 1;
+ }
+
+ ok = ok && gguf_fread_str(file, &info->name, &offset);
+ ok = ok && gguf_fread_el (file, &info->n_dims, sizeof(info->n_dims), &offset);
+
+ ok = ok && (info->n_dims <= GGML_MAX_DIMS);
+
+ for (uint32_t j = 0; j < info->n_dims; ++j) {
+ ok = ok && gguf_fread_el(file, &info->ne[j], sizeof(info->ne[j]), &offset);
+ }
+
+ ok = ok && gguf_fread_el (file, &info->type, sizeof(info->type), &offset);
+ ok = ok && gguf_fread_el (file, &info->offset, sizeof(info->offset), &offset);
+
+ // TODO: return an error instead of crashing with GGML_ASSERT
+ gguf_tensor_info_sanitize(info);
+
+ // make sure there is no duplicated tensor names
+ for (uint64_t j = 0; j < i && ok; ++j) {
+ if (strcmp(info->name.data, ctx->infos[j].name.data) == 0) {
+ fprintf(stderr, "%s: duplicated tensor name %s\n", __func__, info->name.data);
+ ok = false;
+ }
+ }
+
+ if (!ok) {
+ fprintf(stderr, "%s: failed to read tensor info\n", __func__);
+ fclose(file);
+ gguf_free(ctx);
+ return NULL;
+ }
+ }
+ }
+
+ ctx->alignment = GGUF_DEFAULT_ALIGNMENT;
+
+ int alignment_idx = gguf_find_key(ctx, "general.alignment");
+ if (alignment_idx != -1) {
+ ctx->alignment = gguf_get_val_u32(ctx, alignment_idx);
+ }
+
+ // we require the data section to be aligned, so take into account any padding
+ {
+ const size_t offset_pad = offset % ctx->alignment;
+
+ if (offset_pad != 0) {
+ offset += ctx->alignment - offset_pad;
+ fseek(file, offset, SEEK_SET);
+ }
+ }
+
+ // store the current file offset - this is where the data section starts
+ ctx->offset = offset;
+
+ // compute the total size of the data section, taking into account the alignment
+ {
+ ctx->size = 0;
+ for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {
+ struct gguf_tensor_info * info = &ctx->infos[i];
+
+ const int64_t ne =
+ (int64_t) info->ne[0] *
+ (int64_t) info->ne[1] *
+ (int64_t) info->ne[2] *
+ (int64_t) info->ne[3];
+
+ if (ggml_blck_size(info->type) == 0 || ne % ggml_blck_size(info->type) != 0) {
+ fprintf(stderr, "%s: tensor '%s' of type %d (%s) number of elements (%" PRId64 ") is not a multiple of block size (%" PRId64 ")\n",
+ __func__, info->name.data, (int) info->type, ggml_type_name(info->type), ne, ggml_blck_size(info->type));
+ fclose(file);
+ gguf_free(ctx);
+ return NULL;
+ }
+
+ const size_t size_cur = ggml_row_size(info->type, ne);
+
+ ctx->size += GGML_PAD(size_cur, ctx->alignment);
+ }
+ }
+
+ // load the tensor data only if requested
+ if (params.ctx != NULL) {
+ // if the provided gguf_context is no_alloc, then we create "empty" tensors and do not read the binary blob
+ // otherwise, we load the binary blob into the created ggml_context as well, and point the "data" members of
+ // the ggml_tensor structs to the appropriate locations in the binary blob
+
+ // compute the exact size needed for the new ggml_context
+ const size_t mem_size =
+ params.no_alloc ?
+ (ctx->header.n_tensors )*ggml_tensor_overhead() :
+ (ctx->header.n_tensors + 1)*ggml_tensor_overhead() + ctx->size;
+
+ struct ggml_init_params pdata = {
+ .mem_size = mem_size,
+ .mem_buffer = NULL,
+ .no_alloc = params.no_alloc,
+ };
+
+ *params.ctx = ggml_init(pdata);
+ if (*params.ctx == NULL) {
+ fprintf(stderr, "%s: failed to initialize context\n", __func__);
+ fclose(file);
+ gguf_free(ctx);
+ return NULL;
+ }
+
+ struct ggml_context * ctx_data = *params.ctx;
+
+ struct ggml_tensor * data = NULL;
+
+ if (!params.no_alloc) {
+ data = ggml_new_tensor_1d(ctx_data, GGML_TYPE_I8, ctx->size);
+
+ ok = ok && data != NULL;
+
+ // read the binary blob with the tensor data
+ ok = ok && gguf_fread_el(file, data->data, ctx->size, &offset);
+
+ if (!ok) {
+ fprintf(stderr, "%s: failed to read tensor data\n", __func__);
+ fclose(file);
+ ggml_free(ctx_data);
+ gguf_free(ctx);
+ return NULL;
+ }
+
+ ctx->data = data->data;
+ }
+
+ ggml_set_no_alloc(ctx_data, true);
+
+ // create the tensors
+ for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {
+ const int64_t ne[GGML_MAX_DIMS] = {
+ ctx->infos[i].ne[0],
+ ctx->infos[i].ne[1],
+ ctx->infos[i].ne[2],
+ ctx->infos[i].ne[3],
+ };
+
+ struct ggml_tensor * cur = ggml_new_tensor(ctx_data, ctx->infos[i].type, ctx->infos[i].n_dims, ne);
+
+ ok = ok && cur != NULL;
+
+ if (!ok) {
+ break;
+ }
+
+ ggml_set_name(cur, ctx->infos[i].name.data);
+
+ // point the data member to the appropriate location in the binary blob using the tensor infos
+ if (!params.no_alloc) {
+ //cur->data = (char *) data->data + ctx->infos[i].offset - ctx->offset; // offset from start of file
+ cur->data = (char *) data->data + ctx->infos[i].offset; // offset from data
+ }
+ }
+
+ if (!ok) {
+ fprintf(stderr, "%s: failed to read the tensor data\n", __func__);
+ fclose(file);
+ ggml_free(ctx_data);
+ gguf_free(ctx);
+ return NULL;
+ }
+
+ ggml_set_no_alloc(ctx_data, params.no_alloc);
+ }
+
+ fclose(file);
+
+ return ctx;
+}
+
+void gguf_free(struct gguf_context * ctx) {
+ if (ctx == NULL) {
+ return;
+ }
+
+ if (ctx->kv) {
+ // free string memory - not great..
+ for (uint64_t i = 0; i < ctx->header.n_kv; ++i) {
+ gguf_free_kv(&ctx->kv[i]);
+ }
+
+ GGML_FREE(ctx->kv);
+ }
+
+ if (ctx->infos) {
+ for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {
+ struct gguf_tensor_info * info = &ctx->infos[i];
+
+ if (info->name.data) {
+ GGML_FREE(info->name.data);
+ }
+ }
+
+ GGML_FREE(ctx->infos);
+ }
+
+ GGML_FREE(ctx);
+}
+
+const char * gguf_type_name(enum gguf_type type) {
+ return GGUF_TYPE_NAME[type];
+}
+
+int gguf_get_version(const struct gguf_context * ctx) {
+ return ctx->header.version;
+}
+
+size_t gguf_get_alignment(const struct gguf_context * ctx) {
+ return ctx->alignment;
+}
+
+size_t gguf_get_data_offset(const struct gguf_context * ctx) {
+ return ctx->offset;
+}
+
+void * gguf_get_data(const struct gguf_context * ctx) {
+ return ctx->data;
+}
+
+int gguf_get_n_kv(const struct gguf_context * ctx) {
+ return ctx->header.n_kv;
+}
+
+int gguf_find_key(const struct gguf_context * ctx, const char * key) {
+ // return -1 if key not found
+ int keyfound = -1;
+
+ const int n_kv = gguf_get_n_kv(ctx);
+
+ for (int i = 0; i < n_kv; ++i) {
+ if (strcmp(key, gguf_get_key(ctx, i)) == 0) {
+ keyfound = i;
+ break;
+ }
+ }
+
+ return keyfound;
+}
+
+const char * gguf_get_key(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ return ctx->kv[key_id].key.data;
+}
+
+enum gguf_type gguf_get_kv_type(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ return ctx->kv[key_id].type;
+}
+
+enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_ARRAY);
+ return ctx->kv[key_id].value.arr.type;
+}
+
+const void * gguf_get_arr_data(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_ARRAY);
+ return ctx->kv[key_id].value.arr.data;
+}
+
+const char * gguf_get_arr_str(const struct gguf_context * ctx, int key_id, int i) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_ARRAY);
+ struct gguf_kv * kv = &ctx->kv[key_id];
+ struct gguf_str * str = &((struct gguf_str *) kv->value.arr.data)[i];
+ return str->data;
+}
+
+int gguf_get_arr_n(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_ARRAY);
+ return ctx->kv[key_id].value.arr.n;
+}
+
+uint8_t gguf_get_val_u8(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_UINT8);
+ return ctx->kv[key_id].value.uint8;
+}
+
+int8_t gguf_get_val_i8(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_INT8);
+ return ctx->kv[key_id].value.int8;
+}
+
+uint16_t gguf_get_val_u16(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_UINT16);
+ return ctx->kv[key_id].value.uint16;
+}
+
+int16_t gguf_get_val_i16(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_INT16);
+ return ctx->kv[key_id].value.int16;
+}
+
+uint32_t gguf_get_val_u32(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_UINT32);
+ return ctx->kv[key_id].value.uint32;
+}
+
+int32_t gguf_get_val_i32(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_INT32);
+ return ctx->kv[key_id].value.int32;
+}
+
+float gguf_get_val_f32(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_FLOAT32);
+ return ctx->kv[key_id].value.float32;
+}
+
+uint64_t gguf_get_val_u64(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_UINT64);
+ return ctx->kv[key_id].value.uint64;
+}
+
+int64_t gguf_get_val_i64(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_INT64);
+ return ctx->kv[key_id].value.int64;
+}
+
+double gguf_get_val_f64(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_FLOAT64);
+ return ctx->kv[key_id].value.float64;
+}
+
+bool gguf_get_val_bool(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_BOOL);
+ return ctx->kv[key_id].value.bool_;
+}
+
+const char * gguf_get_val_str(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_STRING);
+ return ctx->kv[key_id].value.str.data;
+}
+
+const void * gguf_get_val_data(const struct gguf_context * ctx, int key_id) {
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
+ GGML_ASSERT(ctx->kv[key_id].type != GGUF_TYPE_ARRAY);
+ GGML_ASSERT(ctx->kv[key_id].type != GGUF_TYPE_STRING);
+ return &ctx->kv[key_id].value;
+}
+
+int gguf_get_n_tensors(const struct gguf_context * ctx) {
+ return ctx->header.n_tensors;
+}
+
+int gguf_find_tensor(const struct gguf_context * ctx, const char * name) {
+ // return -1 if tensor not found
+ int tensorfound = -1;
+
+ const int n_tensors = gguf_get_n_tensors(ctx);
+
+ for (int i = 0; i < n_tensors; ++i) {
+ if (strcmp(name, gguf_get_tensor_name(ctx, i)) == 0) {
+ tensorfound = i;
+ break;
+ }
+ }
+
+ return tensorfound;
+}
+
+size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int i) {
+ return ctx->infos[i].offset;
+}
+
+char * gguf_get_tensor_name(const struct gguf_context * ctx, int i) {
+ return ctx->infos[i].name.data;
+}
+
+enum ggml_type gguf_get_tensor_type(const struct gguf_context * ctx, int i) {
+ return ctx->infos[i].type;
+}
+
+// returns the index
+static int gguf_get_or_add_key(struct gguf_context * ctx, const char * key) {
+ const int idx = gguf_find_key(ctx, key);
+ if (idx >= 0) {
+ return idx;
+ }
+
+ const int n_kv = gguf_get_n_kv(ctx);
+
+ ctx->kv = realloc(ctx->kv, (n_kv + 1) * sizeof(struct gguf_kv));
+ ctx->kv[n_kv].key.n = strlen(key);
+ ctx->kv[n_kv].key.data = strdup(key);
+ ctx->header.n_kv++;
+
+ return n_kv;
+}
+
+void gguf_remove_key(struct gguf_context * ctx, const char * key) {
+ const int idx = gguf_find_key(ctx, key);
+ if (idx >= 0) {
+ const int n_kv = gguf_get_n_kv(ctx);
+ gguf_free_kv(&ctx->kv[idx]);
+ for (int i = idx; i < n_kv-1; ++i) {
+ ctx->kv[i] = ctx->kv[i+1];
+ }
+ ctx->kv = realloc(ctx->kv, (n_kv - 1) * sizeof(struct gguf_kv));
+ ctx->header.n_kv--;
+ }
+}
+
+void gguf_set_val_u8(struct gguf_context * ctx, const char * key, uint8_t val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_UINT8;
+ ctx->kv[idx].value.uint8 = val;
+}
+
+void gguf_set_val_i8(struct gguf_context * ctx, const char * key, int8_t val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_INT8;
+ ctx->kv[idx].value.int8 = val;
+}
+
+void gguf_set_val_u16(struct gguf_context * ctx, const char * key, uint16_t val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_UINT16;
+ ctx->kv[idx].value.uint16 = val;
+}
+
+void gguf_set_val_i16(struct gguf_context * ctx, const char * key, int16_t val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_INT16;
+ ctx->kv[idx].value.int16 = val;
+}
+
+void gguf_set_val_u32(struct gguf_context * ctx, const char * key, uint32_t val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_UINT32;
+ ctx->kv[idx].value.uint32 = val;
+}
+
+void gguf_set_val_i32(struct gguf_context * ctx, const char * key, int32_t val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_INT32;
+ ctx->kv[idx].value.int32 = val;
+}
+
+void gguf_set_val_f32(struct gguf_context * ctx, const char * key, float val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_FLOAT32;
+ ctx->kv[idx].value.float32 = val;
+}
+
+void gguf_set_val_u64(struct gguf_context * ctx, const char * key, uint64_t val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_UINT64;
+ ctx->kv[idx].value.uint64 = val;
+}
+
+void gguf_set_val_i64(struct gguf_context * ctx, const char * key, int64_t val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_INT64;
+ ctx->kv[idx].value.int64 = val;
+}
+
+void gguf_set_val_f64(struct gguf_context * ctx, const char * key, double val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_FLOAT64;
+ ctx->kv[idx].value.float64 = val;
+}
+
+void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_BOOL;
+ ctx->kv[idx].value.bool_ = val;
+}
+
+void gguf_set_val_str(struct gguf_context * ctx, const char * key, const char * val) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_STRING;
+ ctx->kv[idx].value.str.n = strlen(val);
+ ctx->kv[idx].value.str.data = strdup(val);
+}
+
+void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, int n) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_ARRAY;
+ ctx->kv[idx].value.arr.type = type;
+ ctx->kv[idx].value.arr.n = n;
+ ctx->kv[idx].value.arr.data = GGML_CALLOC(n, gguf_type_size(type));
+ memcpy(ctx->kv[idx].value.arr.data, data, n*gguf_type_size(type));
+}
+
+void gguf_set_arr_str(struct gguf_context * ctx, const char * key, const char ** data, int n) {
+ const int idx = gguf_get_or_add_key(ctx, key);
+
+ ctx->kv[idx].type = GGUF_TYPE_ARRAY;
+ ctx->kv[idx].value.arr.type = GGUF_TYPE_STRING;
+ ctx->kv[idx].value.arr.n = n;
+ ctx->kv[idx].value.arr.data = GGML_CALLOC(n, sizeof(struct gguf_str));
+ for (int i = 0; i < n; i++) {
+ struct gguf_str * str = &((struct gguf_str *)ctx->kv[idx].value.arr.data)[i];
+ str->n = strlen(data[i]);
+ str->data = strdup(data[i]);
+ }
+}
+
+// set or add KV pairs from another context
+void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src) {
+ for (uint32_t i = 0; i < src->header.n_kv; i++) {
+ switch (src->kv[i].type) {
+ case GGUF_TYPE_UINT8: gguf_set_val_u8 (ctx, src->kv[i].key.data, src->kv[i].value.uint8); break;
+ case GGUF_TYPE_INT8: gguf_set_val_i8 (ctx, src->kv[i].key.data, src->kv[i].value.int8); break;
+ case GGUF_TYPE_UINT16: gguf_set_val_u16 (ctx, src->kv[i].key.data, src->kv[i].value.uint16); break;
+ case GGUF_TYPE_INT16: gguf_set_val_i16 (ctx, src->kv[i].key.data, src->kv[i].value.int16); break;
+ case GGUF_TYPE_UINT32: gguf_set_val_u32 (ctx, src->kv[i].key.data, src->kv[i].value.uint32); break;
+ case GGUF_TYPE_INT32: gguf_set_val_i32 (ctx, src->kv[i].key.data, src->kv[i].value.int32); break;
+ case GGUF_TYPE_FLOAT32: gguf_set_val_f32 (ctx, src->kv[i].key.data, src->kv[i].value.float32); break;
+ case GGUF_TYPE_UINT64: gguf_set_val_u64 (ctx, src->kv[i].key.data, src->kv[i].value.uint64); break;
+ case GGUF_TYPE_INT64: gguf_set_val_i64 (ctx, src->kv[i].key.data, src->kv[i].value.int64); break;
+ case GGUF_TYPE_FLOAT64: gguf_set_val_f64 (ctx, src->kv[i].key.data, src->kv[i].value.float64); break;
+ case GGUF_TYPE_BOOL: gguf_set_val_bool(ctx, src->kv[i].key.data, src->kv[i].value.bool_); break;
+ case GGUF_TYPE_STRING: gguf_set_val_str (ctx, src->kv[i].key.data, src->kv[i].value.str.data); break;
+ case GGUF_TYPE_ARRAY:
+ {
+ if (src->kv[i].value.arr.type == GGUF_TYPE_STRING) {
+ const char ** data = GGML_CALLOC(src->kv[i].value.arr.n, sizeof(char *));
+ for (uint32_t j = 0; j < src->kv[i].value.arr.n; j++) {
+ data[j] = ((struct gguf_str *)src->kv[i].value.arr.data)[j].data;
+ }
+ gguf_set_arr_str(ctx, src->kv[i].key.data, data, src->kv[i].value.arr.n);
+ GGML_FREE((void *)data);
+ } else if (src->kv[i].value.arr.type == GGUF_TYPE_ARRAY) {
+ GGML_ABORT("nested arrays not supported");
+ } else {
+ gguf_set_arr_data(ctx, src->kv[i].key.data, src->kv[i].value.arr.type, src->kv[i].value.arr.data, src->kv[i].value.arr.n);
+ }
+ } break;
+ default: GGML_ABORT("invalid type");
+ }
+ }
+}
+
+void gguf_add_tensor(
+ struct gguf_context * ctx,
+ const struct ggml_tensor * tensor) {
+ GGML_ASSERT(tensor);
+ if (gguf_find_tensor(ctx, tensor->name) != -1) {
+ GGML_ABORT("duplicated tensor name");
+ }
+
+ const int idx = ctx->header.n_tensors;
+ ctx->infos = realloc(ctx->infos, (idx + 1)*sizeof(struct gguf_tensor_info));
+
+ ctx->infos[idx].name.n = strlen(tensor->name);
+ ctx->infos[idx].name.data = strdup(tensor->name);
+
+ for (int i = 0; i < GGML_MAX_DIMS; ++i) {
+ ctx->infos[idx].ne[i] = 1;
+ }
+
+ ctx->infos[idx].n_dims = ggml_n_dims(tensor);
+ for (uint32_t i = 0; i < ctx->infos[idx].n_dims; i++) {
+ ctx->infos[idx].ne[i] = tensor->ne[i];
+ }
+
+ ctx->infos[idx].type = tensor->type;
+ ctx->infos[idx].offset = 0;
+ ctx->infos[idx].data = tensor->data;
+ ctx->infos[idx].size = ggml_nbytes(tensor);
+
+ if (ctx->header.n_tensors > 0) {
+ ctx->infos[idx].offset = ctx->infos[idx - 1].offset + GGML_PAD(ctx->infos[idx - 1].size, ctx->alignment);
+ }
+
+ ctx->header.n_tensors++;
+}
+
+void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type) {
+ const int idx = gguf_find_tensor(ctx, name);
+ if (idx < 0) {
+ GGML_ABORT("tensor not found");
+ }
+
+ ctx->infos[idx].type = type;
+}
+
+void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data, size_t size) {
+ const int idx = gguf_find_tensor(ctx, name);
+ if (idx < 0) {
+ GGML_ABORT("tensor not found");
+ }
+
+ ctx->infos[idx].data = data;
+ ctx->infos[idx].size = size;
+
+ // update offsets
+ for (uint32_t i = idx + 1; i < ctx->header.n_tensors; ++i) {
+ ctx->infos[i].offset = ctx->infos[i - 1].offset + GGML_PAD(ctx->infos[i - 1].size, ctx->alignment);
+ }
+}
+
+//static void gguf_fwrite_str(FILE * file, const struct gguf_str * val) {
+// fwrite(&val->n, sizeof(val->n), 1, file);
+// fwrite(val->data, sizeof(char), val->n, file);
+//}
+//
+//static void gguf_fwrite_el(FILE * file, const void * val, size_t size) {
+// fwrite(val, sizeof(char), size, file);
+//}
+
+struct gguf_buf {
+ void * data;
+ size_t size;
+ size_t offset;
+};
+
+static struct gguf_buf gguf_buf_init(size_t size) {
+ struct gguf_buf buf = {
+ /*buf.data =*/ size == 0 ? NULL : GGML_CALLOC(1, size),
+ /*buf.size =*/ size,
+ /*buf.offset =*/ 0,
+ };
+
+ return buf;
+}
+
+static void gguf_buf_free(struct gguf_buf buf) {
+ if (buf.data) {
+ GGML_FREE(buf.data);
+ }
+}
+
+static void gguf_buf_grow(struct gguf_buf * buf, size_t size) {
+ if (buf->offset + size > buf->size) {
+ buf->size = 1.5*(buf->offset + size);
+ if (buf->data) {
+ buf->data = realloc(buf->data, buf->size);
+ }
+ }
+}
+
+static void gguf_bwrite_str(struct gguf_buf * buf, const struct gguf_str * val) {
+ gguf_buf_grow(buf, sizeof(val->n) + val->n);
+
+ if (buf->data) {
+ memcpy((char *) buf->data + buf->offset, &val->n, sizeof(val->n));
+ }
+ buf->offset += sizeof(val->n);
+
+ if (buf->data) {
+ memcpy((char *) buf->data + buf->offset, val->data, val->n);
+ }
+ buf->offset += val->n;
+}
+
+static void gguf_bwrite_el(struct gguf_buf * buf, const void * val, size_t el_size) {
+ gguf_buf_grow(buf, el_size);
+
+ if (buf->data) {
+ memcpy((char *) buf->data + buf->offset, val, el_size);
+ }
+ buf->offset += el_size;
+}
+
+static void gguf_write_to_buf(const struct gguf_context * ctx, struct gguf_buf * buf, bool only_meta) {
+ // write header
+ gguf_bwrite_el(buf, &ctx->header.magic, sizeof(ctx->header.magic));
+ gguf_bwrite_el(buf, &ctx->header.version, sizeof(ctx->header.version));
+ gguf_bwrite_el(buf, &ctx->header.n_tensors, sizeof(ctx->header.n_tensors));
+ gguf_bwrite_el(buf, &ctx->header.n_kv, sizeof(ctx->header.n_kv));
+
+ // write key-value pairs
+ for (uint32_t i = 0; i < ctx->header.n_kv; ++i) {
+ struct gguf_kv * kv = &ctx->kv[i];
+
+ gguf_bwrite_str(buf, &kv->key);
+ gguf_bwrite_el (buf, &kv->type, sizeof(kv->type));
+
+ switch (kv->type) {
+ case GGUF_TYPE_UINT8: gguf_bwrite_el( buf, &kv->value.uint8, sizeof(kv->value.uint8) ); break;
+ case GGUF_TYPE_INT8: gguf_bwrite_el (buf, &kv->value.int8, sizeof(kv->value.int8) ); break;
+ case GGUF_TYPE_UINT16: gguf_bwrite_el (buf, &kv->value.uint16, sizeof(kv->value.uint16) ); break;
+ case GGUF_TYPE_INT16: gguf_bwrite_el (buf, &kv->value.int16, sizeof(kv->value.int16) ); break;
+ case GGUF_TYPE_UINT32: gguf_bwrite_el (buf, &kv->value.uint32, sizeof(kv->value.uint32) ); break;
+ case GGUF_TYPE_INT32: gguf_bwrite_el (buf, &kv->value.int32, sizeof(kv->value.int32) ); break;
+ case GGUF_TYPE_FLOAT32: gguf_bwrite_el (buf, &kv->value.float32, sizeof(kv->value.float32)); break;
+ case GGUF_TYPE_UINT64: gguf_bwrite_el (buf, &kv->value.uint64, sizeof(kv->value.uint64) ); break;
+ case GGUF_TYPE_INT64: gguf_bwrite_el (buf, &kv->value.int64, sizeof(kv->value.int64) ); break;
+ case GGUF_TYPE_FLOAT64: gguf_bwrite_el (buf, &kv->value.float64, sizeof(kv->value.float64)); break;
+ case GGUF_TYPE_BOOL: gguf_bwrite_el (buf, &kv->value.bool_, sizeof(kv->value.bool_) ); break;
+ case GGUF_TYPE_STRING: gguf_bwrite_str(buf, &kv->value.str ); break;
+ case GGUF_TYPE_ARRAY:
+ {
+ gguf_bwrite_el(buf, &kv->value.arr.type, sizeof(kv->value.arr.type));
+ gguf_bwrite_el(buf, &kv->value.arr.n, sizeof(kv->value.arr.n) );
+
+ switch (kv->value.arr.type) {
+ case GGUF_TYPE_UINT8:
+ case GGUF_TYPE_INT8:
+ case GGUF_TYPE_UINT16:
+ case GGUF_TYPE_INT16:
+ case GGUF_TYPE_UINT32:
+ case GGUF_TYPE_INT32:
+ case GGUF_TYPE_FLOAT32:
+ case GGUF_TYPE_UINT64:
+ case GGUF_TYPE_INT64:
+ case GGUF_TYPE_FLOAT64:
+ case GGUF_TYPE_BOOL:
+ {
+ gguf_bwrite_el(buf, kv->value.arr.data, kv->value.arr.n * gguf_type_size(kv->value.arr.type));
+ } break;
+ case GGUF_TYPE_STRING:
+ {
+ for (uint32_t j = 0; j < kv->value.arr.n; ++j) {
+ gguf_bwrite_str(buf, &((struct gguf_str *) kv->value.arr.data)[j]);
+ }
+ } break;
+ case GGUF_TYPE_ARRAY:
+ default: GGML_ABORT("invalid type");
+ }
+ } break;
+ default: GGML_ABORT("invalid type");
+ }
+ }
+
+ // write tensor infos
+ for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) {
+ struct gguf_tensor_info * info = &ctx->infos[i];
+
+ gguf_bwrite_str(buf, &info->name);
+ gguf_bwrite_el (buf, &info->n_dims, sizeof(info->n_dims));
+ for (uint32_t j = 0; j < info->n_dims; ++j) {
+ gguf_bwrite_el(buf, &info->ne[j], sizeof(info->ne[j]));
+ }
+ gguf_bwrite_el(buf, &info->type, sizeof(info->type));
+ gguf_bwrite_el(buf, &info->offset, sizeof(info->offset));
+ }
+
+ // we require the data section to be aligned, so take into account any padding
+ {
+ const size_t offset = buf->offset;
+ const size_t offset_pad = GGML_PAD(offset, ctx->alignment);
+
+ if (offset_pad != offset) {
+ uint8_t pad = 0;
+ for (size_t i = 0; i < offset_pad - offset; ++i) {
+ gguf_bwrite_el(buf, &pad, sizeof(pad));
+ }
+ }
+ }
+
+ if (only_meta) {
+ return;
+ }
+
+ size_t offset = 0;
+
+ // write tensor data
+ for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) {
+ struct gguf_tensor_info * info = &ctx->infos[i];
+
+ const size_t size = info->size;
+ const size_t size_pad = GGML_PAD(size, ctx->alignment);
+
+ gguf_bwrite_el(buf, info->data, size);
+
+ if (size_pad != size) {
+ uint8_t pad = 0;
+ for (size_t j = 0; j < size_pad - size; ++j) {
+ gguf_bwrite_el(buf, &pad, sizeof(pad));
+ }
+ }
+
+ GGML_ASSERT(offset == info->offset);
+
+ offset += size_pad;
+ }
+}
+
+void gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta) {
+ FILE * file = ggml_fopen(fname, "wb");
+ if (!file) {
+ GGML_ABORT("failed to open file for writing");
+ }
+
+ struct gguf_buf buf = gguf_buf_init(16*1024);
+
+ gguf_write_to_buf(ctx, &buf, only_meta);
+
+ fwrite(buf.data, 1, buf.offset, file);
+
+ gguf_buf_free(buf);
+
+ fclose(file);
+}
+
+size_t gguf_get_meta_size(const struct gguf_context * ctx) {
+ // no allocs - only compute size
+ struct gguf_buf buf = gguf_buf_init(0);
+
+ gguf_write_to_buf(ctx, &buf, true);
+
+ return buf.offset;
+}
+
+void gguf_get_meta_data(const struct gguf_context * ctx, void * data) {
+ struct gguf_buf buf = gguf_buf_init(16*1024);
+
+ gguf_write_to_buf(ctx, &buf, true);
+
+ memcpy(data, buf.data, buf.offset);
+
+ gguf_buf_free(buf);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+int ggml_cpu_has_avx(void) {
+#if defined(__AVX__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_avx_vnni(void) {
+#if defined(__AVXVNNI__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_avx2(void) {
+#if defined(__AVX2__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_avx512(void) {
+#if defined(__AVX512F__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_avx512_vbmi(void) {
+#if defined(__AVX512VBMI__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_avx512_vnni(void) {
+#if defined(__AVX512VNNI__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_avx512_bf16(void) {
+#if defined(__AVX512BF16__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_fma(void) {
+#if defined(__FMA__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_neon(void) {
+#if defined(__ARM_NEON)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_sve(void) {
+#if defined(__ARM_FEATURE_SVE)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_arm_fma(void) {
+#if defined(__ARM_FEATURE_FMA)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_metal(void) {
+#if defined(GGML_USE_METAL)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_f16c(void) {
+#if defined(__F16C__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_fp16_va(void) {
+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_wasm_simd(void) {
+#if defined(__wasm_simd128__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_blas(void) {
+#if defined(GGML_USE_BLAS) || defined(GGML_USE_CUDA) || defined(GGML_USE_VULKAN) || defined(GGML_USE_SYCL)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_cuda(void) {
+#if defined(GGML_USE_CUDA)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_vulkan(void) {
+#if defined(GGML_USE_VULKAN)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_kompute(void) {
+#if defined(GGML_USE_KOMPUTE)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_sycl(void) {
+#if defined(GGML_USE_SYCL)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_rpc(void) {
+#if defined(GGML_USE_RPC)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_cann(void) {
+#if defined(GGML_USE_CANN)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_llamafile(void) {
+#if defined(GGML_USE_LLAMAFILE)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_gpublas(void) {
+ return ggml_cpu_has_cuda() || ggml_cpu_has_vulkan() || ggml_cpu_has_kompute() || ggml_cpu_has_sycl();
+}
+
+int ggml_cpu_has_sse3(void) {
+#if defined(__SSE3__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_ssse3(void) {
+#if defined(__SSSE3__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_vsx(void) {
+#if defined(__POWER9_VECTOR__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+int ggml_cpu_has_matmul_int8(void) {
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+////////////////////////////////////////////////////////////////////////////////
diff --git a/llama/ggml.h b/llama/ggml.h
new file mode 100644
index 00000000..177b527f
--- /dev/null
+++ b/llama/ggml.h
@@ -0,0 +1,2586 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+//
+// GGML Tensor Library
+//
+// This documentation is still a work in progress.
+// If you wish some specific topics to be covered, feel free to drop a comment:
+//
+// https://github.com/ggerganov/whisper.cpp/issues/40
+//
+// ## Overview
+//
+// This library implements:
+//
+// - a set of tensor operations
+// - automatic differentiation
+// - basic optimization algorithms
+//
+// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,
+// but is not limited to, the following:
+//
+// - linear regression
+// - support vector machines
+// - neural networks
+//
+// The library allows the user to define a certain function using the available tensor operations. This function
+// definition is represented internally via a computation graph. Each tensor operation in the function definition
+// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the
+// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized
+// using one of the available optimization algorithms.
+//
+// For example, here we define the function: f(x) = a*x^2 + b
+//
+// {
+// struct ggml_init_params params = {
+// .mem_size = 16*1024*1024,
+// .mem_buffer = NULL,
+// };
+//
+// // memory allocation happens here
+// struct ggml_context * ctx = ggml_init(params);
+//
+// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+//
+// ggml_set_param(ctx, x); // x is an input variable
+//
+// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+// struct ggml_tensor * x2 = ggml_mul(ctx, x, x);
+// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b);
+//
+// ...
+// }
+//
+// Notice that the function definition above does not involve any actual computation. The computation is performed only
+// when the user explicitly requests it. For example, to compute the function's value at x = 2.0:
+//
+// {
+// ...
+//
+// struct ggml_cgraph * gf = ggml_new_graph(ctx);
+// ggml_build_forward_expand(gf, f);
+//
+// // set the input variable and parameter values
+// ggml_set_f32(x, 2.0f);
+// ggml_set_f32(a, 3.0f);
+// ggml_set_f32(b, 4.0f);
+//
+// ggml_graph_compute_with_ctx(ctx, &gf, n_threads);
+//
+// printf("f = %f\n", ggml_get_f32_1d(f, 0));
+//
+// ...
+// }
+//
+// The actual computation is performed in the ggml_graph_compute() function.
+//
+// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the
+// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know
+// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory
+// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was
+// actually needed.
+//
+// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic
+// differentiation and optimization algorithms.
+//
+// The described approach allows to define the function graph once and then compute its forward or backward graphs
+// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way
+// the user can avoid the memory allocation overhead at runtime.
+//
+// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class
+// citizens, but in theory the library can be extended to support FP8 and integer data types.
+//
+// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary
+// and binary operations. Most of the available operations fall into one of these two categories. With time, it became
+// clear that the library needs to support more complex operations. The way to support these operations is not clear
+// yet, but a few examples are demonstrated in the following operations:
+//
+// - ggml_permute()
+// - ggml_conv_1d_1s()
+// - ggml_conv_1d_2s()
+//
+// For each tensor operator, the library implements a forward and backward computation function. The forward function
+// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the
+// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a
+// calculus class, or watch the following video:
+//
+// What is Automatic Differentiation?
+// https://www.youtube.com/watch?v=wG_nF1awSSY
+//
+//
+// ## Tensor data (struct ggml_tensor)
+//
+// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of
+// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains
+// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example:
+//
+// {
+// struct ggml_tensor * c = ggml_add(ctx, a, b);
+//
+// assert(c->src[0] == a);
+// assert(c->src[1] == b);
+// }
+//
+// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the
+// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows
+// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and
+// permutation. All tensor operations have to take the stride into account and not assume that the tensor is
+// contiguous in memory.
+//
+// The data of the tensor is accessed via the "data" pointer. For example:
+//
+// {
+// const int nx = 2;
+// const int ny = 3;
+//
+// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny);
+//
+// for (int y = 0; y < ny; y++) {
+// for (int x = 0; x < nx; x++) {
+// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;
+// }
+// }
+//
+// ...
+// }
+//
+// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used.
+//
+// ## The matrix multiplication operator (ggml_mul_mat)
+//
+// TODO
+//
+//
+// ## Multi-threading
+//
+// TODO
+//
+//
+// ## Overview of ggml.c
+//
+// TODO
+//
+//
+// ## SIMD optimizations
+//
+// TODO
+//
+//
+// ## Debugging ggml
+//
+// TODO
+//
+//
+
+#ifdef GGML_SHARED
+# if defined(_WIN32) && !defined(__MINGW32__)
+# ifdef GGML_BUILD
+# define GGML_API __declspec(dllexport)
+# else
+# define GGML_API __declspec(dllimport)
+# endif
+# else
+# define GGML_API __attribute__ ((visibility ("default")))
+# endif
+#else
+# define GGML_API
+#endif
+
+#ifdef GGML_MULTIPLATFORM
+# if defined(_WIN32)
+# define GGML_CALL
+# else
+# define GGML_CALL __attribute__((__ms_abi__))
+# endif
+#else
+# define GGML_CALL
+#endif
+
+// TODO: support for clang
+#ifdef __GNUC__
+# define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
+#elif defined(_MSC_VER)
+# define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func
+#else
+# define GGML_DEPRECATED(func, hint) func
+#endif
+
+#ifndef __GNUC__
+# define GGML_ATTRIBUTE_FORMAT(...)
+#elif defined(__MINGW32__)
+# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
+#else
+# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
+#endif
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#define GGML_FILE_MAGIC 0x67676d6c // "ggml"
+#define GGML_FILE_VERSION 2
+
+#define GGML_QNT_VERSION 2 // bump this on quantization format changes
+#define GGML_QNT_VERSION_FACTOR 1000 // do not change this
+
+#define GGML_MAX_DIMS 4
+#define GGML_MAX_PARAMS 2048
+#define GGML_MAX_CONTEXTS 64
+#define GGML_MAX_SRC 10
+#ifndef GGML_MAX_NAME
+#define GGML_MAX_NAME 64
+#define GGML_MAX_N_THREADS 512
+
+#endif
+#define GGML_MAX_OP_PARAMS 64
+#define GGML_DEFAULT_N_THREADS 4
+#define GGML_DEFAULT_GRAPH_SIZE 2048
+#if UINTPTR_MAX == 0xFFFFFFFF
+ #define GGML_MEM_ALIGN 4
+#else
+ #define GGML_MEM_ALIGN 16
+#endif
+
+#define GGML_EXIT_SUCCESS 0
+#define GGML_EXIT_ABORTED 1
+
+#define GGML_ROPE_TYPE_NEOX 2
+
+#define GGUF_MAGIC "GGUF"
+
+#define GGUF_VERSION 3
+
+#define GGUF_DEFAULT_ALIGNMENT 32
+
+#define GGML_UNUSED(x) (void)(x)
+
+#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))
+
+#ifndef NDEBUG
+#define GGML_UNREACHABLE() do { fprintf(stderr, "statement should be unreachable\n"); abort(); } while(0)
+#elif defined(__GNUC__)
+#define GGML_UNREACHABLE() __builtin_unreachable()
+#elif defined(_MSC_VER)
+#define GGML_UNREACHABLE() __assume(0)
+#else
+#define GGML_UNREACHABLE() ((void) 0)
+#endif
+
+#ifdef __cplusplus
+#define GGML_NORETURN [[noreturn]]
+#elif defined(_MSC_VER)
+#define GGML_NORETURN __declspec(noreturn)
+#else
+#define GGML_NORETURN _Noreturn
+#endif
+
+#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__)
+#define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x)
+
+// used to copy the number of elements and stride in bytes of tensors into local variables.
+// main purpose is to reduce code duplication and improve readability.
+//
+// example:
+//
+// GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);
+// GGML_TENSOR_LOCALS(size_t, nb1, src1, nb);
+//
+#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \
+ const type prefix##0 = (pointer)->array[0]; \
+ GGML_UNUSED(prefix##0);
+#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \
+ GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \
+ const type prefix##1 = (pointer)->array[1]; \
+ GGML_UNUSED(prefix##1);
+#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \
+ GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \
+ const type prefix##2 = (pointer)->array[2]; \
+ GGML_UNUSED(prefix##2);
+#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \
+ GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \
+ const type prefix##3 = (pointer)->array[3]; \
+ GGML_UNUSED(prefix##3);
+
+#define GGML_TENSOR_UNARY_OP_LOCALS \
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
+#define GGML_TENSOR_BINARY_OP_LOCALS \
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
+ GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
+#define GGML_TENSOR_BINARY_OP_LOCALS01 \
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
+ GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4)
+ GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...);
+
+ enum ggml_status {
+ GGML_STATUS_ALLOC_FAILED = -2,
+ GGML_STATUS_FAILED = -1,
+ GGML_STATUS_SUCCESS = 0,
+ GGML_STATUS_ABORTED = 1,
+ };
+
+ // get ggml_status name string
+ GGML_API GGML_CALL const char * ggml_status_to_string(enum ggml_status status);
+
+ // ieee 754-2008 half-precision float16
+ // todo: make this not an integral type
+ typedef uint16_t ggml_fp16_t;
+ GGML_API float ggml_fp16_to_fp32(ggml_fp16_t);
+ GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);
+ GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);
+ GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);
+
+ // google brain half-precision bfloat16
+ typedef struct { uint16_t bits; } ggml_bf16_t;
+ GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);
+ GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16
+ GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);
+ GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);
+ GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);
+
+ struct ggml_object;
+ struct ggml_context;
+
+ // NOTE: always add types at the end of the enum to keep backward compatibility
+ enum ggml_type {
+ GGML_TYPE_F32 = 0,
+ GGML_TYPE_F16 = 1,
+ GGML_TYPE_Q4_0 = 2,
+ GGML_TYPE_Q4_1 = 3,
+ // GGML_TYPE_Q4_2 = 4, support has been removed
+ // GGML_TYPE_Q4_3 = 5, support has been removed
+ GGML_TYPE_Q5_0 = 6,
+ GGML_TYPE_Q5_1 = 7,
+ GGML_TYPE_Q8_0 = 8,
+ GGML_TYPE_Q8_1 = 9,
+ GGML_TYPE_Q2_K = 10,
+ GGML_TYPE_Q3_K = 11,
+ GGML_TYPE_Q4_K = 12,
+ GGML_TYPE_Q5_K = 13,
+ GGML_TYPE_Q6_K = 14,
+ GGML_TYPE_Q8_K = 15,
+ GGML_TYPE_IQ2_XXS = 16,
+ GGML_TYPE_IQ2_XS = 17,
+ GGML_TYPE_IQ3_XXS = 18,
+ GGML_TYPE_IQ1_S = 19,
+ GGML_TYPE_IQ4_NL = 20,
+ GGML_TYPE_IQ3_S = 21,
+ GGML_TYPE_IQ2_S = 22,
+ GGML_TYPE_IQ4_XS = 23,
+ GGML_TYPE_I8 = 24,
+ GGML_TYPE_I16 = 25,
+ GGML_TYPE_I32 = 26,
+ GGML_TYPE_I64 = 27,
+ GGML_TYPE_F64 = 28,
+ GGML_TYPE_IQ1_M = 29,
+ GGML_TYPE_BF16 = 30,
+ GGML_TYPE_Q4_0_4_4 = 31,
+ GGML_TYPE_Q4_0_4_8 = 32,
+ GGML_TYPE_Q4_0_8_8 = 33,
+ GGML_TYPE_COUNT,
+ };
+
+ // precision
+ enum ggml_prec {
+ GGML_PREC_DEFAULT,
+ GGML_PREC_F32,
+ };
+
+ enum ggml_backend_type {
+ GGML_BACKEND_TYPE_CPU = 0,
+ GGML_BACKEND_TYPE_GPU = 10,
+ GGML_BACKEND_TYPE_GPU_SPLIT = 20,
+ };
+
+ // model file types
+ enum ggml_ftype {
+ GGML_FTYPE_UNKNOWN = -1,
+ GGML_FTYPE_ALL_F32 = 0,
+ GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
+ GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors
+ GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q4_0_4_4 = 25, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q4_0_4_8 = 26, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q4_0_8_8 = 27, // except 1d tensors
+ };
+
+ // available tensor operations:
+ enum ggml_op {
+ GGML_OP_NONE = 0,
+
+ GGML_OP_DUP,
+ GGML_OP_ADD,
+ GGML_OP_ADD1,
+ GGML_OP_ACC,
+ GGML_OP_SUB,
+ GGML_OP_MUL,
+ GGML_OP_DIV,
+ GGML_OP_SQR,
+ GGML_OP_SQRT,
+ GGML_OP_LOG,
+ GGML_OP_SIN,
+ GGML_OP_COS,
+ GGML_OP_SUM,
+ GGML_OP_SUM_ROWS,
+ GGML_OP_MEAN,
+ GGML_OP_ARGMAX,
+ GGML_OP_REPEAT,
+ GGML_OP_REPEAT_BACK,
+ GGML_OP_CONCAT,
+ GGML_OP_SILU_BACK,
+ GGML_OP_NORM, // normalize
+ GGML_OP_RMS_NORM,
+ GGML_OP_RMS_NORM_BACK,
+ GGML_OP_GROUP_NORM,
+
+ GGML_OP_MUL_MAT,
+ GGML_OP_MUL_MAT_ID,
+ GGML_OP_OUT_PROD,
+
+ GGML_OP_SCALE,
+ GGML_OP_SET,
+ GGML_OP_CPY,
+ GGML_OP_CONT,
+ GGML_OP_RESHAPE,
+ GGML_OP_VIEW,
+ GGML_OP_PERMUTE,
+ GGML_OP_TRANSPOSE,
+ GGML_OP_GET_ROWS,
+ GGML_OP_GET_ROWS_BACK,
+ GGML_OP_DIAG,
+ GGML_OP_DIAG_MASK_INF,
+ GGML_OP_DIAG_MASK_ZERO,
+ GGML_OP_SOFT_MAX,
+ GGML_OP_SOFT_MAX_BACK,
+ GGML_OP_ROPE,
+ GGML_OP_ROPE_BACK,
+ GGML_OP_CLAMP,
+ GGML_OP_CONV_TRANSPOSE_1D,
+ GGML_OP_IM2COL,
+ GGML_OP_IM2COL_BACK,
+ GGML_OP_CONV_TRANSPOSE_2D,
+ GGML_OP_POOL_1D,
+ GGML_OP_POOL_2D,
+ GGML_OP_POOL_2D_BACK,
+ GGML_OP_UPSCALE, // nearest interpolate
+ GGML_OP_PAD,
+ GGML_OP_ARANGE,
+ GGML_OP_TIMESTEP_EMBEDDING,
+ GGML_OP_ARGSORT,
+ GGML_OP_LEAKY_RELU,
+
+ GGML_OP_FLASH_ATTN_EXT,
+ GGML_OP_FLASH_ATTN_BACK,
+ GGML_OP_SSM_CONV,
+ GGML_OP_SSM_SCAN,
+ GGML_OP_WIN_PART,
+ GGML_OP_WIN_UNPART,
+ GGML_OP_GET_REL_POS,
+ GGML_OP_ADD_REL_POS,
+ GGML_OP_RWKV_WKV,
+
+ GGML_OP_UNARY,
+
+ GGML_OP_MAP_UNARY,
+ GGML_OP_MAP_BINARY,
+
+ GGML_OP_MAP_CUSTOM1_F32,
+ GGML_OP_MAP_CUSTOM2_F32,
+ GGML_OP_MAP_CUSTOM3_F32,
+
+ GGML_OP_MAP_CUSTOM1,
+ GGML_OP_MAP_CUSTOM2,
+ GGML_OP_MAP_CUSTOM3,
+
+ GGML_OP_CROSS_ENTROPY_LOSS,
+ GGML_OP_CROSS_ENTROPY_LOSS_BACK,
+
+ GGML_OP_COUNT,
+ };
+
+ enum ggml_unary_op {
+ GGML_UNARY_OP_ABS,
+ GGML_UNARY_OP_SGN,
+ GGML_UNARY_OP_NEG,
+ GGML_UNARY_OP_STEP,
+ GGML_UNARY_OP_TANH,
+ GGML_UNARY_OP_ELU,
+ GGML_UNARY_OP_RELU,
+ GGML_UNARY_OP_SIGMOID,
+ GGML_UNARY_OP_GELU,
+ GGML_UNARY_OP_GELU_QUICK,
+ GGML_UNARY_OP_SILU,
+ GGML_UNARY_OP_HARDSWISH,
+ GGML_UNARY_OP_HARDSIGMOID,
+ GGML_UNARY_OP_EXP,
+
+ GGML_UNARY_OP_COUNT,
+ };
+
+ enum ggml_object_type {
+ GGML_OBJECT_TYPE_TENSOR,
+ GGML_OBJECT_TYPE_GRAPH,
+ GGML_OBJECT_TYPE_WORK_BUFFER
+ };
+
+ enum ggml_log_level {
+ GGML_LOG_LEVEL_ERROR = 2,
+ GGML_LOG_LEVEL_WARN = 3,
+ GGML_LOG_LEVEL_INFO = 4,
+ GGML_LOG_LEVEL_DEBUG = 5
+ };
+
+ enum ggml_tensor_flag {
+ GGML_TENSOR_FLAG_INPUT = 1,
+ GGML_TENSOR_FLAG_OUTPUT = 2,
+ GGML_TENSOR_FLAG_PARAM = 4,
+ };
+
+ // ggml object
+ struct ggml_object {
+ size_t offs;
+ size_t size;
+
+ struct ggml_object * next;
+
+ enum ggml_object_type type;
+
+ char padding[4];
+ };
+
+ static const size_t GGML_OBJECT_SIZE = sizeof(struct ggml_object);
+
+ // n-dimensional tensor
+ struct ggml_tensor {
+ enum ggml_type type;
+
+ GGML_DEPRECATED(enum ggml_backend_type backend, "use the buffer type to find the storage location of the tensor");
+
+ struct ggml_backend_buffer * buffer;
+
+ int64_t ne[GGML_MAX_DIMS]; // number of elements
+ size_t nb[GGML_MAX_DIMS]; // stride in bytes:
+ // nb[0] = ggml_type_size(type)
+ // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding
+ // nb[i] = nb[i-1] * ne[i-1]
+
+ // compute data
+ enum ggml_op op;
+
+ // op params - allocated as int32_t for alignment
+ int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];
+
+ int32_t flags;
+
+ struct ggml_tensor * grad;
+ struct ggml_tensor * src[GGML_MAX_SRC];
+
+ // source tensor and offset for views
+ struct ggml_tensor * view_src;
+ size_t view_offs;
+
+ void * data;
+
+ char name[GGML_MAX_NAME];
+
+ void * extra; // extra things e.g. for ggml-cuda.cu
+
+ // char padding[4];
+ };
+
+ static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);
+
+ // Abort callback
+ // If not NULL, called before ggml computation
+ // If it returns true, the computation is aborted
+ typedef bool (*ggml_abort_callback)(void * data);
+
+ // Scheduling priorities
+ enum ggml_sched_priority {
+ GGML_SCHED_PRIO_NORMAL,
+ GGML_SCHED_PRIO_MEDIUM,
+ GGML_SCHED_PRIO_HIGH,
+ GGML_SCHED_PRIO_REALTIME
+ };
+
+ // Threadpool params
+ // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults
+ struct ggml_threadpool_params {
+ bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)
+ int n_threads; // number of threads
+ enum ggml_sched_priority prio; // thread priority
+ uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling)
+ bool strict_cpu; // strict cpu placement
+ bool paused; // start in paused state
+ };
+
+ struct ggml_threadpool; // forward declaration, see ggml.c
+
+ typedef struct ggml_threadpool * ggml_threadpool_t;
+
+ // the compute plan that needs to be prepared for ggml_graph_compute()
+ // since https://github.com/ggerganov/ggml/issues/287
+ struct ggml_cplan {
+ size_t work_size; // size of work buffer, calculated by `ggml_graph_plan()`
+ uint8_t * work_data; // work buffer, to be allocated by caller before calling to `ggml_graph_compute()`
+
+ int n_threads;
+ struct ggml_threadpool * threadpool;
+
+ // abort ggml_graph_compute when true
+ ggml_abort_callback abort_callback;
+ void * abort_callback_data;
+ };
+
+ enum ggml_cgraph_eval_order {
+ GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,
+ GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,
+ GGML_CGRAPH_EVAL_ORDER_COUNT
+ };
+
+ typedef uint32_t ggml_bitset_t;
+
+ struct ggml_hash_set {
+ size_t size;
+ ggml_bitset_t * used;
+ struct ggml_tensor ** keys;
+ };
+
+ // computation graph
+ struct ggml_cgraph {
+ int size;
+ int n_nodes;
+ int n_leafs;
+
+ struct ggml_tensor ** nodes;
+ struct ggml_tensor ** grads;
+ struct ggml_tensor ** leafs;
+
+ struct ggml_hash_set visited_hash_set;
+
+ enum ggml_cgraph_eval_order order;
+ };
+
+ // scratch buffer
+ struct ggml_scratch {
+ size_t offs;
+ size_t size;
+ void * data;
+ };
+
+ struct ggml_init_params {
+ // memory pool
+ size_t mem_size; // bytes
+ void * mem_buffer; // if NULL, memory will be allocated internally
+ bool no_alloc; // don't allocate memory for the tensor data
+ };
+
+ // numa strategies
+ enum ggml_numa_strategy {
+ GGML_NUMA_STRATEGY_DISABLED = 0,
+ GGML_NUMA_STRATEGY_DISTRIBUTE = 1,
+ GGML_NUMA_STRATEGY_ISOLATE = 2,
+ GGML_NUMA_STRATEGY_NUMACTL = 3,
+ GGML_NUMA_STRATEGY_MIRROR = 4,
+ GGML_NUMA_STRATEGY_COUNT
+ };
+
+ //
+ // GUID
+ //
+
+ // GUID types
+ typedef uint8_t ggml_guid[16];
+ typedef ggml_guid * ggml_guid_t;
+
+ GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b);
+
+ // misc
+
+ GGML_API void ggml_time_init(void); // call this once at the beginning of the program
+ GGML_API int64_t ggml_time_ms(void);
+ GGML_API int64_t ggml_time_us(void);
+ GGML_API int64_t ggml_cycles(void);
+ GGML_API int64_t ggml_cycles_per_ms(void);
+
+ // accepts a UTF-8 path, even on Windows
+ GGML_API FILE * ggml_fopen(const char * fname, const char * mode);
+
+ GGML_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems
+ GGML_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node
+
+ GGML_API void ggml_print_object (const struct ggml_object * obj);
+ GGML_API void ggml_print_objects(const struct ggml_context * ctx);
+
+ GGML_API GGML_CALL int64_t ggml_nelements (const struct ggml_tensor * tensor);
+ GGML_API GGML_CALL int64_t ggml_nrows (const struct ggml_tensor * tensor);
+ GGML_API GGML_CALL size_t ggml_nbytes (const struct ggml_tensor * tensor);
+ GGML_API size_t ggml_nbytes_pad (const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN
+
+ GGML_API GGML_CALL int64_t ggml_blck_size(enum ggml_type type);
+ GGML_API GGML_CALL size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block
+ GGML_API GGML_CALL size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row
+
+ GGML_DEPRECATED(
+ GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float
+ "use ggml_row_size() instead");
+
+ GGML_API GGML_CALL const char * ggml_type_name(enum ggml_type type);
+ GGML_API GGML_CALL const char * ggml_op_name (enum ggml_op op);
+ GGML_API const char * ggml_op_symbol(enum ggml_op op);
+
+ GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);
+ GGML_API GGML_CALL const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name
+
+ GGML_API GGML_CALL size_t ggml_element_size(const struct ggml_tensor * tensor);
+
+ GGML_API GGML_CALL bool ggml_is_quantized(enum ggml_type type);
+
+ // TODO: temporary until model loading of ggml examples is refactored
+ GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);
+
+ GGML_API GGML_CALL bool ggml_is_transposed(const struct ggml_tensor * tensor);
+ GGML_API GGML_CALL bool ggml_is_permuted (const struct ggml_tensor * tensor);
+ GGML_API GGML_CALL bool ggml_is_empty (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);
+ GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars
+
+ GGML_API GGML_CALL bool ggml_is_contiguous (const struct ggml_tensor * tensor);
+ GGML_API GGML_CALL bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous()
+ GGML_API GGML_CALL bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1
+ GGML_API GGML_CALL bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2
+
+ GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);
+ GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
+
+ GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
+
+ // use this to compute the memory overhead of a tensor
+ GGML_API size_t ggml_tensor_overhead(void);
+
+ GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);
+
+ // main
+
+ GGML_API struct ggml_context * ggml_init(struct ggml_init_params params);
+ GGML_API void ggml_free(struct ggml_context * ctx);
+
+ GGML_API size_t ggml_used_mem(const struct ggml_context * ctx);
+
+ GGML_API size_t ggml_set_scratch (struct ggml_context * ctx, struct ggml_scratch scratch);
+ GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx);
+ GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);
+
+ GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx);
+ GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx);
+ GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx);
+
+ GGML_API struct ggml_tensor * ggml_new_tensor(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int n_dims,
+ const int64_t *ne);
+
+ GGML_API struct ggml_tensor * ggml_new_tensor_1d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0);
+
+ GGML_API struct ggml_tensor * ggml_new_tensor_2d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0,
+ int64_t ne1);
+
+ GGML_API struct ggml_tensor * ggml_new_tensor_3d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2);
+
+ GGML_API struct ggml_tensor * ggml_new_tensor_4d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3);
+
+ GGML_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value);
+ GGML_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value);
+
+ GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);
+ GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);
+
+ // Context tensor enumeration and lookup
+ GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx);
+ GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);
+ GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);
+
+ GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);
+ GGML_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value);
+ GGML_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value);
+
+ // Converts a flat index into coordinates
+ GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
+
+ GGML_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i);
+ GGML_API void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value);
+
+ GGML_API int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);
+ GGML_API void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value);
+
+ GGML_API float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i);
+ GGML_API void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value);
+
+ GGML_API float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);
+ GGML_API void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);
+
+ GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);
+ GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);
+
+ GGML_API GGML_CALL enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);
+
+ GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor);
+ GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name);
+ GGML_ATTRIBUTE_FORMAT(2, 3)
+ GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...);
+
+ //
+ // operations on tensors with backpropagation
+ //
+
+ GGML_API struct ggml_tensor * ggml_dup(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_dup_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_add(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_add_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_add_cast(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ enum ggml_type type);
+
+ GGML_API struct ggml_tensor * ggml_add1(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_add1_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // dst = a
+ // view(dst, nb1, nb2, nb3, offset) += b
+ // return dst
+ GGML_API struct ggml_tensor * ggml_acc(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_acc_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_sub(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_sub_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_mul(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_mul_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_div(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_div_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_sqr(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sqr_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sqrt(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sqrt_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_log(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_log_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sin(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sin_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_cos(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_cos_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // return scalar
+ GGML_API struct ggml_tensor * ggml_sum(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]
+ GGML_API struct ggml_tensor * ggml_sum_rows(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // mean along rows
+ GGML_API struct ggml_tensor * ggml_mean(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // argmax along rows
+ GGML_API struct ggml_tensor * ggml_argmax(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // if a is the same shape as b, and a is not parameter, return a
+ // otherwise, return a new tensor: repeat(a) to fit in b
+ GGML_API struct ggml_tensor * ggml_repeat(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // sums repetitions in a into shape of b
+ GGML_API struct ggml_tensor * ggml_repeat_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // concat a and b along dim
+ // used in stable-diffusion
+ GGML_API struct ggml_tensor * ggml_concat(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int dim);
+
+ GGML_API struct ggml_tensor * ggml_abs(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_abs_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sgn(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sgn_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_neg(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_neg_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_step(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_step_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_tanh(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_tanh_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_elu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_elu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_relu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_leaky_relu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, float negative_slope, bool inplace);
+
+ GGML_API struct ggml_tensor * ggml_relu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sigmoid(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sigmoid_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_gelu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_gelu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_gelu_quick(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_gelu_quick_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_silu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_silu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // a - x
+ // b - dy
+ GGML_API struct ggml_tensor * ggml_silu_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // hardswish(x) = x * relu6(x + 3) / 6
+ GGML_API struct ggml_tensor * ggml_hardswish(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // hardsigmoid(x) = relu6(x + 3) / 6
+ GGML_API struct ggml_tensor * ggml_hardsigmoid(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_exp(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_exp_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // normalize along rows
+ GGML_API struct ggml_tensor * ggml_norm(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps);
+
+ GGML_API struct ggml_tensor * ggml_norm_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps);
+
+ GGML_API struct ggml_tensor * ggml_rms_norm(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps);
+
+ GGML_API struct ggml_tensor * ggml_rms_norm_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps);
+
+ // group normalize along ne0*ne1*n_groups
+ // used in stable-diffusion
+ GGML_API struct ggml_tensor * ggml_group_norm(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_groups,
+ float eps);
+
+ GGML_API struct ggml_tensor * ggml_group_norm_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_groups,
+ float eps);
+
+ // a - x
+ // b - dy
+ GGML_API struct ggml_tensor * ggml_rms_norm_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ float eps);
+
+ // A: k columns, n rows => [ne03, ne02, n, k]
+ // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
+ // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
+ GGML_API struct ggml_tensor * ggml_mul_mat(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // change the precision of a matrix multiplication
+ // set to GGML_PREC_F32 for higher precision (useful for phi-2)
+ GGML_API void ggml_mul_mat_set_prec(
+ struct ggml_tensor * a,
+ enum ggml_prec prec);
+
+ // indirect matrix multiplication
+ GGML_API struct ggml_tensor * ggml_mul_mat_id(
+ struct ggml_context * ctx,
+ struct ggml_tensor * as,
+ struct ggml_tensor * b,
+ struct ggml_tensor * ids);
+
+ // A: m columns, n rows,
+ // B: p columns, n rows,
+ // result is m columns, p rows
+ GGML_API struct ggml_tensor * ggml_out_prod(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ //
+ // operations on tensors without backpropagation
+ //
+
+ GGML_API struct ggml_tensor * ggml_scale(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float s);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_scale_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float s);
+
+ // b -> view(a,offset,nb1,nb2,3), return modified a
+ GGML_API struct ggml_tensor * ggml_set(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset);
+
+ // b -> view(a,offset,nb1,nb2,3), return view(a)
+ GGML_API struct ggml_tensor * ggml_set_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_set_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_set_1d_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t offset);
+
+ // b -> view(a,offset,nb1,nb2,3), return modified a
+ GGML_API struct ggml_tensor * ggml_set_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t offset);
+
+ // b -> view(a,offset,nb1,nb2,3), return view(a)
+ GGML_API struct ggml_tensor * ggml_set_2d_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t offset);
+
+ // a -> b, return view(b)
+ GGML_API struct ggml_tensor * ggml_cpy(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_cast(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_type type);
+
+ // make contiguous
+ GGML_API struct ggml_tensor * ggml_cont(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // make contiguous, with new shape
+ GGML_API struct ggml_tensor * ggml_cont_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0);
+
+ GGML_API struct ggml_tensor * ggml_cont_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1);
+
+ GGML_API struct ggml_tensor * ggml_cont_3d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2);
+
+ GGML_API struct ggml_tensor * ggml_cont_4d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3);
+
+ // return view(a), b specifies the new shape
+ // TODO: when we start computing gradient, make a copy instead of view
+ GGML_API struct ggml_tensor * ggml_reshape(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // return view(a)
+ // TODO: when we start computing gradient, make a copy instead of view
+ GGML_API struct ggml_tensor * ggml_reshape_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0);
+
+ GGML_API struct ggml_tensor * ggml_reshape_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1);
+
+ // return view(a)
+ // TODO: when we start computing gradient, make a copy instead of view
+ GGML_API struct ggml_tensor * ggml_reshape_3d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2);
+
+ GGML_API struct ggml_tensor * ggml_reshape_4d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3);
+
+ // offset in bytes
+ GGML_API struct ggml_tensor * ggml_view_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_view_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ size_t nb1, // row stride in bytes
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_view_3d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ size_t nb1, // row stride in bytes
+ size_t nb2, // slice stride in bytes
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_view_4d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3,
+ size_t nb1, // row stride in bytes
+ size_t nb2, // slice stride in bytes
+ size_t nb3,
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_permute(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int axis0,
+ int axis1,
+ int axis2,
+ int axis3);
+
+ // alias for ggml_permute(ctx, a, 1, 0, 2, 3)
+ GGML_API struct ggml_tensor * ggml_transpose(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // supports 3D: a->ne[2] == b->ne[1]
+ GGML_API struct ggml_tensor * ggml_get_rows(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_get_rows_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c);
+
+ GGML_API struct ggml_tensor * ggml_diag(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // set elements above the diagonal to -INF
+ GGML_API struct ggml_tensor * ggml_diag_mask_inf(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past);
+
+ // set elements above the diagonal to 0
+ GGML_API struct ggml_tensor * ggml_diag_mask_zero(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past);
+
+ GGML_API struct ggml_tensor * ggml_soft_max(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_soft_max_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // fused soft_max(a*scale + mask*(ALiBi slope))
+ // mask is optional
+ // max_bias = 0.0f for no ALiBi
+ GGML_API struct ggml_tensor * ggml_soft_max_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * mask,
+ float scale,
+ float max_bias);
+
+ GGML_API struct ggml_tensor * ggml_soft_max_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_soft_max_back_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // rotary position embedding
+ // if (mode & 1) - skip n_past elements (NOT SUPPORTED)
+ // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style
+ //
+ // b is an int32 vector with size a->ne[2], it contains the positions
+ GGML_API struct ggml_tensor * ggml_rope(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_rope_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode);
+
+ // custom RoPE
+ // c is freq factors (e.g. phi3-128k), (optional)
+ GGML_API struct ggml_tensor * ggml_rope_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_rope_ext_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow);
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow),
+ "use ggml_rope_ext instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow),
+ "use ggml_rope_ext_inplace instead");
+
+ // compute correction dims for YaRN RoPE scaling
+ GGML_CALL void ggml_rope_yarn_corr_dims(
+ int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);
+
+ // rotary position embedding backward, i.e compute dx from dy
+ // a - dy
+ GGML_API struct ggml_tensor * ggml_rope_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow);
+
+ // clamp
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_clamp(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float min,
+ float max);
+
+ // im2col
+ // converts data into a format that effectively results in a convolution when combined with matrix multiplication
+ GGML_API struct ggml_tensor * ggml_im2col(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride dimension 0
+ int s1, // stride dimension 1
+ int p0, // padding dimension 0
+ int p1, // padding dimension 1
+ int d0, // dilation dimension 0
+ int d1, // dilation dimension 1
+ bool is_2D,
+ enum ggml_type dst_type);
+
+ GGML_API struct ggml_tensor * ggml_im2col_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // gradient of im2col output
+ int64_t * ne, // shape of im2col input
+ int s0, // stride dimension 0
+ int s1, // stride dimension 1
+ int p0, // padding dimension 0
+ int p1, // padding dimension 1
+ int d0, // dilation dimension 0
+ int d1, // dilation dimension 1
+ bool is_2D);
+
+ GGML_API struct ggml_tensor * ggml_conv_depthwise_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride dimension 0
+ int s1, // stride dimension 1
+ int p0, // padding dimension 0
+ int p1, // padding dimension 1
+ int d0, // dilation dimension 0
+ int d1); // dilation dimension 1
+
+ GGML_API struct ggml_tensor * ggml_conv_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride
+ int p0, // padding
+ int d0); // dilation
+
+ // conv_1d with padding = half
+ // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)
+ GGML_API struct ggml_tensor* ggml_conv_1d_ph(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s, // stride
+ int d); // dilation
+
+ GGML_API struct ggml_tensor * ggml_conv_transpose_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride
+ int p0, // padding
+ int d0); // dilation
+
+ GGML_API struct ggml_tensor * ggml_conv_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride dimension 0
+ int s1, // stride dimension 1
+ int p0, // padding dimension 0
+ int p1, // padding dimension 1
+ int d0, // dilation dimension 0
+ int d1); // dilation dimension 1
+
+
+ // kernel size is a->ne[0] x a->ne[1]
+ // stride is equal to kernel size
+ // padding is zero
+ // example:
+ // a: 16 16 3 768
+ // b: 1024 1024 3 1
+ // res: 64 64 768 1
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // kernel size is a->ne[0] x a->ne[1]
+ // stride is 1
+ // padding is half
+ // example:
+ // a: 3 3 256 256
+ // b: 64 64 256 1
+ // res: 64 64 256 1
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int stride);
+
+ enum ggml_op_pool {
+ GGML_OP_POOL_MAX,
+ GGML_OP_POOL_AVG,
+ GGML_OP_POOL_COUNT,
+ };
+
+ GGML_API struct ggml_tensor * ggml_pool_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_op_pool op,
+ int k0, // kernel size
+ int s0, // stride
+ int p0); // padding
+
+ // the result will have 2*p0 padding for the first dimension
+ // and 2*p1 padding for the second dimension
+ GGML_API struct ggml_tensor * ggml_pool_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_op_pool op,
+ int k0,
+ int k1,
+ int s0,
+ int s1,
+ float p0,
+ float p1);
+
+ GGML_API struct ggml_tensor * ggml_pool_2d_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * af, // "a"/input used in forward pass
+ enum ggml_op_pool op,
+ int k0,
+ int k1,
+ int s0,
+ int s1,
+ float p0,
+ float p1);
+
+ // nearest interpolate
+ // multiplies ne0 and ne1 by scale factor
+ // used in stable-diffusion
+ GGML_API struct ggml_tensor * ggml_upscale(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int scale_factor);
+
+ // nearest interpolate
+ // nearest interpolate to specified dimensions
+ // used in tortoise.cpp
+ GGML_API struct ggml_tensor * ggml_upscale_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int ne0,
+ int ne1,
+ int ne2,
+ int ne3);
+
+ // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]
+ GGML_API struct ggml_tensor * ggml_pad(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int p0,
+ int p1,
+ int p2,
+ int p3);
+
+ // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
+ // timesteps: [N,]
+ // return: [N, dim]
+ GGML_API struct ggml_tensor * ggml_timestep_embedding(
+ struct ggml_context * ctx,
+ struct ggml_tensor * timesteps,
+ int dim,
+ int max_period);
+
+ // sort rows
+ enum ggml_sort_order {
+ GGML_SORT_ORDER_ASC,
+ GGML_SORT_ORDER_DESC,
+ };
+
+ GGML_API struct ggml_tensor * ggml_argsort(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_sort_order order);
+
+ GGML_API struct ggml_tensor * ggml_arange(
+ struct ggml_context * ctx,
+ float start,
+ float stop,
+ float step);
+
+ // top k elements per row
+ GGML_API struct ggml_tensor * ggml_top_k(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int k);
+
+#define GGML_KQ_MASK_PAD 32
+
+ // q: [n_embd, n_batch, n_head, 1]
+ // k: [n_embd, n_kv, n_head_kv, 1]
+ // v: [n_embd, n_kv, n_head_kv, 1] !! not transposed !!
+ // mask: [n_kv, n_batch_pad, 1, 1] !! n_batch_pad = GGML_PAD(n_batch, GGML_KQ_MASK_PAD) !!
+ // res: [n_embd, n_head, n_batch, 1] !! permuted !!
+ GGML_API struct ggml_tensor * ggml_flash_attn_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * mask,
+ float scale,
+ float max_bias,
+ float logit_softcap);
+
+ GGML_API void ggml_flash_attn_ext_set_prec(
+ struct ggml_tensor * a,
+ enum ggml_prec prec);
+
+ // TODO: needs to be adapted to ggml_flash_attn_ext
+ GGML_API struct ggml_tensor * ggml_flash_attn_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * d,
+ bool masked);
+
+ GGML_API struct ggml_tensor * ggml_ssm_conv(
+ struct ggml_context * ctx,
+ struct ggml_tensor * sx,
+ struct ggml_tensor * c);
+
+ GGML_API struct ggml_tensor * ggml_ssm_scan(
+ struct ggml_context * ctx,
+ struct ggml_tensor * s,
+ struct ggml_tensor * x,
+ struct ggml_tensor * dt,
+ struct ggml_tensor * A,
+ struct ggml_tensor * B,
+ struct ggml_tensor * C);
+
+ // partition into non-overlapping windows with padding if needed
+ // example:
+ // a: 768 64 64 1
+ // w: 14
+ // res: 768 14 14 25
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_win_part(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int w);
+
+ // reverse of ggml_win_part
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_win_unpart(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int w0,
+ int h0,
+ int w);
+
+ GGML_API struct ggml_tensor * ggml_unary(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_unary_op op);
+
+ GGML_API struct ggml_tensor * ggml_unary_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_unary_op op);
+
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_get_rel_pos(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int qh,
+ int kh);
+
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_add_rel_pos(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * pw,
+ struct ggml_tensor * ph);
+
+ GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * pw,
+ struct ggml_tensor * ph);
+
+ GGML_API struct ggml_tensor * ggml_rwkv_wkv(
+ struct ggml_context * ctx,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * r,
+ struct ggml_tensor * tf,
+ struct ggml_tensor * td,
+ struct ggml_tensor * state);
+
+ // custom operators
+
+ typedef void (*ggml_unary_op_f32_t) (const int, float *, const float *);
+ typedef void (*ggml_binary_op_f32_t)(const int, float *, const float *, const float *);
+
+ typedef void (*ggml_custom1_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *);
+ typedef void (*ggml_custom2_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *);
+ typedef void (*ggml_custom3_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *);
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ ggml_unary_op_f32_t fun),
+ "use ggml_map_custom1 instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_inplace_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ ggml_unary_op_f32_t fun),
+ "use ggml_map_custom1_inplace instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ ggml_binary_op_f32_t fun),
+ "use ggml_map_custom2 instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_inplace_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ ggml_binary_op_f32_t fun),
+ "use ggml_map_custom2_inplace instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ ggml_custom1_op_f32_t fun),
+ "use ggml_map_custom1 instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_inplace_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ ggml_custom1_op_f32_t fun),
+ "use ggml_map_custom1_inplace instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ ggml_custom2_op_f32_t fun),
+ "use ggml_map_custom2 instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_inplace_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ ggml_custom2_op_f32_t fun),
+ "use ggml_map_custom2_inplace instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ ggml_custom3_op_f32_t fun),
+ "use ggml_map_custom3 instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_inplace_f32(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ ggml_custom3_op_f32_t fun),
+ "use ggml_map_custom3_inplace instead");
+
+ // custom operators v2
+
+ typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);
+ typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata);
+ typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata);
+
+ #define GGML_N_TASKS_MAX -1
+
+ GGML_API struct ggml_tensor * ggml_map_custom1(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ ggml_custom1_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_map_custom1_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ ggml_custom1_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_map_custom2(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ ggml_custom2_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_map_custom2_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ ggml_custom2_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_map_custom3(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ ggml_custom3_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_map_custom3_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ ggml_custom3_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ // loss function
+
+ GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c);
+
+ //
+ // automatic differentiation
+ //
+
+ GGML_API void ggml_set_param(
+ struct ggml_context * ctx,
+ struct ggml_tensor * tensor);
+
+
+ GGML_API void ggml_build_forward_expand (struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);
+ GGML_API void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_cgraph * gb, bool keep);
+
+ // graph allocation in a context
+ GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false
+ GGML_API struct ggml_cgraph * ggml_new_graph_custom (struct ggml_context * ctx, size_t size, bool grads);
+ GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph);
+ GGML_API struct ggml_cgraph ggml_graph_view (struct ggml_cgraph * cgraph, int i0, int i1);
+ GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst);
+ GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // zero grads
+ GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph);
+
+ GGML_API size_t ggml_graph_overhead(void);
+ GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads);
+
+ GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);
+ GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params *p, int n_threads);
+ GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params *p0, const struct ggml_threadpool_params *p1);
+ GGML_API struct ggml_threadpool* ggml_threadpool_new (struct ggml_threadpool_params * params);
+ GGML_API void ggml_threadpool_free (struct ggml_threadpool * threadpool);
+ GGML_API int ggml_threadpool_get_n_threads(struct ggml_threadpool * threadpool);
+ GGML_API void ggml_threadpool_pause (struct ggml_threadpool * threadpool);
+ GGML_API void ggml_threadpool_resume (struct ggml_threadpool * threadpool);
+
+ // ggml_graph_plan() has to be called before ggml_graph_compute()
+ // when plan.work_size > 0, caller must allocate memory for plan.work_data
+ GGML_API struct ggml_cplan ggml_graph_plan(
+ const struct ggml_cgraph * cgraph,
+ int n_threads, /* = GGML_DEFAULT_N_THREADS */
+ struct ggml_threadpool * threadpool /* = NULL */ );
+ GGML_API enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);
+
+ // same as ggml_graph_compute() but the work data is allocated as a part of the context
+ // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data
+ GGML_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads);
+
+ GGML_API struct ggml_tensor * ggml_graph_get_tensor(struct ggml_cgraph * cgraph, const char * name);
+
+ GGML_API void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname);
+ GGML_API struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context ** ctx_data, struct ggml_context ** ctx_eval);
+
+ // print info and performance information for the graph
+ GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);
+
+ // dump the graph into a file using the dot format
+ GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename);
+
+ // build gradient checkpointing backward graph gb for gf using provided checkpoints
+ // gb_tmp will contain original backward graph with rewritten backward process nodes,
+ // but without the second forward pass nodes.
+ GGML_API void ggml_build_backward_gradient_checkpointing(
+ struct ggml_context * ctx,
+ struct ggml_cgraph * gf,
+ struct ggml_cgraph * gb,
+ struct ggml_cgraph * gb_tmp,
+ struct ggml_tensor * * checkpoints,
+ int n_checkpoints);
+ //
+ // optimization
+ //
+
+ // optimization methods
+ enum ggml_opt_type {
+ GGML_OPT_TYPE_ADAM,
+ GGML_OPT_TYPE_LBFGS,
+ };
+
+ // linesearch methods
+ enum ggml_linesearch {
+ GGML_LINESEARCH_DEFAULT = 1,
+
+ GGML_LINESEARCH_BACKTRACKING_ARMIJO = 0,
+ GGML_LINESEARCH_BACKTRACKING_WOLFE = 1,
+ GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE = 2,
+ };
+
+ // optimization return values
+ enum ggml_opt_result {
+ GGML_OPT_RESULT_OK = 0,
+ GGML_OPT_RESULT_DID_NOT_CONVERGE,
+ GGML_OPT_RESULT_NO_CONTEXT,
+ GGML_OPT_RESULT_INVALID_WOLFE,
+ GGML_OPT_RESULT_FAIL,
+ GGML_OPT_RESULT_CANCEL,
+
+ GGML_LINESEARCH_FAIL = -128,
+ GGML_LINESEARCH_MINIMUM_STEP,
+ GGML_LINESEARCH_MAXIMUM_STEP,
+ GGML_LINESEARCH_MAXIMUM_ITERATIONS,
+ GGML_LINESEARCH_INVALID_PARAMETERS,
+ };
+
+ typedef void (*ggml_opt_callback)(void * data, int accum_step, float * sched, bool * cancel);
+ typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data);
+
+ // optimization parameters
+ //
+ // see ggml.c (ggml_opt_default_params) for default values
+ //
+ struct ggml_opt_params {
+ enum ggml_opt_type type;
+
+ size_t graph_size;
+
+ int n_threads;
+
+ // delta-based convergence test
+ //
+ // if past == 0 - disabled
+ // if past > 0:
+ // stop if |f(x) - f(x_past)| < delta * max(1, |f(x)|)
+ //
+ int past;
+ float delta;
+
+ // maximum number of iterations without improvement
+ //
+ // if 0 - disabled
+ // if > 0:
+ // assume convergence if no cost improvement in this number of iterations
+ //
+ int max_no_improvement;
+
+ bool print_forward_graph;
+ bool print_backward_graph;
+
+ int n_gradient_accumulation;
+
+ // ADAM parameters
+ struct {
+ int n_iter;
+
+ float sched; // schedule multiplier (fixed, decay or warmup)
+ float decay; // weight decay for AdamW, use 0.0f to disable
+ int decay_min_ndim; // minimum number of tensor dimension to apply weight decay
+ float alpha; // learning rate
+ float beta1;
+ float beta2;
+ float eps; // epsilon for numerical stability
+ float eps_f; // epsilon for convergence test
+ float eps_g; // epsilon for convergence test
+ float gclip; // gradient clipping
+ } adam;
+
+ // LBFGS parameters
+ struct {
+ int m; // number of corrections to approximate the inv. Hessian
+ int n_iter;
+ int max_linesearch;
+
+ float eps; // convergence tolerance
+ float ftol; // line search tolerance
+ float wolfe;
+ float min_step;
+ float max_step;
+
+ enum ggml_linesearch linesearch;
+ } lbfgs;
+ };
+
+ struct ggml_opt_context {
+ struct ggml_context * ctx;
+ struct ggml_opt_params params;
+
+ int iter;
+ int64_t nx; // number of parameter elements
+
+ bool just_initialized;
+
+ float loss_before;
+ float loss_after;
+
+ struct {
+ struct ggml_tensor * g; // current gradient
+ struct ggml_tensor * m; // first moment
+ struct ggml_tensor * v; // second moment
+ struct ggml_tensor * pf; // past function values
+ float fx_best;
+ float fx_prev;
+ int n_no_improvement;
+ } adam;
+
+ struct {
+ struct ggml_tensor * x; // current parameters
+ struct ggml_tensor * xp; // previous parameters
+ struct ggml_tensor * g; // current gradient
+ struct ggml_tensor * gp; // previous gradient
+ struct ggml_tensor * d; // search direction
+ struct ggml_tensor * pf; // past function values
+ struct ggml_tensor * lmal; // the L-BFGS memory alpha
+ struct ggml_tensor * lmys; // the L-BFGS memory ys
+ struct ggml_tensor * lms; // the L-BFGS memory s
+ struct ggml_tensor * lmy; // the L-BFGS memory y
+ float fx_best;
+ float step;
+ int j;
+ int k;
+ int end;
+ int n_no_improvement;
+ } lbfgs;
+ };
+
+ GGML_API struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type);
+
+ // optimize the function defined by the tensor f
+ GGML_API enum ggml_opt_result ggml_opt(
+ struct ggml_context * ctx,
+ struct ggml_opt_params params,
+ struct ggml_tensor * f);
+
+ // initialize optimizer context
+ GGML_API void ggml_opt_init(
+ struct ggml_context * ctx,
+ struct ggml_opt_context * opt,
+ struct ggml_opt_params params,
+ int64_t nx);
+
+ // continue optimizing the function defined by the tensor f
+ GGML_API enum ggml_opt_result ggml_opt_resume(
+ struct ggml_context * ctx,
+ struct ggml_opt_context * opt,
+ struct ggml_tensor * f);
+
+ // continue optimizing the function defined by the tensor f
+ GGML_API enum ggml_opt_result ggml_opt_resume_g(
+ struct ggml_context * ctx,
+ struct ggml_opt_context * opt,
+ struct ggml_tensor * f,
+ struct ggml_cgraph * gf,
+ struct ggml_cgraph * gb,
+ ggml_opt_callback callback,
+ void * callback_data);
+
+ //
+ // tensor flags
+ //
+ GGML_API void ggml_set_input(struct ggml_tensor * tensor);
+ GGML_API void ggml_set_output(struct ggml_tensor * tensor);
+
+ //
+ // quantization
+ //
+
+ // - ggml_quantize_init can be called multiple times with the same type
+ // it will only initialize the quantization tables for the first call or after ggml_quantize_free
+ // automatically called by ggml_quantize_chunk for convenience
+ //
+ // - ggml_quantize_free will free any memory allocated by ggml_quantize_init
+ // call this at the end of the program to avoid memory leaks
+ //
+ // note: these are thread-safe
+ //
+ GGML_API void ggml_quantize_init(enum ggml_type type);
+ GGML_API void ggml_quantize_free(void);
+
+ // some quantization type cannot be used without an importance matrix
+ GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);
+
+ // calls ggml_quantize_init internally (i.e. can allocate memory)
+ GGML_API size_t ggml_quantize_chunk(
+ enum ggml_type type,
+ const float * src,
+ void * dst,
+ int64_t start,
+ int64_t nrows,
+ int64_t n_per_row,
+ const float * imatrix);
+
+ //
+ // gguf
+ //
+
+ enum gguf_type {
+ GGUF_TYPE_UINT8 = 0,
+ GGUF_TYPE_INT8 = 1,
+ GGUF_TYPE_UINT16 = 2,
+ GGUF_TYPE_INT16 = 3,
+ GGUF_TYPE_UINT32 = 4,
+ GGUF_TYPE_INT32 = 5,
+ GGUF_TYPE_FLOAT32 = 6,
+ GGUF_TYPE_BOOL = 7,
+ GGUF_TYPE_STRING = 8,
+ GGUF_TYPE_ARRAY = 9,
+ GGUF_TYPE_UINT64 = 10,
+ GGUF_TYPE_INT64 = 11,
+ GGUF_TYPE_FLOAT64 = 12,
+ GGUF_TYPE_COUNT, // marks the end of the enum
+ };
+
+ struct gguf_context;
+
+ struct gguf_init_params {
+ bool no_alloc;
+
+ // if not NULL, create a ggml_context and allocate the tensor data in it
+ struct ggml_context ** ctx;
+ };
+
+ GGML_API struct gguf_context * gguf_init_empty(void);
+ GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);
+ //GGML_API struct gguf_context * gguf_init_from_buffer(..);
+
+ GGML_API void gguf_free(struct gguf_context * ctx);
+
+ GGML_API const char * gguf_type_name(enum gguf_type type);
+
+ GGML_API int gguf_get_version (const struct gguf_context * ctx);
+ GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx);
+ GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx);
+ GGML_API void * gguf_get_data (const struct gguf_context * ctx);
+
+ GGML_API int gguf_get_n_kv(const struct gguf_context * ctx);
+ GGML_API int gguf_find_key(const struct gguf_context * ctx, const char * key);
+ GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int key_id);
+
+ GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int key_id);
+ GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int key_id);
+
+ // will abort if the wrong type is used for the key
+ GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int key_id);
+ GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int key_id);
+ GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int key_id);
+ GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int key_id);
+ GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int key_id);
+ GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int key_id);
+ GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int key_id);
+ GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int key_id);
+ GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int key_id);
+ GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int key_id);
+ GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int key_id);
+ GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int key_id);
+ GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int key_id);
+ GGML_API int gguf_get_arr_n (const struct gguf_context * ctx, int key_id);
+ GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int key_id);
+ GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int key_id, int i);
+
+ GGML_API int gguf_get_n_tensors (const struct gguf_context * ctx);
+ GGML_API int gguf_find_tensor (const struct gguf_context * ctx, const char * name);
+ GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int i);
+ GGML_API char * gguf_get_tensor_name (const struct gguf_context * ctx, int i);
+ GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int i);
+
+ // removes key if it exists
+ GGML_API void gguf_remove_key(struct gguf_context * ctx, const char * key);
+
+ // overrides existing values or adds a new one
+ GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val);
+ GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val);
+ GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val);
+ GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val);
+ GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val);
+ GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val);
+ GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val);
+ GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val);
+ GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val);
+ GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val);
+ GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val);
+ GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);
+ GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, int n);
+ GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, int n);
+
+ // set or add KV pairs from another context
+ GGML_API void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src);
+
+ // manage tensor info
+ GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor);
+ GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);
+ GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data, size_t size);
+
+ // writing gguf files can be done in 2 ways:
+ //
+ // - write the entire gguf_context to a binary file in a single pass:
+ //
+ // gguf_write_to_file(ctx, fname);
+ //
+ // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:
+ //
+ // FILE * f = fopen(fname, "wb");
+ // fseek(f, gguf_get_meta_size(ctx), SEEK_SET);
+ // fwrite(f, ...);
+ // void * data = gguf_meta_get_meta_data(ctx);
+ // fseek(f, 0, SEEK_SET);
+ // fwrite(f, data, gguf_get_meta_size(ctx));
+ // free(data);
+ // fclose(f);
+ //
+
+ // write the entire context to a binary file
+ GGML_API void gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta);
+
+ // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding
+ GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);
+ GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data);
+
+ //
+ // system info
+ //
+
+ GGML_API int ggml_cpu_has_avx (void);
+ GGML_API int ggml_cpu_has_avx_vnni (void);
+ GGML_API int ggml_cpu_has_avx2 (void);
+ GGML_API int ggml_cpu_has_avx512 (void);
+ GGML_API int ggml_cpu_has_avx512_vbmi(void);
+ GGML_API int ggml_cpu_has_avx512_vnni(void);
+ GGML_API int ggml_cpu_has_avx512_bf16(void);
+ GGML_API int ggml_cpu_has_fma (void);
+ GGML_API int ggml_cpu_has_neon (void);
+ GGML_API int ggml_cpu_has_sve (void);
+ GGML_API int ggml_cpu_has_arm_fma (void);
+ GGML_API int ggml_cpu_has_metal (void);
+ GGML_API int ggml_cpu_has_f16c (void);
+ GGML_API int ggml_cpu_has_fp16_va (void);
+ GGML_API int ggml_cpu_has_wasm_simd (void);
+ GGML_API int ggml_cpu_has_blas (void);
+ GGML_API int ggml_cpu_has_cuda (void);
+ GGML_API int ggml_cpu_has_vulkan (void);
+ GGML_API int ggml_cpu_has_kompute (void);
+ GGML_API int ggml_cpu_has_gpublas (void);
+ GGML_API int ggml_cpu_has_sse3 (void);
+ GGML_API int ggml_cpu_has_ssse3 (void);
+ GGML_API int ggml_cpu_has_sycl (void);
+ GGML_API int ggml_cpu_has_rpc (void);
+ GGML_API int ggml_cpu_has_vsx (void);
+ GGML_API int ggml_cpu_has_matmul_int8(void);
+ GGML_API int ggml_cpu_has_cann (void);
+ GGML_API int ggml_cpu_has_llamafile (void);
+
+ //
+ // Internal types and functions exposed for tests and benchmarks
+ //
+
+#ifdef __cplusplus
+// restrict not standard in C++
+#define GGML_RESTRICT
+#else
+#define GGML_RESTRICT restrict
+#endif
+ typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+ typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+ typedef void (*ggml_from_float_to_mat_t)
+ (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t nr, int64_t k, int64_t bs);
+ typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx,
+ const void * GGML_RESTRICT y, size_t by, int nrc);
+ typedef void (*ggml_gemv_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x,
+ const void * GGML_RESTRICT y, int nr, int nc);
+ typedef void (*ggml_gemm_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x,
+ const void * GGML_RESTRICT y, int nr, int nc);
+
+ typedef struct {
+ const char * type_name;
+ int64_t blck_size;
+ int64_t blck_size_interleave; // interleave elements in blocks
+ size_t type_size;
+ bool is_quantized;
+ ggml_to_float_t to_float;
+ ggml_from_float_t from_float;
+ ggml_from_float_t from_float_ref;
+ ggml_from_float_to_mat_t from_float_to_mat;
+ ggml_vec_dot_t vec_dot;
+ enum ggml_type vec_dot_type;
+ int64_t nrows; // number of rows to process simultaneously
+ int64_t ncols; // number of columns to process simultaneously
+ ggml_gemv_t gemv;
+ ggml_gemm_t gemm;
+ } ggml_type_traits_t;
+
+ GGML_API ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama/grammar-parser.cpp b/llama/grammar-parser.cpp
new file mode 100644
index 00000000..979e681a
--- /dev/null
+++ b/llama/grammar-parser.cpp
@@ -0,0 +1,565 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "grammar-parser.h"
+#include <cstdint>
+#include <cwchar>
+#include <string>
+#include <utility>
+#include <stdexcept>
+#include <exception>
+
+namespace grammar_parser {
+ // NOTE: assumes valid utf8 (but checks for overrun)
+ // copied from llama.cpp
+ static std::pair<uint32_t, const char *> decode_utf8(const char * src) {
+ static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };
+ uint8_t first_byte = static_cast<uint8_t>(*src);
+ uint8_t highbits = first_byte >> 4;
+ int len = lookup[highbits];
+ uint8_t mask = (1 << (8 - len)) - 1;
+ uint32_t value = first_byte & mask;
+ const char * end = src + len; // may overrun!
+ const char * pos = src + 1;
+ for ( ; pos < end && *pos; pos++) {
+ value = (value << 6) + (static_cast<uint8_t>(*pos) & 0x3F);
+ }
+ return std::make_pair(value, pos);
+ }
+
+ static uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) {
+ uint32_t next_id = static_cast<uint32_t>(state.symbol_ids.size());
+ auto result = state.symbol_ids.emplace(std::string(src, len), next_id);
+ return result.first->second;
+ }
+
+ static uint32_t generate_symbol_id(parse_state & state, const std::string & base_name) {
+ uint32_t next_id = static_cast<uint32_t>(state.symbol_ids.size());
+ state.symbol_ids[base_name + '_' + std::to_string(next_id)] = next_id;
+ return next_id;
+ }
+
+ static void add_rule(
+ parse_state & state,
+ uint32_t rule_id,
+ const std::vector<llama_grammar_element> & rule) {
+ if (state.rules.size() <= rule_id) {
+ state.rules.resize(rule_id + 1);
+ }
+ state.rules[rule_id] = rule;
+ }
+
+ static bool is_digit_char(char c) {
+ return '0' <= c && c <= '9';
+ }
+
+ static bool is_word_char(char c) {
+ return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || is_digit_char(c);
+ }
+
+ static std::pair<uint32_t, const char *> parse_hex(const char * src, int size) {
+ const char * pos = src;
+ const char * end = src + size;
+ uint32_t value = 0;
+ for ( ; pos < end && *pos; pos++) {
+ value <<= 4;
+ char c = *pos;
+ if ('a' <= c && c <= 'f') {
+ value += c - 'a' + 10;
+ } else if ('A' <= c && c <= 'F') {
+ value += c - 'A' + 10;
+ } else if ('0' <= c && c <= '9') {
+ value += c - '0';
+ } else {
+ break;
+ }
+ }
+ if (pos != end) {
+ throw std::runtime_error("expecting " + std::to_string(size) + " hex chars at " + src);
+ }
+ return std::make_pair(value, pos);
+ }
+
+ static const char * parse_space(const char * src, bool newline_ok) {
+ const char * pos = src;
+ while (*pos == ' ' || *pos == '\t' || *pos == '#' ||
+ (newline_ok && (*pos == '\r' || *pos == '\n'))) {
+ if (*pos == '#') {
+ while (*pos && *pos != '\r' && *pos != '\n') {
+ pos++;
+ }
+ } else {
+ pos++;
+ }
+ }
+ return pos;
+ }
+
+ static const char * parse_name(const char * src) {
+ const char * pos = src;
+ while (is_word_char(*pos)) {
+ pos++;
+ }
+ if (pos == src) {
+ throw std::runtime_error(std::string("expecting name at ") + src);
+ }
+ return pos;
+ }
+
+ static const char * parse_int(const char * src) {
+ const char * pos = src;
+ while (is_digit_char(*pos)) {
+ pos++;
+ }
+ if (pos == src) {
+ throw std::runtime_error(std::string("expecting integer at ") + src);
+ }
+ return pos;
+ }
+
+ static std::pair<uint32_t, const char *> parse_char(const char * src) {
+ if (*src == '\\') {
+ switch (src[1]) {
+ case 'x': return parse_hex(src + 2, 2);
+ case 'u': return parse_hex(src + 2, 4);
+ case 'U': return parse_hex(src + 2, 8);
+ case 't': return std::make_pair('\t', src + 2);
+ case 'r': return std::make_pair('\r', src + 2);
+ case 'n': return std::make_pair('\n', src + 2);
+ case '\\':
+ case '"':
+ case '[':
+ case ']':
+ return std::make_pair(src[1], src + 2);
+ default:
+ throw std::runtime_error(std::string("unknown escape at ") + src);
+ }
+ } else if (*src) {
+ return decode_utf8(src);
+ }
+ throw std::runtime_error("unexpected end of input");
+ }
+
+ const char * parse_alternates(
+ parse_state & state,
+ const char * src,
+ const std::string & rule_name,
+ uint32_t rule_id,
+ bool is_nested);
+
+ static const char * parse_sequence(
+ parse_state & state,
+ const char * src,
+ const std::string & rule_name,
+ std::vector<llama_grammar_element> & out_elements,
+ bool is_nested) {
+ size_t last_sym_start = out_elements.size();
+ const char * pos = src;
+
+ auto handle_repetitions = [&](int min_times, int max_times) {
+
+ if (last_sym_start == out_elements.size()) {
+ throw std::runtime_error(std::string("expecting preceding item to */+/?/{ at ") + pos);
+ }
+
+ // apply transformation to previous symbol (last_sym_start to end) according to
+ // the following rewrite rules:
+ // S{m,n} --> S S S (m times) S'(n-m)
+ // S'(x) ::= S S'(x-1) |
+ // (... n-m definitions of these S' rules ...)
+ // S'(1) ::= S |
+ // S{m,} --> S S S (m times) S'
+ // S' ::= S S' |
+ // S* --> S{0,}
+ // --> S' ::= S S' |
+ // S+ --> S{1,}
+ // --> S S'
+ // S' ::= S S' |
+ // S? --> S{0,1}
+ // --> S'
+ // S' ::= S |
+
+ std::vector<llama_grammar_element> previous_elements(out_elements.begin() + last_sym_start, out_elements.end());
+ if (min_times == 0) {
+ out_elements.resize(last_sym_start);
+ } else {
+ // Repeat the previous elements (min_times - 1) times
+ for (int i = 1; i < min_times; i++) {
+ out_elements.insert(out_elements.end(), previous_elements.begin(), previous_elements.end());
+ }
+ }
+
+ uint32_t last_rec_rule_id = 0;
+ auto n_opt = max_times < 0 ? 1 : max_times - min_times;
+
+ std::vector<llama_grammar_element> rec_rule(previous_elements);
+ for (int i = 0; i < n_opt; i++) {
+ rec_rule.resize(previous_elements.size());
+ uint32_t rec_rule_id = generate_symbol_id(state, rule_name);
+ if (i > 0 || max_times < 0) {
+ rec_rule.push_back({LLAMA_GRETYPE_RULE_REF, max_times < 0 ? rec_rule_id : last_rec_rule_id});
+ }
+ rec_rule.push_back({LLAMA_GRETYPE_ALT, 0});
+ rec_rule.push_back({LLAMA_GRETYPE_END, 0});
+ add_rule(state, rec_rule_id, rec_rule);
+ last_rec_rule_id = rec_rule_id;
+ }
+ if (n_opt > 0) {
+ out_elements.push_back({LLAMA_GRETYPE_RULE_REF, last_rec_rule_id});
+ }
+ };
+
+ while (*pos) {
+ if (*pos == '"') { // literal string
+ pos++;
+ last_sym_start = out_elements.size();
+ while (*pos != '"') {
+ if (!*pos) {
+ throw std::runtime_error("unexpected end of input");
+ }
+ auto char_pair = parse_char(pos);
+ pos = char_pair.second;
+ out_elements.push_back({LLAMA_GRETYPE_CHAR, char_pair.first});
+ }
+ pos = parse_space(pos + 1, is_nested);
+ } else if (*pos == '[') { // char range(s)
+ pos++;
+ enum llama_gretype start_type = LLAMA_GRETYPE_CHAR;
+ if (*pos == '^') {
+ pos++;
+ start_type = LLAMA_GRETYPE_CHAR_NOT;
+ }
+ last_sym_start = out_elements.size();
+ while (*pos != ']') {
+ if (!*pos) {
+ throw std::runtime_error("unexpected end of input");
+ }
+ auto char_pair = parse_char(pos);
+ pos = char_pair.second;
+ enum llama_gretype type = last_sym_start < out_elements.size()
+ ? LLAMA_GRETYPE_CHAR_ALT
+ : start_type;
+
+ out_elements.push_back({type, char_pair.first});
+ if (pos[0] == '-' && pos[1] != ']') {
+ if (!pos[1]) {
+ throw std::runtime_error("unexpected end of input");
+ }
+ auto endchar_pair = parse_char(pos + 1);
+ pos = endchar_pair.second;
+ out_elements.push_back({LLAMA_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first});
+ }
+ }
+ pos = parse_space(pos + 1, is_nested);
+ } else if (is_word_char(*pos)) { // rule reference
+ const char * name_end = parse_name(pos);
+ uint32_t ref_rule_id = get_symbol_id(state, pos, name_end - pos);
+ pos = parse_space(name_end, is_nested);
+ last_sym_start = out_elements.size();
+ out_elements.push_back({LLAMA_GRETYPE_RULE_REF, ref_rule_id});
+ } else if (*pos == '(') { // grouping
+ // parse nested alternates into synthesized rule
+ pos = parse_space(pos + 1, true);
+ uint32_t sub_rule_id = generate_symbol_id(state, rule_name);
+ pos = parse_alternates(state, pos, rule_name, sub_rule_id, true);
+ last_sym_start = out_elements.size();
+ // output reference to synthesized rule
+ out_elements.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id});
+ if (*pos != ')') {
+ throw std::runtime_error(std::string("expecting ')' at ") + pos);
+ }
+ pos = parse_space(pos + 1, is_nested);
+ } else if (*pos == '.') { // any char
+ last_sym_start = out_elements.size();
+ out_elements.push_back({LLAMA_GRETYPE_CHAR_ANY, 0});
+ pos = parse_space(pos + 1, is_nested);
+ } else if (*pos == '*') {
+ pos = parse_space(pos + 1, is_nested);
+ handle_repetitions(0, -1);
+ } else if (*pos == '+') {
+ pos = parse_space(pos + 1, is_nested);
+ handle_repetitions(1, -1);
+ } else if (*pos == '?') {
+ pos = parse_space(pos + 1, is_nested);
+ handle_repetitions(0, 1);
+ } else if (*pos == '{') {
+ pos = parse_space(pos + 1, is_nested);
+
+ if (!is_digit_char(*pos)) {
+ throw std::runtime_error(std::string("expecting an int at ") + pos);
+ }
+ const char * int_end = parse_int(pos);
+ int min_times = std::stoul(std::string(pos, int_end - pos));
+ pos = parse_space(int_end, is_nested);
+
+ int max_times = -1;
+
+ if (*pos == '}') {
+ max_times = min_times;
+ pos = parse_space(pos + 1, is_nested);
+ } else if (*pos == ',') {
+ pos = parse_space(pos + 1, is_nested);
+
+ if (is_digit_char(*pos)) {
+ const char * int_end = parse_int(pos);
+ max_times = std::stoul(std::string(pos, int_end - pos));
+ pos = parse_space(int_end, is_nested);
+ }
+
+ if (*pos != '}') {
+ throw std::runtime_error(std::string("expecting '}' at ") + pos);
+ }
+ pos = parse_space(pos + 1, is_nested);
+ } else {
+ throw std::runtime_error(std::string("expecting ',' at ") + pos);
+ }
+ handle_repetitions(min_times, max_times);
+ } else {
+ break;
+ }
+ }
+ return pos;
+ }
+
+ const char * parse_alternates(
+ parse_state & state,
+ const char * src,
+ const std::string & rule_name,
+ uint32_t rule_id,
+ bool is_nested) {
+ std::vector<llama_grammar_element> rule;
+ const char * pos = parse_sequence(state, src, rule_name, rule, is_nested);
+ while (*pos == '|') {
+ rule.push_back({LLAMA_GRETYPE_ALT, 0});
+ pos = parse_space(pos + 1, true);
+ pos = parse_sequence(state, pos, rule_name, rule, is_nested);
+ }
+ rule.push_back({LLAMA_GRETYPE_END, 0});
+ add_rule(state, rule_id, rule);
+ return pos;
+ }
+
+ static const char * parse_rule(parse_state & state, const char * src) {
+ const char * name_end = parse_name(src);
+ const char * pos = parse_space(name_end, false);
+ size_t name_len = name_end - src;
+ uint32_t rule_id = get_symbol_id(state, src, name_len);
+ const std::string name(src, name_len);
+
+ if (!(pos[0] == ':' && pos[1] == ':' && pos[2] == '=')) {
+ throw std::runtime_error(std::string("expecting ::= at ") + pos);
+ }
+ pos = parse_space(pos + 3, true);
+
+ pos = parse_alternates(state, pos, name, rule_id, false);
+
+ if (*pos == '\r') {
+ pos += pos[1] == '\n' ? 2 : 1;
+ } else if (*pos == '\n') {
+ pos++;
+ } else if (*pos) {
+ throw std::runtime_error(std::string("expecting newline or end at ") + pos);
+ }
+ return parse_space(pos, true);
+ }
+
+ parse_state parse(const char * src) {
+ try {
+ parse_state state;
+ const char * pos = parse_space(src, true);
+ while (*pos) {
+ pos = parse_rule(state, pos);
+ }
+ // Validate the state to ensure that all rules are defined
+ for (const auto & rule : state.rules) {
+ if (rule.empty()) {
+ throw std::runtime_error("Undefined rule");
+ }
+ for (const auto & elem : rule) {
+ if (elem.type == LLAMA_GRETYPE_RULE_REF) {
+ // Ensure that the rule at that location exists
+ if (elem.value >= state.rules.size() || state.rules[elem.value].empty()) {
+ // Get the name of the rule that is missing
+ for (const auto & kv : state.symbol_ids) {
+ if (kv.second == elem.value) {
+ throw std::runtime_error("Undefined rule identifier '" + kv.first + "'");
+ }
+ }
+ }
+ }
+ }
+ }
+ return state;
+ } catch (const std::exception & err) {
+ fprintf(stderr, "%s: error parsing grammar: %s\n", __func__, err.what());
+ return parse_state();
+ }
+ }
+
+ static void print_grammar_char(FILE * file, uint32_t c) {
+ if (0x20 <= c && c <= 0x7f) {
+ fprintf(file, "%c", static_cast<char>(c));
+ } else {
+ // cop out of encoding UTF-8
+ fprintf(file, "<U+%04X>", c);
+ }
+ }
+
+ static bool is_char_element(llama_grammar_element elem) {
+ switch (elem.type) {
+ case LLAMA_GRETYPE_CHAR: return true;
+ case LLAMA_GRETYPE_CHAR_NOT: return true;
+ case LLAMA_GRETYPE_CHAR_ALT: return true;
+ case LLAMA_GRETYPE_CHAR_RNG_UPPER: return true;
+ case LLAMA_GRETYPE_CHAR_ANY: return true;
+ default: return false;
+ }
+ }
+
+ static void print_rule_binary(FILE * file, const std::vector<llama_grammar_element> & rule) {
+ for (auto elem : rule) {
+ switch (elem.type) {
+ case LLAMA_GRETYPE_END: fprintf(file, "END"); break;
+ case LLAMA_GRETYPE_ALT: fprintf(file, "ALT"); break;
+ case LLAMA_GRETYPE_RULE_REF: fprintf(file, "RULE_REF"); break;
+ case LLAMA_GRETYPE_CHAR: fprintf(file, "CHAR"); break;
+ case LLAMA_GRETYPE_CHAR_NOT: fprintf(file, "CHAR_NOT"); break;
+ case LLAMA_GRETYPE_CHAR_RNG_UPPER: fprintf(file, "CHAR_RNG_UPPER"); break;
+ case LLAMA_GRETYPE_CHAR_ALT: fprintf(file, "CHAR_ALT"); break;
+ case LLAMA_GRETYPE_CHAR_ANY: fprintf(file, "CHAR_ANY"); break;
+ }
+ switch (elem.type) {
+ case LLAMA_GRETYPE_END:
+ case LLAMA_GRETYPE_ALT:
+ case LLAMA_GRETYPE_RULE_REF:
+ fprintf(file, "(%u) ", elem.value);
+ break;
+ case LLAMA_GRETYPE_CHAR:
+ case LLAMA_GRETYPE_CHAR_NOT:
+ case LLAMA_GRETYPE_CHAR_RNG_UPPER:
+ case LLAMA_GRETYPE_CHAR_ALT:
+ case LLAMA_GRETYPE_CHAR_ANY:
+ fprintf(file, "(\"");
+ print_grammar_char(file, elem.value);
+ fprintf(file, "\") ");
+ break;
+ }
+ }
+ fprintf(file, "\n");
+ }
+
+ static void print_rule(
+ FILE * file,
+ uint32_t rule_id,
+ const std::vector<llama_grammar_element> & rule,
+ const std::map<uint32_t, std::string> & symbol_id_names) {
+ if (rule.empty() || rule.back().type != LLAMA_GRETYPE_END) {
+ throw std::runtime_error(
+ "malformed rule, does not end with LLAMA_GRETYPE_END: " + std::to_string(rule_id));
+ }
+ fprintf(file, "%s ::= ", symbol_id_names.at(rule_id).c_str());
+ for (size_t i = 0, end = rule.size() - 1; i < end; i++) {
+ llama_grammar_element elem = rule[i];
+ switch (elem.type) {
+ case LLAMA_GRETYPE_END:
+ throw std::runtime_error(
+ "unexpected end of rule: " + std::to_string(rule_id) + "," +
+ std::to_string(i));
+ case LLAMA_GRETYPE_ALT:
+ fprintf(file, "| ");
+ break;
+ case LLAMA_GRETYPE_RULE_REF:
+ fprintf(file, "%s ", symbol_id_names.at(elem.value).c_str());
+ break;
+ case LLAMA_GRETYPE_CHAR:
+ fprintf(file, "[");
+ print_grammar_char(file, elem.value);
+ break;
+ case LLAMA_GRETYPE_CHAR_NOT:
+ fprintf(file, "[^");
+ print_grammar_char(file, elem.value);
+ break;
+ case LLAMA_GRETYPE_CHAR_RNG_UPPER:
+ if (i == 0 || !is_char_element(rule[i - 1])) {
+ throw std::runtime_error(
+ "LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: " +
+ std::to_string(rule_id) + "," + std::to_string(i));
+ }
+ fprintf(file, "-");
+ print_grammar_char(file, elem.value);
+ break;
+ case LLAMA_GRETYPE_CHAR_ALT:
+ if (i == 0 || !is_char_element(rule[i - 1])) {
+ throw std::runtime_error(
+ "LLAMA_GRETYPE_CHAR_ALT without preceding char: " +
+ std::to_string(rule_id) + "," + std::to_string(i));
+ }
+ print_grammar_char(file, elem.value);
+ break;
+ case LLAMA_GRETYPE_CHAR_ANY:
+ fprintf(file, ".");
+ break;
+ }
+ if (is_char_element(elem)) {
+ switch (rule[i + 1].type) {
+ case LLAMA_GRETYPE_CHAR_ALT:
+ case LLAMA_GRETYPE_CHAR_RNG_UPPER:
+ case LLAMA_GRETYPE_CHAR_ANY:
+ break;
+ default:
+ fprintf(file, "] ");
+ }
+ }
+ }
+ fprintf(file, "\n");
+ }
+
+ void print_grammar(FILE * file, const parse_state & state) {
+ try {
+ std::map<uint32_t, std::string> symbol_id_names;
+ for (const auto & kv : state.symbol_ids) {
+ symbol_id_names[kv.second] = kv.first;
+ }
+ for (size_t i = 0, end = state.rules.size(); i < end; i++) {
+ // fprintf(file, "%zu: ", i);
+ // print_rule_binary(file, state.rules[i]);
+ print_rule(file, uint32_t(i), state.rules[i], symbol_id_names);
+ // fprintf(file, "\n");
+ }
+ } catch (const std::exception & err) {
+ fprintf(stderr, "\n%s: error printing grammar: %s\n", __func__, err.what());
+ }
+ }
+
+ std::vector<const llama_grammar_element *> parse_state::c_rules() {
+ std::vector<const llama_grammar_element *> ret;
+ ret.reserve(rules.size());
+ for (const auto & rule : rules) {
+ ret.push_back(rule.data());
+ }
+ return ret;
+ }
+}
diff --git a/llama/grammar-parser.h b/llama/grammar-parser.h
new file mode 100644
index 00000000..51f98664
--- /dev/null
+++ b/llama/grammar-parser.h
@@ -0,0 +1,55 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// Implements a parser for an extended Backus-Naur form (BNF), producing the
+// binary context-free grammar format specified by llama.h. Supports character
+// ranges, grouping, and repetition operators. As an example, a grammar for
+// arithmetic might look like:
+//
+// root ::= expr
+// expr ::= term ([-+*/] term)*
+// term ::= num | "(" space expr ")" space
+// num ::= [0-9]+ space
+// space ::= [ \t\n]*
+
+#pragma once
+#include "llama.h"
+#include <vector>
+#include <map>
+#include <cstdint>
+#include <string>
+
+namespace grammar_parser {
+ struct parse_state {
+ std::map<std::string, uint32_t> symbol_ids;
+ std::vector<std::vector<llama_grammar_element>> rules;
+
+ std::vector<const llama_grammar_element *> c_rules();
+ };
+
+ parse_state parse(const char * src);
+ void print_grammar(FILE * file, const parse_state & state);
+}
diff --git a/llama/json-schema-to-grammar.cpp b/llama/json-schema-to-grammar.cpp
new file mode 100644
index 00000000..9623d710
--- /dev/null
+++ b/llama/json-schema-to-grammar.cpp
@@ -0,0 +1,1071 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "json-schema-to-grammar.h"
+#include <algorithm>
+#include <fstream>
+#include <map>
+#include <regex>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+using json = nlohmann::ordered_json;
+
+template <typename Iterator>
+static std::string join(Iterator begin, Iterator end, const std::string & separator);
+
+static std::string repeat(const std::string & str, size_t n);
+
+static std::string build_repetition(const std::string & item_rule, int min_items, int max_items, const std::string & separator_rule = "") {
+ auto has_max = max_items != std::numeric_limits<int>::max();
+
+ if (min_items == 0 && max_items == 1) {
+ return item_rule + "?";
+ }
+
+ if (separator_rule.empty()) {
+ if (min_items == 1 && !has_max) {
+ return item_rule + "+";
+ } else if (min_items == 0 && !has_max) {
+ return item_rule + "*";
+ } else {
+ return item_rule + "{" + std::to_string(min_items) + "," + (has_max ? std::to_string(max_items) : "") + "}";
+ }
+ }
+
+ auto result = item_rule + " " + build_repetition("(" + separator_rule + " " + item_rule + ")", min_items == 0 ? 0 : min_items - 1, has_max ? max_items - 1 : max_items);
+ if (min_items == 0) {
+ result = "(" + result + ")?";
+ }
+ return result;
+}
+
+/* Minimalistic replacement for std::string_view, which is only available from C++17 onwards */
+class string_view {
+ const std::string & _str;
+ const size_t _start;
+ const size_t _end;
+public:
+ string_view(const std::string & str, size_t start = 0, size_t end = std::string::npos) : _str(str), _start(start), _end(end == std::string::npos ? str.length() : end) {}
+
+ size_t size() const {
+ return _end - _start;
+ }
+
+ size_t length() const {
+ return size();
+ }
+
+ operator std::string() const {
+ return str();
+ }
+
+ std::string str() const {
+ return _str.substr(_start, _end - _start);
+ }
+
+ string_view substr(size_t pos, size_t len = std::string::npos) const {
+ return string_view(_str, _start + pos, len == std::string::npos ? _end : _start + pos + len);
+ }
+
+ char operator[](size_t pos) const {
+ auto index = _start + pos;
+ if (index >= _end) {
+ throw std::out_of_range("string_view index out of range");
+ }
+ return _str[_start + pos];
+ }
+
+ bool operator==(const string_view & other) const {
+ std::string this_str = *this;
+ std::string other_str = other;
+ return this_str == other_str;
+ }
+};
+
+static void _build_min_max_int(int min_value, int max_value, std::stringstream & out, int decimals_left = 16, bool top_level = true) {
+ auto has_min = min_value != std::numeric_limits<int>::min();
+ auto has_max = max_value != std::numeric_limits<int>::max();
+
+ auto digit_range = [&](char from, char to) {
+ out << "[";
+ if (from == to) {
+ out << from;
+ } else {
+ out << from << "-" << to;
+ }
+ out << "]";
+ };
+ auto more_digits = [&](int min_digits, int max_digits) {
+ out << "[0-9]";
+ if (min_digits == max_digits && min_digits == 1) {
+ return;
+ }
+ out << "{";
+ out << min_digits;
+ if (max_digits != min_digits) {
+ out << ",";
+ if (max_digits != std::numeric_limits<int>::max()) {
+ out << max_digits;
+ }
+ }
+ out << "}";
+ };
+ std::function<void(const string_view &, const string_view &)> uniform_range =
+ [&](const string_view & from, const string_view & to) {
+ size_t i = 0;
+ while (i < from.length() && i < to.length() && from[i] == to[i]) {
+ i++;
+ }
+ if (i > 0) {
+ out << "\"" << from.substr(0, i).str() << "\"";
+ }
+ if (i < from.length() && i < to.length()) {
+ if (i > 0) {
+ out << " ";
+ }
+ auto sub_len = from.length() - i - 1;
+ if (sub_len > 0) {
+ auto from_sub = from.substr(i + 1);
+ auto to_sub = to.substr(i + 1);
+ auto sub_zeros = repeat("0", sub_len);
+ auto sub_nines = repeat("9", sub_len);
+
+ auto to_reached = false;
+ out << "(";
+ if (from_sub == sub_zeros) {
+ digit_range(from[i], to[i] - 1);
+ out << " ";
+ more_digits(sub_len, sub_len);
+ } else {
+ out << "[" << from[i] << "] ";
+ out << "(";
+ uniform_range(from_sub, sub_nines);
+ out << ")";
+ if (from[i] < to[i] - 1) {
+ out << " | ";
+ if (to_sub == sub_nines) {
+ digit_range(from[i] + 1, to[i]);
+ to_reached = true;
+ } else {
+ digit_range(from[i] + 1, to[i] - 1);
+ }
+ out << " ";
+ more_digits(sub_len, sub_len);
+ }
+ }
+ if (!to_reached) {
+ out << " | ";
+ digit_range(to[i], to[i]);
+ out << " ";
+ uniform_range(sub_zeros, to_sub);
+ }
+ out << ")";
+ } else {
+ out << "[" << from[i] << "-" << to[i] << "]";
+ }
+ }
+ };
+
+ if (has_min && has_max) {
+ if (min_value < 0 && max_value < 0) {
+ out << "\"-\" (";
+ _build_min_max_int(-max_value, -min_value, out, decimals_left, /* top_level= */ true);
+ out << ")";
+ return;
+ }
+
+ if (min_value < 0) {
+ out << "\"-\" (";
+ _build_min_max_int(0, -min_value, out, decimals_left, /* top_level= */ true);
+ out << ") | ";
+ min_value = 0;
+ }
+
+ auto min_s = std::to_string(min_value);
+ auto max_s = std::to_string(max_value);
+ auto min_digits = min_s.length();
+ auto max_digits = max_s.length();
+
+ for (auto digits = min_digits; digits < max_digits; digits++) {
+ uniform_range(min_s, repeat("9", digits));
+ min_s = "1" + repeat("0", digits);
+ out << " | ";
+ }
+ uniform_range(min_s, max_s);
+ return;
+ }
+
+ auto less_decimals = std::max(decimals_left - 1, 1);
+
+ if (has_min) {
+ if (min_value < 0) {
+ out << "\"-\" (";
+ _build_min_max_int(std::numeric_limits<int>::min(), -min_value, out, decimals_left, /* top_level= */ false);
+ out << ") | [0] | [1-9] ";
+ more_digits(0, decimals_left - 1);
+ } else if (min_value == 0) {
+ if (top_level) {
+ out << "[0] | [1-9] ";
+ more_digits(0, less_decimals);
+ } else {
+ more_digits(1, decimals_left);
+ }
+ } else if (min_value <= 9) {
+ char c = '0' + min_value;
+ auto range_start = top_level ? '1' : '0';
+ if (c > range_start) {
+ digit_range(range_start, c - 1);
+ out << " ";
+ more_digits(1, less_decimals);
+ out << " | ";
+ }
+ digit_range(c, '9');
+ out << " ";
+ more_digits(0, less_decimals);
+ } else {
+ auto min_s = std::to_string(min_value);
+ auto len = min_s.length();
+ auto c = min_s[0];
+
+ if (c > '1') {
+ digit_range(top_level ? '1' : '0', c - 1);
+ out << " ";
+ more_digits(len, less_decimals);
+ out << " | ";
+ }
+ digit_range(c, c);
+ out << " (";
+ _build_min_max_int(std::stoi(min_s.substr(1)), std::numeric_limits<int>::max(), out, less_decimals, /* top_level= */ false);
+ out << ")";
+ if (c < '9') {
+ out << " | ";
+ digit_range(c + 1, '9');
+ out << " ";
+ more_digits(len - 1, less_decimals);
+ }
+ }
+ return;
+ }
+
+ if (has_max) {
+ if (max_value >= 0) {
+ if (top_level) {
+ out << "\"-\" [1-9] ";
+ more_digits(0, less_decimals);
+ out << " | ";
+ }
+ _build_min_max_int(0, max_value, out, decimals_left, /* top_level= */ true);
+ } else {
+ out << "\"-\" (";
+ _build_min_max_int(-max_value, std::numeric_limits<int>::max(), out, decimals_left, /* top_level= */ false);
+ out << ")";
+ }
+ return;
+ }
+
+ throw std::runtime_error("At least one of min_value or max_value must be set");
+}
+
+const std::string SPACE_RULE = "| \" \" | \"\\n\" [ \\t]{0,20}";
+
+struct BuiltinRule {
+ std::string content;
+ std::vector<std::string> deps;
+};
+
+std::unordered_map<std::string, BuiltinRule> PRIMITIVE_RULES = {
+ {"boolean", {"(\"true\" | \"false\") space", {}}},
+ {"decimal-part", {"[0-9]{1,16}", {}}},
+ {"integral-part", {"[0] | [1-9] [0-9]{0,15}", {}}},
+ {"number", {"(\"-\"? integral-part) (\".\" decimal-part)? ([eE] [-+]? integral-part)? space", {"integral-part", "decimal-part"}}},
+ {"integer", {"(\"-\"? integral-part) space", {"integral-part"}}},
+ {"value", {"object | array | string | number | boolean | null", {"object", "array", "string", "number", "boolean", "null"}}},
+ {"object", {"\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? \"}\" space", {"string", "value"}}},
+ {"array", {"\"[\" space ( value (\",\" space value)* )? \"]\" space", {"value"}}},
+ {"uuid", {"\"\\\"\" [0-9a-fA-F]{8} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{12} \"\\\"\" space", {}}},
+ {"char", {"[^\"\\\\\\x7F\\x00-\\x1F] | [\\\\] ([\"\\\\bfnrt] | \"u\" [0-9a-fA-F]{4})", {}}},
+ {"string", {"\"\\\"\" char* \"\\\"\" space", {"char"}}},
+ {"null", {"\"null\" space", {}}},
+};
+
+std::unordered_map<std::string, BuiltinRule> STRING_FORMAT_RULES = {
+ {"date", {"[0-9]{4} \"-\" ( \"0\" [1-9] | \"1\" [0-2] ) \"-\" ( \"0\" [1-9] | [1-2] [0-9] | \"3\" [0-1] )", {}}},
+ {"time", {"([01] [0-9] | \"2\" [0-3]) \":\" [0-5] [0-9] \":\" [0-5] [0-9] ( \".\" [0-9]{3} )? ( \"Z\" | ( \"+\" | \"-\" ) ( [01] [0-9] | \"2\" [0-3] ) \":\" [0-5] [0-9] )", {}}},
+ {"date-time", {"date \"T\" time", {"date", "time"}}},
+ {"date-string", {"\"\\\"\" date \"\\\"\" space", {"date"}}},
+ {"time-string", {"\"\\\"\" time \"\\\"\" space", {"time"}}},
+ {"date-time-string", {"\"\\\"\" date-time \"\\\"\" space", {"date-time"}}}
+};
+
+static bool is_reserved_name(const std::string & name) {
+ static std::unordered_set<std::string> RESERVED_NAMES;
+ if (RESERVED_NAMES.empty()) {
+ RESERVED_NAMES.insert("root");
+ for (const auto &p : PRIMITIVE_RULES) RESERVED_NAMES.insert(p.first);
+ for (const auto &p : STRING_FORMAT_RULES) RESERVED_NAMES.insert(p.first);
+ }
+ return RESERVED_NAMES.find(name) != RESERVED_NAMES.end();
+}
+
+std::regex INVALID_RULE_CHARS_RE("[^a-zA-Z0-9-]+");
+std::regex GRAMMAR_LITERAL_ESCAPE_RE("[\r\n\"]");
+std::regex GRAMMAR_RANGE_LITERAL_ESCAPE_RE("[\r\n\"\\]\\-\\\\]");
+std::unordered_map<char, std::string> GRAMMAR_LITERAL_ESCAPES = {
+ {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}
+};
+
+std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'};
+std::unordered_set<char> ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'^', '$', '.', '[', ']', '(', ')', '|', '{', '}', '*', '+', '?'};
+
+template <typename Iterator>
+std::string join(Iterator begin, Iterator end, const std::string & separator) {
+ std::ostringstream result;
+ if (begin != end) {
+ result << *begin;
+ for (Iterator it = begin + 1; it != end; ++it) {
+ result << separator << *it;
+ }
+ }
+ return result.str();
+}
+
+static std::vector<std::string> split(const std::string & str, const std::string & delimiter) {
+ std::vector<std::string> tokens;
+ size_t start = 0;
+ size_t end = str.find(delimiter);
+
+ while (end != std::string::npos) {
+ tokens.push_back(str.substr(start, end - start));
+ start = end + delimiter.length();
+ end = str.find(delimiter, start);
+ }
+
+ tokens.push_back(str.substr(start));
+
+ return tokens;
+}
+
+static std::string repeat(const std::string & str, size_t n) {
+ if (n == 0) {
+ return "";
+ }
+
+ std::string result;
+ result.reserve(str.length() * n);
+
+ for (size_t i = 0; i < n; ++i) {
+ result += str;
+ }
+
+ return result;
+}
+
+static std::string replacePattern(const std::string & input, const std::regex & regex, const std::function<std::string(const std::smatch &)> & replacement) {
+ std::smatch match;
+ std::string result;
+
+ std::string::const_iterator searchStart(input.cbegin());
+ std::string::const_iterator searchEnd(input.cend());
+
+ while (std::regex_search(searchStart, searchEnd, match, regex)) {
+ result.append(searchStart, searchStart + match.position());
+ result.append(replacement(match));
+ searchStart = match.suffix().first;
+ }
+
+ result.append(searchStart, searchEnd);
+
+ return result;
+}
+
+static std::string format_literal(const std::string & literal) {
+ std::string escaped = replacePattern(literal, GRAMMAR_LITERAL_ESCAPE_RE, [&](const std::smatch & match) {
+ char c = match.str()[0];
+ return GRAMMAR_LITERAL_ESCAPES.at(c);
+ });
+ return "\"" + escaped + "\"";
+}
+
+class SchemaConverter {
+private:
+ std::function<json(const std::string &)> _fetch_json;
+ bool _dotall;
+ std::map<std::string, std::string> _rules;
+ std::unordered_map<std::string, json> _refs;
+ std::unordered_set<std::string> _refs_being_resolved;
+ std::vector<std::string> _errors;
+ std::vector<std::string> _warnings;
+
+ std::string _add_rule(const std::string & name, const std::string & rule) {
+ std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
+ if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
+ _rules[esc_name] = rule;
+ return esc_name;
+ } else {
+ int i = 0;
+ while (_rules.find(esc_name + std::to_string(i)) != _rules.end() && _rules[esc_name + std::to_string(i)] != rule) {
+ i++;
+ }
+ std::string key = esc_name + std::to_string(i);
+ _rules[key] = rule;
+ return key;
+ }
+ }
+
+ std::string _generate_union_rule(const std::string & name, const std::vector<json> & alt_schemas) {
+ std::vector<std::string> rules;
+ for (size_t i = 0; i < alt_schemas.size(); i++) {
+ rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i)));
+ }
+ return join(rules.begin(), rules.end(), " | ");
+ }
+
+ std::string _visit_pattern(const std::string & pattern, const std::string & name) {
+ if (!(pattern.front() == '^' && pattern.back() == '$')) {
+ _errors.push_back("Pattern must start with '^' and end with '$'");
+ return "";
+ }
+ std::string sub_pattern = pattern.substr(1, pattern.length() - 2);
+ std::unordered_map<std::string, std::string> sub_rule_ids;
+
+ size_t i = 0;
+ size_t length = sub_pattern.length();
+
+ using literal_or_rule = std::pair<std::string, bool>;
+ auto to_rule = [&](const literal_or_rule & ls) {
+ auto is_literal = ls.second;
+ auto s = ls.first;
+ return is_literal ? "\"" + s + "\"" : s;
+ };
+ std::function<literal_or_rule()> transform = [&]() -> literal_or_rule {
+ size_t start = i;
+ std::vector<literal_or_rule> seq;
+
+ auto get_dot = [&]() {
+ std::string rule;
+ if (_dotall) {
+ rule = "[\\U00000000-\\U0010FFFF]";
+ } else {
+ rule = "[^\\x0A\\x0D]";
+ }
+ return _add_rule("dot", rule);
+ };
+
+ // Joins the sequence, merging consecutive literals together.
+ auto join_seq = [&]() {
+ std::vector<literal_or_rule> ret;
+
+ std::string literal;
+ auto flush_literal = [&]() {
+ if (literal.empty()) {
+ return false;
+ }
+ ret.emplace_back(literal, true);
+ literal.clear();
+ return true;
+ };
+
+ for (const auto & item : seq) {
+ auto is_literal = item.second;
+ if (is_literal) {
+ literal += item.first;
+ } else {
+ flush_literal();
+ ret.push_back(item);
+ }
+ }
+ flush_literal();
+
+ std::vector<std::string> results;
+ for (const auto & item : ret) {
+ results.push_back(to_rule(item));
+ }
+ return std::make_pair(join(results.begin(), results.end(), " "), false);
+ };
+
+ while (i < length) {
+ char c = sub_pattern[i];
+ if (c == '.') {
+ seq.emplace_back(get_dot(), false);
+ i++;
+ } else if (c == '(') {
+ i++;
+ if (i < length) {
+ if (sub_pattern[i] == '?') {
+ _warnings.push_back("Unsupported pattern syntax");
+ }
+ }
+ seq.emplace_back("(" + to_rule(transform()) + ")", false);
+ } else if (c == ')') {
+ i++;
+ if (start > 0 && sub_pattern[start - 1] != '(') {
+ _errors.push_back("Unbalanced parentheses");
+ }
+ return join_seq();
+ } else if (c == '[') {
+ std::string square_brackets = std::string(1, c);
+ i++;
+ while (i < length && sub_pattern[i] != ']') {
+ if (sub_pattern[i] == '\\') {
+ square_brackets += sub_pattern.substr(i, 2);
+ i += 2;
+ } else {
+ square_brackets += sub_pattern[i];
+ i++;
+ }
+ }
+ if (i >= length) {
+ _errors.push_back("Unbalanced square brackets");
+ }
+ square_brackets += ']';
+ i++;
+ seq.emplace_back(square_brackets, false);
+ } else if (c == '|') {
+ seq.emplace_back("|", false);
+ i++;
+ } else if (c == '*' || c == '+' || c == '?') {
+ seq.back() = std::make_pair(to_rule(seq.back()) + c, false);
+ i++;
+ } else if (c == '{') {
+ std::string curly_brackets = std::string(1, c);
+ i++;
+ while (i < length && sub_pattern[i] != '}') {
+ curly_brackets += sub_pattern[i];
+ i++;
+ }
+ if (i >= length) {
+ _errors.push_back("Unbalanced curly brackets");
+ }
+ curly_brackets += '}';
+ i++;
+ auto nums = split(curly_brackets.substr(1, curly_brackets.length() - 2), ",");
+ int min_times = 0;
+ int max_times = std::numeric_limits<int>::max();
+ try {
+ if (nums.size() == 1) {
+ min_times = max_times = std::stoi(nums[0]);
+ } else if (nums.size() != 2) {
+ _errors.push_back("Wrong number of values in curly brackets");
+ } else {
+ if (!nums[0].empty()) {
+ min_times = std::stoi(nums[0]);
+ }
+ if (!nums[1].empty()) {
+ max_times = std::stoi(nums[1]);
+ }
+ }
+ } catch (const std::invalid_argument & e) {
+ _errors.push_back("Invalid number in curly brackets");
+ return std::make_pair("", false);
+ }
+ auto &last = seq.back();
+ auto &sub = last.first;
+ auto sub_is_literal = last.second;
+
+ if (!sub_is_literal) {
+ std::string & sub_id = sub_rule_ids[sub];
+ if (sub_id.empty()) {
+ sub_id = _add_rule(name + "-" + std::to_string(sub_rule_ids.size()), sub);
+ }
+ sub = sub_id;
+ }
+ seq.back().first = build_repetition(
+ sub_is_literal ? "\"" + sub + "\"" : sub,
+ min_times,
+ max_times,
+ ""
+ );
+ seq.back().second = false;
+ } else {
+ std::string literal;
+ auto is_non_literal = [&](char c) {
+ return NON_LITERAL_SET.find(c) != NON_LITERAL_SET.end();
+ };
+ while (i < length) {
+ if (sub_pattern[i] == '\\' && i < length - 1) {
+ char next = sub_pattern[i + 1];
+ if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(next) != ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end()) {
+ i++;
+ literal += sub_pattern[i];
+ i++;
+ } else {
+ literal += sub_pattern.substr(i, 2);
+ i += 2;
+ }
+ } else if (sub_pattern[i] == '"') {
+ literal += "\\\"";
+ i++;
+ } else if (!is_non_literal(sub_pattern[i]) &&
+ (i == length - 1 || literal.empty() || sub_pattern[i + 1] == '.' || !is_non_literal(sub_pattern[i + 1]))) {
+ literal += sub_pattern[i];
+ i++;
+ } else {
+ break;
+ }
+ }
+ if (!literal.empty()) {
+ seq.emplace_back(literal, true);
+ }
+ }
+ }
+ return join_seq();
+ };
+ return _add_rule(name, "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space");
+ }
+
+ /*
+ Returns a rule that matches a JSON string that is none of the provided strings
+
+ not_strings({"a"})
+ -> ["] ( [a] char+ | [^"a] char* )? ["] space
+ not_strings({"and", "also"})
+ -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["] space
+ */
+ std::string _not_strings(const std::vector<std::string> & strings) {
+
+ struct TrieNode {
+ std::map<char, TrieNode> children;
+ bool is_end_of_string;
+
+ TrieNode() : is_end_of_string(false) {}
+
+ void insert(const std::string & string) {
+ auto node = this;
+ for (char c : string) {
+ node = &node->children[c];
+ }
+ node->is_end_of_string = true;
+ }
+ };
+
+ TrieNode trie;
+ for (const auto & s : strings) {
+ trie.insert(s);
+ }
+
+ std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
+ std::ostringstream out;
+ out << "[\"] ( ";
+ std::function<void(const TrieNode &)> visit = [&](const TrieNode & node) {
+ std::ostringstream rejects;
+ auto first = true;
+ for (const auto & kv : node.children) {
+ rejects << kv.first;
+ if (first) {
+ first = false;
+ } else {
+ out << " | ";
+ }
+ out << "[" << kv.first << "]";
+ if (!kv.second.children.empty()) {
+ out << " (";
+ visit(kv.second);
+ out << ")";
+ } else if (kv.second.is_end_of_string) {
+ out << " " << char_rule << "+";
+ }
+ }
+ if (!node.children.empty()) {
+ if (!first) {
+ out << " | ";
+ }
+ out << "[^\"" << rejects.str() << "] " << char_rule << "*";
+ }
+ };
+ visit(trie);
+
+ out << " )";
+ if (!trie.is_end_of_string) {
+ out << "?";
+ }
+ out << " [\"] space";
+ return out.str();
+ }
+
+ std::string _resolve_ref(const std::string & ref) {
+ std::string ref_name = ref.substr(ref.find_last_of('/') + 1);
+ if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) {
+ _refs_being_resolved.insert(ref);
+ json resolved = _refs[ref];
+ ref_name = visit(resolved, ref_name);
+ _refs_being_resolved.erase(ref);
+ }
+ return ref_name;
+ }
+
+ std::string _build_object_rule(
+ const std::vector<std::pair<std::string, json>> & properties,
+ const std::unordered_set<std::string> & required,
+ const std::string & name,
+ const json & additional_properties)
+ {
+ std::vector<std::string> required_props;
+ std::vector<std::string> optional_props;
+ std::unordered_map<std::string, std::string> prop_kv_rule_names;
+ std::vector<std::string> prop_names;
+ for (const auto & kv : properties) {
+ const auto &prop_name = kv.first;
+ const auto &prop_schema = kv.second;
+
+ std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name);
+ prop_kv_rule_names[prop_name] = _add_rule(
+ name + (name.empty() ? "" : "-") + prop_name + "-kv",
+ format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name
+ );
+ if (required.find(prop_name) != required.end()) {
+ required_props.push_back(prop_name);
+ } else {
+ optional_props.push_back(prop_name);
+ }
+ prop_names.push_back(prop_name);
+ }
+ if ((additional_properties.is_boolean() && additional_properties.get<bool>()) || additional_properties.is_object()) {
+ std::string sub_name = name + (name.empty() ? "" : "-") + "additional";
+ std::string value_rule =
+ additional_properties.is_object() ? visit(additional_properties, sub_name + "-value")
+ : _add_primitive("value", PRIMITIVE_RULES.at("value"));
+
+ auto key_rule =
+ prop_names.empty() ? _add_primitive("string", PRIMITIVE_RULES.at("string"))
+ : _add_rule(sub_name + "-k", _not_strings(prop_names));
+ std::string kv_rule = _add_rule(sub_name + "-kv", key_rule + " \":\" space " + value_rule);
+ prop_kv_rule_names["*"] = kv_rule;
+ optional_props.push_back("*");
+ }
+
+ std::string rule = "\"{\" space ";
+ for (size_t i = 0; i < required_props.size(); i++) {
+ if (i > 0) {
+ rule += " \",\" space ";
+ }
+ rule += prop_kv_rule_names[required_props[i]];
+ }
+
+ if (!optional_props.empty()) {
+ rule += " (";
+ if (!required_props.empty()) {
+ rule += " \",\" space ( ";
+ }
+
+ std::function<std::string(const std::vector<std::string> &, bool)> get_recursive_refs = [&](const std::vector<std::string> & ks, bool first_is_optional) {
+ std::string res;
+ if (ks.empty()) {
+ return res;
+ }
+ std::string k = ks[0];
+ std::string kv_rule_name = prop_kv_rule_names[k];
+ std::string comma_ref = "( \",\" space " + kv_rule_name + " )";
+ if (first_is_optional) {
+ res = comma_ref + (k == "*" ? "*" : "?");
+ } else {
+ res = kv_rule_name + (k == "*" ? " " + comma_ref + "*" : "");
+ }
+ if (ks.size() > 1) {
+ res += " " + _add_rule(
+ name + (name.empty() ? "" : "-") + k + "-rest",
+ get_recursive_refs(std::vector<std::string>(ks.begin() + 1, ks.end()), true)
+ );
+ }
+ return res;
+ };
+
+ for (size_t i = 0; i < optional_props.size(); i++) {
+ if (i > 0) {
+ rule += " | ";
+ }
+ rule += get_recursive_refs(std::vector<std::string>(optional_props.begin() + i, optional_props.end()), false);
+ }
+ if (!required_props.empty()) {
+ rule += " )";
+ }
+ rule += " )?";
+ }
+
+ rule += " \"}\" space";
+
+ return rule;
+ }
+
+ std::string _add_primitive(const std::string & name, const BuiltinRule & rule) {
+ auto n = _add_rule(name, rule.content);
+ for (const auto & dep : rule.deps) {
+ BuiltinRule dep_rule;
+ auto it = PRIMITIVE_RULES.find(dep);
+ if (it == PRIMITIVE_RULES.end()) {
+ it = STRING_FORMAT_RULES.find(dep);
+ if (it == STRING_FORMAT_RULES.end()) {
+ _errors.push_back("Rule " + dep + " not known");
+ continue;
+ }
+ }
+ if (_rules.find(dep) == _rules.end()) {
+ _add_primitive(dep, it->second);
+ }
+ }
+ return n;
+ }
+
+public:
+ SchemaConverter(
+ const std::function<json(const std::string &)> & fetch_json,
+ bool dotall)
+ : _fetch_json(fetch_json), _dotall(dotall)
+ {
+ _rules["space"] = SPACE_RULE;
+ }
+
+ void resolve_refs(json & schema, const std::string & url) {
+ /*
+ * Resolves all $ref fields in the given schema, fetching any remote schemas,
+ * replacing each $ref with absolute reference URL and populates _refs with the
+ * respective referenced (sub)schema dictionaries.
+ */
+ std::function<void(json &)> visit_refs = [&](json & n) {
+ if (n.is_array()) {
+ for (auto & x : n) {
+ visit_refs(x);
+ }
+ } else if (n.is_object()) {
+ if (n.contains("$ref")) {
+ std::string ref = n["$ref"];
+ if (_refs.find(ref) == _refs.end()) {
+ json target;
+ if (ref.find("https://") == 0) {
+ std::string base_url = ref.substr(0, ref.find('#'));
+ auto it = _refs.find(base_url);
+ if (it != _refs.end()) {
+ target = it->second;
+ } else {
+ // Fetch the referenced schema and resolve its refs
+ auto referenced = _fetch_json(ref);
+ resolve_refs(referenced, base_url);
+ _refs[base_url] = referenced;
+ }
+ if (ref.find('#') == std::string::npos || ref.substr(ref.find('#') + 1).empty()) {
+ return;
+ }
+ } else if (ref.find("#/") == 0) {
+ target = schema;
+ n["$ref"] = url + ref;
+ ref = url + ref;
+ } else {
+ _errors.push_back("Unsupported ref: " + ref);
+ return;
+ }
+ std::string pointer = ref.substr(ref.find('#') + 1);
+ std::vector<std::string> tokens = split(pointer, "/");
+ for (size_t i = 1; i < tokens.size(); ++i) {
+ std::string sel = tokens[i];
+ if (target.is_null() || !target.contains(sel)) {
+ _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
+ return;
+ }
+ target = target[sel];
+ }
+ _refs[ref] = target;
+ }
+ } else {
+ for (auto & kv : n.items()) {
+ visit_refs(kv.value());
+ }
+ }
+ }
+ };
+
+ visit_refs(schema);
+ }
+
+ std::string _generate_constant_rule(const json & value) {
+ return format_literal(value.dump());
+ }
+
+ std::string visit(const json & schema, const std::string & name) {
+ json schema_type = schema.contains("type") ? schema["type"] : json();
+ std::string schema_format = schema.contains("format") ? schema["format"].get<std::string>() : "";
+ std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
+
+ if (schema.contains("$ref")) {
+ return _add_rule(rule_name, _resolve_ref(schema["$ref"]));
+ } else if (schema.contains("oneOf") || schema.contains("anyOf")) {
+ std::vector<json> alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get<std::vector<json>>() : schema["anyOf"].get<std::vector<json>>();
+ return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));
+ } else if (schema_type.is_array()) {
+ std::vector<json> schema_types;
+ for (const auto & t : schema_type) {
+ json schema_copy(schema);
+ schema_copy["type"] = t;
+ schema_types.push_back(schema_copy);
+ }
+ return _add_rule(rule_name, _generate_union_rule(name, schema_types));
+ } else if (schema.contains("const")) {
+ return _add_rule(rule_name, _generate_constant_rule(schema["const"]) + " space");
+ } else if (schema.contains("enum")) {
+ std::vector<std::string> enum_values;
+ for (const auto & v : schema["enum"]) {
+ enum_values.push_back(_generate_constant_rule(v));
+ }
+ return _add_rule(rule_name, "(" + join(enum_values.begin(), enum_values.end(), " | ") + ") space");
+ } else if ((schema_type.is_null() || schema_type == "object")
+ && (schema.contains("properties") ||
+ (schema.contains("additionalProperties") && schema["additionalProperties"] != true))) {
+ std::unordered_set<std::string> required;
+ if (schema.contains("required") && schema["required"].is_array()) {
+ for (const auto & item : schema["required"]) {
+ if (item.is_string()) {
+ required.insert(item.get<std::string>());
+ }
+ }
+ }
+ std::vector<std::pair<std::string, json>> properties;
+ if (schema.contains("properties")) {
+ for (const auto & prop : schema["properties"].items()) {
+ properties.emplace_back(prop.key(), prop.value());
+ }
+ }
+ return _add_rule(rule_name,
+ _build_object_rule(
+ properties, required, name,
+ schema.contains("additionalProperties") ? schema["additionalProperties"] : json()));
+ } else if ((schema_type.is_null() || schema_type == "object") && schema.contains("allOf")) {
+ std::unordered_set<std::string> required;
+ std::vector<std::pair<std::string, json>> properties;
+ std::string hybrid_name = name;
+ std::function<void(const json &, bool)> add_component = [&](const json & comp_schema, bool is_required) {
+ if (comp_schema.contains("$ref")) {
+ add_component(_refs[comp_schema["$ref"]], is_required);
+ } else if (comp_schema.contains("properties")) {
+ for (const auto & prop : comp_schema["properties"].items()) {
+ properties.emplace_back(prop.key(), prop.value());
+ if (is_required) {
+ required.insert(prop.key());
+ }
+ }
+ } else {
+ // todo warning
+ }
+ };
+ for (auto & t : schema["allOf"]) {
+ if (t.contains("anyOf")) {
+ for (auto & tt : t["anyOf"]) {
+ add_component(tt, false);
+ }
+ } else {
+ add_component(t, true);
+ }
+ }
+ return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json()));
+ } else if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) {
+ json items = schema.contains("items") ? schema["items"] : schema["prefixItems"];
+ if (items.is_array()) {
+ std::string rule = "\"[\" space ";
+ for (size_t i = 0; i < items.size(); i++) {
+ if (i > 0) {
+ rule += " \",\" space ";
+ }
+ rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i));
+ }
+ rule += " \"]\" space";
+ return _add_rule(rule_name, rule);
+ } else {
+ std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item");
+ int min_items = schema.contains("minItems") ? schema["minItems"].get<int>() : 0;
+ json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json();
+ int max_items = max_items_json.is_number_integer() ? max_items_json.get<int>() : std::numeric_limits<int>::max();
+
+ return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " \"]\" space");
+ }
+ } else if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) {
+ return _visit_pattern(schema["pattern"], rule_name);
+ } else if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) {
+ return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid"));
+ } else if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) {
+ auto prim_name = schema_format + "-string";
+ return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
+ } else if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) {
+ std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
+ int min_len = schema.contains("minLength") ? schema["minLength"].get<int>() : 0;
+ int max_len = schema.contains("maxLength") ? schema["maxLength"].get<int>() : std::numeric_limits<int>::max();
+ return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space");
+ } else if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) {
+ int min_value = std::numeric_limits<int>::min();
+ int max_value = std::numeric_limits<int>::max();
+ if (schema.contains("minimum")) {
+ min_value = schema["minimum"].get<int>();
+ } else if (schema.contains("exclusiveMinimum")) {
+ min_value = schema["exclusiveMinimum"].get<int>() + 1;
+ }
+ if (schema.contains("maximum")) {
+ max_value = schema["maximum"].get<int>();
+ } else if (schema.contains("exclusiveMaximum")) {
+ max_value = schema["exclusiveMaximum"].get<int>() - 1;
+ }
+ std::stringstream out;
+ out << "(";
+ _build_min_max_int(min_value, max_value, out);
+ out << ") space";
+ return _add_rule(rule_name, out.str());
+ } else if (schema.empty() || schema_type == "object") {
+ return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
+ } else {
+ if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get<std::string>()) == PRIMITIVE_RULES.end()) {
+ _errors.push_back("Unrecognized schema: " + schema.dump());
+ return "";
+ }
+ // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero
+ return _add_primitive(rule_name == "root" ? "root" : schema_type.get<std::string>(), PRIMITIVE_RULES.at(schema_type.get<std::string>()));
+ }
+ }
+
+ void check_errors() {
+ if (!_errors.empty()) {
+ throw std::runtime_error("JSON schema conversion failed:\n" + join(_errors.begin(), _errors.end(), "\n"));
+ }
+ if (!_warnings.empty()) {
+ fprintf(stderr, "WARNING: JSON schema conversion was incomplete: %s\n", join(_warnings.begin(), _warnings.end(), "; ").c_str());
+ }
+ }
+
+ std::string format_grammar() {
+ std::stringstream ss;
+ for (const auto & kv : _rules) {
+ ss << kv.first << " ::= " << kv.second << std::endl;
+ }
+ return ss.str();
+ }
+};
+
+std::string json_schema_to_grammar(const json & schema) {
+ SchemaConverter converter([](const std::string &) { return json::object(); }, /* dotall= */ false);
+ auto copy = schema;
+ converter.resolve_refs(copy, "input");
+ converter.visit(copy, "");
+ converter.check_errors();
+ return converter.format_grammar();
+}
diff --git a/llama/json-schema-to-grammar.h b/llama/json-schema-to-grammar.h
new file mode 100644
index 00000000..01f56761
--- /dev/null
+++ b/llama/json-schema-to-grammar.h
@@ -0,0 +1,34 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "ggml.h"
+// Change JSON_ASSERT from assert() to GGML_ASSERT:
+#define JSON_ASSERT GGML_ASSERT
+#include "json.hpp"
+
+std::string json_schema_to_grammar(const nlohmann::ordered_json& schema);
diff --git a/llama/json.hpp b/llama/json.hpp
new file mode 100644
index 00000000..a858728c
--- /dev/null
+++ b/llama/json.hpp
@@ -0,0 +1,24766 @@
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+/****************************************************************************\
+ * Note on documentation: The source files contain links to the online *
+ * documentation of the public API at https://json.nlohmann.me. This URL *
+ * contains the most recent documentation and should also be applicable to *
+ * previous versions; documentation for deprecated functions is not *
+ * removed, but marked deprecated. See "Generate documentation" section in *
+ * file docs/README.md. *
+\****************************************************************************/
+
+#ifndef INCLUDE_NLOHMANN_JSON_HPP_
+#define INCLUDE_NLOHMANN_JSON_HPP_
+
+#include <algorithm> // all_of, find, for_each
+#include <cstddef> // nullptr_t, ptrdiff_t, size_t
+#include <functional> // hash, less
+#include <initializer_list> // initializer_list
+#ifndef JSON_NO_IO
+ #include <iosfwd> // istream, ostream
+#endif // JSON_NO_IO
+#include <iterator> // random_access_iterator_tag
+#include <memory> // unique_ptr
+#include <string> // string, stoi, to_string
+#include <utility> // declval, forward, move, pair, swap
+#include <vector> // vector
+
+// #include <nlohmann/adl_serializer.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <utility>
+
+// #include <nlohmann/detail/abi_macros.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+// This file contains all macro definitions affecting or depending on the ABI
+
+#ifndef JSON_SKIP_LIBRARY_VERSION_CHECK
+ #if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH)
+ #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 11 || NLOHMANN_JSON_VERSION_PATCH != 3
+ #warning "Already included a different version of the library!"
+ #endif
+ #endif
+#endif
+
+#define NLOHMANN_JSON_VERSION_MAJOR 3 // NOLINT(modernize-macro-to-enum)
+#define NLOHMANN_JSON_VERSION_MINOR 11 // NOLINT(modernize-macro-to-enum)
+#define NLOHMANN_JSON_VERSION_PATCH 3 // NOLINT(modernize-macro-to-enum)
+
+#ifndef JSON_DIAGNOSTICS
+ #define JSON_DIAGNOSTICS 0
+#endif
+
+#ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
+ #define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0
+#endif
+
+#if JSON_DIAGNOSTICS
+ #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag
+#else
+ #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS
+#endif
+
+#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
+ #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp
+#else
+ #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON
+#endif
+
+#ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION
+ #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0
+#endif
+
+// Construct the namespace ABI tags component
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b) json_abi ## a ## b
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b) \
+ NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b)
+
+#define NLOHMANN_JSON_ABI_TAGS \
+ NLOHMANN_JSON_ABI_TAGS_CONCAT( \
+ NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS, \
+ NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON)
+
+// Construct the namespace version component
+#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \
+ _v ## major ## _ ## minor ## _ ## patch
+#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(major, minor, patch) \
+ NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch)
+
+#if NLOHMANN_JSON_NAMESPACE_NO_VERSION
+#define NLOHMANN_JSON_NAMESPACE_VERSION
+#else
+#define NLOHMANN_JSON_NAMESPACE_VERSION \
+ NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(NLOHMANN_JSON_VERSION_MAJOR, \
+ NLOHMANN_JSON_VERSION_MINOR, \
+ NLOHMANN_JSON_VERSION_PATCH)
+#endif
+
+// Combine namespace components
+#define NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) a ## b
+#define NLOHMANN_JSON_NAMESPACE_CONCAT(a, b) \
+ NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b)
+
+#ifndef NLOHMANN_JSON_NAMESPACE
+#define NLOHMANN_JSON_NAMESPACE \
+ nlohmann::NLOHMANN_JSON_NAMESPACE_CONCAT( \
+ NLOHMANN_JSON_ABI_TAGS, \
+ NLOHMANN_JSON_NAMESPACE_VERSION)
+#endif
+
+#ifndef NLOHMANN_JSON_NAMESPACE_BEGIN
+#define NLOHMANN_JSON_NAMESPACE_BEGIN \
+ namespace nlohmann \
+ { \
+ inline namespace NLOHMANN_JSON_NAMESPACE_CONCAT( \
+ NLOHMANN_JSON_ABI_TAGS, \
+ NLOHMANN_JSON_NAMESPACE_VERSION) \
+ {
+#endif
+
+#ifndef NLOHMANN_JSON_NAMESPACE_END
+#define NLOHMANN_JSON_NAMESPACE_END \
+ } /* namespace (inline namespace) NOLINT(readability/namespace) */ \
+ } // namespace nlohmann
+#endif
+
+// #include <nlohmann/detail/conversions/from_json.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <algorithm> // transform
+#include <array> // array
+#include <forward_list> // forward_list
+#include <iterator> // inserter, front_inserter, end
+#include <map> // map
+#include <string> // string
+#include <tuple> // tuple, make_tuple
+#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible
+#include <unordered_map> // unordered_map
+#include <utility> // pair, declval
+#include <valarray> // valarray
+
+// #include <nlohmann/detail/exceptions.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <cstddef> // nullptr_t
+#include <exception> // exception
+#if JSON_DIAGNOSTICS
+ #include <numeric> // accumulate
+#endif
+#include <stdexcept> // runtime_error
+#include <string> // to_string
+#include <vector> // vector
+
+// #include <nlohmann/detail/value_t.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <array> // array
+#include <cstddef> // size_t
+#include <cstdint> // uint8_t
+#include <string> // string
+
+// #include <nlohmann/detail/macro_scope.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <utility> // declval, pair
+// #include <nlohmann/detail/meta/detected.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <type_traits>
+
+// #include <nlohmann/detail/meta/void_t.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+template<typename ...Ts> struct make_void
+{
+ using type = void;
+};
+template<typename ...Ts> using void_t = typename make_void<Ts...>::type;
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+// https://en.cppreference.com/w/cpp/experimental/is_detected
+struct nonesuch
+{
+ nonesuch() = delete;
+ ~nonesuch() = delete;
+ nonesuch(nonesuch const&) = delete;
+ nonesuch(nonesuch const&&) = delete;
+ void operator=(nonesuch const&) = delete;
+ void operator=(nonesuch&&) = delete;
+};
+
+template<class Default,
+ class AlwaysVoid,
+ template<class...> class Op,
+ class... Args>
+struct detector
+{
+ using value_t = std::false_type;
+ using type = Default;
+};
+
+template<class Default, template<class...> class Op, class... Args>
+struct detector<Default, void_t<Op<Args...>>, Op, Args...>
+{
+ using value_t = std::true_type;
+ using type = Op<Args...>;
+};
+
+template<template<class...> class Op, class... Args>
+using is_detected = typename detector<nonesuch, void, Op, Args...>::value_t;
+
+template<template<class...> class Op, class... Args>
+struct is_detected_lazy : is_detected<Op, Args...> { };
+
+template<template<class...> class Op, class... Args>
+using detected_t = typename detector<nonesuch, void, Op, Args...>::type;
+
+template<class Default, template<class...> class Op, class... Args>
+using detected_or = detector<Default, void, Op, Args...>;
+
+template<class Default, template<class...> class Op, class... Args>
+using detected_or_t = typename detected_or<Default, Op, Args...>::type;
+
+template<class Expected, template<class...> class Op, class... Args>
+using is_detected_exact = std::is_same<Expected, detected_t<Op, Args...>>;
+
+template<class To, template<class...> class Op, class... Args>
+using is_detected_convertible =
+ std::is_convertible<detected_t<Op, Args...>, To>;
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/thirdparty/hedley/hedley.hpp>
+
+
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-FileCopyrightText: 2016-2021 Evan Nemerson <evan@nemerson.com>
+// SPDX-License-Identifier: MIT
+
+/* Hedley - https://nemequ.github.io/hedley
+ * Created by Evan Nemerson <evan@nemerson.com>
+ */
+
+#if !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < 15)
+#if defined(JSON_HEDLEY_VERSION)
+ #undef JSON_HEDLEY_VERSION
+#endif
+#define JSON_HEDLEY_VERSION 15
+
+#if defined(JSON_HEDLEY_STRINGIFY_EX)
+ #undef JSON_HEDLEY_STRINGIFY_EX
+#endif
+#define JSON_HEDLEY_STRINGIFY_EX(x) #x
+
+#if defined(JSON_HEDLEY_STRINGIFY)
+ #undef JSON_HEDLEY_STRINGIFY
+#endif
+#define JSON_HEDLEY_STRINGIFY(x) JSON_HEDLEY_STRINGIFY_EX(x)
+
+#if defined(JSON_HEDLEY_CONCAT_EX)
+ #undef JSON_HEDLEY_CONCAT_EX
+#endif
+#define JSON_HEDLEY_CONCAT_EX(a,b) a##b
+
+#if defined(JSON_HEDLEY_CONCAT)
+ #undef JSON_HEDLEY_CONCAT
+#endif
+#define JSON_HEDLEY_CONCAT(a,b) JSON_HEDLEY_CONCAT_EX(a,b)
+
+#if defined(JSON_HEDLEY_CONCAT3_EX)
+ #undef JSON_HEDLEY_CONCAT3_EX
+#endif
+#define JSON_HEDLEY_CONCAT3_EX(a,b,c) a##b##c
+
+#if defined(JSON_HEDLEY_CONCAT3)
+ #undef JSON_HEDLEY_CONCAT3
+#endif
+#define JSON_HEDLEY_CONCAT3(a,b,c) JSON_HEDLEY_CONCAT3_EX(a,b,c)
+
+#if defined(JSON_HEDLEY_VERSION_ENCODE)
+ #undef JSON_HEDLEY_VERSION_ENCODE
+#endif
+#define JSON_HEDLEY_VERSION_ENCODE(major,minor,revision) (((major) * 1000000) + ((minor) * 1000) + (revision))
+
+#if defined(JSON_HEDLEY_VERSION_DECODE_MAJOR)
+ #undef JSON_HEDLEY_VERSION_DECODE_MAJOR
+#endif
+#define JSON_HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000)
+
+#if defined(JSON_HEDLEY_VERSION_DECODE_MINOR)
+ #undef JSON_HEDLEY_VERSION_DECODE_MINOR
+#endif
+#define JSON_HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000)
+
+#if defined(JSON_HEDLEY_VERSION_DECODE_REVISION)
+ #undef JSON_HEDLEY_VERSION_DECODE_REVISION
+#endif
+#define JSON_HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000)
+
+#if defined(JSON_HEDLEY_GNUC_VERSION)
+ #undef JSON_HEDLEY_GNUC_VERSION
+#endif
+#if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__)
+ #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__)
+#elif defined(__GNUC__)
+ #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0)
+#endif
+
+#if defined(JSON_HEDLEY_GNUC_VERSION_CHECK)
+ #undef JSON_HEDLEY_GNUC_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_GNUC_VERSION)
+ #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_MSVC_VERSION)
+ #undef JSON_HEDLEY_MSVC_VERSION
+#endif
+#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) && !defined(__ICL)
+ #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, (_MSC_FULL_VER % 100000) / 100)
+#elif defined(_MSC_FULL_VER) && !defined(__ICL)
+ #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10)
+#elif defined(_MSC_VER) && !defined(__ICL)
+ #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0)
+#endif
+
+#if defined(JSON_HEDLEY_MSVC_VERSION_CHECK)
+ #undef JSON_HEDLEY_MSVC_VERSION_CHECK
+#endif
+#if !defined(JSON_HEDLEY_MSVC_VERSION)
+ #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (0)
+#elif defined(_MSC_VER) && (_MSC_VER >= 1400)
+ #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch)))
+#elif defined(_MSC_VER) && (_MSC_VER >= 1200)
+ #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch)))
+#else
+ #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_VER >= ((major * 100) + (minor)))
+#endif
+
+#if defined(JSON_HEDLEY_INTEL_VERSION)
+ #undef JSON_HEDLEY_INTEL_VERSION
+#endif
+#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && !defined(__ICL)
+ #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE)
+#elif defined(__INTEL_COMPILER) && !defined(__ICL)
+ #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0)
+#endif
+
+#if defined(JSON_HEDLEY_INTEL_VERSION_CHECK)
+ #undef JSON_HEDLEY_INTEL_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_INTEL_VERSION)
+ #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_INTEL_CL_VERSION)
+ #undef JSON_HEDLEY_INTEL_CL_VERSION
+#endif
+#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && defined(__ICL)
+ #define JSON_HEDLEY_INTEL_CL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER, __INTEL_COMPILER_UPDATE, 0)
+#endif
+
+#if defined(JSON_HEDLEY_INTEL_CL_VERSION_CHECK)
+ #undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_INTEL_CL_VERSION)
+ #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_CL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_PGI_VERSION)
+ #undef JSON_HEDLEY_PGI_VERSION
+#endif
+#if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__)
+ #define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__)
+#endif
+
+#if defined(JSON_HEDLEY_PGI_VERSION_CHECK)
+ #undef JSON_HEDLEY_PGI_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_PGI_VERSION)
+ #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_SUNPRO_VERSION)
+ #undef JSON_HEDLEY_SUNPRO_VERSION
+#endif
+#if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000)
+ #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10)
+#elif defined(__SUNPRO_C)
+ #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C) & 0xf)
+#elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000)
+ #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), (__SUNPRO_CC & 0xf) * 10)
+#elif defined(__SUNPRO_CC)
+ #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC) & 0xf)
+#endif
+
+#if defined(JSON_HEDLEY_SUNPRO_VERSION_CHECK)
+ #undef JSON_HEDLEY_SUNPRO_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_SUNPRO_VERSION)
+ #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION)
+ #undef JSON_HEDLEY_EMSCRIPTEN_VERSION
+#endif
+#if defined(__EMSCRIPTEN__)
+ #define JSON_HEDLEY_EMSCRIPTEN_VERSION JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__)
+#endif
+
+#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK)
+ #undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION)
+ #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_ARM_VERSION)
+ #undef JSON_HEDLEY_ARM_VERSION
+#endif
+#if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION)
+ #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, (__ARMCOMPILER_VERSION % 10000) / 100)
+#elif defined(__CC_ARM) && defined(__ARMCC_VERSION)
+ #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, (__ARMCC_VERSION % 10000) / 100)
+#endif
+
+#if defined(JSON_HEDLEY_ARM_VERSION_CHECK)
+ #undef JSON_HEDLEY_ARM_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_ARM_VERSION)
+ #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_IBM_VERSION)
+ #undef JSON_HEDLEY_IBM_VERSION
+#endif
+#if defined(__ibmxl__)
+ #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__)
+#elif defined(__xlC__) && defined(__xlC_ver__)
+ #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff)
+#elif defined(__xlC__)
+ #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0)
+#endif
+
+#if defined(JSON_HEDLEY_IBM_VERSION_CHECK)
+ #undef JSON_HEDLEY_IBM_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_IBM_VERSION)
+ #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_TI_VERSION)
+ #undef JSON_HEDLEY_TI_VERSION
+#endif
+#if \
+ defined(__TI_COMPILER_VERSION__) && \
+ ( \
+ defined(__TMS470__) || defined(__TI_ARM__) || \
+ defined(__MSP430__) || \
+ defined(__TMS320C2000__) \
+ )
+#if (__TI_COMPILER_VERSION__ >= 16000000)
+ #define JSON_HEDLEY_TI_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))
+#endif
+#endif
+
+#if defined(JSON_HEDLEY_TI_VERSION_CHECK)
+ #undef JSON_HEDLEY_TI_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_TI_VERSION)
+ #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_TI_CL2000_VERSION)
+ #undef JSON_HEDLEY_TI_CL2000_VERSION
+#endif
+#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C2000__)
+ #define JSON_HEDLEY_TI_CL2000_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))
+#endif
+
+#if defined(JSON_HEDLEY_TI_CL2000_VERSION_CHECK)
+ #undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_TI_CL2000_VERSION)
+ #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL2000_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_TI_CL430_VERSION)
+ #undef JSON_HEDLEY_TI_CL430_VERSION
+#endif
+#if defined(__TI_COMPILER_VERSION__) && defined(__MSP430__)
+ #define JSON_HEDLEY_TI_CL430_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))
+#endif
+
+#if defined(JSON_HEDLEY_TI_CL430_VERSION_CHECK)
+ #undef JSON_HEDLEY_TI_CL430_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_TI_CL430_VERSION)
+ #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL430_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_TI_ARMCL_VERSION)
+ #undef JSON_HEDLEY_TI_ARMCL_VERSION
+#endif
+#if defined(__TI_COMPILER_VERSION__) && (defined(__TMS470__) || defined(__TI_ARM__))
+ #define JSON_HEDLEY_TI_ARMCL_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))
+#endif
+
+#if defined(JSON_HEDLEY_TI_ARMCL_VERSION_CHECK)
+ #undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_TI_ARMCL_VERSION)
+ #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_ARMCL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_TI_CL6X_VERSION)
+ #undef JSON_HEDLEY_TI_CL6X_VERSION
+#endif
+#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C6X__)
+ #define JSON_HEDLEY_TI_CL6X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))
+#endif
+
+#if defined(JSON_HEDLEY_TI_CL6X_VERSION_CHECK)
+ #undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_TI_CL6X_VERSION)
+ #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL6X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_TI_CL7X_VERSION)
+ #undef JSON_HEDLEY_TI_CL7X_VERSION
+#endif
+#if defined(__TI_COMPILER_VERSION__) && defined(__C7000__)
+ #define JSON_HEDLEY_TI_CL7X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))
+#endif
+
+#if defined(JSON_HEDLEY_TI_CL7X_VERSION_CHECK)
+ #undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_TI_CL7X_VERSION)
+ #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL7X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_TI_CLPRU_VERSION)
+ #undef JSON_HEDLEY_TI_CLPRU_VERSION
+#endif
+#if defined(__TI_COMPILER_VERSION__) && defined(__PRU__)
+ #define JSON_HEDLEY_TI_CLPRU_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))
+#endif
+
+#if defined(JSON_HEDLEY_TI_CLPRU_VERSION_CHECK)
+ #undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_TI_CLPRU_VERSION)
+ #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CLPRU_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_CRAY_VERSION)
+ #undef JSON_HEDLEY_CRAY_VERSION
+#endif
+#if defined(_CRAYC)
+ #if defined(_RELEASE_PATCHLEVEL)
+ #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL)
+ #else
+ #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0)
+ #endif
+#endif
+
+#if defined(JSON_HEDLEY_CRAY_VERSION_CHECK)
+ #undef JSON_HEDLEY_CRAY_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_CRAY_VERSION)
+ #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_IAR_VERSION)
+ #undef JSON_HEDLEY_IAR_VERSION
+#endif
+#if defined(__IAR_SYSTEMS_ICC__)
+ #if __VER__ > 1000
+ #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000))
+ #else
+ #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(__VER__ / 100, __VER__ % 100, 0)
+ #endif
+#endif
+
+#if defined(JSON_HEDLEY_IAR_VERSION_CHECK)
+ #undef JSON_HEDLEY_IAR_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_IAR_VERSION)
+ #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_TINYC_VERSION)
+ #undef JSON_HEDLEY_TINYC_VERSION
+#endif
+#if defined(__TINYC__)
+ #define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100)
+#endif
+
+#if defined(JSON_HEDLEY_TINYC_VERSION_CHECK)
+ #undef JSON_HEDLEY_TINYC_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_TINYC_VERSION)
+ #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_DMC_VERSION)
+ #undef JSON_HEDLEY_DMC_VERSION
+#endif
+#if defined(__DMC__)
+ #define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf)
+#endif
+
+#if defined(JSON_HEDLEY_DMC_VERSION_CHECK)
+ #undef JSON_HEDLEY_DMC_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_DMC_VERSION)
+ #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_COMPCERT_VERSION)
+ #undef JSON_HEDLEY_COMPCERT_VERSION
+#endif
+#if defined(__COMPCERT_VERSION__)
+ #define JSON_HEDLEY_COMPCERT_VERSION JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, __COMPCERT_VERSION__ % 100)
+#endif
+
+#if defined(JSON_HEDLEY_COMPCERT_VERSION_CHECK)
+ #undef JSON_HEDLEY_COMPCERT_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_COMPCERT_VERSION)
+ #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_PELLES_VERSION)
+ #undef JSON_HEDLEY_PELLES_VERSION
+#endif
+#if defined(__POCC__)
+ #define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0)
+#endif
+
+#if defined(JSON_HEDLEY_PELLES_VERSION_CHECK)
+ #undef JSON_HEDLEY_PELLES_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_PELLES_VERSION)
+ #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_MCST_LCC_VERSION)
+ #undef JSON_HEDLEY_MCST_LCC_VERSION
+#endif
+#if defined(__LCC__) && defined(__LCC_MINOR__)
+ #define JSON_HEDLEY_MCST_LCC_VERSION JSON_HEDLEY_VERSION_ENCODE(__LCC__ / 100, __LCC__ % 100, __LCC_MINOR__)
+#endif
+
+#if defined(JSON_HEDLEY_MCST_LCC_VERSION_CHECK)
+ #undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_MCST_LCC_VERSION)
+ #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_MCST_LCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_GCC_VERSION)
+ #undef JSON_HEDLEY_GCC_VERSION
+#endif
+#if \
+ defined(JSON_HEDLEY_GNUC_VERSION) && \
+ !defined(__clang__) && \
+ !defined(JSON_HEDLEY_INTEL_VERSION) && \
+ !defined(JSON_HEDLEY_PGI_VERSION) && \
+ !defined(JSON_HEDLEY_ARM_VERSION) && \
+ !defined(JSON_HEDLEY_CRAY_VERSION) && \
+ !defined(JSON_HEDLEY_TI_VERSION) && \
+ !defined(JSON_HEDLEY_TI_ARMCL_VERSION) && \
+ !defined(JSON_HEDLEY_TI_CL430_VERSION) && \
+ !defined(JSON_HEDLEY_TI_CL2000_VERSION) && \
+ !defined(JSON_HEDLEY_TI_CL6X_VERSION) && \
+ !defined(JSON_HEDLEY_TI_CL7X_VERSION) && \
+ !defined(JSON_HEDLEY_TI_CLPRU_VERSION) && \
+ !defined(__COMPCERT__) && \
+ !defined(JSON_HEDLEY_MCST_LCC_VERSION)
+ #define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION
+#endif
+
+#if defined(JSON_HEDLEY_GCC_VERSION_CHECK)
+ #undef JSON_HEDLEY_GCC_VERSION_CHECK
+#endif
+#if defined(JSON_HEDLEY_GCC_VERSION)
+ #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))
+#else
+ #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (0)
+#endif
+
+#if defined(JSON_HEDLEY_HAS_ATTRIBUTE)
+ #undef JSON_HEDLEY_HAS_ATTRIBUTE
+#endif
+#if \
+ defined(__has_attribute) && \
+ ( \
+ (!defined(JSON_HEDLEY_IAR_VERSION) || JSON_HEDLEY_IAR_VERSION_CHECK(8,5,9)) \
+ )
+# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute)
+#else
+# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0)
+#endif
+
+#if defined(JSON_HEDLEY_GNUC_HAS_ATTRIBUTE)
+ #undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE
+#endif
+#if defined(__has_attribute)
+ #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute)
+#else
+ #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_GCC_HAS_ATTRIBUTE)
+ #undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE
+#endif
+#if defined(__has_attribute)
+ #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute)
+#else
+ #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE)
+ #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE
+#endif
+#if \
+ defined(__has_cpp_attribute) && \
+ defined(__cplusplus) && \
+ (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0))
+ #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute)
+#else
+ #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0)
+#endif
+
+#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS)
+ #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS
+#endif
+#if !defined(__cplusplus) || !defined(__has_cpp_attribute)
+ #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0)
+#elif \
+ !defined(JSON_HEDLEY_PGI_VERSION) && \
+ !defined(JSON_HEDLEY_IAR_VERSION) && \
+ (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) && \
+ (!defined(JSON_HEDLEY_MSVC_VERSION) || JSON_HEDLEY_MSVC_VERSION_CHECK(19,20,0))
+ #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(ns::attribute)
+#else
+ #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0)
+#endif
+
+#if defined(JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE)
+ #undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE
+#endif
+#if defined(__has_cpp_attribute) && defined(__cplusplus)
+ #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute)
+#else
+ #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE)
+ #undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE
+#endif
+#if defined(__has_cpp_attribute) && defined(__cplusplus)
+ #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute)
+#else
+ #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_HAS_BUILTIN)
+ #undef JSON_HEDLEY_HAS_BUILTIN
+#endif
+#if defined(__has_builtin)
+ #define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin)
+#else
+ #define JSON_HEDLEY_HAS_BUILTIN(builtin) (0)
+#endif
+
+#if defined(JSON_HEDLEY_GNUC_HAS_BUILTIN)
+ #undef JSON_HEDLEY_GNUC_HAS_BUILTIN
+#endif
+#if defined(__has_builtin)
+ #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin)
+#else
+ #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_GCC_HAS_BUILTIN)
+ #undef JSON_HEDLEY_GCC_HAS_BUILTIN
+#endif
+#if defined(__has_builtin)
+ #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin)
+#else
+ #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_HAS_FEATURE)
+ #undef JSON_HEDLEY_HAS_FEATURE
+#endif
+#if defined(__has_feature)
+ #define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature)
+#else
+ #define JSON_HEDLEY_HAS_FEATURE(feature) (0)
+#endif
+
+#if defined(JSON_HEDLEY_GNUC_HAS_FEATURE)
+ #undef JSON_HEDLEY_GNUC_HAS_FEATURE
+#endif
+#if defined(__has_feature)
+ #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature)
+#else
+ #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_GCC_HAS_FEATURE)
+ #undef JSON_HEDLEY_GCC_HAS_FEATURE
+#endif
+#if defined(__has_feature)
+ #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature)
+#else
+ #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_HAS_EXTENSION)
+ #undef JSON_HEDLEY_HAS_EXTENSION
+#endif
+#if defined(__has_extension)
+ #define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension)
+#else
+ #define JSON_HEDLEY_HAS_EXTENSION(extension) (0)
+#endif
+
+#if defined(JSON_HEDLEY_GNUC_HAS_EXTENSION)
+ #undef JSON_HEDLEY_GNUC_HAS_EXTENSION
+#endif
+#if defined(__has_extension)
+ #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension)
+#else
+ #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_GCC_HAS_EXTENSION)
+ #undef JSON_HEDLEY_GCC_HAS_EXTENSION
+#endif
+#if defined(__has_extension)
+ #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension)
+#else
+ #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE)
+ #undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE
+#endif
+#if defined(__has_declspec_attribute)
+ #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute)
+#else
+ #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0)
+#endif
+
+#if defined(JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE)
+ #undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE
+#endif
+#if defined(__has_declspec_attribute)
+ #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute)
+#else
+ #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE)
+ #undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE
+#endif
+#if defined(__has_declspec_attribute)
+ #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute)
+#else
+ #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_HAS_WARNING)
+ #undef JSON_HEDLEY_HAS_WARNING
+#endif
+#if defined(__has_warning)
+ #define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning)
+#else
+ #define JSON_HEDLEY_HAS_WARNING(warning) (0)
+#endif
+
+#if defined(JSON_HEDLEY_GNUC_HAS_WARNING)
+ #undef JSON_HEDLEY_GNUC_HAS_WARNING
+#endif
+#if defined(__has_warning)
+ #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning)
+#else
+ #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_GCC_HAS_WARNING)
+ #undef JSON_HEDLEY_GCC_HAS_WARNING
+#endif
+#if defined(__has_warning)
+ #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning)
+#else
+ #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if \
+ (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \
+ defined(__clang__) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \
+ JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,0,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) || \
+ JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,17) || \
+ JSON_HEDLEY_SUNPRO_VERSION_CHECK(8,0,0) || \
+ (JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) && defined(__C99_PRAGMA_OPERATOR))
+ #define JSON_HEDLEY_PRAGMA(value) _Pragma(#value)
+#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0)
+ #define JSON_HEDLEY_PRAGMA(value) __pragma(value)
+#else
+ #define JSON_HEDLEY_PRAGMA(value)
+#endif
+
+#if defined(JSON_HEDLEY_DIAGNOSTIC_PUSH)
+ #undef JSON_HEDLEY_DIAGNOSTIC_PUSH
+#endif
+#if defined(JSON_HEDLEY_DIAGNOSTIC_POP)
+ #undef JSON_HEDLEY_DIAGNOSTIC_POP
+#endif
+#if defined(__clang__)
+ #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push")
+ #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop")
+#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)")
+ #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)")
+#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push")
+ #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop")
+#elif \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push))
+ #define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop))
+#elif JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("push")
+ #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("pop")
+#elif \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,4,0) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push")
+ #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop")
+#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)")
+ #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)")
+#else
+ #define JSON_HEDLEY_DIAGNOSTIC_PUSH
+ #define JSON_HEDLEY_DIAGNOSTIC_POP
+#endif
+
+/* JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ is for
+ HEDLEY INTERNAL USE ONLY. API subject to change without notice. */
+#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_)
+ #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_
+#endif
+#if defined(__cplusplus)
+# if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat")
+# if JSON_HEDLEY_HAS_WARNING("-Wc++17-extensions")
+# if JSON_HEDLEY_HAS_WARNING("-Wc++1z-extensions")
+# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \
+ JSON_HEDLEY_DIAGNOSTIC_PUSH \
+ _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \
+ _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \
+ _Pragma("clang diagnostic ignored \"-Wc++1z-extensions\"") \
+ xpr \
+ JSON_HEDLEY_DIAGNOSTIC_POP
+# else
+# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \
+ JSON_HEDLEY_DIAGNOSTIC_PUSH \
+ _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \
+ _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \
+ xpr \
+ JSON_HEDLEY_DIAGNOSTIC_POP
+# endif
+# else
+# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \
+ JSON_HEDLEY_DIAGNOSTIC_PUSH \
+ _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \
+ xpr \
+ JSON_HEDLEY_DIAGNOSTIC_POP
+# endif
+# endif
+#endif
+#if !defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(x) x
+#endif
+
+#if defined(JSON_HEDLEY_CONST_CAST)
+ #undef JSON_HEDLEY_CONST_CAST
+#endif
+#if defined(__cplusplus)
+# define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast<T>(expr))
+#elif \
+ JSON_HEDLEY_HAS_WARNING("-Wcast-qual") || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)
+# define JSON_HEDLEY_CONST_CAST(T, expr) (__extension__ ({ \
+ JSON_HEDLEY_DIAGNOSTIC_PUSH \
+ JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \
+ ((T) (expr)); \
+ JSON_HEDLEY_DIAGNOSTIC_POP \
+ }))
+#else
+# define JSON_HEDLEY_CONST_CAST(T, expr) ((T) (expr))
+#endif
+
+#if defined(JSON_HEDLEY_REINTERPRET_CAST)
+ #undef JSON_HEDLEY_REINTERPRET_CAST
+#endif
+#if defined(__cplusplus)
+ #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast<T>(expr))
+#else
+ #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) ((T) (expr))
+#endif
+
+#if defined(JSON_HEDLEY_STATIC_CAST)
+ #undef JSON_HEDLEY_STATIC_CAST
+#endif
+#if defined(__cplusplus)
+ #define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast<T>(expr))
+#else
+ #define JSON_HEDLEY_STATIC_CAST(T, expr) ((T) (expr))
+#endif
+
+#if defined(JSON_HEDLEY_CPP_CAST)
+ #undef JSON_HEDLEY_CPP_CAST
+#endif
+#if defined(__cplusplus)
+# if JSON_HEDLEY_HAS_WARNING("-Wold-style-cast")
+# define JSON_HEDLEY_CPP_CAST(T, expr) \
+ JSON_HEDLEY_DIAGNOSTIC_PUSH \
+ _Pragma("clang diagnostic ignored \"-Wold-style-cast\"") \
+ ((T) (expr)) \
+ JSON_HEDLEY_DIAGNOSTIC_POP
+# elif JSON_HEDLEY_IAR_VERSION_CHECK(8,3,0)
+# define JSON_HEDLEY_CPP_CAST(T, expr) \
+ JSON_HEDLEY_DIAGNOSTIC_PUSH \
+ _Pragma("diag_suppress=Pe137") \
+ JSON_HEDLEY_DIAGNOSTIC_POP
+# else
+# define JSON_HEDLEY_CPP_CAST(T, expr) ((T) (expr))
+# endif
+#else
+# define JSON_HEDLEY_CPP_CAST(T, expr) (expr)
+#endif
+
+#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED)
+ #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED
+#endif
+#if JSON_HEDLEY_HAS_WARNING("-Wdeprecated-declarations")
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"")
+#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)")
+#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:1478 1786))
+#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1216,1444,1445")
+#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444")
+#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"")
+#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:4996))
+#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444")
+#elif \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \
+ (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718")
+#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && !defined(__cplusplus)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)")
+#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && defined(__cplusplus)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)")
+#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215")
+#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)")
+#else
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED
+#endif
+
+#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS)
+ #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS
+#endif
+#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas")
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"")
+#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)")
+#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:161))
+#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675")
+#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"")
+#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:4068))
+#elif \
+ JSON_HEDLEY_TI_VERSION_CHECK(16,9,0) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163")
+#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163")
+#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161")
+#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 161")
+#else
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS
+#endif
+
+#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES)
+ #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES
+#endif
+#if JSON_HEDLEY_HAS_WARNING("-Wunknown-attributes")
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("clang diagnostic ignored \"-Wunknown-attributes\"")
+#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"")
+#elif JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("warning(disable:1292)")
+#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:1292))
+#elif JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:5030))
+#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097,1098")
+#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097")
+#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("error_messages(off,attrskipunsup)")
+#elif \
+ JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1173")
+#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress=Pe1097")
+#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097")
+#else
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES
+#endif
+
+#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL)
+ #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL
+#endif
+#if JSON_HEDLEY_HAS_WARNING("-Wcast-qual")
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"")
+#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)")
+#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"")
+#else
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL
+#endif
+
+#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION)
+ #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION
+#endif
+#if JSON_HEDLEY_HAS_WARNING("-Wunused-function")
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("clang diagnostic ignored \"-Wunused-function\"")
+#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("GCC diagnostic ignored \"-Wunused-function\"")
+#elif JSON_HEDLEY_MSVC_VERSION_CHECK(1,0,0)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION __pragma(warning(disable:4505))
+#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("diag_suppress 3142")
+#else
+ #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION
+#endif
+
+#if defined(JSON_HEDLEY_DEPRECATED)
+ #undef JSON_HEDLEY_DEPRECATED
+#endif
+#if defined(JSON_HEDLEY_DEPRECATED_FOR)
+ #undef JSON_HEDLEY_DEPRECATED_FOR
+#endif
+#if \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+ #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " # since))
+ #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement))
+#elif \
+ (JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) && !defined(JSON_HEDLEY_IAR_VERSION)) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \
+ JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) || \
+ JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(18,1,0) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since)))
+ #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__("Since " #since "; use " #replacement)))
+#elif defined(__cplusplus) && (__cplusplus >= 201402L)
+ #define JSON_HEDLEY_DEPRECATED(since) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since)]])
+ #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since "; use " #replacement)]])
+#elif \
+ JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \
+ (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \
+ JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0)
+ #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__))
+ #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__))
+#elif \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \
+ JSON_HEDLEY_PELLES_VERSION_CHECK(6,50,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+ #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated)
+ #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated)
+#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)
+ #define JSON_HEDLEY_DEPRECATED(since) _Pragma("deprecated")
+ #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated")
+#else
+ #define JSON_HEDLEY_DEPRECATED(since)
+ #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement)
+#endif
+
+#if defined(JSON_HEDLEY_UNAVAILABLE)
+ #undef JSON_HEDLEY_UNAVAILABLE
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(warning) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since)))
+#else
+ #define JSON_HEDLEY_UNAVAILABLE(available_since)
+#endif
+
+#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT)
+ #undef JSON_HEDLEY_WARN_UNUSED_RESULT
+#endif
+#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT_MSG)
+ #undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \
+ (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \
+ JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__))
+ #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) __attribute__((__warn_unused_result__))
+#elif (JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) >= 201907L)
+ #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]])
+ #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard(msg)]])
+#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard)
+ #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]])
+ #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]])
+#elif defined(_Check_return_) /* SAL */
+ #define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_
+ #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) _Check_return_
+#else
+ #define JSON_HEDLEY_WARN_UNUSED_RESULT
+ #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg)
+#endif
+
+#if defined(JSON_HEDLEY_SENTINEL)
+ #undef JSON_HEDLEY_SENTINEL
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position)))
+#else
+ #define JSON_HEDLEY_SENTINEL(position)
+#endif
+
+#if defined(JSON_HEDLEY_NO_RETURN)
+ #undef JSON_HEDLEY_NO_RETURN
+#endif
+#if JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)
+ #define JSON_HEDLEY_NO_RETURN __noreturn
+#elif \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__))
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+ #define JSON_HEDLEY_NO_RETURN _Noreturn
+#elif defined(__cplusplus) && (__cplusplus >= 201103L)
+ #define JSON_HEDLEY_NO_RETURN JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[noreturn]])
+#elif \
+ JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,2,0) || \
+ JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \
+ (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0)
+ #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__))
+#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0)
+ #define JSON_HEDLEY_NO_RETURN _Pragma("does_not_return")
+#elif \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+ #define JSON_HEDLEY_NO_RETURN __declspec(noreturn)
+#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus)
+ #define JSON_HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;")
+#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0)
+ #define JSON_HEDLEY_NO_RETURN __attribute((noreturn))
+#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0)
+ #define JSON_HEDLEY_NO_RETURN __declspec(noreturn)
+#else
+ #define JSON_HEDLEY_NO_RETURN
+#endif
+
+#if defined(JSON_HEDLEY_NO_ESCAPE)
+ #undef JSON_HEDLEY_NO_ESCAPE
+#endif
+#if JSON_HEDLEY_HAS_ATTRIBUTE(noescape)
+ #define JSON_HEDLEY_NO_ESCAPE __attribute__((__noescape__))
+#else
+ #define JSON_HEDLEY_NO_ESCAPE
+#endif
+
+#if defined(JSON_HEDLEY_UNREACHABLE)
+ #undef JSON_HEDLEY_UNREACHABLE
+#endif
+#if defined(JSON_HEDLEY_UNREACHABLE_RETURN)
+ #undef JSON_HEDLEY_UNREACHABLE_RETURN
+#endif
+#if defined(JSON_HEDLEY_ASSUME)
+ #undef JSON_HEDLEY_ASSUME
+#endif
+#if \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+ #define JSON_HEDLEY_ASSUME(expr) __assume(expr)
+#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_assume)
+ #define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr)
+#elif \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0)
+ #if defined(__cplusplus)
+ #define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr)
+ #else
+ #define JSON_HEDLEY_ASSUME(expr) _nassert(expr)
+ #endif
+#endif
+#if \
+ (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \
+ JSON_HEDLEY_PGI_VERSION_CHECK(18,10,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(13,1,5) || \
+ JSON_HEDLEY_CRAY_VERSION_CHECK(10,0,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable()
+#elif defined(JSON_HEDLEY_ASSUME)
+ #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0)
+#endif
+#if !defined(JSON_HEDLEY_ASSUME)
+ #if defined(JSON_HEDLEY_UNREACHABLE)
+ #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, ((expr) ? 1 : (JSON_HEDLEY_UNREACHABLE(), 1)))
+ #else
+ #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, expr)
+ #endif
+#endif
+#if defined(JSON_HEDLEY_UNREACHABLE)
+ #if \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0)
+ #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (JSON_HEDLEY_STATIC_CAST(void, JSON_HEDLEY_ASSUME(0)), (value))
+ #else
+ #define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE()
+ #endif
+#else
+ #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (value)
+#endif
+#if !defined(JSON_HEDLEY_UNREACHABLE)
+ #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0)
+#endif
+
+JSON_HEDLEY_DIAGNOSTIC_PUSH
+#if JSON_HEDLEY_HAS_WARNING("-Wpedantic")
+ #pragma clang diagnostic ignored "-Wpedantic"
+#endif
+#if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat-pedantic") && defined(__cplusplus)
+ #pragma clang diagnostic ignored "-Wc++98-compat-pedantic"
+#endif
+#if JSON_HEDLEY_GCC_HAS_WARNING("-Wvariadic-macros",4,0,0)
+ #if defined(__clang__)
+ #pragma clang diagnostic ignored "-Wvariadic-macros"
+ #elif defined(JSON_HEDLEY_GCC_VERSION)
+ #pragma GCC diagnostic ignored "-Wvariadic-macros"
+ #endif
+#endif
+#if defined(JSON_HEDLEY_NON_NULL)
+ #undef JSON_HEDLEY_NON_NULL
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0)
+ #define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__)))
+#else
+ #define JSON_HEDLEY_NON_NULL(...)
+#endif
+JSON_HEDLEY_DIAGNOSTIC_POP
+
+#if defined(JSON_HEDLEY_PRINTF_FORMAT)
+ #undef JSON_HEDLEY_PRINTF_FORMAT
+#endif
+#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && !defined(__USE_MINGW_ANSI_STDIO)
+ #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(ms_printf, string_idx, first_to_check)))
+#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && defined(__USE_MINGW_ANSI_STDIO)
+ #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(gnu_printf, string_idx, first_to_check)))
+#elif \
+ JSON_HEDLEY_HAS_ATTRIBUTE(format) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \
+ (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(__printf__, string_idx, first_to_check)))
+#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6,0,0)
+ #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __declspec(vaformat(printf,string_idx,first_to_check))
+#else
+ #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check)
+#endif
+
+#if defined(JSON_HEDLEY_CONSTEXPR)
+ #undef JSON_HEDLEY_CONSTEXPR
+#endif
+#if defined(__cplusplus)
+ #if __cplusplus >= 201103L
+ #define JSON_HEDLEY_CONSTEXPR JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(constexpr)
+ #endif
+#endif
+#if !defined(JSON_HEDLEY_CONSTEXPR)
+ #define JSON_HEDLEY_CONSTEXPR
+#endif
+
+#if defined(JSON_HEDLEY_PREDICT)
+ #undef JSON_HEDLEY_PREDICT
+#endif
+#if defined(JSON_HEDLEY_LIKELY)
+ #undef JSON_HEDLEY_LIKELY
+#endif
+#if defined(JSON_HEDLEY_UNLIKELY)
+ #undef JSON_HEDLEY_UNLIKELY
+#endif
+#if defined(JSON_HEDLEY_UNPREDICTABLE)
+ #undef JSON_HEDLEY_UNPREDICTABLE
+#endif
+#if JSON_HEDLEY_HAS_BUILTIN(__builtin_unpredictable)
+ #define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable((expr))
+#endif
+#if \
+ (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) && !defined(JSON_HEDLEY_PGI_VERSION)) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(9,0,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+# define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability( (expr), (value), (probability))
+# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1 , (probability))
+# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0 , (probability))
+# define JSON_HEDLEY_LIKELY(expr) __builtin_expect (!!(expr), 1 )
+# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect (!!(expr), 0 )
+#elif \
+ (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,27) || \
+ JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+# define JSON_HEDLEY_PREDICT(expr, expected, probability) \
+ (((probability) >= 0.9) ? __builtin_expect((expr), (expected)) : (JSON_HEDLEY_STATIC_CAST(void, expected), (expr)))
+# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \
+ (__extension__ ({ \
+ double hedley_probability_ = (probability); \
+ ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \
+ }))
+# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \
+ (__extension__ ({ \
+ double hedley_probability_ = (probability); \
+ ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \
+ }))
+# define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1)
+# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0)
+#else
+# define JSON_HEDLEY_PREDICT(expr, expected, probability) (JSON_HEDLEY_STATIC_CAST(void, expected), (expr))
+# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr))
+# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr))
+# define JSON_HEDLEY_LIKELY(expr) (!!(expr))
+# define JSON_HEDLEY_UNLIKELY(expr) (!!(expr))
+#endif
+#if !defined(JSON_HEDLEY_UNPREDICTABLE)
+ #define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5)
+#endif
+
+#if defined(JSON_HEDLEY_MALLOC)
+ #undef JSON_HEDLEY_MALLOC
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \
+ (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_MALLOC __attribute__((__malloc__))
+#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0)
+ #define JSON_HEDLEY_MALLOC _Pragma("returns_new_memory")
+#elif \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+ #define JSON_HEDLEY_MALLOC __declspec(restrict)
+#else
+ #define JSON_HEDLEY_MALLOC
+#endif
+
+#if defined(JSON_HEDLEY_PURE)
+ #undef JSON_HEDLEY_PURE
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(pure) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(2,96,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \
+ (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+# define JSON_HEDLEY_PURE __attribute__((__pure__))
+#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0)
+# define JSON_HEDLEY_PURE _Pragma("does_not_write_global_data")
+#elif defined(__cplusplus) && \
+ ( \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) \
+ )
+# define JSON_HEDLEY_PURE _Pragma("FUNC_IS_PURE;")
+#else
+# define JSON_HEDLEY_PURE
+#endif
+
+#if defined(JSON_HEDLEY_CONST)
+ #undef JSON_HEDLEY_CONST
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(const) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(2,5,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \
+ (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_CONST __attribute__((__const__))
+#elif \
+ JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0)
+ #define JSON_HEDLEY_CONST _Pragma("no_side_effect")
+#else
+ #define JSON_HEDLEY_CONST JSON_HEDLEY_PURE
+#endif
+
+#if defined(JSON_HEDLEY_RESTRICT)
+ #undef JSON_HEDLEY_RESTRICT
+#endif
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus)
+ #define JSON_HEDLEY_RESTRICT restrict
+#elif \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \
+ JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,4) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)) || \
+ JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \
+ defined(__clang__) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_RESTRICT __restrict
+#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,3,0) && !defined(__cplusplus)
+ #define JSON_HEDLEY_RESTRICT _Restrict
+#else
+ #define JSON_HEDLEY_RESTRICT
+#endif
+
+#if defined(JSON_HEDLEY_INLINE)
+ #undef JSON_HEDLEY_INLINE
+#endif
+#if \
+ (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \
+ (defined(__cplusplus) && (__cplusplus >= 199711L))
+ #define JSON_HEDLEY_INLINE inline
+#elif \
+ defined(JSON_HEDLEY_GCC_VERSION) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(6,2,0)
+ #define JSON_HEDLEY_INLINE __inline__
+#elif \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,1,0) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_INLINE __inline
+#else
+ #define JSON_HEDLEY_INLINE
+#endif
+
+#if defined(JSON_HEDLEY_ALWAYS_INLINE)
+ #undef JSON_HEDLEY_ALWAYS_INLINE
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \
+ (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \
+ JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0)
+# define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE
+#elif \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+# define JSON_HEDLEY_ALWAYS_INLINE __forceinline
+#elif defined(__cplusplus) && \
+ ( \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) \
+ )
+# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;")
+#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)
+# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("inline=forced")
+#else
+# define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE
+#endif
+
+#if defined(JSON_HEDLEY_NEVER_INLINE)
+ #undef JSON_HEDLEY_NEVER_INLINE
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \
+ JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \
+ (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \
+ (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \
+ (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \
+ (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \
+ JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \
+ JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \
+ JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0)
+ #define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__))
+#elif \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+ #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline)
+#elif JSON_HEDLEY_PGI_VERSION_CHECK(10,2,0)
+ #define JSON_HEDLEY_NEVER_INLINE _Pragma("noinline")
+#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus)
+ #define JSON_HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;")
+#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)
+ #define JSON_HEDLEY_NEVER_INLINE _Pragma("inline=never")
+#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0)
+ #define JSON_HEDLEY_NEVER_INLINE __attribute((noinline))
+#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0)
+ #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline)
+#else
+ #define JSON_HEDLEY_NEVER_INLINE
+#endif
+
+#if defined(JSON_HEDLEY_PRIVATE)
+ #undef JSON_HEDLEY_PRIVATE
+#endif
+#if defined(JSON_HEDLEY_PUBLIC)
+ #undef JSON_HEDLEY_PUBLIC
+#endif
+#if defined(JSON_HEDLEY_IMPORT)
+ #undef JSON_HEDLEY_IMPORT
+#endif
+#if defined(_WIN32) || defined(__CYGWIN__)
+# define JSON_HEDLEY_PRIVATE
+# define JSON_HEDLEY_PUBLIC __declspec(dllexport)
+# define JSON_HEDLEY_IMPORT __declspec(dllimport)
+#else
+# if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \
+ JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \
+ ( \
+ defined(__TI_EABI__) && \
+ ( \
+ (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) \
+ ) \
+ ) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+# define JSON_HEDLEY_PRIVATE __attribute__((__visibility__("hidden")))
+# define JSON_HEDLEY_PUBLIC __attribute__((__visibility__("default")))
+# else
+# define JSON_HEDLEY_PRIVATE
+# define JSON_HEDLEY_PUBLIC
+# endif
+# define JSON_HEDLEY_IMPORT extern
+#endif
+
+#if defined(JSON_HEDLEY_NO_THROW)
+ #undef JSON_HEDLEY_NO_THROW
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__))
+#elif \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(13,1,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0)
+ #define JSON_HEDLEY_NO_THROW __declspec(nothrow)
+#else
+ #define JSON_HEDLEY_NO_THROW
+#endif
+
+#if defined(JSON_HEDLEY_FALL_THROUGH)
+ #undef JSON_HEDLEY_FALL_THROUGH
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(fallthrough) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(7,0,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__))
+#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(clang,fallthrough)
+ #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[clang::fallthrough]])
+#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough)
+ #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[fallthrough]])
+#elif defined(__fallthrough) /* SAL */
+ #define JSON_HEDLEY_FALL_THROUGH __fallthrough
+#else
+ #define JSON_HEDLEY_FALL_THROUGH
+#endif
+
+#if defined(JSON_HEDLEY_RETURNS_NON_NULL)
+ #undef JSON_HEDLEY_RETURNS_NON_NULL
+#endif
+#if \
+ JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__))
+#elif defined(_Ret_notnull_) /* SAL */
+ #define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_
+#else
+ #define JSON_HEDLEY_RETURNS_NON_NULL
+#endif
+
+#if defined(JSON_HEDLEY_ARRAY_PARAM)
+ #undef JSON_HEDLEY_ARRAY_PARAM
+#endif
+#if \
+ defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
+ !defined(__STDC_NO_VLA__) && \
+ !defined(__cplusplus) && \
+ !defined(JSON_HEDLEY_PGI_VERSION) && \
+ !defined(JSON_HEDLEY_TINYC_VERSION)
+ #define JSON_HEDLEY_ARRAY_PARAM(name) (name)
+#else
+ #define JSON_HEDLEY_ARRAY_PARAM(name)
+#endif
+
+#if defined(JSON_HEDLEY_IS_CONSTANT)
+ #undef JSON_HEDLEY_IS_CONSTANT
+#endif
+#if defined(JSON_HEDLEY_REQUIRE_CONSTEXPR)
+ #undef JSON_HEDLEY_REQUIRE_CONSTEXPR
+#endif
+/* JSON_HEDLEY_IS_CONSTEXPR_ is for
+ HEDLEY INTERNAL USE ONLY. API subject to change without notice. */
+#if defined(JSON_HEDLEY_IS_CONSTEXPR_)
+ #undef JSON_HEDLEY_IS_CONSTEXPR_
+#endif
+#if \
+ JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,19) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \
+ JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \
+ (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) && !defined(__cplusplus)) || \
+ JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \
+ JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)
+ #define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr)
+#endif
+#if !defined(__cplusplus)
+# if \
+ JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \
+ JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \
+ JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,24)
+#if defined(__INTPTR_TYPE__)
+ #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0)), int*)
+#else
+ #include <stdint.h>
+ #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((intptr_t) ((expr) * 0)) : (int*) 0)), int*)
+#endif
+# elif \
+ ( \
+ defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \
+ !defined(JSON_HEDLEY_SUNPRO_VERSION) && \
+ !defined(JSON_HEDLEY_PGI_VERSION) && \
+ !defined(JSON_HEDLEY_IAR_VERSION)) || \
+ (JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) && !defined(JSON_HEDLEY_IAR_VERSION)) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) || \
+ JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \
+ JSON_HEDLEY_ARM_VERSION_CHECK(5,3,0)
+#if defined(__INTPTR_TYPE__)
+ #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0), int*: 1, void*: 0)
+#else
+ #include <stdint.h>
+ #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((intptr_t) * 0) : (int*) 0), int*: 1, void*: 0)
+#endif
+# elif \
+ defined(JSON_HEDLEY_GCC_VERSION) || \
+ defined(JSON_HEDLEY_INTEL_VERSION) || \
+ defined(JSON_HEDLEY_TINYC_VERSION) || \
+ defined(JSON_HEDLEY_TI_ARMCL_VERSION) || \
+ JSON_HEDLEY_TI_CL430_VERSION_CHECK(18,12,0) || \
+ defined(JSON_HEDLEY_TI_CL2000_VERSION) || \
+ defined(JSON_HEDLEY_TI_CL6X_VERSION) || \
+ defined(JSON_HEDLEY_TI_CL7X_VERSION) || \
+ defined(JSON_HEDLEY_TI_CLPRU_VERSION) || \
+ defined(__clang__)
+# define JSON_HEDLEY_IS_CONSTEXPR_(expr) ( \
+ sizeof(void) != \
+ sizeof(*( \
+ 1 ? \
+ ((void*) ((expr) * 0L) ) : \
+((struct { char v[sizeof(void) * 2]; } *) 1) \
+ ) \
+ ) \
+ )
+# endif
+#endif
+#if defined(JSON_HEDLEY_IS_CONSTEXPR_)
+ #if !defined(JSON_HEDLEY_IS_CONSTANT)
+ #define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY_IS_CONSTEXPR_(expr)
+ #endif
+ #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY_IS_CONSTEXPR_(expr) ? (expr) : (-1))
+#else
+ #if !defined(JSON_HEDLEY_IS_CONSTANT)
+ #define JSON_HEDLEY_IS_CONSTANT(expr) (0)
+ #endif
+ #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr)
+#endif
+
+#if defined(JSON_HEDLEY_BEGIN_C_DECLS)
+ #undef JSON_HEDLEY_BEGIN_C_DECLS
+#endif
+#if defined(JSON_HEDLEY_END_C_DECLS)
+ #undef JSON_HEDLEY_END_C_DECLS
+#endif
+#if defined(JSON_HEDLEY_C_DECL)
+ #undef JSON_HEDLEY_C_DECL
+#endif
+#if defined(__cplusplus)
+ #define JSON_HEDLEY_BEGIN_C_DECLS extern "C" {
+ #define JSON_HEDLEY_END_C_DECLS }
+ #define JSON_HEDLEY_C_DECL extern "C"
+#else
+ #define JSON_HEDLEY_BEGIN_C_DECLS
+ #define JSON_HEDLEY_END_C_DECLS
+ #define JSON_HEDLEY_C_DECL
+#endif
+
+#if defined(JSON_HEDLEY_STATIC_ASSERT)
+ #undef JSON_HEDLEY_STATIC_ASSERT
+#endif
+#if \
+ !defined(__cplusplus) && ( \
+ (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || \
+ (JSON_HEDLEY_HAS_FEATURE(c_static_assert) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \
+ JSON_HEDLEY_GCC_VERSION_CHECK(6,0,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \
+ defined(_Static_assert) \
+ )
+# define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message)
+#elif \
+ (defined(__cplusplus) && (__cplusplus >= 201103L)) || \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(16,0,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+# define JSON_HEDLEY_STATIC_ASSERT(expr, message) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(static_assert(expr, message))
+#else
+# define JSON_HEDLEY_STATIC_ASSERT(expr, message)
+#endif
+
+#if defined(JSON_HEDLEY_NULL)
+ #undef JSON_HEDLEY_NULL
+#endif
+#if defined(__cplusplus)
+ #if __cplusplus >= 201103L
+ #define JSON_HEDLEY_NULL JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(nullptr)
+ #elif defined(NULL)
+ #define JSON_HEDLEY_NULL NULL
+ #else
+ #define JSON_HEDLEY_NULL JSON_HEDLEY_STATIC_CAST(void*, 0)
+ #endif
+#elif defined(NULL)
+ #define JSON_HEDLEY_NULL NULL
+#else
+ #define JSON_HEDLEY_NULL ((void*) 0)
+#endif
+
+#if defined(JSON_HEDLEY_MESSAGE)
+ #undef JSON_HEDLEY_MESSAGE
+#endif
+#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas")
+# define JSON_HEDLEY_MESSAGE(msg) \
+ JSON_HEDLEY_DIAGNOSTIC_PUSH \
+ JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \
+ JSON_HEDLEY_PRAGMA(message msg) \
+ JSON_HEDLEY_DIAGNOSTIC_POP
+#elif \
+ JSON_HEDLEY_GCC_VERSION_CHECK(4,4,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)
+# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg)
+#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0)
+# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg)
+#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)
+# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg))
+#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,0,0)
+# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg))
+#else
+# define JSON_HEDLEY_MESSAGE(msg)
+#endif
+
+#if defined(JSON_HEDLEY_WARNING)
+ #undef JSON_HEDLEY_WARNING
+#endif
+#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas")
+# define JSON_HEDLEY_WARNING(msg) \
+ JSON_HEDLEY_DIAGNOSTIC_PUSH \
+ JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \
+ JSON_HEDLEY_PRAGMA(clang warning msg) \
+ JSON_HEDLEY_DIAGNOSTIC_POP
+#elif \
+ JSON_HEDLEY_GCC_VERSION_CHECK(4,8,0) || \
+ JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \
+ JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)
+# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg)
+#elif \
+ JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg))
+#else
+# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg)
+#endif
+
+#if defined(JSON_HEDLEY_REQUIRE)
+ #undef JSON_HEDLEY_REQUIRE
+#endif
+#if defined(JSON_HEDLEY_REQUIRE_MSG)
+ #undef JSON_HEDLEY_REQUIRE_MSG
+#endif
+#if JSON_HEDLEY_HAS_ATTRIBUTE(diagnose_if)
+# if JSON_HEDLEY_HAS_WARNING("-Wgcc-compat")
+# define JSON_HEDLEY_REQUIRE(expr) \
+ JSON_HEDLEY_DIAGNOSTIC_PUSH \
+ _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \
+ __attribute__((diagnose_if(!(expr), #expr, "error"))) \
+ JSON_HEDLEY_DIAGNOSTIC_POP
+# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) \
+ JSON_HEDLEY_DIAGNOSTIC_PUSH \
+ _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \
+ __attribute__((diagnose_if(!(expr), msg, "error"))) \
+ JSON_HEDLEY_DIAGNOSTIC_POP
+# else
+# define JSON_HEDLEY_REQUIRE(expr) __attribute__((diagnose_if(!(expr), #expr, "error")))
+# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) __attribute__((diagnose_if(!(expr), msg, "error")))
+# endif
+#else
+# define JSON_HEDLEY_REQUIRE(expr)
+# define JSON_HEDLEY_REQUIRE_MSG(expr,msg)
+#endif
+
+#if defined(JSON_HEDLEY_FLAGS)
+ #undef JSON_HEDLEY_FLAGS
+#endif
+#if JSON_HEDLEY_HAS_ATTRIBUTE(flag_enum) && (!defined(__cplusplus) || JSON_HEDLEY_HAS_WARNING("-Wbitfield-enum-conversion"))
+ #define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__))
+#else
+ #define JSON_HEDLEY_FLAGS
+#endif
+
+#if defined(JSON_HEDLEY_FLAGS_CAST)
+ #undef JSON_HEDLEY_FLAGS_CAST
+#endif
+#if JSON_HEDLEY_INTEL_VERSION_CHECK(19,0,0)
+# define JSON_HEDLEY_FLAGS_CAST(T, expr) (__extension__ ({ \
+ JSON_HEDLEY_DIAGNOSTIC_PUSH \
+ _Pragma("warning(disable:188)") \
+ ((T) (expr)); \
+ JSON_HEDLEY_DIAGNOSTIC_POP \
+ }))
+#else
+# define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr)
+#endif
+
+#if defined(JSON_HEDLEY_EMPTY_BASES)
+ #undef JSON_HEDLEY_EMPTY_BASES
+#endif
+#if \
+ (JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,23918) && !JSON_HEDLEY_MSVC_VERSION_CHECK(20,0,0)) || \
+ JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)
+ #define JSON_HEDLEY_EMPTY_BASES __declspec(empty_bases)
+#else
+ #define JSON_HEDLEY_EMPTY_BASES
+#endif
+
+/* Remaining macros are deprecated. */
+
+#if defined(JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK)
+ #undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK
+#endif
+#if defined(__clang__)
+ #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) (0)
+#else
+ #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)
+#endif
+
+#if defined(JSON_HEDLEY_CLANG_HAS_ATTRIBUTE)
+ #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE
+#endif
+#define JSON_HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_ATTRIBUTE(attribute)
+
+#if defined(JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE)
+ #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE
+#endif
+#define JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute)
+
+#if defined(JSON_HEDLEY_CLANG_HAS_BUILTIN)
+ #undef JSON_HEDLEY_CLANG_HAS_BUILTIN
+#endif
+#define JSON_HEDLEY_CLANG_HAS_BUILTIN(builtin) JSON_HEDLEY_HAS_BUILTIN(builtin)
+
+#if defined(JSON_HEDLEY_CLANG_HAS_FEATURE)
+ #undef JSON_HEDLEY_CLANG_HAS_FEATURE
+#endif
+#define JSON_HEDLEY_CLANG_HAS_FEATURE(feature) JSON_HEDLEY_HAS_FEATURE(feature)
+
+#if defined(JSON_HEDLEY_CLANG_HAS_EXTENSION)
+ #undef JSON_HEDLEY_CLANG_HAS_EXTENSION
+#endif
+#define JSON_HEDLEY_CLANG_HAS_EXTENSION(extension) JSON_HEDLEY_HAS_EXTENSION(extension)
+
+#if defined(JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE)
+ #undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE
+#endif
+#define JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute)
+
+#if defined(JSON_HEDLEY_CLANG_HAS_WARNING)
+ #undef JSON_HEDLEY_CLANG_HAS_WARNING
+#endif
+#define JSON_HEDLEY_CLANG_HAS_WARNING(warning) JSON_HEDLEY_HAS_WARNING(warning)
+
+#endif /* !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < X) */
+
+
+// This file contains all internal macro definitions (except those affecting ABI)
+// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+
+// exclude unsupported compilers
+#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK)
+ #if defined(__clang__)
+ #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400
+ #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers"
+ #endif
+ #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER))
+ #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800
+ #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers"
+ #endif
+ #endif
+#endif
+
+// C++ language standard detection
+// if the user manually specified the used c++ version this is skipped
+#if !defined(JSON_HAS_CPP_20) && !defined(JSON_HAS_CPP_17) && !defined(JSON_HAS_CPP_14) && !defined(JSON_HAS_CPP_11)
+ #if (defined(__cplusplus) && __cplusplus >= 202002L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L)
+ #define JSON_HAS_CPP_20
+ #define JSON_HAS_CPP_17
+ #define JSON_HAS_CPP_14
+ #elif (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464
+ #define JSON_HAS_CPP_17
+ #define JSON_HAS_CPP_14
+ #elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1)
+ #define JSON_HAS_CPP_14
+ #endif
+ // the cpp 11 flag is always specified because it is the minimal required version
+ #define JSON_HAS_CPP_11
+#endif
+
+#ifdef __has_include
+ #if __has_include(<version>)
+ #include <version>
+ #endif
+#endif
+
+#if !defined(JSON_HAS_FILESYSTEM) && !defined(JSON_HAS_EXPERIMENTAL_FILESYSTEM)
+ #ifdef JSON_HAS_CPP_17
+ #if defined(__cpp_lib_filesystem)
+ #define JSON_HAS_FILESYSTEM 1
+ #elif defined(__cpp_lib_experimental_filesystem)
+ #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1
+ #elif !defined(__has_include)
+ #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1
+ #elif __has_include(<filesystem>)
+ #define JSON_HAS_FILESYSTEM 1
+ #elif __has_include(<experimental/filesystem>)
+ #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1
+ #endif
+
+ // std::filesystem does not work on MinGW GCC 8: https://sourceforge.net/p/mingw-w64/bugs/737/
+ #if defined(__MINGW32__) && defined(__GNUC__) && __GNUC__ == 8
+ #undef JSON_HAS_FILESYSTEM
+ #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM
+ #endif
+
+ // no filesystem support before GCC 8: https://en.cppreference.com/w/cpp/compiler_support
+ #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 8
+ #undef JSON_HAS_FILESYSTEM
+ #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM
+ #endif
+
+ // no filesystem support before Clang 7: https://en.cppreference.com/w/cpp/compiler_support
+ #if defined(__clang_major__) && __clang_major__ < 7
+ #undef JSON_HAS_FILESYSTEM
+ #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM
+ #endif
+
+ // no filesystem support before MSVC 19.14: https://en.cppreference.com/w/cpp/compiler_support
+ #if defined(_MSC_VER) && _MSC_VER < 1914
+ #undef JSON_HAS_FILESYSTEM
+ #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM
+ #endif
+
+ // no filesystem support before iOS 13
+ #if defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED < 130000
+ #undef JSON_HAS_FILESYSTEM
+ #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM
+ #endif
+
+ // no filesystem support before macOS Catalina
+ #if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500
+ #undef JSON_HAS_FILESYSTEM
+ #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM
+ #endif
+ #endif
+#endif
+
+#ifndef JSON_HAS_EXPERIMENTAL_FILESYSTEM
+ #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 0
+#endif
+
+#ifndef JSON_HAS_FILESYSTEM
+ #define JSON_HAS_FILESYSTEM 0
+#endif
+
+#ifndef JSON_HAS_THREE_WAY_COMPARISON
+ #if defined(__cpp_impl_three_way_comparison) && __cpp_impl_three_way_comparison >= 201907L \
+ && defined(__cpp_lib_three_way_comparison) && __cpp_lib_three_way_comparison >= 201907L
+ #define JSON_HAS_THREE_WAY_COMPARISON 1
+ #else
+ #define JSON_HAS_THREE_WAY_COMPARISON 0
+ #endif
+#endif
+
+#ifndef JSON_HAS_RANGES
+ // ranges header shipping in GCC 11.1.0 (released 2021-04-27) has syntax error
+ #if defined(__GLIBCXX__) && __GLIBCXX__ == 20210427
+ #define JSON_HAS_RANGES 0
+ #elif defined(__cpp_lib_ranges)
+ #define JSON_HAS_RANGES 1
+ #else
+ #define JSON_HAS_RANGES 0
+ #endif
+#endif
+
+#ifndef JSON_HAS_STATIC_RTTI
+ #if !defined(_HAS_STATIC_RTTI) || _HAS_STATIC_RTTI != 0
+ #define JSON_HAS_STATIC_RTTI 1
+ #else
+ #define JSON_HAS_STATIC_RTTI 0
+ #endif
+#endif
+
+#ifdef JSON_HAS_CPP_17
+ #define JSON_INLINE_VARIABLE inline
+#else
+ #define JSON_INLINE_VARIABLE
+#endif
+
+#if JSON_HEDLEY_HAS_ATTRIBUTE(no_unique_address)
+ #define JSON_NO_UNIQUE_ADDRESS [[no_unique_address]]
+#else
+ #define JSON_NO_UNIQUE_ADDRESS
+#endif
+
+// disable documentation warnings on clang
+#if defined(__clang__)
+ #pragma clang diagnostic push
+ #pragma clang diagnostic ignored "-Wdocumentation"
+ #pragma clang diagnostic ignored "-Wdocumentation-unknown-command"
+#endif
+
+// allow disabling exceptions
+#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION)
+ #define JSON_THROW(exception) throw exception
+ #define JSON_TRY try
+ #define JSON_CATCH(exception) catch(exception)
+ #define JSON_INTERNAL_CATCH(exception) catch(exception)
+#else
+ #include <cstdlib>
+ #define JSON_THROW(exception) std::abort()
+ #define JSON_TRY if(true)
+ #define JSON_CATCH(exception) if(false)
+ #define JSON_INTERNAL_CATCH(exception) if(false)
+#endif
+
+// override exception macros
+#if defined(JSON_THROW_USER)
+ #undef JSON_THROW
+ #define JSON_THROW JSON_THROW_USER
+#endif
+#if defined(JSON_TRY_USER)
+ #undef JSON_TRY
+ #define JSON_TRY JSON_TRY_USER
+#endif
+#if defined(JSON_CATCH_USER)
+ #undef JSON_CATCH
+ #define JSON_CATCH JSON_CATCH_USER
+ #undef JSON_INTERNAL_CATCH
+ #define JSON_INTERNAL_CATCH JSON_CATCH_USER
+#endif
+#if defined(JSON_INTERNAL_CATCH_USER)
+ #undef JSON_INTERNAL_CATCH
+ #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER
+#endif
+
+// allow overriding assert
+#if !defined(JSON_ASSERT)
+ #include <cassert> // assert
+ #define JSON_ASSERT(x) assert(x)
+#endif
+
+// allow to access some private functions (needed by the test suite)
+#if defined(JSON_TESTS_PRIVATE)
+ #define JSON_PRIVATE_UNLESS_TESTED public
+#else
+ #define JSON_PRIVATE_UNLESS_TESTED private
+#endif
+
+/*!
+@brief macro to briefly define a mapping between an enum and JSON
+@def NLOHMANN_JSON_SERIALIZE_ENUM
+@since version 3.4.0
+*/
+#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \
+ template<typename BasicJsonType> \
+ inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \
+ { \
+ static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!"); \
+ static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__; \
+ auto it = std::find_if(std::begin(m), std::end(m), \
+ [e](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \
+ { \
+ return ej_pair.first == e; \
+ }); \
+ j = ((it != std::end(m)) ? it : std::begin(m))->second; \
+ } \
+ template<typename BasicJsonType> \
+ inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \
+ { \
+ static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!"); \
+ static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__; \
+ auto it = std::find_if(std::begin(m), std::end(m), \
+ [&j](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \
+ { \
+ return ej_pair.second == j; \
+ }); \
+ e = ((it != std::end(m)) ? it : std::begin(m))->first; \
+ }
+
+// Ugly macros to avoid uglier copy-paste when specializing basic_json. They
+// may be removed in the future once the class is split.
+
+#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \
+ template<template<typename, typename, typename...> class ObjectType, \
+ template<typename, typename...> class ArrayType, \
+ class StringType, class BooleanType, class NumberIntegerType, \
+ class NumberUnsignedType, class NumberFloatType, \
+ template<typename> class AllocatorType, \
+ template<typename, typename = void> class JSONSerializer, \
+ class BinaryType, \
+ class CustomBaseClass>
+
+#define NLOHMANN_BASIC_JSON_TPL \
+ basic_json<ObjectType, ArrayType, StringType, BooleanType, \
+ NumberIntegerType, NumberUnsignedType, NumberFloatType, \
+ AllocatorType, JSONSerializer, BinaryType, CustomBaseClass>
+
+// Macros to simplify conversion from/to types
+
+#define NLOHMANN_JSON_EXPAND( x ) x
+#define NLOHMANN_JSON_GET_MACRO(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, _64, NAME,...) NAME
+#define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \
+ NLOHMANN_JSON_PASTE64, \
+ NLOHMANN_JSON_PASTE63, \
+ NLOHMANN_JSON_PASTE62, \
+ NLOHMANN_JSON_PASTE61, \
+ NLOHMANN_JSON_PASTE60, \
+ NLOHMANN_JSON_PASTE59, \
+ NLOHMANN_JSON_PASTE58, \
+ NLOHMANN_JSON_PASTE57, \
+ NLOHMANN_JSON_PASTE56, \
+ NLOHMANN_JSON_PASTE55, \
+ NLOHMANN_JSON_PASTE54, \
+ NLOHMANN_JSON_PASTE53, \
+ NLOHMANN_JSON_PASTE52, \
+ NLOHMANN_JSON_PASTE51, \
+ NLOHMANN_JSON_PASTE50, \
+ NLOHMANN_JSON_PASTE49, \
+ NLOHMANN_JSON_PASTE48, \
+ NLOHMANN_JSON_PASTE47, \
+ NLOHMANN_JSON_PASTE46, \
+ NLOHMANN_JSON_PASTE45, \
+ NLOHMANN_JSON_PASTE44, \
+ NLOHMANN_JSON_PASTE43, \
+ NLOHMANN_JSON_PASTE42, \
+ NLOHMANN_JSON_PASTE41, \
+ NLOHMANN_JSON_PASTE40, \
+ NLOHMANN_JSON_PASTE39, \
+ NLOHMANN_JSON_PASTE38, \
+ NLOHMANN_JSON_PASTE37, \
+ NLOHMANN_JSON_PASTE36, \
+ NLOHMANN_JSON_PASTE35, \
+ NLOHMANN_JSON_PASTE34, \
+ NLOHMANN_JSON_PASTE33, \
+ NLOHMANN_JSON_PASTE32, \
+ NLOHMANN_JSON_PASTE31, \
+ NLOHMANN_JSON_PASTE30, \
+ NLOHMANN_JSON_PASTE29, \
+ NLOHMANN_JSON_PASTE28, \
+ NLOHMANN_JSON_PASTE27, \
+ NLOHMANN_JSON_PASTE26, \
+ NLOHMANN_JSON_PASTE25, \
+ NLOHMANN_JSON_PASTE24, \
+ NLOHMANN_JSON_PASTE23, \
+ NLOHMANN_JSON_PASTE22, \
+ NLOHMANN_JSON_PASTE21, \
+ NLOHMANN_JSON_PASTE20, \
+ NLOHMANN_JSON_PASTE19, \
+ NLOHMANN_JSON_PASTE18, \
+ NLOHMANN_JSON_PASTE17, \
+ NLOHMANN_JSON_PASTE16, \
+ NLOHMANN_JSON_PASTE15, \
+ NLOHMANN_JSON_PASTE14, \
+ NLOHMANN_JSON_PASTE13, \
+ NLOHMANN_JSON_PASTE12, \
+ NLOHMANN_JSON_PASTE11, \
+ NLOHMANN_JSON_PASTE10, \
+ NLOHMANN_JSON_PASTE9, \
+ NLOHMANN_JSON_PASTE8, \
+ NLOHMANN_JSON_PASTE7, \
+ NLOHMANN_JSON_PASTE6, \
+ NLOHMANN_JSON_PASTE5, \
+ NLOHMANN_JSON_PASTE4, \
+ NLOHMANN_JSON_PASTE3, \
+ NLOHMANN_JSON_PASTE2, \
+ NLOHMANN_JSON_PASTE1)(__VA_ARGS__))
+#define NLOHMANN_JSON_PASTE2(func, v1) func(v1)
+#define NLOHMANN_JSON_PASTE3(func, v1, v2) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE2(func, v2)
+#define NLOHMANN_JSON_PASTE4(func, v1, v2, v3) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE3(func, v2, v3)
+#define NLOHMANN_JSON_PASTE5(func, v1, v2, v3, v4) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE4(func, v2, v3, v4)
+#define NLOHMANN_JSON_PASTE6(func, v1, v2, v3, v4, v5) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE5(func, v2, v3, v4, v5)
+#define NLOHMANN_JSON_PASTE7(func, v1, v2, v3, v4, v5, v6) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE6(func, v2, v3, v4, v5, v6)
+#define NLOHMANN_JSON_PASTE8(func, v1, v2, v3, v4, v5, v6, v7) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE7(func, v2, v3, v4, v5, v6, v7)
+#define NLOHMANN_JSON_PASTE9(func, v1, v2, v3, v4, v5, v6, v7, v8) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE8(func, v2, v3, v4, v5, v6, v7, v8)
+#define NLOHMANN_JSON_PASTE10(func, v1, v2, v3, v4, v5, v6, v7, v8, v9) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE9(func, v2, v3, v4, v5, v6, v7, v8, v9)
+#define NLOHMANN_JSON_PASTE11(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE10(func, v2, v3, v4, v5, v6, v7, v8, v9, v10)
+#define NLOHMANN_JSON_PASTE12(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE11(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11)
+#define NLOHMANN_JSON_PASTE13(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE12(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12)
+#define NLOHMANN_JSON_PASTE14(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE13(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13)
+#define NLOHMANN_JSON_PASTE15(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE14(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14)
+#define NLOHMANN_JSON_PASTE16(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE15(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15)
+#define NLOHMANN_JSON_PASTE17(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE16(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16)
+#define NLOHMANN_JSON_PASTE18(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE17(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17)
+#define NLOHMANN_JSON_PASTE19(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE18(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18)
+#define NLOHMANN_JSON_PASTE20(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE19(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19)
+#define NLOHMANN_JSON_PASTE21(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE20(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20)
+#define NLOHMANN_JSON_PASTE22(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE21(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21)
+#define NLOHMANN_JSON_PASTE23(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE22(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22)
+#define NLOHMANN_JSON_PASTE24(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE23(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23)
+#define NLOHMANN_JSON_PASTE25(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE24(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24)
+#define NLOHMANN_JSON_PASTE26(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE25(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25)
+#define NLOHMANN_JSON_PASTE27(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE26(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26)
+#define NLOHMANN_JSON_PASTE28(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE27(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27)
+#define NLOHMANN_JSON_PASTE29(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE28(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28)
+#define NLOHMANN_JSON_PASTE30(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE29(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29)
+#define NLOHMANN_JSON_PASTE31(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE30(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30)
+#define NLOHMANN_JSON_PASTE32(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE31(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31)
+#define NLOHMANN_JSON_PASTE33(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE32(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32)
+#define NLOHMANN_JSON_PASTE34(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE33(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33)
+#define NLOHMANN_JSON_PASTE35(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE34(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34)
+#define NLOHMANN_JSON_PASTE36(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE35(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35)
+#define NLOHMANN_JSON_PASTE37(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE36(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36)
+#define NLOHMANN_JSON_PASTE38(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE37(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37)
+#define NLOHMANN_JSON_PASTE39(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE38(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38)
+#define NLOHMANN_JSON_PASTE40(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE39(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39)
+#define NLOHMANN_JSON_PASTE41(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE40(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40)
+#define NLOHMANN_JSON_PASTE42(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE41(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41)
+#define NLOHMANN_JSON_PASTE43(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE42(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42)
+#define NLOHMANN_JSON_PASTE44(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE43(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43)
+#define NLOHMANN_JSON_PASTE45(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE44(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44)
+#define NLOHMANN_JSON_PASTE46(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE45(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45)
+#define NLOHMANN_JSON_PASTE47(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE46(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46)
+#define NLOHMANN_JSON_PASTE48(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE47(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47)
+#define NLOHMANN_JSON_PASTE49(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE48(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48)
+#define NLOHMANN_JSON_PASTE50(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE49(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49)
+#define NLOHMANN_JSON_PASTE51(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE50(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50)
+#define NLOHMANN_JSON_PASTE52(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE51(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51)
+#define NLOHMANN_JSON_PASTE53(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE52(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52)
+#define NLOHMANN_JSON_PASTE54(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE53(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53)
+#define NLOHMANN_JSON_PASTE55(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE54(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54)
+#define NLOHMANN_JSON_PASTE56(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE55(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55)
+#define NLOHMANN_JSON_PASTE57(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE56(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56)
+#define NLOHMANN_JSON_PASTE58(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE57(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57)
+#define NLOHMANN_JSON_PASTE59(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE58(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58)
+#define NLOHMANN_JSON_PASTE60(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE59(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59)
+#define NLOHMANN_JSON_PASTE61(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE60(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60)
+#define NLOHMANN_JSON_PASTE62(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE61(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61)
+#define NLOHMANN_JSON_PASTE63(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE62(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62)
+#define NLOHMANN_JSON_PASTE64(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE63(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63)
+
+#define NLOHMANN_JSON_TO(v1) nlohmann_json_j[#v1] = nlohmann_json_t.v1;
+#define NLOHMANN_JSON_FROM(v1) nlohmann_json_j.at(#v1).get_to(nlohmann_json_t.v1);
+#define NLOHMANN_JSON_FROM_WITH_DEFAULT(v1) nlohmann_json_t.v1 = nlohmann_json_j.value(#v1, nlohmann_json_default_obj.v1);
+
+/*!
+@brief macro
+@def NLOHMANN_DEFINE_TYPE_INTRUSIVE
+@since version 3.9.0
+*/
+#define NLOHMANN_DEFINE_TYPE_INTRUSIVE(Type, ...) \
+ friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \
+ friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) }
+
+#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Type, ...) \
+ friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \
+ friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) }
+
+#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \
+ friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) }
+
+/*!
+@brief macro
+@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE
+@since version 3.9.0
+*/
+#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Type, ...) \
+ inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \
+ inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) }
+
+#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \
+ inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) }
+
+#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, ...) \
+ inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \
+ inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) }
+
+// inspired from https://stackoverflow.com/a/26745591
+// allows to call any std function as if (e.g. with begin):
+// using std::begin; begin(x);
+//
+// it allows using the detected idiom to retrieve the return type
+// of such an expression
+#define NLOHMANN_CAN_CALL_STD_FUNC_IMPL(std_name) \
+ namespace detail { \
+ using std::std_name; \
+ \
+ template<typename... T> \
+ using result_of_##std_name = decltype(std_name(std::declval<T>()...)); \
+ } \
+ \
+ namespace detail2 { \
+ struct std_name##_tag \
+ { \
+ }; \
+ \
+ template<typename... T> \
+ std_name##_tag std_name(T&&...); \
+ \
+ template<typename... T> \
+ using result_of_##std_name = decltype(std_name(std::declval<T>()...)); \
+ \
+ template<typename... T> \
+ struct would_call_std_##std_name \
+ { \
+ static constexpr auto const value = ::nlohmann::detail:: \
+ is_detected_exact<std_name##_tag, result_of_##std_name, T...>::value; \
+ }; \
+ } /* namespace detail2 */ \
+ \
+ template<typename... T> \
+ struct would_call_std_##std_name : detail2::would_call_std_##std_name<T...> \
+ { \
+ }
+
+#ifndef JSON_USE_IMPLICIT_CONVERSIONS
+ #define JSON_USE_IMPLICIT_CONVERSIONS 1
+#endif
+
+#if JSON_USE_IMPLICIT_CONVERSIONS
+ #define JSON_EXPLICIT
+#else
+ #define JSON_EXPLICIT explicit
+#endif
+
+#ifndef JSON_DISABLE_ENUM_SERIALIZATION
+ #define JSON_DISABLE_ENUM_SERIALIZATION 0
+#endif
+
+#ifndef JSON_USE_GLOBAL_UDLS
+ #define JSON_USE_GLOBAL_UDLS 1
+#endif
+
+#if JSON_HAS_THREE_WAY_COMPARISON
+ #include <compare> // partial_ordering
+#endif
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+///////////////////////////
+// JSON type enumeration //
+///////////////////////////
+
+/*!
+@brief the JSON type enumeration
+
+This enumeration collects the different JSON types. It is internally used to
+distinguish the stored values, and the functions @ref basic_json::is_null(),
+@ref basic_json::is_object(), @ref basic_json::is_array(),
+@ref basic_json::is_string(), @ref basic_json::is_boolean(),
+@ref basic_json::is_number() (with @ref basic_json::is_number_integer(),
+@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()),
+@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and
+@ref basic_json::is_structured() rely on it.
+
+@note There are three enumeration entries (number_integer, number_unsigned, and
+number_float), because the library distinguishes these three types for numbers:
+@ref basic_json::number_unsigned_t is used for unsigned integers,
+@ref basic_json::number_integer_t is used for signed integers, and
+@ref basic_json::number_float_t is used for floating-point numbers or to
+approximate integers which do not fit in the limits of their respective type.
+
+@sa see @ref basic_json::basic_json(const value_t value_type) -- create a JSON
+value with the default value for a given type
+
+@since version 1.0.0
+*/
+enum class value_t : std::uint8_t
+{
+ null, ///< null value
+ object, ///< object (unordered set of name/value pairs)
+ array, ///< array (ordered collection of values)
+ string, ///< string value
+ boolean, ///< boolean value
+ number_integer, ///< number value (signed integer)
+ number_unsigned, ///< number value (unsigned integer)
+ number_float, ///< number value (floating-point)
+ binary, ///< binary array (ordered collection of bytes)
+ discarded ///< discarded by the parser callback function
+};
+
+/*!
+@brief comparison operator for JSON types
+
+Returns an ordering that is similar to Python:
+- order: null < boolean < number < object < array < string < binary
+- furthermore, each type is not smaller than itself
+- discarded values are not comparable
+- binary is represented as a b"" string in python and directly comparable to a
+ string; however, making a binary array directly comparable with a string would
+ be surprising behavior in a JSON file.
+
+@since version 1.0.0
+*/
+#if JSON_HAS_THREE_WAY_COMPARISON
+ inline std::partial_ordering operator<=>(const value_t lhs, const value_t rhs) noexcept // *NOPAD*
+#else
+ inline bool operator<(const value_t lhs, const value_t rhs) noexcept
+#endif
+{
+ static constexpr std::array<std::uint8_t, 9> order = {{
+ 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */,
+ 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */,
+ 6 /* binary */
+ }
+ };
+
+ const auto l_index = static_cast<std::size_t>(lhs);
+ const auto r_index = static_cast<std::size_t>(rhs);
+#if JSON_HAS_THREE_WAY_COMPARISON
+ if (l_index < order.size() && r_index < order.size())
+ {
+ return order[l_index] <=> order[r_index]; // *NOPAD*
+ }
+ return std::partial_ordering::unordered;
+#else
+ return l_index < order.size() && r_index < order.size() && order[l_index] < order[r_index];
+#endif
+}
+
+// GCC selects the built-in operator< over an operator rewritten from
+// a user-defined spaceship operator
+// Clang, MSVC, and ICC select the rewritten candidate
+// (see GCC bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105200)
+#if JSON_HAS_THREE_WAY_COMPARISON && defined(__GNUC__)
+inline bool operator<(const value_t lhs, const value_t rhs) noexcept
+{
+ return std::is_lt(lhs <=> rhs); // *NOPAD*
+}
+#endif
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/string_escape.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+/*!
+@brief replace all occurrences of a substring by another string
+
+@param[in,out] s the string to manipulate; changed so that all
+ occurrences of @a f are replaced with @a t
+@param[in] f the substring to replace with @a t
+@param[in] t the string to replace @a f
+
+@pre The search string @a f must not be empty. **This precondition is
+enforced with an assertion.**
+
+@since version 2.0.0
+*/
+template<typename StringType>
+inline void replace_substring(StringType& s, const StringType& f,
+ const StringType& t)
+{
+ JSON_ASSERT(!f.empty());
+ for (auto pos = s.find(f); // find first occurrence of f
+ pos != StringType::npos; // make sure f was found
+ s.replace(pos, f.size(), t), // replace with t, and
+ pos = s.find(f, pos + t.size())) // find next occurrence of f
+ {}
+}
+
+/*!
+ * @brief string escaping as described in RFC 6901 (Sect. 4)
+ * @param[in] s string to escape
+ * @return escaped string
+ *
+ * Note the order of escaping "~" to "~0" and "/" to "~1" is important.
+ */
+template<typename StringType>
+inline StringType escape(StringType s)
+{
+ replace_substring(s, StringType{"~"}, StringType{"~0"});
+ replace_substring(s, StringType{"/"}, StringType{"~1"});
+ return s;
+}
+
+/*!
+ * @brief string unescaping as described in RFC 6901 (Sect. 4)
+ * @param[in] s string to unescape
+ * @return unescaped string
+ *
+ * Note the order of escaping "~1" to "/" and "~0" to "~" is important.
+ */
+template<typename StringType>
+static void unescape(StringType& s)
+{
+ replace_substring(s, StringType{"~1"}, StringType{"/"});
+ replace_substring(s, StringType{"~0"}, StringType{"~"});
+}
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/input/position_t.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <cstddef> // size_t
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+/// struct to capture the start position of the current token
+struct position_t
+{
+ /// the total number of characters read
+ std::size_t chars_read_total = 0;
+ /// the number of characters read in the current line
+ std::size_t chars_read_current_line = 0;
+ /// the number of lines read
+ std::size_t lines_read = 0;
+
+ /// conversion to size_t to preserve SAX interface
+ constexpr operator size_t() const
+ {
+ return chars_read_total;
+ }
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-FileCopyrightText: 2018 The Abseil Authors
+// SPDX-License-Identifier: MIT
+
+
+
+#include <array> // array
+#include <cstddef> // size_t
+#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type
+#include <utility> // index_sequence, make_index_sequence, index_sequence_for
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+template<typename T>
+using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
+
+#ifdef JSON_HAS_CPP_14
+
+// the following utilities are natively available in C++14
+using std::enable_if_t;
+using std::index_sequence;
+using std::make_index_sequence;
+using std::index_sequence_for;
+
+#else
+
+// alias templates to reduce boilerplate
+template<bool B, typename T = void>
+using enable_if_t = typename std::enable_if<B, T>::type;
+
+// The following code is taken from https://github.com/abseil/abseil-cpp/blob/10cb35e459f5ecca5b2ff107635da0bfa41011b4/absl/utility/utility.h
+// which is part of Google Abseil (https://github.com/abseil/abseil-cpp), licensed under the Apache License 2.0.
+
+//// START OF CODE FROM GOOGLE ABSEIL
+
+// integer_sequence
+//
+// Class template representing a compile-time integer sequence. An instantiation
+// of `integer_sequence<T, Ints...>` has a sequence of integers encoded in its
+// type through its template arguments (which is a common need when
+// working with C++11 variadic templates). `absl::integer_sequence` is designed
+// to be a drop-in replacement for C++14's `std::integer_sequence`.
+//
+// Example:
+//
+// template< class T, T... Ints >
+// void user_function(integer_sequence<T, Ints...>);
+//
+// int main()
+// {
+// // user_function's `T` will be deduced to `int` and `Ints...`
+// // will be deduced to `0, 1, 2, 3, 4`.
+// user_function(make_integer_sequence<int, 5>());
+// }
+template <typename T, T... Ints>
+struct integer_sequence
+{
+ using value_type = T;
+ static constexpr std::size_t size() noexcept
+ {
+ return sizeof...(Ints);
+ }
+};
+
+// index_sequence
+//
+// A helper template for an `integer_sequence` of `size_t`,
+// `absl::index_sequence` is designed to be a drop-in replacement for C++14's
+// `std::index_sequence`.
+template <size_t... Ints>
+using index_sequence = integer_sequence<size_t, Ints...>;
+
+namespace utility_internal
+{
+
+template <typename Seq, size_t SeqSize, size_t Rem>
+struct Extend;
+
+// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency.
+template <typename T, T... Ints, size_t SeqSize>
+struct Extend<integer_sequence<T, Ints...>, SeqSize, 0>
+{
+ using type = integer_sequence < T, Ints..., (Ints + SeqSize)... >;
+};
+
+template <typename T, T... Ints, size_t SeqSize>
+struct Extend<integer_sequence<T, Ints...>, SeqSize, 1>
+{
+ using type = integer_sequence < T, Ints..., (Ints + SeqSize)..., 2 * SeqSize >;
+};
+
+// Recursion helper for 'make_integer_sequence<T, N>'.
+// 'Gen<T, N>::type' is an alias for 'integer_sequence<T, 0, 1, ... N-1>'.
+template <typename T, size_t N>
+struct Gen
+{
+ using type =
+ typename Extend < typename Gen < T, N / 2 >::type, N / 2, N % 2 >::type;
+};
+
+template <typename T>
+struct Gen<T, 0>
+{
+ using type = integer_sequence<T>;
+};
+
+} // namespace utility_internal
+
+// Compile-time sequences of integers
+
+// make_integer_sequence
+//
+// This template alias is equivalent to
+// `integer_sequence<int, 0, 1, ..., N-1>`, and is designed to be a drop-in
+// replacement for C++14's `std::make_integer_sequence`.
+template <typename T, T N>
+using make_integer_sequence = typename utility_internal::Gen<T, N>::type;
+
+// make_index_sequence
+//
+// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`,
+// and is designed to be a drop-in replacement for C++14's
+// `std::make_index_sequence`.
+template <size_t N>
+using make_index_sequence = make_integer_sequence<size_t, N>;
+
+// index_sequence_for
+//
+// Converts a typename pack into an index sequence of the same length, and
+// is designed to be a drop-in replacement for C++14's
+// `std::index_sequence_for()`
+template <typename... Ts>
+using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
+
+//// END OF CODE FROM GOOGLE ABSEIL
+
+#endif
+
+// dispatch utility (taken from ranges-v3)
+template<unsigned N> struct priority_tag : priority_tag < N - 1 > {};
+template<> struct priority_tag<0> {};
+
+// taken from ranges-v3
+template<typename T>
+struct static_const
+{
+ static JSON_INLINE_VARIABLE constexpr T value{};
+};
+
+#ifndef JSON_HAS_CPP_17
+ template<typename T>
+ constexpr T static_const<T>::value;
+#endif
+
+template<typename T, typename... Args>
+inline constexpr std::array<T, sizeof...(Args)> make_array(Args&& ... args)
+{
+ return std::array<T, sizeof...(Args)> {{static_cast<T>(std::forward<Args>(args))...}};
+}
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <limits> // numeric_limits
+#include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type
+#include <utility> // declval
+#include <tuple> // tuple
+#include <string> // char_traits
+
+// #include <nlohmann/detail/iterators/iterator_traits.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <iterator> // random_access_iterator_tag
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+// #include <nlohmann/detail/meta/void_t.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+template<typename It, typename = void>
+struct iterator_types {};
+
+template<typename It>
+struct iterator_types <
+ It,
+ void_t<typename It::difference_type, typename It::value_type, typename It::pointer,
+ typename It::reference, typename It::iterator_category >>
+{
+ using difference_type = typename It::difference_type;
+ using value_type = typename It::value_type;
+ using pointer = typename It::pointer;
+ using reference = typename It::reference;
+ using iterator_category = typename It::iterator_category;
+};
+
+// This is required as some compilers implement std::iterator_traits in a way that
+// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341.
+template<typename T, typename = void>
+struct iterator_traits
+{
+};
+
+template<typename T>
+struct iterator_traits < T, enable_if_t < !std::is_pointer<T>::value >>
+ : iterator_types<T>
+{
+};
+
+template<typename T>
+struct iterator_traits<T*, enable_if_t<std::is_object<T>::value>>
+{
+ using iterator_category = std::random_access_iterator_tag;
+ using value_type = T;
+ using difference_type = ptrdiff_t;
+ using pointer = T*;
+ using reference = T&;
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/call_std/begin.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+
+NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin);
+
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/meta/call_std/end.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+
+NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end);
+
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/detected.hpp>
+
+// #include <nlohmann/json_fwd.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_
+ #define INCLUDE_NLOHMANN_JSON_FWD_HPP_
+
+ #include <cstdint> // int64_t, uint64_t
+ #include <map> // map
+ #include <memory> // allocator
+ #include <string> // string
+ #include <vector> // vector
+
+ // #include <nlohmann/detail/abi_macros.hpp>
+
+
+ /*!
+ @brief namespace for Niels Lohmann
+ @see https://github.com/nlohmann
+ @since version 1.0.0
+ */
+ NLOHMANN_JSON_NAMESPACE_BEGIN
+
+ /*!
+ @brief default JSONSerializer template argument
+
+ This serializer ignores the template arguments and uses ADL
+ ([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl))
+ for serialization.
+ */
+ template<typename T = void, typename SFINAE = void>
+ struct adl_serializer;
+
+ /// a class to store JSON values
+ /// @sa https://json.nlohmann.me/api/basic_json/
+ template<template<typename U, typename V, typename... Args> class ObjectType =
+ std::map,
+ template<typename U, typename... Args> class ArrayType = std::vector,
+ class StringType = std::string, class BooleanType = bool,
+ class NumberIntegerType = std::int64_t,
+ class NumberUnsignedType = std::uint64_t,
+ class NumberFloatType = double,
+ template<typename U> class AllocatorType = std::allocator,
+ template<typename T, typename SFINAE = void> class JSONSerializer =
+ adl_serializer,
+ class BinaryType = std::vector<std::uint8_t>, // cppcheck-suppress syntaxError
+ class CustomBaseClass = void>
+ class basic_json;
+
+ /// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document
+ /// @sa https://json.nlohmann.me/api/json_pointer/
+ template<typename RefStringType>
+ class json_pointer;
+
+ /*!
+ @brief default specialization
+ @sa https://json.nlohmann.me/api/json/
+ */
+ using json = basic_json<>;
+
+ /// @brief a minimal map-like container that preserves insertion order
+ /// @sa https://json.nlohmann.me/api/ordered_map/
+ template<class Key, class T, class IgnoredLess, class Allocator>
+ struct ordered_map;
+
+ /// @brief specialization that maintains the insertion order of object keys
+ /// @sa https://json.nlohmann.me/api/ordered_json/
+ using ordered_json = basic_json<nlohmann::ordered_map>;
+
+ NLOHMANN_JSON_NAMESPACE_END
+
+#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+/*!
+@brief detail namespace with internal helper functions
+
+This namespace collects functions that should not be exposed,
+implementations of some @ref basic_json methods, and meta-programming helpers.
+
+@since version 2.1.0
+*/
+namespace detail
+{
+
+/////////////
+// helpers //
+/////////////
+
+// Note to maintainers:
+//
+// Every trait in this file expects a non CV-qualified type.
+// The only exceptions are in the 'aliases for detected' section
+// (i.e. those of the form: decltype(T::member_function(std::declval<T>())))
+//
+// In this case, T has to be properly CV-qualified to constraint the function arguments
+// (e.g. to_json(BasicJsonType&, const T&))
+
+template<typename> struct is_basic_json : std::false_type {};
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {};
+
+// used by exceptions create() member functions
+// true_type for pointer to possibly cv-qualified basic_json or std::nullptr_t
+// false_type otherwise
+template<typename BasicJsonContext>
+struct is_basic_json_context :
+ std::integral_constant < bool,
+ is_basic_json<typename std::remove_cv<typename std::remove_pointer<BasicJsonContext>::type>::type>::value
+ || std::is_same<BasicJsonContext, std::nullptr_t>::value >
+{};
+
+//////////////////////
+// json_ref helpers //
+//////////////////////
+
+template<typename>
+class json_ref;
+
+template<typename>
+struct is_json_ref : std::false_type {};
+
+template<typename T>
+struct is_json_ref<json_ref<T>> : std::true_type {};
+
+//////////////////////////
+// aliases for detected //
+//////////////////////////
+
+template<typename T>
+using mapped_type_t = typename T::mapped_type;
+
+template<typename T>
+using key_type_t = typename T::key_type;
+
+template<typename T>
+using value_type_t = typename T::value_type;
+
+template<typename T>
+using difference_type_t = typename T::difference_type;
+
+template<typename T>
+using pointer_t = typename T::pointer;
+
+template<typename T>
+using reference_t = typename T::reference;
+
+template<typename T>
+using iterator_category_t = typename T::iterator_category;
+
+template<typename T, typename... Args>
+using to_json_function = decltype(T::to_json(std::declval<Args>()...));
+
+template<typename T, typename... Args>
+using from_json_function = decltype(T::from_json(std::declval<Args>()...));
+
+template<typename T, typename U>
+using get_template_function = decltype(std::declval<T>().template get<U>());
+
+// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists
+template<typename BasicJsonType, typename T, typename = void>
+struct has_from_json : std::false_type {};
+
+// trait checking if j.get<T> is valid
+// use this trait instead of std::is_constructible or std::is_convertible,
+// both rely on, or make use of implicit conversions, and thus fail when T
+// has several constructors/operator= (see https://github.com/nlohmann/json/issues/958)
+template <typename BasicJsonType, typename T>
+struct is_getable
+{
+ static constexpr bool value = is_detected<get_template_function, const BasicJsonType&, T>::value;
+};
+
+template<typename BasicJsonType, typename T>
+struct has_from_json < BasicJsonType, T, enable_if_t < !is_basic_json<T>::value >>
+{
+ using serializer = typename BasicJsonType::template json_serializer<T, void>;
+
+ static constexpr bool value =
+ is_detected_exact<void, from_json_function, serializer,
+ const BasicJsonType&, T&>::value;
+};
+
+// This trait checks if JSONSerializer<T>::from_json(json const&) exists
+// this overload is used for non-default-constructible user-defined-types
+template<typename BasicJsonType, typename T, typename = void>
+struct has_non_default_from_json : std::false_type {};
+
+template<typename BasicJsonType, typename T>
+struct has_non_default_from_json < BasicJsonType, T, enable_if_t < !is_basic_json<T>::value >>
+{
+ using serializer = typename BasicJsonType::template json_serializer<T, void>;
+
+ static constexpr bool value =
+ is_detected_exact<T, from_json_function, serializer,
+ const BasicJsonType&>::value;
+};
+
+// This trait checks if BasicJsonType::json_serializer<T>::to_json exists
+// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion.
+template<typename BasicJsonType, typename T, typename = void>
+struct has_to_json : std::false_type {};
+
+template<typename BasicJsonType, typename T>
+struct has_to_json < BasicJsonType, T, enable_if_t < !is_basic_json<T>::value >>
+{
+ using serializer = typename BasicJsonType::template json_serializer<T, void>;
+
+ static constexpr bool value =
+ is_detected_exact<void, to_json_function, serializer, BasicJsonType&,
+ T>::value;
+};
+
+template<typename T>
+using detect_key_compare = typename T::key_compare;
+
+template<typename T>
+struct has_key_compare : std::integral_constant<bool, is_detected<detect_key_compare, T>::value> {};
+
+// obtains the actual object key comparator
+template<typename BasicJsonType>
+struct actual_object_comparator
+{
+ using object_t = typename BasicJsonType::object_t;
+ using object_comparator_t = typename BasicJsonType::default_object_comparator_t;
+ using type = typename std::conditional < has_key_compare<object_t>::value,
+ typename object_t::key_compare, object_comparator_t>::type;
+};
+
+template<typename BasicJsonType>
+using actual_object_comparator_t = typename actual_object_comparator<BasicJsonType>::type;
+
+/////////////////
+// char_traits //
+/////////////////
+
+// Primary template of char_traits calls std char_traits
+template<typename T>
+struct char_traits : std::char_traits<T>
+{};
+
+// Explicitly define char traits for unsigned char since it is not standard
+template<>
+struct char_traits<unsigned char> : std::char_traits<char>
+{
+ using char_type = unsigned char;
+ using int_type = uint64_t;
+
+ // Redefine to_int_type function
+ static int_type to_int_type(char_type c) noexcept
+ {
+ return static_cast<int_type>(c);
+ }
+
+ static char_type to_char_type(int_type i) noexcept
+ {
+ return static_cast<char_type>(i);
+ }
+
+ static constexpr int_type eof() noexcept
+ {
+ return static_cast<int_type>(EOF);
+ }
+};
+
+// Explicitly define char traits for signed char since it is not standard
+template<>
+struct char_traits<signed char> : std::char_traits<char>
+{
+ using char_type = signed char;
+ using int_type = uint64_t;
+
+ // Redefine to_int_type function
+ static int_type to_int_type(char_type c) noexcept
+ {
+ return static_cast<int_type>(c);
+ }
+
+ static char_type to_char_type(int_type i) noexcept
+ {
+ return static_cast<char_type>(i);
+ }
+
+ static constexpr int_type eof() noexcept
+ {
+ return static_cast<int_type>(EOF);
+ }
+};
+
+///////////////////
+// is_ functions //
+///////////////////
+
+// https://en.cppreference.com/w/cpp/types/conjunction
+template<class...> struct conjunction : std::true_type { };
+template<class B> struct conjunction<B> : B { };
+template<class B, class... Bn>
+struct conjunction<B, Bn...>
+: std::conditional<static_cast<bool>(B::value), conjunction<Bn...>, B>::type {};
+
+// https://en.cppreference.com/w/cpp/types/negation
+template<class B> struct negation : std::integral_constant < bool, !B::value > { };
+
+// Reimplementation of is_constructible and is_default_constructible, due to them being broken for
+// std::pair and std::tuple until LWG 2367 fix (see https://cplusplus.github.io/LWG/lwg-defects.html#2367).
+// This causes compile errors in e.g. clang 3.5 or gcc 4.9.
+template <typename T>
+struct is_default_constructible : std::is_default_constructible<T> {};
+
+template <typename T1, typename T2>
+struct is_default_constructible<std::pair<T1, T2>>
+ : conjunction<is_default_constructible<T1>, is_default_constructible<T2>> {};
+
+template <typename T1, typename T2>
+struct is_default_constructible<const std::pair<T1, T2>>
+ : conjunction<is_default_constructible<T1>, is_default_constructible<T2>> {};
+
+template <typename... Ts>
+struct is_default_constructible<std::tuple<Ts...>>
+ : conjunction<is_default_constructible<Ts>...> {};
+
+template <typename... Ts>
+struct is_default_constructible<const std::tuple<Ts...>>
+ : conjunction<is_default_constructible<Ts>...> {};
+
+template <typename T, typename... Args>
+struct is_constructible : std::is_constructible<T, Args...> {};
+
+template <typename T1, typename T2>
+struct is_constructible<std::pair<T1, T2>> : is_default_constructible<std::pair<T1, T2>> {};
+
+template <typename T1, typename T2>
+struct is_constructible<const std::pair<T1, T2>> : is_default_constructible<const std::pair<T1, T2>> {};
+
+template <typename... Ts>
+struct is_constructible<std::tuple<Ts...>> : is_default_constructible<std::tuple<Ts...>> {};
+
+template <typename... Ts>
+struct is_constructible<const std::tuple<Ts...>> : is_default_constructible<const std::tuple<Ts...>> {};
+
+template<typename T, typename = void>
+struct is_iterator_traits : std::false_type {};
+
+template<typename T>
+struct is_iterator_traits<iterator_traits<T>>
+{
+ private:
+ using traits = iterator_traits<T>;
+
+ public:
+ static constexpr auto value =
+ is_detected<value_type_t, traits>::value &&
+ is_detected<difference_type_t, traits>::value &&
+ is_detected<pointer_t, traits>::value &&
+ is_detected<iterator_category_t, traits>::value &&
+ is_detected<reference_t, traits>::value;
+};
+
+template<typename T>
+struct is_range
+{
+ private:
+ using t_ref = typename std::add_lvalue_reference<T>::type;
+
+ using iterator = detected_t<result_of_begin, t_ref>;
+ using sentinel = detected_t<result_of_end, t_ref>;
+
+ // to be 100% correct, it should use https://en.cppreference.com/w/cpp/iterator/input_or_output_iterator
+ // and https://en.cppreference.com/w/cpp/iterator/sentinel_for
+ // but reimplementing these would be too much work, as a lot of other concepts are used underneath
+ static constexpr auto is_iterator_begin =
+ is_iterator_traits<iterator_traits<iterator>>::value;
+
+ public:
+ static constexpr bool value = !std::is_same<iterator, nonesuch>::value && !std::is_same<sentinel, nonesuch>::value && is_iterator_begin;
+};
+
+template<typename R>
+using iterator_t = enable_if_t<is_range<R>::value, result_of_begin<decltype(std::declval<R&>())>>;
+
+template<typename T>
+using range_value_t = value_type_t<iterator_traits<iterator_t<T>>>;
+
+// The following implementation of is_complete_type is taken from
+// https://blogs.msdn.microsoft.com/vcblog/2015/12/02/partial-support-for-expression-sfinae-in-vs-2015-update-1/
+// and is written by Xiang Fan who agreed to using it in this library.
+
+template<typename T, typename = void>
+struct is_complete_type : std::false_type {};
+
+template<typename T>
+struct is_complete_type<T, decltype(void(sizeof(T)))> : std::true_type {};
+
+template<typename BasicJsonType, typename CompatibleObjectType,
+ typename = void>
+struct is_compatible_object_type_impl : std::false_type {};
+
+template<typename BasicJsonType, typename CompatibleObjectType>
+struct is_compatible_object_type_impl <
+ BasicJsonType, CompatibleObjectType,
+ enable_if_t < is_detected<mapped_type_t, CompatibleObjectType>::value&&
+ is_detected<key_type_t, CompatibleObjectType>::value >>
+{
+ using object_t = typename BasicJsonType::object_t;
+
+ // macOS's is_constructible does not play well with nonesuch...
+ static constexpr bool value =
+ is_constructible<typename object_t::key_type,
+ typename CompatibleObjectType::key_type>::value &&
+ is_constructible<typename object_t::mapped_type,
+ typename CompatibleObjectType::mapped_type>::value;
+};
+
+template<typename BasicJsonType, typename CompatibleObjectType>
+struct is_compatible_object_type
+ : is_compatible_object_type_impl<BasicJsonType, CompatibleObjectType> {};
+
+template<typename BasicJsonType, typename ConstructibleObjectType,
+ typename = void>
+struct is_constructible_object_type_impl : std::false_type {};
+
+template<typename BasicJsonType, typename ConstructibleObjectType>
+struct is_constructible_object_type_impl <
+ BasicJsonType, ConstructibleObjectType,
+ enable_if_t < is_detected<mapped_type_t, ConstructibleObjectType>::value&&
+ is_detected<key_type_t, ConstructibleObjectType>::value >>
+{
+ using object_t = typename BasicJsonType::object_t;
+
+ static constexpr bool value =
+ (is_default_constructible<ConstructibleObjectType>::value &&
+ (std::is_move_assignable<ConstructibleObjectType>::value ||
+ std::is_copy_assignable<ConstructibleObjectType>::value) &&
+ (is_constructible<typename ConstructibleObjectType::key_type,
+ typename object_t::key_type>::value &&
+ std::is_same <
+ typename object_t::mapped_type,
+ typename ConstructibleObjectType::mapped_type >::value)) ||
+ (has_from_json<BasicJsonType,
+ typename ConstructibleObjectType::mapped_type>::value ||
+ has_non_default_from_json <
+ BasicJsonType,
+ typename ConstructibleObjectType::mapped_type >::value);
+};
+
+template<typename BasicJsonType, typename ConstructibleObjectType>
+struct is_constructible_object_type
+ : is_constructible_object_type_impl<BasicJsonType,
+ ConstructibleObjectType> {};
+
+template<typename BasicJsonType, typename CompatibleStringType>
+struct is_compatible_string_type
+{
+ static constexpr auto value =
+ is_constructible<typename BasicJsonType::string_t, CompatibleStringType>::value;
+};
+
+template<typename BasicJsonType, typename ConstructibleStringType>
+struct is_constructible_string_type
+{
+ // launder type through decltype() to fix compilation failure on ICPC
+#ifdef __INTEL_COMPILER
+ using laundered_type = decltype(std::declval<ConstructibleStringType>());
+#else
+ using laundered_type = ConstructibleStringType;
+#endif
+
+ static constexpr auto value =
+ conjunction <
+ is_constructible<laundered_type, typename BasicJsonType::string_t>,
+ is_detected_exact<typename BasicJsonType::string_t::value_type,
+ value_type_t, laundered_type >>::value;
+};
+
+template<typename BasicJsonType, typename CompatibleArrayType, typename = void>
+struct is_compatible_array_type_impl : std::false_type {};
+
+template<typename BasicJsonType, typename CompatibleArrayType>
+struct is_compatible_array_type_impl <
+ BasicJsonType, CompatibleArrayType,
+ enable_if_t <
+ is_detected<iterator_t, CompatibleArrayType>::value&&
+ is_iterator_traits<iterator_traits<detected_t<iterator_t, CompatibleArrayType>>>::value&&
+// special case for types like std::filesystem::path whose iterator's value_type are themselves
+// c.f. https://github.com/nlohmann/json/pull/3073
+ !std::is_same<CompatibleArrayType, detected_t<range_value_t, CompatibleArrayType>>::value >>
+{
+ static constexpr bool value =
+ is_constructible<BasicJsonType,
+ range_value_t<CompatibleArrayType>>::value;
+};
+
+template<typename BasicJsonType, typename CompatibleArrayType>
+struct is_compatible_array_type
+ : is_compatible_array_type_impl<BasicJsonType, CompatibleArrayType> {};
+
+template<typename BasicJsonType, typename ConstructibleArrayType, typename = void>
+struct is_constructible_array_type_impl : std::false_type {};
+
+template<typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type_impl <
+ BasicJsonType, ConstructibleArrayType,
+ enable_if_t<std::is_same<ConstructibleArrayType,
+ typename BasicJsonType::value_type>::value >>
+ : std::true_type {};
+
+template<typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type_impl <
+ BasicJsonType, ConstructibleArrayType,
+ enable_if_t < !std::is_same<ConstructibleArrayType,
+ typename BasicJsonType::value_type>::value&&
+ !is_compatible_string_type<BasicJsonType, ConstructibleArrayType>::value&&
+ is_default_constructible<ConstructibleArrayType>::value&&
+(std::is_move_assignable<ConstructibleArrayType>::value ||
+ std::is_copy_assignable<ConstructibleArrayType>::value)&&
+is_detected<iterator_t, ConstructibleArrayType>::value&&
+is_iterator_traits<iterator_traits<detected_t<iterator_t, ConstructibleArrayType>>>::value&&
+is_detected<range_value_t, ConstructibleArrayType>::value&&
+// special case for types like std::filesystem::path whose iterator's value_type are themselves
+// c.f. https://github.com/nlohmann/json/pull/3073
+!std::is_same<ConstructibleArrayType, detected_t<range_value_t, ConstructibleArrayType>>::value&&
+ is_complete_type <
+ detected_t<range_value_t, ConstructibleArrayType >>::value >>
+{
+ using value_type = range_value_t<ConstructibleArrayType>;
+
+ static constexpr bool value =
+ std::is_same<value_type,
+ typename BasicJsonType::array_t::value_type>::value ||
+ has_from_json<BasicJsonType,
+ value_type>::value ||
+ has_non_default_from_json <
+ BasicJsonType,
+ value_type >::value;
+};
+
+template<typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type
+ : is_constructible_array_type_impl<BasicJsonType, ConstructibleArrayType> {};
+
+template<typename RealIntegerType, typename CompatibleNumberIntegerType,
+ typename = void>
+struct is_compatible_integer_type_impl : std::false_type {};
+
+template<typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type_impl <
+ RealIntegerType, CompatibleNumberIntegerType,
+ enable_if_t < std::is_integral<RealIntegerType>::value&&
+ std::is_integral<CompatibleNumberIntegerType>::value&&
+ !std::is_same<bool, CompatibleNumberIntegerType>::value >>
+{
+ // is there an assert somewhere on overflows?
+ using RealLimits = std::numeric_limits<RealIntegerType>;
+ using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;
+
+ static constexpr auto value =
+ is_constructible<RealIntegerType,
+ CompatibleNumberIntegerType>::value &&
+ CompatibleLimits::is_integer &&
+ RealLimits::is_signed == CompatibleLimits::is_signed;
+};
+
+template<typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type
+ : is_compatible_integer_type_impl<RealIntegerType,
+ CompatibleNumberIntegerType> {};
+
+template<typename BasicJsonType, typename CompatibleType, typename = void>
+struct is_compatible_type_impl: std::false_type {};
+
+template<typename BasicJsonType, typename CompatibleType>
+struct is_compatible_type_impl <
+ BasicJsonType, CompatibleType,
+ enable_if_t<is_complete_type<CompatibleType>::value >>
+{
+ static constexpr bool value =
+ has_to_json<BasicJsonType, CompatibleType>::value;
+};
+
+template<typename BasicJsonType, typename CompatibleType>
+struct is_compatible_type
+ : is_compatible_type_impl<BasicJsonType, CompatibleType> {};
+
+template<typename T1, typename T2>
+struct is_constructible_tuple : std::false_type {};
+
+template<typename T1, typename... Args>
+struct is_constructible_tuple<T1, std::tuple<Args...>> : conjunction<is_constructible<T1, Args>...> {};
+
+template<typename BasicJsonType, typename T>
+struct is_json_iterator_of : std::false_type {};
+
+template<typename BasicJsonType>
+struct is_json_iterator_of<BasicJsonType, typename BasicJsonType::iterator> : std::true_type {};
+
+template<typename BasicJsonType>
+struct is_json_iterator_of<BasicJsonType, typename BasicJsonType::const_iterator> : std::true_type
+{};
+
+// checks if a given type T is a template specialization of Primary
+template<template <typename...> class Primary, typename T>
+struct is_specialization_of : std::false_type {};
+
+template<template <typename...> class Primary, typename... Args>
+struct is_specialization_of<Primary, Primary<Args...>> : std::true_type {};
+
+template<typename T>
+using is_json_pointer = is_specialization_of<::nlohmann::json_pointer, uncvref_t<T>>;
+
+// checks if A and B are comparable using Compare functor
+template<typename Compare, typename A, typename B, typename = void>
+struct is_comparable : std::false_type {};
+
+template<typename Compare, typename A, typename B>
+struct is_comparable<Compare, A, B, void_t<
+decltype(std::declval<Compare>()(std::declval<A>(), std::declval<B>())),
+decltype(std::declval<Compare>()(std::declval<B>(), std::declval<A>()))
+>> : std::true_type {};
+
+template<typename T>
+using detect_is_transparent = typename T::is_transparent;
+
+// type trait to check if KeyType can be used as object key (without a BasicJsonType)
+// see is_usable_as_basic_json_key_type below
+template<typename Comparator, typename ObjectKeyType, typename KeyTypeCVRef, bool RequireTransparentComparator = true,
+ bool ExcludeObjectKeyType = RequireTransparentComparator, typename KeyType = uncvref_t<KeyTypeCVRef>>
+using is_usable_as_key_type = typename std::conditional <
+ is_comparable<Comparator, ObjectKeyType, KeyTypeCVRef>::value
+ && !(ExcludeObjectKeyType && std::is_same<KeyType,
+ ObjectKeyType>::value)
+ && (!RequireTransparentComparator
+ || is_detected <detect_is_transparent, Comparator>::value)
+ && !is_json_pointer<KeyType>::value,
+ std::true_type,
+ std::false_type >::type;
+
+// type trait to check if KeyType can be used as object key
+// true if:
+// - KeyType is comparable with BasicJsonType::object_t::key_type
+// - if ExcludeObjectKeyType is true, KeyType is not BasicJsonType::object_t::key_type
+// - the comparator is transparent or RequireTransparentComparator is false
+// - KeyType is not a JSON iterator or json_pointer
+template<typename BasicJsonType, typename KeyTypeCVRef, bool RequireTransparentComparator = true,
+ bool ExcludeObjectKeyType = RequireTransparentComparator, typename KeyType = uncvref_t<KeyTypeCVRef>>
+using is_usable_as_basic_json_key_type = typename std::conditional <
+ is_usable_as_key_type<typename BasicJsonType::object_comparator_t,
+ typename BasicJsonType::object_t::key_type, KeyTypeCVRef,
+ RequireTransparentComparator, ExcludeObjectKeyType>::value
+ && !is_json_iterator_of<BasicJsonType, KeyType>::value,
+ std::true_type,
+ std::false_type >::type;
+
+template<typename ObjectType, typename KeyType>
+using detect_erase_with_key_type = decltype(std::declval<ObjectType&>().erase(std::declval<KeyType>()));
+
+// type trait to check if object_t has an erase() member functions accepting KeyType
+template<typename BasicJsonType, typename KeyType>
+using has_erase_with_key_type = typename std::conditional <
+ is_detected <
+ detect_erase_with_key_type,
+ typename BasicJsonType::object_t, KeyType >::value,
+ std::true_type,
+ std::false_type >::type;
+
+// a naive helper to check if a type is an ordered_map (exploits the fact that
+// ordered_map inherits capacity() from std::vector)
+template <typename T>
+struct is_ordered_map
+{
+ using one = char;
+
+ struct two
+ {
+ char x[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+ };
+
+ template <typename C> static one test( decltype(&C::capacity) ) ;
+ template <typename C> static two test(...);
+
+ enum { value = sizeof(test<T>(nullptr)) == sizeof(char) }; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+};
+
+// to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324)
+template < typename T, typename U, enable_if_t < !std::is_same<T, U>::value, int > = 0 >
+T conditional_static_cast(U value)
+{
+ return static_cast<T>(value);
+}
+
+template<typename T, typename U, enable_if_t<std::is_same<T, U>::value, int> = 0>
+T conditional_static_cast(U value)
+{
+ return value;
+}
+
+template<typename... Types>
+using all_integral = conjunction<std::is_integral<Types>...>;
+
+template<typename... Types>
+using all_signed = conjunction<std::is_signed<Types>...>;
+
+template<typename... Types>
+using all_unsigned = conjunction<std::is_unsigned<Types>...>;
+
+// there's a disjunction trait in another PR; replace when merged
+template<typename... Types>
+using same_sign = std::integral_constant < bool,
+ all_signed<Types...>::value || all_unsigned<Types...>::value >;
+
+template<typename OfType, typename T>
+using never_out_of_range = std::integral_constant < bool,
+ (std::is_signed<OfType>::value && (sizeof(T) < sizeof(OfType)))
+ || (same_sign<OfType, T>::value && sizeof(OfType) == sizeof(T)) >;
+
+template<typename OfType, typename T,
+ bool OfTypeSigned = std::is_signed<OfType>::value,
+ bool TSigned = std::is_signed<T>::value>
+struct value_in_range_of_impl2;
+
+template<typename OfType, typename T>
+struct value_in_range_of_impl2<OfType, T, false, false>
+{
+ static constexpr bool test(T val)
+ {
+ using CommonType = typename std::common_type<OfType, T>::type;
+ return static_cast<CommonType>(val) <= static_cast<CommonType>((std::numeric_limits<OfType>::max)());
+ }
+};
+
+template<typename OfType, typename T>
+struct value_in_range_of_impl2<OfType, T, true, false>
+{
+ static constexpr bool test(T val)
+ {
+ using CommonType = typename std::common_type<OfType, T>::type;
+ return static_cast<CommonType>(val) <= static_cast<CommonType>((std::numeric_limits<OfType>::max)());
+ }
+};
+
+template<typename OfType, typename T>
+struct value_in_range_of_impl2<OfType, T, false, true>
+{
+ static constexpr bool test(T val)
+ {
+ using CommonType = typename std::common_type<OfType, T>::type;
+ return val >= 0 && static_cast<CommonType>(val) <= static_cast<CommonType>((std::numeric_limits<OfType>::max)());
+ }
+};
+
+template<typename OfType, typename T>
+struct value_in_range_of_impl2<OfType, T, true, true>
+{
+ static constexpr bool test(T val)
+ {
+ using CommonType = typename std::common_type<OfType, T>::type;
+ return static_cast<CommonType>(val) >= static_cast<CommonType>((std::numeric_limits<OfType>::min)())
+ && static_cast<CommonType>(val) <= static_cast<CommonType>((std::numeric_limits<OfType>::max)());
+ }
+};
+
+template<typename OfType, typename T,
+ bool NeverOutOfRange = never_out_of_range<OfType, T>::value,
+ typename = detail::enable_if_t<all_integral<OfType, T>::value>>
+struct value_in_range_of_impl1;
+
+template<typename OfType, typename T>
+struct value_in_range_of_impl1<OfType, T, false>
+{
+ static constexpr bool test(T val)
+ {
+ return value_in_range_of_impl2<OfType, T>::test(val);
+ }
+};
+
+template<typename OfType, typename T>
+struct value_in_range_of_impl1<OfType, T, true>
+{
+ static constexpr bool test(T /*val*/)
+ {
+ return true;
+ }
+};
+
+template<typename OfType, typename T>
+inline constexpr bool value_in_range_of(T val)
+{
+ return value_in_range_of_impl1<OfType, T>::test(val);
+}
+
+template<bool Value>
+using bool_constant = std::integral_constant<bool, Value>;
+
+///////////////////////////////////////////////////////////////////////////////
+// is_c_string
+///////////////////////////////////////////////////////////////////////////////
+
+namespace impl
+{
+
+template<typename T>
+inline constexpr bool is_c_string()
+{
+ using TUnExt = typename std::remove_extent<T>::type;
+ using TUnCVExt = typename std::remove_cv<TUnExt>::type;
+ using TUnPtr = typename std::remove_pointer<T>::type;
+ using TUnCVPtr = typename std::remove_cv<TUnPtr>::type;
+ return
+ (std::is_array<T>::value && std::is_same<TUnCVExt, char>::value)
+ || (std::is_pointer<T>::value && std::is_same<TUnCVPtr, char>::value);
+}
+
+} // namespace impl
+
+// checks whether T is a [cv] char */[cv] char[] C string
+template<typename T>
+struct is_c_string : bool_constant<impl::is_c_string<T>()> {};
+
+template<typename T>
+using is_c_string_uncvref = is_c_string<uncvref_t<T>>;
+
+///////////////////////////////////////////////////////////////////////////////
+// is_transparent
+///////////////////////////////////////////////////////////////////////////////
+
+namespace impl
+{
+
+template<typename T>
+inline constexpr bool is_transparent()
+{
+ return is_detected<detect_is_transparent, T>::value;
+}
+
+} // namespace impl
+
+// checks whether T has a member named is_transparent
+template<typename T>
+struct is_transparent : bool_constant<impl::is_transparent<T>()> {};
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/string_concat.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <cstring> // strlen
+#include <string> // string
+#include <utility> // forward
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/detected.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+inline std::size_t concat_length()
+{
+ return 0;
+}
+
+template<typename... Args>
+inline std::size_t concat_length(const char* cstr, const Args& ... rest);
+
+template<typename StringType, typename... Args>
+inline std::size_t concat_length(const StringType& str, const Args& ... rest);
+
+template<typename... Args>
+inline std::size_t concat_length(const char /*c*/, const Args& ... rest)
+{
+ return 1 + concat_length(rest...);
+}
+
+template<typename... Args>
+inline std::size_t concat_length(const char* cstr, const Args& ... rest)
+{
+ // cppcheck-suppress ignoredReturnValue
+ return ::strlen(cstr) + concat_length(rest...);
+}
+
+template<typename StringType, typename... Args>
+inline std::size_t concat_length(const StringType& str, const Args& ... rest)
+{
+ return str.size() + concat_length(rest...);
+}
+
+template<typename OutStringType>
+inline void concat_into(OutStringType& /*out*/)
+{}
+
+template<typename StringType, typename Arg>
+using string_can_append = decltype(std::declval<StringType&>().append(std::declval < Arg && > ()));
+
+template<typename StringType, typename Arg>
+using detect_string_can_append = is_detected<string_can_append, StringType, Arg>;
+
+template<typename StringType, typename Arg>
+using string_can_append_op = decltype(std::declval<StringType&>() += std::declval < Arg && > ());
+
+template<typename StringType, typename Arg>
+using detect_string_can_append_op = is_detected<string_can_append_op, StringType, Arg>;
+
+template<typename StringType, typename Arg>
+using string_can_append_iter = decltype(std::declval<StringType&>().append(std::declval<const Arg&>().begin(), std::declval<const Arg&>().end()));
+
+template<typename StringType, typename Arg>
+using detect_string_can_append_iter = is_detected<string_can_append_iter, StringType, Arg>;
+
+template<typename StringType, typename Arg>
+using string_can_append_data = decltype(std::declval<StringType&>().append(std::declval<const Arg&>().data(), std::declval<const Arg&>().size()));
+
+template<typename StringType, typename Arg>
+using detect_string_can_append_data = is_detected<string_can_append_data, StringType, Arg>;
+
+template < typename OutStringType, typename Arg, typename... Args,
+ enable_if_t < !detect_string_can_append<OutStringType, Arg>::value
+ && detect_string_can_append_op<OutStringType, Arg>::value, int > = 0 >
+inline void concat_into(OutStringType& out, Arg && arg, Args && ... rest);
+
+template < typename OutStringType, typename Arg, typename... Args,
+ enable_if_t < !detect_string_can_append<OutStringType, Arg>::value
+ && !detect_string_can_append_op<OutStringType, Arg>::value
+ && detect_string_can_append_iter<OutStringType, Arg>::value, int > = 0 >
+inline void concat_into(OutStringType& out, const Arg& arg, Args && ... rest);
+
+template < typename OutStringType, typename Arg, typename... Args,
+ enable_if_t < !detect_string_can_append<OutStringType, Arg>::value
+ && !detect_string_can_append_op<OutStringType, Arg>::value
+ && !detect_string_can_append_iter<OutStringType, Arg>::value
+ && detect_string_can_append_data<OutStringType, Arg>::value, int > = 0 >
+inline void concat_into(OutStringType& out, const Arg& arg, Args && ... rest);
+
+template<typename OutStringType, typename Arg, typename... Args,
+ enable_if_t<detect_string_can_append<OutStringType, Arg>::value, int> = 0>
+inline void concat_into(OutStringType& out, Arg && arg, Args && ... rest)
+{
+ out.append(std::forward<Arg>(arg));
+ concat_into(out, std::forward<Args>(rest)...);
+}
+
+template < typename OutStringType, typename Arg, typename... Args,
+ enable_if_t < !detect_string_can_append<OutStringType, Arg>::value
+ && detect_string_can_append_op<OutStringType, Arg>::value, int > >
+inline void concat_into(OutStringType& out, Arg&& arg, Args&& ... rest)
+{
+ out += std::forward<Arg>(arg);
+ concat_into(out, std::forward<Args>(rest)...);
+}
+
+template < typename OutStringType, typename Arg, typename... Args,
+ enable_if_t < !detect_string_can_append<OutStringType, Arg>::value
+ && !detect_string_can_append_op<OutStringType, Arg>::value
+ && detect_string_can_append_iter<OutStringType, Arg>::value, int > >
+inline void concat_into(OutStringType& out, const Arg& arg, Args&& ... rest)
+{
+ out.append(arg.begin(), arg.end());
+ concat_into(out, std::forward<Args>(rest)...);
+}
+
+template < typename OutStringType, typename Arg, typename... Args,
+ enable_if_t < !detect_string_can_append<OutStringType, Arg>::value
+ && !detect_string_can_append_op<OutStringType, Arg>::value
+ && !detect_string_can_append_iter<OutStringType, Arg>::value
+ && detect_string_can_append_data<OutStringType, Arg>::value, int > >
+inline void concat_into(OutStringType& out, const Arg& arg, Args&& ... rest)
+{
+ out.append(arg.data(), arg.size());
+ concat_into(out, std::forward<Args>(rest)...);
+}
+
+template<typename OutStringType = std::string, typename... Args>
+inline OutStringType concat(Args && ... args)
+{
+ OutStringType str;
+ str.reserve(concat_length(args...));
+ concat_into(str, std::forward<Args>(args)...);
+ return str;
+}
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+////////////////
+// exceptions //
+////////////////
+
+/// @brief general exception of the @ref basic_json class
+/// @sa https://json.nlohmann.me/api/basic_json/exception/
+class exception : public std::exception
+{
+ public:
+ /// returns the explanatory string
+ const char* what() const noexcept override
+ {
+ return m.what();
+ }
+
+ /// the id of the exception
+ const int id; // NOLINT(cppcoreguidelines-non-private-member-variables-in-classes)
+
+ protected:
+ JSON_HEDLEY_NON_NULL(3)
+ exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} // NOLINT(bugprone-throw-keyword-missing)
+
+ static std::string name(const std::string& ename, int id_)
+ {
+ return concat("[json.exception.", ename, '.', std::to_string(id_), "] ");
+ }
+
+ static std::string diagnostics(std::nullptr_t /*leaf_element*/)
+ {
+ return "";
+ }
+
+ template<typename BasicJsonType>
+ static std::string diagnostics(const BasicJsonType* leaf_element)
+ {
+#if JSON_DIAGNOSTICS
+ std::vector<std::string> tokens;
+ for (const auto* current = leaf_element; current != nullptr && current->m_parent != nullptr; current = current->m_parent)
+ {
+ switch (current->m_parent->type())
+ {
+ case value_t::array:
+ {
+ for (std::size_t i = 0; i < current->m_parent->m_data.m_value.array->size(); ++i)
+ {
+ if (¤t->m_parent->m_data.m_value.array->operator[](i) == current)
+ {
+ tokens.emplace_back(std::to_string(i));
+ break;
+ }
+ }
+ break;
+ }
+
+ case value_t::object:
+ {
+ for (const auto& element : *current->m_parent->m_data.m_value.object)
+ {
+ if (&element.second == current)
+ {
+ tokens.emplace_back(element.first.c_str());
+ break;
+ }
+ }
+ break;
+ }
+
+ case value_t::null: // LCOV_EXCL_LINE
+ case value_t::string: // LCOV_EXCL_LINE
+ case value_t::boolean: // LCOV_EXCL_LINE
+ case value_t::number_integer: // LCOV_EXCL_LINE
+ case value_t::number_unsigned: // LCOV_EXCL_LINE
+ case value_t::number_float: // LCOV_EXCL_LINE
+ case value_t::binary: // LCOV_EXCL_LINE
+ case value_t::discarded: // LCOV_EXCL_LINE
+ default: // LCOV_EXCL_LINE
+ break; // LCOV_EXCL_LINE
+ }
+ }
+
+ if (tokens.empty())
+ {
+ return "";
+ }
+
+ auto str = std::accumulate(tokens.rbegin(), tokens.rend(), std::string{},
+ [](const std::string & a, const std::string & b)
+ {
+ return concat(a, '/', detail::escape(b));
+ });
+ return concat('(', str, ") ");
+#else
+ static_cast<void>(leaf_element);
+ return "";
+#endif
+ }
+
+ private:
+ /// an exception object as storage for error messages
+ std::runtime_error m;
+};
+
+/// @brief exception indicating a parse error
+/// @sa https://json.nlohmann.me/api/basic_json/parse_error/
+class parse_error : public exception
+{
+ public:
+ /*!
+ @brief create a parse error exception
+ @param[in] id_ the id of the exception
+ @param[in] pos the position where the error occurred (or with
+ chars_read_total=0 if the position cannot be
+ determined)
+ @param[in] what_arg the explanatory string
+ @return parse_error object
+ */
+ template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>
+ static parse_error create(int id_, const position_t& pos, const std::string& what_arg, BasicJsonContext context)
+ {
+ const std::string w = concat(exception::name("parse_error", id_), "parse error",
+ position_string(pos), ": ", exception::diagnostics(context), what_arg);
+ return {id_, pos.chars_read_total, w.c_str()};
+ }
+
+ template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>
+ static parse_error create(int id_, std::size_t byte_, const std::string& what_arg, BasicJsonContext context)
+ {
+ const std::string w = concat(exception::name("parse_error", id_), "parse error",
+ (byte_ != 0 ? (concat(" at byte ", std::to_string(byte_))) : ""),
+ ": ", exception::diagnostics(context), what_arg);
+ return {id_, byte_, w.c_str()};
+ }
+
+ /*!
+ @brief byte index of the parse error
+
+ The byte index of the last read character in the input file.
+
+ @note For an input with n bytes, 1 is the index of the first character and
+ n+1 is the index of the terminating null byte or the end of file.
+ This also holds true when reading a byte vector (CBOR or MessagePack).
+ */
+ const std::size_t byte;
+
+ private:
+ parse_error(int id_, std::size_t byte_, const char* what_arg)
+ : exception(id_, what_arg), byte(byte_) {}
+
+ static std::string position_string(const position_t& pos)
+ {
+ return concat(" at line ", std::to_string(pos.lines_read + 1),
+ ", column ", std::to_string(pos.chars_read_current_line));
+ }
+};
+
+/// @brief exception indicating errors with iterators
+/// @sa https://json.nlohmann.me/api/basic_json/invalid_iterator/
+class invalid_iterator : public exception
+{
+ public:
+ template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>
+ static invalid_iterator create(int id_, const std::string& what_arg, BasicJsonContext context)
+ {
+ const std::string w = concat(exception::name("invalid_iterator", id_), exception::diagnostics(context), what_arg);
+ return {id_, w.c_str()};
+ }
+
+ private:
+ JSON_HEDLEY_NON_NULL(3)
+ invalid_iterator(int id_, const char* what_arg)
+ : exception(id_, what_arg) {}
+};
+
+/// @brief exception indicating executing a member function with a wrong type
+/// @sa https://json.nlohmann.me/api/basic_json/type_error/
+class type_error : public exception
+{
+ public:
+ template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>
+ static type_error create(int id_, const std::string& what_arg, BasicJsonContext context)
+ {
+ const std::string w = concat(exception::name("type_error", id_), exception::diagnostics(context), what_arg);
+ return {id_, w.c_str()};
+ }
+
+ private:
+ JSON_HEDLEY_NON_NULL(3)
+ type_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
+
+/// @brief exception indicating access out of the defined range
+/// @sa https://json.nlohmann.me/api/basic_json/out_of_range/
+class out_of_range : public exception
+{
+ public:
+ template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>
+ static out_of_range create(int id_, const std::string& what_arg, BasicJsonContext context)
+ {
+ const std::string w = concat(exception::name("out_of_range", id_), exception::diagnostics(context), what_arg);
+ return {id_, w.c_str()};
+ }
+
+ private:
+ JSON_HEDLEY_NON_NULL(3)
+ out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
+
+/// @brief exception indicating other library errors
+/// @sa https://json.nlohmann.me/api/basic_json/other_error/
+class other_error : public exception
+{
+ public:
+ template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>
+ static other_error create(int id_, const std::string& what_arg, BasicJsonContext context)
+ {
+ const std::string w = concat(exception::name("other_error", id_), exception::diagnostics(context), what_arg);
+ return {id_, w.c_str()};
+ }
+
+ private:
+ JSON_HEDLEY_NON_NULL(3)
+ other_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/identity_tag.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+// dispatching helper struct
+template <class T> struct identity_tag {};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/meta/std_fs.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+#if JSON_HAS_EXPERIMENTAL_FILESYSTEM
+#include <experimental/filesystem>
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+namespace std_fs = std::experimental::filesystem;
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+#elif JSON_HAS_FILESYSTEM
+#include <filesystem>
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+namespace std_fs = std::filesystem;
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+#endif
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+// #include <nlohmann/detail/string_concat.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+template<typename BasicJsonType>
+inline void from_json(const BasicJsonType& j, typename std::nullptr_t& n)
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_null()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be null, but is ", j.type_name()), &j));
+ }
+ n = nullptr;
+}
+
+// overloads for basic_json template parameters
+template < typename BasicJsonType, typename ArithmeticType,
+ enable_if_t < std::is_arithmetic<ArithmeticType>::value&&
+ !std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
+ int > = 0 >
+void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val)
+{
+ switch (static_cast<value_t>(j))
+ {
+ case value_t::number_unsigned:
+ {
+ val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
+ break;
+ }
+ case value_t::number_integer:
+ {
+ val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
+ break;
+ }
+ case value_t::number_float:
+ {
+ val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
+ break;
+ }
+
+ case value_t::null:
+ case value_t::object:
+ case value_t::array:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ JSON_THROW(type_error::create(302, concat("type must be number, but is ", j.type_name()), &j));
+ }
+}
+
+template<typename BasicJsonType>
+inline void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b)
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_boolean()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be boolean, but is ", j.type_name()), &j));
+ }
+ b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>();
+}
+
+template<typename BasicJsonType>
+inline void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s)
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_string()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be string, but is ", j.type_name()), &j));
+ }
+ s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
+}
+
+template <
+ typename BasicJsonType, typename StringType,
+ enable_if_t <
+ std::is_assignable<StringType&, const typename BasicJsonType::string_t>::value
+ && is_detected_exact<typename BasicJsonType::string_t::value_type, value_type_t, StringType>::value
+ && !std::is_same<typename BasicJsonType::string_t, StringType>::value
+ && !is_json_ref<StringType>::value, int > = 0 >
+inline void from_json(const BasicJsonType& j, StringType& s)
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_string()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be string, but is ", j.type_name()), &j));
+ }
+
+ s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
+}
+
+template<typename BasicJsonType>
+inline void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val)
+{
+ get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType>
+inline void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val)
+{
+ get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType>
+inline void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val)
+{
+ get_arithmetic_value(j, val);
+}
+
+#if !JSON_DISABLE_ENUM_SERIALIZATION
+template<typename BasicJsonType, typename EnumType,
+ enable_if_t<std::is_enum<EnumType>::value, int> = 0>
+inline void from_json(const BasicJsonType& j, EnumType& e)
+{
+ typename std::underlying_type<EnumType>::type val;
+ get_arithmetic_value(j, val);
+ e = static_cast<EnumType>(val);
+}
+#endif // JSON_DISABLE_ENUM_SERIALIZATION
+
+// forward_list doesn't have an insert method
+template<typename BasicJsonType, typename T, typename Allocator,
+ enable_if_t<is_getable<BasicJsonType, T>::value, int> = 0>
+inline void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l)
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_array()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be array, but is ", j.type_name()), &j));
+ }
+ l.clear();
+ std::transform(j.rbegin(), j.rend(),
+ std::front_inserter(l), [](const BasicJsonType & i)
+ {
+ return i.template get<T>();
+ });
+}
+
+// valarray doesn't have an insert method
+template<typename BasicJsonType, typename T,
+ enable_if_t<is_getable<BasicJsonType, T>::value, int> = 0>
+inline void from_json(const BasicJsonType& j, std::valarray<T>& l)
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_array()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be array, but is ", j.type_name()), &j));
+ }
+ l.resize(j.size());
+ std::transform(j.begin(), j.end(), std::begin(l),
+ [](const BasicJsonType & elem)
+ {
+ return elem.template get<T>();
+ });
+}
+
+template<typename BasicJsonType, typename T, std::size_t N>
+auto from_json(const BasicJsonType& j, T (&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+-> decltype(j.template get<T>(), void())
+{
+ for (std::size_t i = 0; i < N; ++i)
+ {
+ arr[i] = j.at(i).template get<T>();
+ }
+}
+
+template<typename BasicJsonType>
+inline void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/)
+{
+ arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();
+}
+
+template<typename BasicJsonType, typename T, std::size_t N>
+auto from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr,
+ priority_tag<2> /*unused*/)
+-> decltype(j.template get<T>(), void())
+{
+ for (std::size_t i = 0; i < N; ++i)
+ {
+ arr[i] = j.at(i).template get<T>();
+ }
+}
+
+template<typename BasicJsonType, typename ConstructibleArrayType,
+ enable_if_t<
+ std::is_assignable<ConstructibleArrayType&, ConstructibleArrayType>::value,
+ int> = 0>
+auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/)
+-> decltype(
+ arr.reserve(std::declval<typename ConstructibleArrayType::size_type>()),
+ j.template get<typename ConstructibleArrayType::value_type>(),
+ void())
+{
+ using std::end;
+
+ ConstructibleArrayType ret;
+ ret.reserve(j.size());
+ std::transform(j.begin(), j.end(),
+ std::inserter(ret, end(ret)), [](const BasicJsonType & i)
+ {
+ // get<BasicJsonType>() returns *this, this won't call a from_json
+ // method when value_type is BasicJsonType
+ return i.template get<typename ConstructibleArrayType::value_type>();
+ });
+ arr = std::move(ret);
+}
+
+template<typename BasicJsonType, typename ConstructibleArrayType,
+ enable_if_t<
+ std::is_assignable<ConstructibleArrayType&, ConstructibleArrayType>::value,
+ int> = 0>
+inline void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr,
+ priority_tag<0> /*unused*/)
+{
+ using std::end;
+
+ ConstructibleArrayType ret;
+ std::transform(
+ j.begin(), j.end(), std::inserter(ret, end(ret)),
+ [](const BasicJsonType & i)
+ {
+ // get<BasicJsonType>() returns *this, this won't call a from_json
+ // method when value_type is BasicJsonType
+ return i.template get<typename ConstructibleArrayType::value_type>();
+ });
+ arr = std::move(ret);
+}
+
+template < typename BasicJsonType, typename ConstructibleArrayType,
+ enable_if_t <
+ is_constructible_array_type<BasicJsonType, ConstructibleArrayType>::value&&
+ !is_constructible_object_type<BasicJsonType, ConstructibleArrayType>::value&&
+ !is_constructible_string_type<BasicJsonType, ConstructibleArrayType>::value&&
+ !std::is_same<ConstructibleArrayType, typename BasicJsonType::binary_t>::value&&
+ !is_basic_json<ConstructibleArrayType>::value,
+ int > = 0 >
+auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr)
+-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}),
+j.template get<typename ConstructibleArrayType::value_type>(),
+void())
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_array()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be array, but is ", j.type_name()), &j));
+ }
+
+ from_json_array_impl(j, arr, priority_tag<3> {});
+}
+
+template < typename BasicJsonType, typename T, std::size_t... Idx >
+std::array<T, sizeof...(Idx)> from_json_inplace_array_impl(BasicJsonType&& j,
+ identity_tag<std::array<T, sizeof...(Idx)>> /*unused*/, index_sequence<Idx...> /*unused*/)
+{
+ return { { std::forward<BasicJsonType>(j).at(Idx).template get<T>()... } };
+}
+
+template < typename BasicJsonType, typename T, std::size_t N >
+auto from_json(BasicJsonType&& j, identity_tag<std::array<T, N>> tag)
+-> decltype(from_json_inplace_array_impl(std::forward<BasicJsonType>(j), tag, make_index_sequence<N> {}))
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_array()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be array, but is ", j.type_name()), &j));
+ }
+
+ return from_json_inplace_array_impl(std::forward<BasicJsonType>(j), tag, make_index_sequence<N> {});
+}
+
+template<typename BasicJsonType>
+inline void from_json(const BasicJsonType& j, typename BasicJsonType::binary_t& bin)
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_binary()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be binary, but is ", j.type_name()), &j));
+ }
+
+ bin = *j.template get_ptr<const typename BasicJsonType::binary_t*>();
+}
+
+template<typename BasicJsonType, typename ConstructibleObjectType,
+ enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0>
+inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_object()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be object, but is ", j.type_name()), &j));
+ }
+
+ ConstructibleObjectType ret;
+ const auto* inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
+ using value_type = typename ConstructibleObjectType::value_type;
+ std::transform(
+ inner_object->begin(), inner_object->end(),
+ std::inserter(ret, ret.begin()),
+ [](typename BasicJsonType::object_t::value_type const & p)
+ {
+ return value_type(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
+ });
+ obj = std::move(ret);
+}
+
+// overload for arithmetic types, not chosen for basic_json template arguments
+// (BooleanType, etc..); note: Is it really necessary to provide explicit
+// overloads for boolean_t etc. in case of a custom BooleanType which is not
+// an arithmetic type?
+template < typename BasicJsonType, typename ArithmeticType,
+ enable_if_t <
+ std::is_arithmetic<ArithmeticType>::value&&
+ !std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value&&
+ !std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value&&
+ !std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value&&
+ !std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
+ int > = 0 >
+inline void from_json(const BasicJsonType& j, ArithmeticType& val)
+{
+ switch (static_cast<value_t>(j))
+ {
+ case value_t::number_unsigned:
+ {
+ val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
+ break;
+ }
+ case value_t::number_integer:
+ {
+ val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
+ break;
+ }
+ case value_t::number_float:
+ {
+ val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
+ break;
+ }
+ case value_t::boolean:
+ {
+ val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>());
+ break;
+ }
+
+ case value_t::null:
+ case value_t::object:
+ case value_t::array:
+ case value_t::string:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ JSON_THROW(type_error::create(302, concat("type must be number, but is ", j.type_name()), &j));
+ }
+}
+
+template<typename BasicJsonType, typename... Args, std::size_t... Idx>
+std::tuple<Args...> from_json_tuple_impl_base(BasicJsonType&& j, index_sequence<Idx...> /*unused*/)
+{
+ return std::make_tuple(std::forward<BasicJsonType>(j).at(Idx).template get<Args>()...);
+}
+
+template < typename BasicJsonType, class A1, class A2 >
+std::pair<A1, A2> from_json_tuple_impl(BasicJsonType&& j, identity_tag<std::pair<A1, A2>> /*unused*/, priority_tag<0> /*unused*/)
+{
+ return {std::forward<BasicJsonType>(j).at(0).template get<A1>(),
+ std::forward<BasicJsonType>(j).at(1).template get<A2>()};
+}
+
+template<typename BasicJsonType, typename A1, typename A2>
+inline void from_json_tuple_impl(BasicJsonType&& j, std::pair<A1, A2>& p, priority_tag<1> /*unused*/)
+{
+ p = from_json_tuple_impl(std::forward<BasicJsonType>(j), identity_tag<std::pair<A1, A2>> {}, priority_tag<0> {});
+}
+
+template<typename BasicJsonType, typename... Args>
+std::tuple<Args...> from_json_tuple_impl(BasicJsonType&& j, identity_tag<std::tuple<Args...>> /*unused*/, priority_tag<2> /*unused*/)
+{
+ return from_json_tuple_impl_base<BasicJsonType, Args...>(std::forward<BasicJsonType>(j), index_sequence_for<Args...> {});
+}
+
+template<typename BasicJsonType, typename... Args>
+inline void from_json_tuple_impl(BasicJsonType&& j, std::tuple<Args...>& t, priority_tag<3> /*unused*/)
+{
+ t = from_json_tuple_impl_base<BasicJsonType, Args...>(std::forward<BasicJsonType>(j), index_sequence_for<Args...> {});
+}
+
+template<typename BasicJsonType, typename TupleRelated>
+auto from_json(BasicJsonType&& j, TupleRelated&& t)
+-> decltype(from_json_tuple_impl(std::forward<BasicJsonType>(j), std::forward<TupleRelated>(t), priority_tag<3> {}))
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_array()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be array, but is ", j.type_name()), &j));
+ }
+
+ return from_json_tuple_impl(std::forward<BasicJsonType>(j), std::forward<TupleRelated>(t), priority_tag<3> {});
+}
+
+template < typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator,
+ typename = enable_if_t < !std::is_constructible <
+ typename BasicJsonType::string_t, Key >::value >>
+inline void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m)
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_array()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be array, but is ", j.type_name()), &j));
+ }
+ m.clear();
+ for (const auto& p : j)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!p.is_array()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be array, but is ", p.type_name()), &j));
+ }
+ m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
+ }
+}
+
+template < typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator,
+ typename = enable_if_t < !std::is_constructible <
+ typename BasicJsonType::string_t, Key >::value >>
+inline void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m)
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_array()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be array, but is ", j.type_name()), &j));
+ }
+ m.clear();
+ for (const auto& p : j)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!p.is_array()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be array, but is ", p.type_name()), &j));
+ }
+ m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
+ }
+}
+
+#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
+template<typename BasicJsonType>
+inline void from_json(const BasicJsonType& j, std_fs::path& p)
+{
+ if (JSON_HEDLEY_UNLIKELY(!j.is_string()))
+ {
+ JSON_THROW(type_error::create(302, concat("type must be string, but is ", j.type_name()), &j));
+ }
+ p = *j.template get_ptr<const typename BasicJsonType::string_t*>();
+}
+#endif
+
+struct from_json_fn
+{
+ template<typename BasicJsonType, typename T>
+ auto operator()(const BasicJsonType& j, T&& val) const
+ noexcept(noexcept(from_json(j, std::forward<T>(val))))
+ -> decltype(from_json(j, std::forward<T>(val)))
+ {
+ return from_json(j, std::forward<T>(val));
+ }
+};
+
+} // namespace detail
+
+#ifndef JSON_HAS_CPP_17
+/// namespace to hold default `from_json` function
+/// to see why this is required:
+/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html
+namespace // NOLINT(cert-dcl59-cpp,fuchsia-header-anon-namespaces,google-build-namespaces)
+{
+#endif
+JSON_INLINE_VARIABLE constexpr const auto& from_json = // NOLINT(misc-definitions-in-headers)
+ detail::static_const<detail::from_json_fn>::value;
+#ifndef JSON_HAS_CPP_17
+} // namespace
+#endif
+
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/conversions/to_json.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <algorithm> // copy
+#include <iterator> // begin, end
+#include <string> // string
+#include <tuple> // tuple, get
+#include <type_traits> // is_same, is_constructible, is_floating_point, is_enum, underlying_type
+#include <utility> // move, forward, declval, pair
+#include <valarray> // valarray
+#include <vector> // vector
+
+// #include <nlohmann/detail/iterators/iteration_proxy.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <cstddef> // size_t
+#include <iterator> // input_iterator_tag
+#include <string> // string, to_string
+#include <tuple> // tuple_size, get, tuple_element
+#include <utility> // move
+
+#if JSON_HAS_RANGES
+ #include <ranges> // enable_borrowed_range
+#endif
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+template<typename string_type>
+void int_to_string( string_type& target, std::size_t value )
+{
+ // For ADL
+ using std::to_string;
+ target = to_string(value);
+}
+template<typename IteratorType> class iteration_proxy_value
+{
+ public:
+ using difference_type = std::ptrdiff_t;
+ using value_type = iteration_proxy_value;
+ using pointer = value_type *;
+ using reference = value_type &;
+ using iterator_category = std::input_iterator_tag;
+ using string_type = typename std::remove_cv< typename std::remove_reference<decltype( std::declval<IteratorType>().key() ) >::type >::type;
+
+ private:
+ /// the iterator
+ IteratorType anchor{};
+ /// an index for arrays (used to create key names)
+ std::size_t array_index = 0;
+ /// last stringified array index
+ mutable std::size_t array_index_last = 0;
+ /// a string representation of the array index
+ mutable string_type array_index_str = "0";
+ /// an empty string (to return a reference for primitive values)
+ string_type empty_str{};
+
+ public:
+ explicit iteration_proxy_value() = default;
+ explicit iteration_proxy_value(IteratorType it, std::size_t array_index_ = 0)
+ noexcept(std::is_nothrow_move_constructible<IteratorType>::value
+ && std::is_nothrow_default_constructible<string_type>::value)
+ : anchor(std::move(it))
+ , array_index(array_index_)
+ {}
+
+ iteration_proxy_value(iteration_proxy_value const&) = default;
+ iteration_proxy_value& operator=(iteration_proxy_value const&) = default;
+ // older GCCs are a bit fussy and require explicit noexcept specifiers on defaulted functions
+ iteration_proxy_value(iteration_proxy_value&&)
+ noexcept(std::is_nothrow_move_constructible<IteratorType>::value
+ && std::is_nothrow_move_constructible<string_type>::value) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor,cppcoreguidelines-noexcept-move-operations)
+ iteration_proxy_value& operator=(iteration_proxy_value&&)
+ noexcept(std::is_nothrow_move_assignable<IteratorType>::value
+ && std::is_nothrow_move_assignable<string_type>::value) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor,cppcoreguidelines-noexcept-move-operations)
+ ~iteration_proxy_value() = default;
+
+ /// dereference operator (needed for range-based for)
+ const iteration_proxy_value& operator*() const
+ {
+ return *this;
+ }
+
+ /// increment operator (needed for range-based for)
+ iteration_proxy_value& operator++()
+ {
+ ++anchor;
+ ++array_index;
+
+ return *this;
+ }
+
+ iteration_proxy_value operator++(int)& // NOLINT(cert-dcl21-cpp)
+ {
+ auto tmp = iteration_proxy_value(anchor, array_index);
+ ++anchor;
+ ++array_index;
+ return tmp;
+ }
+
+ /// equality operator (needed for InputIterator)
+ bool operator==(const iteration_proxy_value& o) const
+ {
+ return anchor == o.anchor;
+ }
+
+ /// inequality operator (needed for range-based for)
+ bool operator!=(const iteration_proxy_value& o) const
+ {
+ return anchor != o.anchor;
+ }
+
+ /// return key of the iterator
+ const string_type& key() const
+ {
+ JSON_ASSERT(anchor.m_object != nullptr);
+
+ switch (anchor.m_object->type())
+ {
+ // use integer array index as key
+ case value_t::array:
+ {
+ if (array_index != array_index_last)
+ {
+ int_to_string( array_index_str, array_index );
+ array_index_last = array_index;
+ }
+ return array_index_str;
+ }
+
+ // use key from the object
+ case value_t::object:
+ return anchor.key();
+
+ // use an empty key for all primitive types
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ return empty_str;
+ }
+ }
+
+ /// return value of the iterator
+ typename IteratorType::reference value() const
+ {
+ return anchor.value();
+ }
+};
+
+/// proxy class for the items() function
+template<typename IteratorType> class iteration_proxy
+{
+ private:
+ /// the container to iterate
+ typename IteratorType::pointer container = nullptr;
+
+ public:
+ explicit iteration_proxy() = default;
+
+ /// construct iteration proxy from a container
+ explicit iteration_proxy(typename IteratorType::reference cont) noexcept
+ : container(&cont) {}
+
+ iteration_proxy(iteration_proxy const&) = default;
+ iteration_proxy& operator=(iteration_proxy const&) = default;
+ iteration_proxy(iteration_proxy&&) noexcept = default;
+ iteration_proxy& operator=(iteration_proxy&&) noexcept = default;
+ ~iteration_proxy() = default;
+
+ /// return iterator begin (needed for range-based for)
+ iteration_proxy_value<IteratorType> begin() const noexcept
+ {
+ return iteration_proxy_value<IteratorType>(container->begin());
+ }
+
+ /// return iterator end (needed for range-based for)
+ iteration_proxy_value<IteratorType> end() const noexcept
+ {
+ return iteration_proxy_value<IteratorType>(container->end());
+ }
+};
+
+// Structured Bindings Support
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+template<std::size_t N, typename IteratorType, enable_if_t<N == 0, int> = 0>
+auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.key())
+{
+ return i.key();
+}
+// Structured Bindings Support
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+template<std::size_t N, typename IteratorType, enable_if_t<N == 1, int> = 0>
+auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.value())
+{
+ return i.value();
+}
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// The Addition to the STD Namespace is required to add
+// Structured Bindings Support to the iteration_proxy_value class
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+namespace std
+{
+
+#if defined(__clang__)
+ // Fix: https://github.com/nlohmann/json/issues/1401
+ #pragma clang diagnostic push
+ #pragma clang diagnostic ignored "-Wmismatched-tags"
+#endif
+template<typename IteratorType>
+class tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>> // NOLINT(cert-dcl58-cpp)
+ : public std::integral_constant<std::size_t, 2> {};
+
+template<std::size_t N, typename IteratorType>
+class tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >> // NOLINT(cert-dcl58-cpp)
+{
+ public:
+ using type = decltype(
+ get<N>(std::declval <
+ ::nlohmann::detail::iteration_proxy_value<IteratorType >> ()));
+};
+#if defined(__clang__)
+ #pragma clang diagnostic pop
+#endif
+
+} // namespace std
+
+#if JSON_HAS_RANGES
+ template <typename IteratorType>
+ inline constexpr bool ::std::ranges::enable_borrowed_range<::nlohmann::detail::iteration_proxy<IteratorType>> = true;
+#endif
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/std_fs.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+//////////////////
+// constructors //
+//////////////////
+
+/*
+ * Note all external_constructor<>::construct functions need to call
+ * j.m_data.m_value.destroy(j.m_data.m_type) to avoid a memory leak in case j contains an
+ * allocated value (e.g., a string). See bug issue
+ * https://github.com/nlohmann/json/issues/2865 for more information.
+ */
+
+template<value_t> struct external_constructor;
+
+template<>
+struct external_constructor<value_t::boolean>
+{
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::boolean;
+ j.m_data.m_value = b;
+ j.assert_invariant();
+ }
+};
+
+template<>
+struct external_constructor<value_t::string>
+{
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s)
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::string;
+ j.m_data.m_value = s;
+ j.assert_invariant();
+ }
+
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s)
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::string;
+ j.m_data.m_value = std::move(s);
+ j.assert_invariant();
+ }
+
+ template < typename BasicJsonType, typename CompatibleStringType,
+ enable_if_t < !std::is_same<CompatibleStringType, typename BasicJsonType::string_t>::value,
+ int > = 0 >
+ static void construct(BasicJsonType& j, const CompatibleStringType& str)
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::string;
+ j.m_data.m_value.string = j.template create<typename BasicJsonType::string_t>(str);
+ j.assert_invariant();
+ }
+};
+
+template<>
+struct external_constructor<value_t::binary>
+{
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b)
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::binary;
+ j.m_data.m_value = typename BasicJsonType::binary_t(b);
+ j.assert_invariant();
+ }
+
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b)
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::binary;
+ j.m_data.m_value = typename BasicJsonType::binary_t(std::move(b));
+ j.assert_invariant();
+ }
+};
+
+template<>
+struct external_constructor<value_t::number_float>
+{
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::number_float;
+ j.m_data.m_value = val;
+ j.assert_invariant();
+ }
+};
+
+template<>
+struct external_constructor<value_t::number_unsigned>
+{
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::number_unsigned;
+ j.m_data.m_value = val;
+ j.assert_invariant();
+ }
+};
+
+template<>
+struct external_constructor<value_t::number_integer>
+{
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::number_integer;
+ j.m_data.m_value = val;
+ j.assert_invariant();
+ }
+};
+
+template<>
+struct external_constructor<value_t::array>
+{
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr)
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::array;
+ j.m_data.m_value = arr;
+ j.set_parents();
+ j.assert_invariant();
+ }
+
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::array;
+ j.m_data.m_value = std::move(arr);
+ j.set_parents();
+ j.assert_invariant();
+ }
+
+ template < typename BasicJsonType, typename CompatibleArrayType,
+ enable_if_t < !std::is_same<CompatibleArrayType, typename BasicJsonType::array_t>::value,
+ int > = 0 >
+ static void construct(BasicJsonType& j, const CompatibleArrayType& arr)
+ {
+ using std::begin;
+ using std::end;
+
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::array;
+ j.m_data.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));
+ j.set_parents();
+ j.assert_invariant();
+ }
+
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, const std::vector<bool>& arr)
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::array;
+ j.m_data.m_value = value_t::array;
+ j.m_data.m_value.array->reserve(arr.size());
+ for (const bool x : arr)
+ {
+ j.m_data.m_value.array->push_back(x);
+ j.set_parent(j.m_data.m_value.array->back());
+ }
+ j.assert_invariant();
+ }
+
+ template<typename BasicJsonType, typename T,
+ enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
+ static void construct(BasicJsonType& j, const std::valarray<T>& arr)
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::array;
+ j.m_data.m_value = value_t::array;
+ j.m_data.m_value.array->resize(arr.size());
+ if (arr.size() > 0)
+ {
+ std::copy(std::begin(arr), std::end(arr), j.m_data.m_value.array->begin());
+ }
+ j.set_parents();
+ j.assert_invariant();
+ }
+};
+
+template<>
+struct external_constructor<value_t::object>
+{
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj)
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::object;
+ j.m_data.m_value = obj;
+ j.set_parents();
+ j.assert_invariant();
+ }
+
+ template<typename BasicJsonType>
+ static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
+ {
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::object;
+ j.m_data.m_value = std::move(obj);
+ j.set_parents();
+ j.assert_invariant();
+ }
+
+ template < typename BasicJsonType, typename CompatibleObjectType,
+ enable_if_t < !std::is_same<CompatibleObjectType, typename BasicJsonType::object_t>::value, int > = 0 >
+ static void construct(BasicJsonType& j, const CompatibleObjectType& obj)
+ {
+ using std::begin;
+ using std::end;
+
+ j.m_data.m_value.destroy(j.m_data.m_type);
+ j.m_data.m_type = value_t::object;
+ j.m_data.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj));
+ j.set_parents();
+ j.assert_invariant();
+ }
+};
+
+/////////////
+// to_json //
+/////////////
+
+template<typename BasicJsonType, typename T,
+ enable_if_t<std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0>
+inline void to_json(BasicJsonType& j, T b) noexcept
+{
+ external_constructor<value_t::boolean>::construct(j, b);
+}
+
+template < typename BasicJsonType, typename BoolRef,
+ enable_if_t <
+ ((std::is_same<std::vector<bool>::reference, BoolRef>::value
+ && !std::is_same <std::vector<bool>::reference, typename BasicJsonType::boolean_t&>::value)
+ || (std::is_same<std::vector<bool>::const_reference, BoolRef>::value
+ && !std::is_same <detail::uncvref_t<std::vector<bool>::const_reference>,
+ typename BasicJsonType::boolean_t >::value))
+ && std::is_convertible<const BoolRef&, typename BasicJsonType::boolean_t>::value, int > = 0 >
+inline void to_json(BasicJsonType& j, const BoolRef& b) noexcept
+{
+ external_constructor<value_t::boolean>::construct(j, static_cast<typename BasicJsonType::boolean_t>(b));
+}
+
+template<typename BasicJsonType, typename CompatibleString,
+ enable_if_t<std::is_constructible<typename BasicJsonType::string_t, CompatibleString>::value, int> = 0>
+inline void to_json(BasicJsonType& j, const CompatibleString& s)
+{
+ external_constructor<value_t::string>::construct(j, s);
+}
+
+template<typename BasicJsonType>
+inline void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s)
+{
+ external_constructor<value_t::string>::construct(j, std::move(s));
+}
+
+template<typename BasicJsonType, typename FloatType,
+ enable_if_t<std::is_floating_point<FloatType>::value, int> = 0>
+inline void to_json(BasicJsonType& j, FloatType val) noexcept
+{
+ external_constructor<value_t::number_float>::construct(j, static_cast<typename BasicJsonType::number_float_t>(val));
+}
+
+template<typename BasicJsonType, typename CompatibleNumberUnsignedType,
+ enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t, CompatibleNumberUnsignedType>::value, int> = 0>
+inline void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept
+{
+ external_constructor<value_t::number_unsigned>::construct(j, static_cast<typename BasicJsonType::number_unsigned_t>(val));
+}
+
+template<typename BasicJsonType, typename CompatibleNumberIntegerType,
+ enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t, CompatibleNumberIntegerType>::value, int> = 0>
+inline void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept
+{
+ external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val));
+}
+
+#if !JSON_DISABLE_ENUM_SERIALIZATION
+template<typename BasicJsonType, typename EnumType,
+ enable_if_t<std::is_enum<EnumType>::value, int> = 0>
+inline void to_json(BasicJsonType& j, EnumType e) noexcept
+{
+ using underlying_type = typename std::underlying_type<EnumType>::type;
+ static constexpr value_t integral_value_t = std::is_unsigned<underlying_type>::value ? value_t::number_unsigned : value_t::number_integer;
+ external_constructor<integral_value_t>::construct(j, static_cast<underlying_type>(e));
+}
+#endif // JSON_DISABLE_ENUM_SERIALIZATION
+
+template<typename BasicJsonType>
+inline void to_json(BasicJsonType& j, const std::vector<bool>& e)
+{
+ external_constructor<value_t::array>::construct(j, e);
+}
+
+template < typename BasicJsonType, typename CompatibleArrayType,
+ enable_if_t < is_compatible_array_type<BasicJsonType,
+ CompatibleArrayType>::value&&
+ !is_compatible_object_type<BasicJsonType, CompatibleArrayType>::value&&
+ !is_compatible_string_type<BasicJsonType, CompatibleArrayType>::value&&
+ !std::is_same<typename BasicJsonType::binary_t, CompatibleArrayType>::value&&
+ !is_basic_json<CompatibleArrayType>::value,
+ int > = 0 >
+inline void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
+{
+ external_constructor<value_t::array>::construct(j, arr);
+}
+
+template<typename BasicJsonType>
+inline void to_json(BasicJsonType& j, const typename BasicJsonType::binary_t& bin)
+{
+ external_constructor<value_t::binary>::construct(j, bin);
+}
+
+template<typename BasicJsonType, typename T,
+ enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
+inline void to_json(BasicJsonType& j, const std::valarray<T>& arr)
+{
+ external_constructor<value_t::array>::construct(j, std::move(arr));
+}
+
+template<typename BasicJsonType>
+inline void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
+{
+ external_constructor<value_t::array>::construct(j, std::move(arr));
+}
+
+template < typename BasicJsonType, typename CompatibleObjectType,
+ enable_if_t < is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value&& !is_basic_json<CompatibleObjectType>::value, int > = 0 >
+inline void to_json(BasicJsonType& j, const CompatibleObjectType& obj)
+{
+ external_constructor<value_t::object>::construct(j, obj);
+}
+
+template<typename BasicJsonType>
+inline void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
+{
+ external_constructor<value_t::object>::construct(j, std::move(obj));
+}
+
+template <
+ typename BasicJsonType, typename T, std::size_t N,
+ enable_if_t < !std::is_constructible<typename BasicJsonType::string_t,
+ const T(&)[N]>::value, // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+ int > = 0 >
+inline void to_json(BasicJsonType& j, const T(&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+{
+ external_constructor<value_t::array>::construct(j, arr);
+}
+
+template < typename BasicJsonType, typename T1, typename T2, enable_if_t < std::is_constructible<BasicJsonType, T1>::value&& std::is_constructible<BasicJsonType, T2>::value, int > = 0 >
+inline void to_json(BasicJsonType& j, const std::pair<T1, T2>& p)
+{
+ j = { p.first, p.second };
+}
+
+// for https://github.com/nlohmann/json/pull/1134
+template<typename BasicJsonType, typename T,
+ enable_if_t<std::is_same<T, iteration_proxy_value<typename BasicJsonType::iterator>>::value, int> = 0>
+inline void to_json(BasicJsonType& j, const T& b)
+{
+ j = { {b.key(), b.value()} };
+}
+
+template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
+inline void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...> /*unused*/)
+{
+ j = { std::get<Idx>(t)... };
+}
+
+template<typename BasicJsonType, typename T, enable_if_t<is_constructible_tuple<BasicJsonType, T>::value, int > = 0>
+inline void to_json(BasicJsonType& j, const T& t)
+{
+ to_json_tuple_impl(j, t, make_index_sequence<std::tuple_size<T>::value> {});
+}
+
+#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
+template<typename BasicJsonType>
+inline void to_json(BasicJsonType& j, const std_fs::path& p)
+{
+ j = p.string();
+}
+#endif
+
+struct to_json_fn
+{
+ template<typename BasicJsonType, typename T>
+ auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward<T>(val))))
+ -> decltype(to_json(j, std::forward<T>(val)), void())
+ {
+ return to_json(j, std::forward<T>(val));
+ }
+};
+} // namespace detail
+
+#ifndef JSON_HAS_CPP_17
+/// namespace to hold default `to_json` function
+/// to see why this is required:
+/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html
+namespace // NOLINT(cert-dcl59-cpp,fuchsia-header-anon-namespaces,google-build-namespaces)
+{
+#endif
+JSON_INLINE_VARIABLE constexpr const auto& to_json = // NOLINT(misc-definitions-in-headers)
+ detail::static_const<detail::to_json_fn>::value;
+#ifndef JSON_HAS_CPP_17
+} // namespace
+#endif
+
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/meta/identity_tag.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+
+/// @sa https://json.nlohmann.me/api/adl_serializer/
+template<typename ValueType, typename>
+struct adl_serializer
+{
+ /// @brief convert a JSON value to any value type
+ /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/
+ template<typename BasicJsonType, typename TargetType = ValueType>
+ static auto from_json(BasicJsonType && j, TargetType& val) noexcept(
+ noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val)))
+ -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), val), void())
+ {
+ ::nlohmann::from_json(std::forward<BasicJsonType>(j), val);
+ }
+
+ /// @brief convert a JSON value to any value type
+ /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/
+ template<typename BasicJsonType, typename TargetType = ValueType>
+ static auto from_json(BasicJsonType && j) noexcept(
+ noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), detail::identity_tag<TargetType> {})))
+ -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), detail::identity_tag<TargetType> {}))
+ {
+ return ::nlohmann::from_json(std::forward<BasicJsonType>(j), detail::identity_tag<TargetType> {});
+ }
+
+ /// @brief convert any value type to a JSON value
+ /// @sa https://json.nlohmann.me/api/adl_serializer/to_json/
+ template<typename BasicJsonType, typename TargetType = ValueType>
+ static auto to_json(BasicJsonType& j, TargetType && val) noexcept(
+ noexcept(::nlohmann::to_json(j, std::forward<TargetType>(val))))
+ -> decltype(::nlohmann::to_json(j, std::forward<TargetType>(val)), void())
+ {
+ ::nlohmann::to_json(j, std::forward<TargetType>(val));
+ }
+};
+
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/byte_container_with_subtype.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <cstdint> // uint8_t, uint64_t
+#include <tuple> // tie
+#include <utility> // move
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+
+/// @brief an internal type for a backed binary type
+/// @sa https://json.nlohmann.me/api/byte_container_with_subtype/
+template<typename BinaryType>
+class byte_container_with_subtype : public BinaryType
+{
+ public:
+ using container_type = BinaryType;
+ using subtype_type = std::uint64_t;
+
+ /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/
+ byte_container_with_subtype() noexcept(noexcept(container_type()))
+ : container_type()
+ {}
+
+ /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/
+ byte_container_with_subtype(const container_type& b) noexcept(noexcept(container_type(b)))
+ : container_type(b)
+ {}
+
+ /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/
+ byte_container_with_subtype(container_type&& b) noexcept(noexcept(container_type(std::move(b))))
+ : container_type(std::move(b))
+ {}
+
+ /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/
+ byte_container_with_subtype(const container_type& b, subtype_type subtype_) noexcept(noexcept(container_type(b)))
+ : container_type(b)
+ , m_subtype(subtype_)
+ , m_has_subtype(true)
+ {}
+
+ /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/
+ byte_container_with_subtype(container_type&& b, subtype_type subtype_) noexcept(noexcept(container_type(std::move(b))))
+ : container_type(std::move(b))
+ , m_subtype(subtype_)
+ , m_has_subtype(true)
+ {}
+
+ bool operator==(const byte_container_with_subtype& rhs) const
+ {
+ return std::tie(static_cast<const BinaryType&>(*this), m_subtype, m_has_subtype) ==
+ std::tie(static_cast<const BinaryType&>(rhs), rhs.m_subtype, rhs.m_has_subtype);
+ }
+
+ bool operator!=(const byte_container_with_subtype& rhs) const
+ {
+ return !(rhs == *this);
+ }
+
+ /// @brief sets the binary subtype
+ /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/set_subtype/
+ void set_subtype(subtype_type subtype_) noexcept
+ {
+ m_subtype = subtype_;
+ m_has_subtype = true;
+ }
+
+ /// @brief return the binary subtype
+ /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/subtype/
+ constexpr subtype_type subtype() const noexcept
+ {
+ return m_has_subtype ? m_subtype : static_cast<subtype_type>(-1);
+ }
+
+ /// @brief return whether the value has a subtype
+ /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/has_subtype/
+ constexpr bool has_subtype() const noexcept
+ {
+ return m_has_subtype;
+ }
+
+ /// @brief clears the binary subtype
+ /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/clear_subtype/
+ void clear_subtype() noexcept
+ {
+ m_subtype = 0;
+ m_has_subtype = false;
+ }
+
+ private:
+ subtype_type m_subtype = 0;
+ bool m_has_subtype = false;
+};
+
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/conversions/from_json.hpp>
+
+// #include <nlohmann/detail/conversions/to_json.hpp>
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/hash.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <cstdint> // uint8_t
+#include <cstddef> // size_t
+#include <functional> // hash
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+// boost::hash_combine
+inline std::size_t combine(std::size_t seed, std::size_t h) noexcept
+{
+ seed ^= h + 0x9e3779b9 + (seed << 6U) + (seed >> 2U);
+ return seed;
+}
+
+/*!
+@brief hash a JSON value
+
+The hash function tries to rely on std::hash where possible. Furthermore, the
+type of the JSON value is taken into account to have different hash values for
+null, 0, 0U, and false, etc.
+
+@tparam BasicJsonType basic_json specialization
+@param j JSON value to hash
+@return hash value of j
+*/
+template<typename BasicJsonType>
+std::size_t hash(const BasicJsonType& j)
+{
+ using string_t = typename BasicJsonType::string_t;
+ using number_integer_t = typename BasicJsonType::number_integer_t;
+ using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+
+ const auto type = static_cast<std::size_t>(j.type());
+ switch (j.type())
+ {
+ case BasicJsonType::value_t::null:
+ case BasicJsonType::value_t::discarded:
+ {
+ return combine(type, 0);
+ }
+
+ case BasicJsonType::value_t::object:
+ {
+ auto seed = combine(type, j.size());
+ for (const auto& element : j.items())
+ {
+ const auto h = std::hash<string_t> {}(element.key());
+ seed = combine(seed, h);
+ seed = combine(seed, hash(element.value()));
+ }
+ return seed;
+ }
+
+ case BasicJsonType::value_t::array:
+ {
+ auto seed = combine(type, j.size());
+ for (const auto& element : j)
+ {
+ seed = combine(seed, hash(element));
+ }
+ return seed;
+ }
+
+ case BasicJsonType::value_t::string:
+ {
+ const auto h = std::hash<string_t> {}(j.template get_ref<const string_t&>());
+ return combine(type, h);
+ }
+
+ case BasicJsonType::value_t::boolean:
+ {
+ const auto h = std::hash<bool> {}(j.template get<bool>());
+ return combine(type, h);
+ }
+
+ case BasicJsonType::value_t::number_integer:
+ {
+ const auto h = std::hash<number_integer_t> {}(j.template get<number_integer_t>());
+ return combine(type, h);
+ }
+
+ case BasicJsonType::value_t::number_unsigned:
+ {
+ const auto h = std::hash<number_unsigned_t> {}(j.template get<number_unsigned_t>());
+ return combine(type, h);
+ }
+
+ case BasicJsonType::value_t::number_float:
+ {
+ const auto h = std::hash<number_float_t> {}(j.template get<number_float_t>());
+ return combine(type, h);
+ }
+
+ case BasicJsonType::value_t::binary:
+ {
+ auto seed = combine(type, j.get_binary().size());
+ const auto h = std::hash<bool> {}(j.get_binary().has_subtype());
+ seed = combine(seed, h);
+ seed = combine(seed, static_cast<std::size_t>(j.get_binary().subtype()));
+ for (const auto byte : j.get_binary())
+ {
+ seed = combine(seed, std::hash<std::uint8_t> {}(byte));
+ }
+ return seed;
+ }
+
+ default: // LCOV_EXCL_LINE
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
+ return 0; // LCOV_EXCL_LINE
+ }
+}
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/input/binary_reader.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <algorithm> // generate_n
+#include <array> // array
+#include <cmath> // ldexp
+#include <cstddef> // size_t
+#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
+#include <cstdio> // snprintf
+#include <cstring> // memcpy
+#include <iterator> // back_inserter
+#include <limits> // numeric_limits
+#include <string> // char_traits, string
+#include <utility> // make_pair, move
+#include <vector> // vector
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <array> // array
+#include <cstddef> // size_t
+#include <cstring> // strlen
+#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next
+#include <memory> // shared_ptr, make_shared, addressof
+#include <numeric> // accumulate
+#include <string> // string, char_traits
+#include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer
+#include <utility> // pair, declval
+
+#ifndef JSON_NO_IO
+ #include <cstdio> // FILE *
+ #include <istream> // istream
+#endif // JSON_NO_IO
+
+// #include <nlohmann/detail/iterators/iterator_traits.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+/// the supported input formats
+enum class input_format_t { json, cbor, msgpack, ubjson, bson, bjdata };
+
+////////////////////
+// input adapters //
+////////////////////
+
+#ifndef JSON_NO_IO
+/*!
+Input adapter for stdio file access. This adapter read only 1 byte and do not use any
+ buffer. This adapter is a very low level adapter.
+*/
+class file_input_adapter
+{
+ public:
+ using char_type = char;
+
+ JSON_HEDLEY_NON_NULL(2)
+ explicit file_input_adapter(std::FILE* f) noexcept
+ : m_file(f)
+ {
+ JSON_ASSERT(m_file != nullptr);
+ }
+
+ // make class move-only
+ file_input_adapter(const file_input_adapter&) = delete;
+ file_input_adapter(file_input_adapter&&) noexcept = default;
+ file_input_adapter& operator=(const file_input_adapter&) = delete;
+ file_input_adapter& operator=(file_input_adapter&&) = delete;
+ ~file_input_adapter() = default;
+
+ std::char_traits<char>::int_type get_character() noexcept
+ {
+ return std::fgetc(m_file);
+ }
+
+ private:
+ /// the file pointer to read from
+ std::FILE* m_file;
+};
+
+/*!
+Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at
+beginning of input. Does not support changing the underlying std::streambuf
+in mid-input. Maintains underlying std::istream and std::streambuf to support
+subsequent use of standard std::istream operations to process any input
+characters following those used in parsing the JSON input. Clears the
+std::istream flags; any input errors (e.g., EOF) will be detected by the first
+subsequent call for input from the std::istream.
+*/
+class input_stream_adapter
+{
+ public:
+ using char_type = char;
+
+ ~input_stream_adapter()
+ {
+ // clear stream flags; we use underlying streambuf I/O, do not
+ // maintain ifstream flags, except eof
+ if (is != nullptr)
+ {
+ is->clear(is->rdstate() & std::ios::eofbit);
+ }
+ }
+
+ explicit input_stream_adapter(std::istream& i)
+ : is(&i), sb(i.rdbuf())
+ {}
+
+ // delete because of pointer members
+ input_stream_adapter(const input_stream_adapter&) = delete;
+ input_stream_adapter& operator=(input_stream_adapter&) = delete;
+ input_stream_adapter& operator=(input_stream_adapter&&) = delete;
+
+ input_stream_adapter(input_stream_adapter&& rhs) noexcept
+ : is(rhs.is), sb(rhs.sb)
+ {
+ rhs.is = nullptr;
+ rhs.sb = nullptr;
+ }
+
+ // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to
+ // ensure that std::char_traits<char>::eof() and the character 0xFF do not
+ // end up as the same value, e.g. 0xFFFFFFFF.
+ std::char_traits<char>::int_type get_character()
+ {
+ auto res = sb->sbumpc();
+ // set eof manually, as we don't use the istream interface.
+ if (JSON_HEDLEY_UNLIKELY(res == std::char_traits<char>::eof()))
+ {
+ is->clear(is->rdstate() | std::ios::eofbit);
+ }
+ return res;
+ }
+
+ private:
+ /// the associated input stream
+ std::istream* is = nullptr;
+ std::streambuf* sb = nullptr;
+};
+#endif // JSON_NO_IO
+
+// General-purpose iterator-based adapter. It might not be as fast as
+// theoretically possible for some containers, but it is extremely versatile.
+template<typename IteratorType>
+class iterator_input_adapter
+{
+ public:
+ using char_type = typename std::iterator_traits<IteratorType>::value_type;
+
+ iterator_input_adapter(IteratorType first, IteratorType last)
+ : current(std::move(first)), end(std::move(last))
+ {}
+
+ typename char_traits<char_type>::int_type get_character()
+ {
+ if (JSON_HEDLEY_LIKELY(current != end))
+ {
+ auto result = char_traits<char_type>::to_int_type(*current);
+ std::advance(current, 1);
+ return result;
+ }
+
+ return char_traits<char_type>::eof();
+ }
+
+ private:
+ IteratorType current;
+ IteratorType end;
+
+ template<typename BaseInputAdapter, size_t T>
+ friend struct wide_string_input_helper;
+
+ bool empty() const
+ {
+ return current == end;
+ }
+};
+
+template<typename BaseInputAdapter, size_t T>
+struct wide_string_input_helper;
+
+template<typename BaseInputAdapter>
+struct wide_string_input_helper<BaseInputAdapter, 4>
+{
+ // UTF-32
+ static void fill_buffer(BaseInputAdapter& input,
+ std::array<std::char_traits<char>::int_type, 4>& utf8_bytes,
+ size_t& utf8_bytes_index,
+ size_t& utf8_bytes_filled)
+ {
+ utf8_bytes_index = 0;
+
+ if (JSON_HEDLEY_UNLIKELY(input.empty()))
+ {
+ utf8_bytes[0] = std::char_traits<char>::eof();
+ utf8_bytes_filled = 1;
+ }
+ else
+ {
+ // get the current character
+ const auto wc = input.get_character();
+
+ // UTF-32 to UTF-8 encoding
+ if (wc < 0x80)
+ {
+ utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);
+ utf8_bytes_filled = 1;
+ }
+ else if (wc <= 0x7FF)
+ {
+ utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xC0u | ((static_cast<unsigned int>(wc) >> 6u) & 0x1Fu));
+ utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | (static_cast<unsigned int>(wc) & 0x3Fu));
+ utf8_bytes_filled = 2;
+ }
+ else if (wc <= 0xFFFF)
+ {
+ utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xE0u | ((static_cast<unsigned int>(wc) >> 12u) & 0x0Fu));
+ utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((static_cast<unsigned int>(wc) >> 6u) & 0x3Fu));
+ utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | (static_cast<unsigned int>(wc) & 0x3Fu));
+ utf8_bytes_filled = 3;
+ }
+ else if (wc <= 0x10FFFF)
+ {
+ utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xF0u | ((static_cast<unsigned int>(wc) >> 18u) & 0x07u));
+ utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((static_cast<unsigned int>(wc) >> 12u) & 0x3Fu));
+ utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | ((static_cast<unsigned int>(wc) >> 6u) & 0x3Fu));
+ utf8_bytes[3] = static_cast<std::char_traits<char>::int_type>(0x80u | (static_cast<unsigned int>(wc) & 0x3Fu));
+ utf8_bytes_filled = 4;
+ }
+ else
+ {
+ // unknown character
+ utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);
+ utf8_bytes_filled = 1;
+ }
+ }
+ }
+};
+
+template<typename BaseInputAdapter>
+struct wide_string_input_helper<BaseInputAdapter, 2>
+{
+ // UTF-16
+ static void fill_buffer(BaseInputAdapter& input,
+ std::array<std::char_traits<char>::int_type, 4>& utf8_bytes,
+ size_t& utf8_bytes_index,
+ size_t& utf8_bytes_filled)
+ {
+ utf8_bytes_index = 0;
+
+ if (JSON_HEDLEY_UNLIKELY(input.empty()))
+ {
+ utf8_bytes[0] = std::char_traits<char>::eof();
+ utf8_bytes_filled = 1;
+ }
+ else
+ {
+ // get the current character
+ const auto wc = input.get_character();
+
+ // UTF-16 to UTF-8 encoding
+ if (wc < 0x80)
+ {
+ utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);
+ utf8_bytes_filled = 1;
+ }
+ else if (wc <= 0x7FF)
+ {
+ utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xC0u | ((static_cast<unsigned int>(wc) >> 6u)));
+ utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | (static_cast<unsigned int>(wc) & 0x3Fu));
+ utf8_bytes_filled = 2;
+ }
+ else if (0xD800 > wc || wc >= 0xE000)
+ {
+ utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xE0u | ((static_cast<unsigned int>(wc) >> 12u)));
+ utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((static_cast<unsigned int>(wc) >> 6u) & 0x3Fu));
+ utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | (static_cast<unsigned int>(wc) & 0x3Fu));
+ utf8_bytes_filled = 3;
+ }
+ else
+ {
+ if (JSON_HEDLEY_UNLIKELY(!input.empty()))
+ {
+ const auto wc2 = static_cast<unsigned int>(input.get_character());
+ const auto charcode = 0x10000u + (((static_cast<unsigned int>(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu));
+ utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xF0u | (charcode >> 18u));
+ utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu));
+ utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | ((charcode >> 6u) & 0x3Fu));
+ utf8_bytes[3] = static_cast<std::char_traits<char>::int_type>(0x80u | (charcode & 0x3Fu));
+ utf8_bytes_filled = 4;
+ }
+ else
+ {
+ utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);
+ utf8_bytes_filled = 1;
+ }
+ }
+ }
+ }
+};
+
+// Wraps another input adapter to convert wide character types into individual bytes.
+template<typename BaseInputAdapter, typename WideCharType>
+class wide_string_input_adapter
+{
+ public:
+ using char_type = char;
+
+ wide_string_input_adapter(BaseInputAdapter base)
+ : base_adapter(base) {}
+
+ typename std::char_traits<char>::int_type get_character() noexcept
+ {
+ // check if buffer needs to be filled
+ if (utf8_bytes_index == utf8_bytes_filled)
+ {
+ fill_buffer<sizeof(WideCharType)>();
+
+ JSON_ASSERT(utf8_bytes_filled > 0);
+ JSON_ASSERT(utf8_bytes_index == 0);
+ }
+
+ // use buffer
+ JSON_ASSERT(utf8_bytes_filled > 0);
+ JSON_ASSERT(utf8_bytes_index < utf8_bytes_filled);
+ return utf8_bytes[utf8_bytes_index++];
+ }
+
+ private:
+ BaseInputAdapter base_adapter;
+
+ template<size_t T>
+ void fill_buffer()
+ {
+ wide_string_input_helper<BaseInputAdapter, T>::fill_buffer(base_adapter, utf8_bytes, utf8_bytes_index, utf8_bytes_filled);
+ }
+
+ /// a buffer for UTF-8 bytes
+ std::array<std::char_traits<char>::int_type, 4> utf8_bytes = {{0, 0, 0, 0}};
+
+ /// index to the utf8_codes array for the next valid byte
+ std::size_t utf8_bytes_index = 0;
+ /// number of valid bytes in the utf8_codes array
+ std::size_t utf8_bytes_filled = 0;
+};
+
+template<typename IteratorType, typename Enable = void>
+struct iterator_input_adapter_factory
+{
+ using iterator_type = IteratorType;
+ using char_type = typename std::iterator_traits<iterator_type>::value_type;
+ using adapter_type = iterator_input_adapter<iterator_type>;
+
+ static adapter_type create(IteratorType first, IteratorType last)
+ {
+ return adapter_type(std::move(first), std::move(last));
+ }
+};
+
+template<typename T>
+struct is_iterator_of_multibyte
+{
+ using value_type = typename std::iterator_traits<T>::value_type;
+ enum
+ {
+ value = sizeof(value_type) > 1
+ };
+};
+
+template<typename IteratorType>
+struct iterator_input_adapter_factory<IteratorType, enable_if_t<is_iterator_of_multibyte<IteratorType>::value>>
+{
+ using iterator_type = IteratorType;
+ using char_type = typename std::iterator_traits<iterator_type>::value_type;
+ using base_adapter_type = iterator_input_adapter<iterator_type>;
+ using adapter_type = wide_string_input_adapter<base_adapter_type, char_type>;
+
+ static adapter_type create(IteratorType first, IteratorType last)
+ {
+ return adapter_type(base_adapter_type(std::move(first), std::move(last)));
+ }
+};
+
+// General purpose iterator-based input
+template<typename IteratorType>
+typename iterator_input_adapter_factory<IteratorType>::adapter_type input_adapter(IteratorType first, IteratorType last)
+{
+ using factory_type = iterator_input_adapter_factory<IteratorType>;
+ return factory_type::create(first, last);
+}
+
+// Convenience shorthand from container to iterator
+// Enables ADL on begin(container) and end(container)
+// Encloses the using declarations in namespace for not to leak them to outside scope
+
+namespace container_input_adapter_factory_impl
+{
+
+using std::begin;
+using std::end;
+
+template<typename ContainerType, typename Enable = void>
+struct container_input_adapter_factory {};
+
+template<typename ContainerType>
+struct container_input_adapter_factory< ContainerType,
+ void_t<decltype(begin(std::declval<ContainerType>()), end(std::declval<ContainerType>()))>>
+ {
+ using adapter_type = decltype(input_adapter(begin(std::declval<ContainerType>()), end(std::declval<ContainerType>())));
+
+ static adapter_type create(const ContainerType& container)
+{
+ return input_adapter(begin(container), end(container));
+}
+ };
+
+} // namespace container_input_adapter_factory_impl
+
+template<typename ContainerType>
+typename container_input_adapter_factory_impl::container_input_adapter_factory<ContainerType>::adapter_type input_adapter(const ContainerType& container)
+{
+ return container_input_adapter_factory_impl::container_input_adapter_factory<ContainerType>::create(container);
+}
+
+#ifndef JSON_NO_IO
+// Special cases with fast paths
+inline file_input_adapter input_adapter(std::FILE* file)
+{
+ return file_input_adapter(file);
+}
+
+inline input_stream_adapter input_adapter(std::istream& stream)
+{
+ return input_stream_adapter(stream);
+}
+
+inline input_stream_adapter input_adapter(std::istream&& stream)
+{
+ return input_stream_adapter(stream);
+}
+#endif // JSON_NO_IO
+
+using contiguous_bytes_input_adapter = decltype(input_adapter(std::declval<const char*>(), std::declval<const char*>()));
+
+// Null-delimited strings, and the like.
+template < typename CharT,
+ typename std::enable_if <
+ std::is_pointer<CharT>::value&&
+ !std::is_array<CharT>::value&&
+ std::is_integral<typename std::remove_pointer<CharT>::type>::value&&
+ sizeof(typename std::remove_pointer<CharT>::type) == 1,
+ int >::type = 0 >
+contiguous_bytes_input_adapter input_adapter(CharT b)
+{
+ auto length = std::strlen(reinterpret_cast<const char*>(b));
+ const auto* ptr = reinterpret_cast<const char*>(b);
+ return input_adapter(ptr, ptr + length);
+}
+
+template<typename T, std::size_t N>
+auto input_adapter(T (&array)[N]) -> decltype(input_adapter(array, array + N)) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+{
+ return input_adapter(array, array + N);
+}
+
+// This class only handles inputs of input_buffer_adapter type.
+// It's required so that expressions like {ptr, len} can be implicitly cast
+// to the correct adapter.
+class span_input_adapter
+{
+ public:
+ template < typename CharT,
+ typename std::enable_if <
+ std::is_pointer<CharT>::value&&
+ std::is_integral<typename std::remove_pointer<CharT>::type>::value&&
+ sizeof(typename std::remove_pointer<CharT>::type) == 1,
+ int >::type = 0 >
+ span_input_adapter(CharT b, std::size_t l)
+ : ia(reinterpret_cast<const char*>(b), reinterpret_cast<const char*>(b) + l) {}
+
+ template<class IteratorType,
+ typename std::enable_if<
+ std::is_same<typename iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,
+ int>::type = 0>
+ span_input_adapter(IteratorType first, IteratorType last)
+ : ia(input_adapter(first, last)) {}
+
+ contiguous_bytes_input_adapter&& get()
+ {
+ return std::move(ia); // NOLINT(hicpp-move-const-arg,performance-move-const-arg)
+ }
+
+ private:
+ contiguous_bytes_input_adapter ia;
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/input/json_sax.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <cstddef>
+#include <string> // string
+#include <utility> // move
+#include <vector> // vector
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/string_concat.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+
+/*!
+@brief SAX interface
+
+This class describes the SAX interface used by @ref nlohmann::json::sax_parse.
+Each function is called in different situations while the input is parsed. The
+boolean return value informs the parser whether to continue processing the
+input.
+*/
+template<typename BasicJsonType>
+struct json_sax
+{
+ using number_integer_t = typename BasicJsonType::number_integer_t;
+ using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+ using string_t = typename BasicJsonType::string_t;
+ using binary_t = typename BasicJsonType::binary_t;
+
+ /*!
+ @brief a null value was read
+ @return whether parsing should proceed
+ */
+ virtual bool null() = 0;
+
+ /*!
+ @brief a boolean value was read
+ @param[in] val boolean value
+ @return whether parsing should proceed
+ */
+ virtual bool boolean(bool val) = 0;
+
+ /*!
+ @brief an integer number was read
+ @param[in] val integer value
+ @return whether parsing should proceed
+ */
+ virtual bool number_integer(number_integer_t val) = 0;
+
+ /*!
+ @brief an unsigned integer number was read
+ @param[in] val unsigned integer value
+ @return whether parsing should proceed
+ */
+ virtual bool number_unsigned(number_unsigned_t val) = 0;
+
+ /*!
+ @brief a floating-point number was read
+ @param[in] val floating-point value
+ @param[in] s raw token value
+ @return whether parsing should proceed
+ */
+ virtual bool number_float(number_float_t val, const string_t& s) = 0;
+
+ /*!
+ @brief a string value was read
+ @param[in] val string value
+ @return whether parsing should proceed
+ @note It is safe to move the passed string value.
+ */
+ virtual bool string(string_t& val) = 0;
+
+ /*!
+ @brief a binary value was read
+ @param[in] val binary value
+ @return whether parsing should proceed
+ @note It is safe to move the passed binary value.
+ */
+ virtual bool binary(binary_t& val) = 0;
+
+ /*!
+ @brief the beginning of an object was read
+ @param[in] elements number of object elements or -1 if unknown
+ @return whether parsing should proceed
+ @note binary formats may report the number of elements
+ */
+ virtual bool start_object(std::size_t elements) = 0;
+
+ /*!
+ @brief an object key was read
+ @param[in] val object key
+ @return whether parsing should proceed
+ @note It is safe to move the passed string.
+ */
+ virtual bool key(string_t& val) = 0;
+
+ /*!
+ @brief the end of an object was read
+ @return whether parsing should proceed
+ */
+ virtual bool end_object() = 0;
+
+ /*!
+ @brief the beginning of an array was read
+ @param[in] elements number of array elements or -1 if unknown
+ @return whether parsing should proceed
+ @note binary formats may report the number of elements
+ */
+ virtual bool start_array(std::size_t elements) = 0;
+
+ /*!
+ @brief the end of an array was read
+ @return whether parsing should proceed
+ */
+ virtual bool end_array() = 0;
+
+ /*!
+ @brief a parse error occurred
+ @param[in] position the position in the input where the error occurs
+ @param[in] last_token the last read token
+ @param[in] ex an exception object describing the error
+ @return whether parsing should proceed (must return false)
+ */
+ virtual bool parse_error(std::size_t position,
+ const std::string& last_token,
+ const detail::exception& ex) = 0;
+
+ json_sax() = default;
+ json_sax(const json_sax&) = default;
+ json_sax(json_sax&&) noexcept = default;
+ json_sax& operator=(const json_sax&) = default;
+ json_sax& operator=(json_sax&&) noexcept = default;
+ virtual ~json_sax() = default;
+};
+
+namespace detail
+{
+/*!
+@brief SAX implementation to create a JSON value from SAX events
+
+This class implements the @ref json_sax interface and processes the SAX events
+to create a JSON value which makes it basically a DOM parser. The structure or
+hierarchy of the JSON value is managed by the stack `ref_stack` which contains
+a pointer to the respective array or object for each recursion depth.
+
+After successful parsing, the value that is passed by reference to the
+constructor contains the parsed value.
+
+@tparam BasicJsonType the JSON type
+*/
+template<typename BasicJsonType>
+class json_sax_dom_parser
+{
+ public:
+ using number_integer_t = typename BasicJsonType::number_integer_t;
+ using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+ using string_t = typename BasicJsonType::string_t;
+ using binary_t = typename BasicJsonType::binary_t;
+
+ /*!
+ @param[in,out] r reference to a JSON value that is manipulated while
+ parsing
+ @param[in] allow_exceptions_ whether parse errors yield exceptions
+ */
+ explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true)
+ : root(r), allow_exceptions(allow_exceptions_)
+ {}
+
+ // make class move-only
+ json_sax_dom_parser(const json_sax_dom_parser&) = delete;
+ json_sax_dom_parser(json_sax_dom_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)
+ json_sax_dom_parser& operator=(const json_sax_dom_parser&) = delete;
+ json_sax_dom_parser& operator=(json_sax_dom_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)
+ ~json_sax_dom_parser() = default;
+
+ bool null()
+ {
+ handle_value(nullptr);
+ return true;
+ }
+
+ bool boolean(bool val)
+ {
+ handle_value(val);
+ return true;
+ }
+
+ bool number_integer(number_integer_t val)
+ {
+ handle_value(val);
+ return true;
+ }
+
+ bool number_unsigned(number_unsigned_t val)
+ {
+ handle_value(val);
+ return true;
+ }
+
+ bool number_float(number_float_t val, const string_t& /*unused*/)
+ {
+ handle_value(val);
+ return true;
+ }
+
+ bool string(string_t& val)
+ {
+ handle_value(val);
+ return true;
+ }
+
+ bool binary(binary_t& val)
+ {
+ handle_value(std::move(val));
+ return true;
+ }
+
+ bool start_object(std::size_t len)
+ {
+ ref_stack.push_back(handle_value(BasicJsonType::value_t::object));
+
+ if (JSON_HEDLEY_UNLIKELY(len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size()))
+ {
+ JSON_THROW(out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back()));
+ }
+
+ return true;
+ }
+
+ bool key(string_t& val)
+ {
+ JSON_ASSERT(!ref_stack.empty());
+ JSON_ASSERT(ref_stack.back()->is_object());
+
+ // add null at given key and store the reference for later
+ object_element = &(ref_stack.back()->m_data.m_value.object->operator[](val));
+ return true;
+ }
+
+ bool end_object()
+ {
+ JSON_ASSERT(!ref_stack.empty());
+ JSON_ASSERT(ref_stack.back()->is_object());
+
+ ref_stack.back()->set_parents();
+ ref_stack.pop_back();
+ return true;
+ }
+
+ bool start_array(std::size_t len)
+ {
+ ref_stack.push_back(handle_value(BasicJsonType::value_t::array));
+
+ if (JSON_HEDLEY_UNLIKELY(len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size()))
+ {
+ JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
+ }
+
+ return true;
+ }
+
+ bool end_array()
+ {
+ JSON_ASSERT(!ref_stack.empty());
+ JSON_ASSERT(ref_stack.back()->is_array());
+
+ ref_stack.back()->set_parents();
+ ref_stack.pop_back();
+ return true;
+ }
+
+ template<class Exception>
+ bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,
+ const Exception& ex)
+ {
+ errored = true;
+ static_cast<void>(ex);
+ if (allow_exceptions)
+ {
+ JSON_THROW(ex);
+ }
+ return false;
+ }
+
+ constexpr bool is_errored() const
+ {
+ return errored;
+ }
+
+ private:
+ /*!
+ @invariant If the ref stack is empty, then the passed value will be the new
+ root.
+ @invariant If the ref stack contains a value, then it is an array or an
+ object to which we can add elements
+ */
+ template<typename Value>
+ JSON_HEDLEY_RETURNS_NON_NULL
+ BasicJsonType* handle_value(Value&& v)
+ {
+ if (ref_stack.empty())
+ {
+ root = BasicJsonType(std::forward<Value>(v));
+ return &root;
+ }
+
+ JSON_ASSERT(ref_stack.back()->is_array() || ref_stack.back()->is_object());
+
+ if (ref_stack.back()->is_array())
+ {
+ ref_stack.back()->m_data.m_value.array->emplace_back(std::forward<Value>(v));
+ return &(ref_stack.back()->m_data.m_value.array->back());
+ }
+
+ JSON_ASSERT(ref_stack.back()->is_object());
+ JSON_ASSERT(object_element);
+ *object_element = BasicJsonType(std::forward<Value>(v));
+ return object_element;
+ }
+
+ /// the parsed JSON value
+ BasicJsonType& root;
+ /// stack to model hierarchy of values
+ std::vector<BasicJsonType*> ref_stack {};
+ /// helper to hold the reference for the next object element
+ BasicJsonType* object_element = nullptr;
+ /// whether a syntax error occurred
+ bool errored = false;
+ /// whether to throw exceptions in case of errors
+ const bool allow_exceptions = true;
+};
+
+template<typename BasicJsonType>
+class json_sax_dom_callback_parser
+{
+ public:
+ using number_integer_t = typename BasicJsonType::number_integer_t;
+ using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+ using string_t = typename BasicJsonType::string_t;
+ using binary_t = typename BasicJsonType::binary_t;
+ using parser_callback_t = typename BasicJsonType::parser_callback_t;
+ using parse_event_t = typename BasicJsonType::parse_event_t;
+
+ json_sax_dom_callback_parser(BasicJsonType& r,
+ const parser_callback_t cb,
+ const bool allow_exceptions_ = true)
+ : root(r), callback(cb), allow_exceptions(allow_exceptions_)
+ {
+ keep_stack.push_back(true);
+ }
+
+ // make class move-only
+ json_sax_dom_callback_parser(const json_sax_dom_callback_parser&) = delete;
+ json_sax_dom_callback_parser(json_sax_dom_callback_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)
+ json_sax_dom_callback_parser& operator=(const json_sax_dom_callback_parser&) = delete;
+ json_sax_dom_callback_parser& operator=(json_sax_dom_callback_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)
+ ~json_sax_dom_callback_parser() = default;
+
+ bool null()
+ {
+ handle_value(nullptr);
+ return true;
+ }
+
+ bool boolean(bool val)
+ {
+ handle_value(val);
+ return true;
+ }
+
+ bool number_integer(number_integer_t val)
+ {
+ handle_value(val);
+ return true;
+ }
+
+ bool number_unsigned(number_unsigned_t val)
+ {
+ handle_value(val);
+ return true;
+ }
+
+ bool number_float(number_float_t val, const string_t& /*unused*/)
+ {
+ handle_value(val);
+ return true;
+ }
+
+ bool string(string_t& val)
+ {
+ handle_value(val);
+ return true;
+ }
+
+ bool binary(binary_t& val)
+ {
+ handle_value(std::move(val));
+ return true;
+ }
+
+ bool start_object(std::size_t len)
+ {
+ // check callback for object start
+ const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded);
+ keep_stack.push_back(keep);
+
+ auto val = handle_value(BasicJsonType::value_t::object, true);
+ ref_stack.push_back(val.second);
+
+ // check object limit
+ if (ref_stack.back() && JSON_HEDLEY_UNLIKELY(len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size()))
+ {
+ JSON_THROW(out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back()));
+ }
+
+ return true;
+ }
+
+ bool key(string_t& val)
+ {
+ BasicJsonType k = BasicJsonType(val);
+
+ // check callback for key
+ const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k);
+ key_keep_stack.push_back(keep);
+
+ // add discarded value at given key and store the reference for later
+ if (keep && ref_stack.back())
+ {
+ object_element = &(ref_stack.back()->m_data.m_value.object->operator[](val) = discarded);
+ }
+
+ return true;
+ }
+
+ bool end_object()
+ {
+ if (ref_stack.back())
+ {
+ if (!callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back()))
+ {
+ // discard object
+ *ref_stack.back() = discarded;
+ }
+ else
+ {
+ ref_stack.back()->set_parents();
+ }
+ }
+
+ JSON_ASSERT(!ref_stack.empty());
+ JSON_ASSERT(!keep_stack.empty());
+ ref_stack.pop_back();
+ keep_stack.pop_back();
+
+ if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured())
+ {
+ // remove discarded value
+ for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it)
+ {
+ if (it->is_discarded())
+ {
+ ref_stack.back()->erase(it);
+ break;
+ }
+ }
+ }
+
+ return true;
+ }
+
+ bool start_array(std::size_t len)
+ {
+ const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded);
+ keep_stack.push_back(keep);
+
+ auto val = handle_value(BasicJsonType::value_t::array, true);
+ ref_stack.push_back(val.second);
+
+ // check array limit
+ if (ref_stack.back() && JSON_HEDLEY_UNLIKELY(len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size()))
+ {
+ JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
+ }
+
+ return true;
+ }
+
+ bool end_array()
+ {
+ bool keep = true;
+
+ if (ref_stack.back())
+ {
+ keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back());
+ if (keep)
+ {
+ ref_stack.back()->set_parents();
+ }
+ else
+ {
+ // discard array
+ *ref_stack.back() = discarded;
+ }
+ }
+
+ JSON_ASSERT(!ref_stack.empty());
+ JSON_ASSERT(!keep_stack.empty());
+ ref_stack.pop_back();
+ keep_stack.pop_back();
+
+ // remove discarded value
+ if (!keep && !ref_stack.empty() && ref_stack.back()->is_array())
+ {
+ ref_stack.back()->m_data.m_value.array->pop_back();
+ }
+
+ return true;
+ }
+
+ template<class Exception>
+ bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,
+ const Exception& ex)
+ {
+ errored = true;
+ static_cast<void>(ex);
+ if (allow_exceptions)
+ {
+ JSON_THROW(ex);
+ }
+ return false;
+ }
+
+ constexpr bool is_errored() const
+ {
+ return errored;
+ }
+
+ private:
+ /*!
+ @param[in] v value to add to the JSON value we build during parsing
+ @param[in] skip_callback whether we should skip calling the callback
+ function; this is required after start_array() and
+ start_object() SAX events, because otherwise we would call the
+ callback function with an empty array or object, respectively.
+
+ @invariant If the ref stack is empty, then the passed value will be the new
+ root.
+ @invariant If the ref stack contains a value, then it is an array or an
+ object to which we can add elements
+
+ @return pair of boolean (whether value should be kept) and pointer (to the
+ passed value in the ref_stack hierarchy; nullptr if not kept)
+ */
+ template<typename Value>
+ std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false)
+ {
+ JSON_ASSERT(!keep_stack.empty());
+
+ // do not handle this value if we know it would be added to a discarded
+ // container
+ if (!keep_stack.back())
+ {
+ return {false, nullptr};
+ }
+
+ // create value
+ auto value = BasicJsonType(std::forward<Value>(v));
+
+ // check callback
+ const bool keep = skip_callback || callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value);
+
+ // do not handle this value if we just learnt it shall be discarded
+ if (!keep)
+ {
+ return {false, nullptr};
+ }
+
+ if (ref_stack.empty())
+ {
+ root = std::move(value);
+ return {true, & root};
+ }
+
+ // skip this value if we already decided to skip the parent
+ // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360)
+ if (!ref_stack.back())
+ {
+ return {false, nullptr};
+ }
+
+ // we now only expect arrays and objects
+ JSON_ASSERT(ref_stack.back()->is_array() || ref_stack.back()->is_object());
+
+ // array
+ if (ref_stack.back()->is_array())
+ {
+ ref_stack.back()->m_data.m_value.array->emplace_back(std::move(value));
+ return {true, & (ref_stack.back()->m_data.m_value.array->back())};
+ }
+
+ // object
+ JSON_ASSERT(ref_stack.back()->is_object());
+ // check if we should store an element for the current key
+ JSON_ASSERT(!key_keep_stack.empty());
+ const bool store_element = key_keep_stack.back();
+ key_keep_stack.pop_back();
+
+ if (!store_element)
+ {
+ return {false, nullptr};
+ }
+
+ JSON_ASSERT(object_element);
+ *object_element = std::move(value);
+ return {true, object_element};
+ }
+
+ /// the parsed JSON value
+ BasicJsonType& root;
+ /// stack to model hierarchy of values
+ std::vector<BasicJsonType*> ref_stack {};
+ /// stack to manage which values to keep
+ std::vector<bool> keep_stack {}; // NOLINT(readability-redundant-member-init)
+ /// stack to manage which object keys to keep
+ std::vector<bool> key_keep_stack {}; // NOLINT(readability-redundant-member-init)
+ /// helper to hold the reference for the next object element
+ BasicJsonType* object_element = nullptr;
+ /// whether a syntax error occurred
+ bool errored = false;
+ /// callback function
+ const parser_callback_t callback = nullptr;
+ /// whether to throw exceptions in case of errors
+ const bool allow_exceptions = true;
+ /// a discarded value for the callback
+ BasicJsonType discarded = BasicJsonType::value_t::discarded;
+};
+
+template<typename BasicJsonType>
+class json_sax_acceptor
+{
+ public:
+ using number_integer_t = typename BasicJsonType::number_integer_t;
+ using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+ using string_t = typename BasicJsonType::string_t;
+ using binary_t = typename BasicJsonType::binary_t;
+
+ bool null()
+ {
+ return true;
+ }
+
+ bool boolean(bool /*unused*/)
+ {
+ return true;
+ }
+
+ bool number_integer(number_integer_t /*unused*/)
+ {
+ return true;
+ }
+
+ bool number_unsigned(number_unsigned_t /*unused*/)
+ {
+ return true;
+ }
+
+ bool number_float(number_float_t /*unused*/, const string_t& /*unused*/)
+ {
+ return true;
+ }
+
+ bool string(string_t& /*unused*/)
+ {
+ return true;
+ }
+
+ bool binary(binary_t& /*unused*/)
+ {
+ return true;
+ }
+
+ bool start_object(std::size_t /*unused*/ = static_cast<std::size_t>(-1))
+ {
+ return true;
+ }
+
+ bool key(string_t& /*unused*/)
+ {
+ return true;
+ }
+
+ bool end_object()
+ {
+ return true;
+ }
+
+ bool start_array(std::size_t /*unused*/ = static_cast<std::size_t>(-1))
+ {
+ return true;
+ }
+
+ bool end_array()
+ {
+ return true;
+ }
+
+ bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/)
+ {
+ return false;
+ }
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/input/lexer.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <array> // array
+#include <clocale> // localeconv
+#include <cstddef> // size_t
+#include <cstdio> // snprintf
+#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull
+#include <initializer_list> // initializer_list
+#include <string> // char_traits, string
+#include <utility> // move
+#include <vector> // vector
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+// #include <nlohmann/detail/input/position_t.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+///////////
+// lexer //
+///////////
+
+template<typename BasicJsonType>
+class lexer_base
+{
+ public:
+ /// token types for the parser
+ enum class token_type
+ {
+ uninitialized, ///< indicating the scanner is uninitialized
+ literal_true, ///< the `true` literal
+ literal_false, ///< the `false` literal
+ literal_null, ///< the `null` literal
+ value_string, ///< a string -- use get_string() for actual value
+ value_unsigned, ///< an unsigned integer -- use get_number_unsigned() for actual value
+ value_integer, ///< a signed integer -- use get_number_integer() for actual value
+ value_float, ///< an floating point number -- use get_number_float() for actual value
+ begin_array, ///< the character for array begin `[`
+ begin_object, ///< the character for object begin `{`
+ end_array, ///< the character for array end `]`
+ end_object, ///< the character for object end `}`
+ name_separator, ///< the name separator `:`
+ value_separator, ///< the value separator `,`
+ parse_error, ///< indicating a parse error
+ end_of_input, ///< indicating the end of the input buffer
+ literal_or_value ///< a literal or the begin of a value (only for diagnostics)
+ };
+
+ /// return name of values of type token_type (only used for errors)
+ JSON_HEDLEY_RETURNS_NON_NULL
+ JSON_HEDLEY_CONST
+ static const char* token_type_name(const token_type t) noexcept
+ {
+ switch (t)
+ {
+ case token_type::uninitialized:
+ return "<uninitialized>";
+ case token_type::literal_true:
+ return "true literal";
+ case token_type::literal_false:
+ return "false literal";
+ case token_type::literal_null:
+ return "null literal";
+ case token_type::value_string:
+ return "string literal";
+ case token_type::value_unsigned:
+ case token_type::value_integer:
+ case token_type::value_float:
+ return "number literal";
+ case token_type::begin_array:
+ return "'['";
+ case token_type::begin_object:
+ return "'{'";
+ case token_type::end_array:
+ return "']'";
+ case token_type::end_object:
+ return "'}'";
+ case token_type::name_separator:
+ return "':'";
+ case token_type::value_separator:
+ return "','";
+ case token_type::parse_error:
+ return "<parse error>";
+ case token_type::end_of_input:
+ return "end of input";
+ case token_type::literal_or_value:
+ return "'[', '{', or a literal";
+ // LCOV_EXCL_START
+ default: // catch non-enum values
+ return "unknown token";
+ // LCOV_EXCL_STOP
+ }
+ }
+};
+/*!
+@brief lexical analysis
+
+This class organizes the lexical analysis during JSON deserialization.
+*/
+template<typename BasicJsonType, typename InputAdapterType>
+class lexer : public lexer_base<BasicJsonType>
+{
+ using number_integer_t = typename BasicJsonType::number_integer_t;
+ using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+ using string_t = typename BasicJsonType::string_t;
+ using char_type = typename InputAdapterType::char_type;
+ using char_int_type = typename char_traits<char_type>::int_type;
+
+ public:
+ using token_type = typename lexer_base<BasicJsonType>::token_type;
+
+ explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept
+ : ia(std::move(adapter))
+ , ignore_comments(ignore_comments_)
+ , decimal_point_char(static_cast<char_int_type>(get_decimal_point()))
+ {}
+
+ // delete because of pointer members
+ lexer(const lexer&) = delete;
+ lexer(lexer&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)
+ lexer& operator=(lexer&) = delete;
+ lexer& operator=(lexer&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)
+ ~lexer() = default;
+
+ private:
+ /////////////////////
+ // locales
+ /////////////////////
+
+ /// return the locale-dependent decimal point
+ JSON_HEDLEY_PURE
+ static char get_decimal_point() noexcept
+ {
+ const auto* loc = localeconv();
+ JSON_ASSERT(loc != nullptr);
+ return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point);
+ }
+
+ /////////////////////
+ // scan functions
+ /////////////////////
+
+ /*!
+ @brief get codepoint from 4 hex characters following `\u`
+
+ For input "\u c1 c2 c3 c4" the codepoint is:
+ (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4
+ = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0)
+
+ Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f'
+ must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The
+ conversion is done by subtracting the offset (0x30, 0x37, and 0x57)
+ between the ASCII value of the character and the desired integer value.
+
+ @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or
+ non-hex character)
+ */
+ int get_codepoint()
+ {
+ // this function only makes sense after reading `\u`
+ JSON_ASSERT(current == 'u');
+ int codepoint = 0;
+
+ const auto factors = { 12u, 8u, 4u, 0u };
+ for (const auto factor : factors)
+ {
+ get();
+
+ if (current >= '0' && current <= '9')
+ {
+ codepoint += static_cast<int>((static_cast<unsigned int>(current) - 0x30u) << factor);
+ }
+ else if (current >= 'A' && current <= 'F')
+ {
+ codepoint += static_cast<int>((static_cast<unsigned int>(current) - 0x37u) << factor);
+ }
+ else if (current >= 'a' && current <= 'f')
+ {
+ codepoint += static_cast<int>((static_cast<unsigned int>(current) - 0x57u) << factor);
+ }
+ else
+ {
+ return -1;
+ }
+ }
+
+ JSON_ASSERT(0x0000 <= codepoint && codepoint <= 0xFFFF);
+ return codepoint;
+ }
+
+ /*!
+ @brief check if the next byte(s) are inside a given range
+
+ Adds the current byte and, for each passed range, reads a new byte and
+ checks if it is inside the range. If a violation was detected, set up an
+ error message and return false. Otherwise, return true.
+
+ @param[in] ranges list of integers; interpreted as list of pairs of
+ inclusive lower and upper bound, respectively
+
+ @pre The passed list @a ranges must have 2, 4, or 6 elements; that is,
+ 1, 2, or 3 pairs. This precondition is enforced by an assertion.
+
+ @return true if and only if no range violation was detected
+ */
+ bool next_byte_in_range(std::initializer_list<char_int_type> ranges)
+ {
+ JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);
+ add(current);
+
+ for (auto range = ranges.begin(); range != ranges.end(); ++range)
+ {
+ get();
+ if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions)
+ {
+ add(current);
+ }
+ else
+ {
+ error_message = "invalid string: ill-formed UTF-8 byte";
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /*!
+ @brief scan a string literal
+
+ This function scans a string according to Sect. 7 of RFC 8259. While
+ scanning, bytes are escaped and copied into buffer token_buffer. Then the
+ function returns successfully, token_buffer is *not* null-terminated (as it
+ may contain \0 bytes), and token_buffer.size() is the number of bytes in the
+ string.
+
+ @return token_type::value_string if string could be successfully scanned,
+ token_type::parse_error otherwise
+
+ @note In case of errors, variable error_message contains a textual
+ description.
+ */
+ token_type scan_string()
+ {
+ // reset token_buffer (ignore opening quote)
+ reset();
+
+ // we entered the function by reading an open quote
+ JSON_ASSERT(current == '\"');
+
+ while (true)
+ {
+ // get next character
+ switch (get())
+ {
+ // end of file while parsing string
+ case char_traits<char_type>::eof():
+ {
+ error_message = "invalid string: missing closing quote";
+ return token_type::parse_error;
+ }
+
+ // closing quote
+ case '\"':
+ {
+ return token_type::value_string;
+ }
+
+ // escapes
+ case '\\':
+ {
+ switch (get())
+ {
+ // quotation mark
+ case '\"':
+ add('\"');
+ break;
+ // reverse solidus
+ case '\\':
+ add('\\');
+ break;
+ // solidus
+ case '/':
+ add('/');
+ break;
+ // backspace
+ case 'b':
+ add('\b');
+ break;
+ // form feed
+ case 'f':
+ add('\f');
+ break;
+ // line feed
+ case 'n':
+ add('\n');
+ break;
+ // carriage return
+ case 'r':
+ add('\r');
+ break;
+ // tab
+ case 't':
+ add('\t');
+ break;
+
+ // unicode escapes
+ case 'u':
+ {
+ const int codepoint1 = get_codepoint();
+ int codepoint = codepoint1; // start with codepoint1
+
+ if (JSON_HEDLEY_UNLIKELY(codepoint1 == -1))
+ {
+ error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+ return token_type::parse_error;
+ }
+
+ // check if code point is a high surrogate
+ if (0xD800 <= codepoint1 && codepoint1 <= 0xDBFF)
+ {
+ // expect next \uxxxx entry
+ if (JSON_HEDLEY_LIKELY(get() == '\\' && get() == 'u'))
+ {
+ const int codepoint2 = get_codepoint();
+
+ if (JSON_HEDLEY_UNLIKELY(codepoint2 == -1))
+ {
+ error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+ return token_type::parse_error;
+ }
+
+ // check if codepoint2 is a low surrogate
+ if (JSON_HEDLEY_LIKELY(0xDC00 <= codepoint2 && codepoint2 <= 0xDFFF))
+ {
+ // overwrite codepoint
+ codepoint = static_cast<int>(
+ // high surrogate occupies the most significant 22 bits
+ (static_cast<unsigned int>(codepoint1) << 10u)
+ // low surrogate occupies the least significant 15 bits
+ + static_cast<unsigned int>(codepoint2)
+ // there is still the 0xD800, 0xDC00 and 0x10000 noise
+ // in the result, so we have to subtract with:
+ // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00
+ - 0x35FDC00u);
+ }
+ else
+ {
+ error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF";
+ return token_type::parse_error;
+ }
+ }
+ else
+ {
+ error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF";
+ return token_type::parse_error;
+ }
+ }
+ else
+ {
+ if (JSON_HEDLEY_UNLIKELY(0xDC00 <= codepoint1 && codepoint1 <= 0xDFFF))
+ {
+ error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF";
+ return token_type::parse_error;
+ }
+ }
+
+ // result of the above calculation yields a proper codepoint
+ JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);
+
+ // translate codepoint into bytes
+ if (codepoint < 0x80)
+ {
+ // 1-byte characters: 0xxxxxxx (ASCII)
+ add(static_cast<char_int_type>(codepoint));
+ }
+ else if (codepoint <= 0x7FF)
+ {
+ // 2-byte characters: 110xxxxx 10xxxxxx
+ add(static_cast<char_int_type>(0xC0u | (static_cast<unsigned int>(codepoint) >> 6u)));
+ add(static_cast<char_int_type>(0x80u | (static_cast<unsigned int>(codepoint) & 0x3Fu)));
+ }
+ else if (codepoint <= 0xFFFF)
+ {
+ // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx
+ add(static_cast<char_int_type>(0xE0u | (static_cast<unsigned int>(codepoint) >> 12u)));
+ add(static_cast<char_int_type>(0x80u | ((static_cast<unsigned int>(codepoint) >> 6u) & 0x3Fu)));
+ add(static_cast<char_int_type>(0x80u | (static_cast<unsigned int>(codepoint) & 0x3Fu)));
+ }
+ else
+ {
+ // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+ add(static_cast<char_int_type>(0xF0u | (static_cast<unsigned int>(codepoint) >> 18u)));
+ add(static_cast<char_int_type>(0x80u | ((static_cast<unsigned int>(codepoint) >> 12u) & 0x3Fu)));
+ add(static_cast<char_int_type>(0x80u | ((static_cast<unsigned int>(codepoint) >> 6u) & 0x3Fu)));
+ add(static_cast<char_int_type>(0x80u | (static_cast<unsigned int>(codepoint) & 0x3Fu)));
+ }
+
+ break;
+ }
+
+ // other characters after escape
+ default:
+ error_message = "invalid string: forbidden character after backslash";
+ return token_type::parse_error;
+ }
+
+ break;
+ }
+
+ // invalid control characters
+ case 0x00:
+ {
+ error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000";
+ return token_type::parse_error;
+ }
+
+ case 0x01:
+ {
+ error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001";
+ return token_type::parse_error;
+ }
+
+ case 0x02:
+ {
+ error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002";
+ return token_type::parse_error;
+ }
+
+ case 0x03:
+ {
+ error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003";
+ return token_type::parse_error;
+ }
+
+ case 0x04:
+ {
+ error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004";
+ return token_type::parse_error;
+ }
+
+ case 0x05:
+ {
+ error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005";
+ return token_type::parse_error;
+ }
+
+ case 0x06:
+ {
+ error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006";
+ return token_type::parse_error;
+ }
+
+ case 0x07:
+ {
+ error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007";
+ return token_type::parse_error;
+ }
+
+ case 0x08:
+ {
+ error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b";
+ return token_type::parse_error;
+ }
+
+ case 0x09:
+ {
+ error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t";
+ return token_type::parse_error;
+ }
+
+ case 0x0A:
+ {
+ error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n";
+ return token_type::parse_error;
+ }
+
+ case 0x0B:
+ {
+ error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B";
+ return token_type::parse_error;
+ }
+
+ case 0x0C:
+ {
+ error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f";
+ return token_type::parse_error;
+ }
+
+ case 0x0D:
+ {
+ error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r";
+ return token_type::parse_error;
+ }
+
+ case 0x0E:
+ {
+ error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E";
+ return token_type::parse_error;
+ }
+
+ case 0x0F:
+ {
+ error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F";
+ return token_type::parse_error;
+ }
+
+ case 0x10:
+ {
+ error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010";
+ return token_type::parse_error;
+ }
+
+ case 0x11:
+ {
+ error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011";
+ return token_type::parse_error;
+ }
+
+ case 0x12:
+ {
+ error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012";
+ return token_type::parse_error;
+ }
+
+ case 0x13:
+ {
+ error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013";
+ return token_type::parse_error;
+ }
+
+ case 0x14:
+ {
+ error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014";
+ return token_type::parse_error;
+ }
+
+ case 0x15:
+ {
+ error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015";
+ return token_type::parse_error;
+ }
+
+ case 0x16:
+ {
+ error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016";
+ return token_type::parse_error;
+ }
+
+ case 0x17:
+ {
+ error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017";
+ return token_type::parse_error;
+ }
+
+ case 0x18:
+ {
+ error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018";
+ return token_type::parse_error;
+ }
+
+ case 0x19:
+ {
+ error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019";
+ return token_type::parse_error;
+ }
+
+ case 0x1A:
+ {
+ error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A";
+ return token_type::parse_error;
+ }
+
+ case 0x1B:
+ {
+ error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B";
+ return token_type::parse_error;
+ }
+
+ case 0x1C:
+ {
+ error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C";
+ return token_type::parse_error;
+ }
+
+ case 0x1D:
+ {
+ error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D";
+ return token_type::parse_error;
+ }
+
+ case 0x1E:
+ {
+ error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E";
+ return token_type::parse_error;
+ }
+
+ case 0x1F:
+ {
+ error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F";
+ return token_type::parse_error;
+ }
+
+ // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace))
+ case 0x20:
+ case 0x21:
+ case 0x23:
+ case 0x24:
+ case 0x25:
+ case 0x26:
+ case 0x27:
+ case 0x28:
+ case 0x29:
+ case 0x2A:
+ case 0x2B:
+ case 0x2C:
+ case 0x2D:
+ case 0x2E:
+ case 0x2F:
+ case 0x30:
+ case 0x31:
+ case 0x32:
+ case 0x33:
+ case 0x34:
+ case 0x35:
+ case 0x36:
+ case 0x37:
+ case 0x38:
+ case 0x39:
+ case 0x3A:
+ case 0x3B:
+ case 0x3C:
+ case 0x3D:
+ case 0x3E:
+ case 0x3F:
+ case 0x40:
+ case 0x41:
+ case 0x42:
+ case 0x43:
+ case 0x44:
+ case 0x45:
+ case 0x46:
+ case 0x47:
+ case 0x48:
+ case 0x49:
+ case 0x4A:
+ case 0x4B:
+ case 0x4C:
+ case 0x4D:
+ case 0x4E:
+ case 0x4F:
+ case 0x50:
+ case 0x51:
+ case 0x52:
+ case 0x53:
+ case 0x54:
+ case 0x55:
+ case 0x56:
+ case 0x57:
+ case 0x58:
+ case 0x59:
+ case 0x5A:
+ case 0x5B:
+ case 0x5D:
+ case 0x5E:
+ case 0x5F:
+ case 0x60:
+ case 0x61:
+ case 0x62:
+ case 0x63:
+ case 0x64:
+ case 0x65:
+ case 0x66:
+ case 0x67:
+ case 0x68:
+ case 0x69:
+ case 0x6A:
+ case 0x6B:
+ case 0x6C:
+ case 0x6D:
+ case 0x6E:
+ case 0x6F:
+ case 0x70:
+ case 0x71:
+ case 0x72:
+ case 0x73:
+ case 0x74:
+ case 0x75:
+ case 0x76:
+ case 0x77:
+ case 0x78:
+ case 0x79:
+ case 0x7A:
+ case 0x7B:
+ case 0x7C:
+ case 0x7D:
+ case 0x7E:
+ case 0x7F:
+ {
+ add(current);
+ break;
+ }
+
+ // U+0080..U+07FF: bytes C2..DF 80..BF
+ case 0xC2:
+ case 0xC3:
+ case 0xC4:
+ case 0xC5:
+ case 0xC6:
+ case 0xC7:
+ case 0xC8:
+ case 0xC9:
+ case 0xCA:
+ case 0xCB:
+ case 0xCC:
+ case 0xCD:
+ case 0xCE:
+ case 0xCF:
+ case 0xD0:
+ case 0xD1:
+ case 0xD2:
+ case 0xD3:
+ case 0xD4:
+ case 0xD5:
+ case 0xD6:
+ case 0xD7:
+ case 0xD8:
+ case 0xD9:
+ case 0xDA:
+ case 0xDB:
+ case 0xDC:
+ case 0xDD:
+ case 0xDE:
+ case 0xDF:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF})))
+ {
+ return token_type::parse_error;
+ }
+ break;
+ }
+
+ // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
+ case 0xE0:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
+ {
+ return token_type::parse_error;
+ }
+ break;
+ }
+
+ // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF
+ // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF
+ case 0xE1:
+ case 0xE2:
+ case 0xE3:
+ case 0xE4:
+ case 0xE5:
+ case 0xE6:
+ case 0xE7:
+ case 0xE8:
+ case 0xE9:
+ case 0xEA:
+ case 0xEB:
+ case 0xEC:
+ case 0xEE:
+ case 0xEF:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
+ {
+ return token_type::parse_error;
+ }
+ break;
+ }
+
+ // U+D000..U+D7FF: bytes ED 80..9F 80..BF
+ case 0xED:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
+ {
+ return token_type::parse_error;
+ }
+ break;
+ }
+
+ // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
+ case 0xF0:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+ {
+ return token_type::parse_error;
+ }
+ break;
+ }
+
+ // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
+ case 0xF1:
+ case 0xF2:
+ case 0xF3:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+ {
+ return token_type::parse_error;
+ }
+ break;
+ }
+
+ // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
+ case 0xF4:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
+ {
+ return token_type::parse_error;
+ }
+ break;
+ }
+
+ // remaining bytes (80..C1 and F5..FF) are ill-formed
+ default:
+ {
+ error_message = "invalid string: ill-formed UTF-8 byte";
+ return token_type::parse_error;
+ }
+ }
+ }
+ }
+
+ /*!
+ * @brief scan a comment
+ * @return whether comment could be scanned successfully
+ */
+ bool scan_comment()
+ {
+ switch (get())
+ {
+ // single-line comments skip input until a newline or EOF is read
+ case '/':
+ {
+ while (true)
+ {
+ switch (get())
+ {
+ case '\n':
+ case '\r':
+ case char_traits<char_type>::eof():
+ case '\0':
+ return true;
+
+ default:
+ break;
+ }
+ }
+ }
+
+ // multi-line comments skip input until */ is read
+ case '*':
+ {
+ while (true)
+ {
+ switch (get())
+ {
+ case char_traits<char_type>::eof():
+ case '\0':
+ {
+ error_message = "invalid comment; missing closing '*/'";
+ return false;
+ }
+
+ case '*':
+ {
+ switch (get())
+ {
+ case '/':
+ return true;
+
+ default:
+ {
+ unget();
+ continue;
+ }
+ }
+ }
+
+ default:
+ continue;
+ }
+ }
+ }
+
+ // unexpected character after reading '/'
+ default:
+ {
+ error_message = "invalid comment; expecting '/' or '*' after '/'";
+ return false;
+ }
+ }
+ }
+
+ JSON_HEDLEY_NON_NULL(2)
+ static void strtof(float& f, const char* str, char** endptr) noexcept
+ {
+ f = std::strtof(str, endptr);
+ }
+
+ JSON_HEDLEY_NON_NULL(2)
+ static void strtof(double& f, const char* str, char** endptr) noexcept
+ {
+ f = std::strtod(str, endptr);
+ }
+
+ JSON_HEDLEY_NON_NULL(2)
+ static void strtof(long double& f, const char* str, char** endptr) noexcept
+ {
+ f = std::strtold(str, endptr);
+ }
+
+ /*!
+ @brief scan a number literal
+
+ This function scans a string according to Sect. 6 of RFC 8259.
+
+ The function is realized with a deterministic finite state machine derived
+ from the grammar described in RFC 8259. Starting in state "init", the
+ input is read and used to determined the next state. Only state "done"
+ accepts the number. State "error" is a trap state to model errors. In the
+ table below, "anything" means any character but the ones listed before.
+
+ state | 0 | 1-9 | e E | + | - | . | anything
+ ---------|----------|----------|----------|---------|---------|----------|-----------
+ init | zero | any1 | [error] | [error] | minus | [error] | [error]
+ minus | zero | any1 | [error] | [error] | [error] | [error] | [error]
+ zero | done | done | exponent | done | done | decimal1 | done
+ any1 | any1 | any1 | exponent | done | done | decimal1 | done
+ decimal1 | decimal2 | decimal2 | [error] | [error] | [error] | [error] | [error]
+ decimal2 | decimal2 | decimal2 | exponent | done | done | done | done
+ exponent | any2 | any2 | [error] | sign | sign | [error] | [error]
+ sign | any2 | any2 | [error] | [error] | [error] | [error] | [error]
+ any2 | any2 | any2 | done | done | done | done | done
+
+ The state machine is realized with one label per state (prefixed with
+ "scan_number_") and `goto` statements between them. The state machine
+ contains cycles, but any cycle can be left when EOF is read. Therefore,
+ the function is guaranteed to terminate.
+
+ During scanning, the read bytes are stored in token_buffer. This string is
+ then converted to a signed integer, an unsigned integer, or a
+ floating-point number.
+
+ @return token_type::value_unsigned, token_type::value_integer, or
+ token_type::value_float if number could be successfully scanned,
+ token_type::parse_error otherwise
+
+ @note The scanner is independent of the current locale. Internally, the
+ locale's decimal point is used instead of `.` to work with the
+ locale-dependent converters.
+ */
+ token_type scan_number() // lgtm [cpp/use-of-goto]
+ {
+ // reset token_buffer to store the number's bytes
+ reset();
+
+ // the type of the parsed number; initially set to unsigned; will be
+ // changed if minus sign, decimal point or exponent is read
+ token_type number_type = token_type::value_unsigned;
+
+ // state (init): we just found out we need to scan a number
+ switch (current)
+ {
+ case '-':
+ {
+ add(current);
+ goto scan_number_minus;
+ }
+
+ case '0':
+ {
+ add(current);
+ goto scan_number_zero;
+ }
+
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ {
+ add(current);
+ goto scan_number_any1;
+ }
+
+ // all other characters are rejected outside scan_number()
+ default: // LCOV_EXCL_LINE
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
+ }
+
+scan_number_minus:
+ // state: we just parsed a leading minus sign
+ number_type = token_type::value_integer;
+ switch (get())
+ {
+ case '0':
+ {
+ add(current);
+ goto scan_number_zero;
+ }
+
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ {
+ add(current);
+ goto scan_number_any1;
+ }
+
+ default:
+ {
+ error_message = "invalid number; expected digit after '-'";
+ return token_type::parse_error;
+ }
+ }
+
+scan_number_zero:
+ // state: we just parse a zero (maybe with a leading minus sign)
+ switch (get())
+ {
+ case '.':
+ {
+ add(decimal_point_char);
+ goto scan_number_decimal1;
+ }
+
+ case 'e':
+ case 'E':
+ {
+ add(current);
+ goto scan_number_exponent;
+ }
+
+ default:
+ goto scan_number_done;
+ }
+
+scan_number_any1:
+ // state: we just parsed a number 0-9 (maybe with a leading minus sign)
+ switch (get())
+ {
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ {
+ add(current);
+ goto scan_number_any1;
+ }
+
+ case '.':
+ {
+ add(decimal_point_char);
+ goto scan_number_decimal1;
+ }
+
+ case 'e':
+ case 'E':
+ {
+ add(current);
+ goto scan_number_exponent;
+ }
+
+ default:
+ goto scan_number_done;
+ }
+
+scan_number_decimal1:
+ // state: we just parsed a decimal point
+ number_type = token_type::value_float;
+ switch (get())
+ {
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ {
+ add(current);
+ goto scan_number_decimal2;
+ }
+
+ default:
+ {
+ error_message = "invalid number; expected digit after '.'";
+ return token_type::parse_error;
+ }
+ }
+
+scan_number_decimal2:
+ // we just parsed at least one number after a decimal point
+ switch (get())
+ {
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ {
+ add(current);
+ goto scan_number_decimal2;
+ }
+
+ case 'e':
+ case 'E':
+ {
+ add(current);
+ goto scan_number_exponent;
+ }
+
+ default:
+ goto scan_number_done;
+ }
+
+scan_number_exponent:
+ // we just parsed an exponent
+ number_type = token_type::value_float;
+ switch (get())
+ {
+ case '+':
+ case '-':
+ {
+ add(current);
+ goto scan_number_sign;
+ }
+
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ {
+ add(current);
+ goto scan_number_any2;
+ }
+
+ default:
+ {
+ error_message =
+ "invalid number; expected '+', '-', or digit after exponent";
+ return token_type::parse_error;
+ }
+ }
+
+scan_number_sign:
+ // we just parsed an exponent sign
+ switch (get())
+ {
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ {
+ add(current);
+ goto scan_number_any2;
+ }
+
+ default:
+ {
+ error_message = "invalid number; expected digit after exponent sign";
+ return token_type::parse_error;
+ }
+ }
+
+scan_number_any2:
+ // we just parsed a number after the exponent or exponent sign
+ switch (get())
+ {
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ {
+ add(current);
+ goto scan_number_any2;
+ }
+
+ default:
+ goto scan_number_done;
+ }
+
+scan_number_done:
+ // unget the character after the number (we only read it to know that
+ // we are done scanning a number)
+ unget();
+
+ char* endptr = nullptr; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+ errno = 0;
+
+ // try to parse integers first and fall back to floats
+ if (number_type == token_type::value_unsigned)
+ {
+ const auto x = std::strtoull(token_buffer.data(), &endptr, 10);
+
+ // we checked the number format before
+ JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
+
+ if (errno == 0)
+ {
+ value_unsigned = static_cast<number_unsigned_t>(x);
+ if (value_unsigned == x)
+ {
+ return token_type::value_unsigned;
+ }
+ }
+ }
+ else if (number_type == token_type::value_integer)
+ {
+ const auto x = std::strtoll(token_buffer.data(), &endptr, 10);
+
+ // we checked the number format before
+ JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
+
+ if (errno == 0)
+ {
+ value_integer = static_cast<number_integer_t>(x);
+ if (value_integer == x)
+ {
+ return token_type::value_integer;
+ }
+ }
+ }
+
+ // this code is reached if we parse a floating-point number or if an
+ // integer conversion above failed
+ strtof(value_float, token_buffer.data(), &endptr);
+
+ // we checked the number format before
+ JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
+
+ return token_type::value_float;
+ }
+
+ /*!
+ @param[in] literal_text the literal text to expect
+ @param[in] length the length of the passed literal text
+ @param[in] return_type the token type to return on success
+ */
+ JSON_HEDLEY_NON_NULL(2)
+ token_type scan_literal(const char_type* literal_text, const std::size_t length,
+ token_type return_type)
+ {
+ JSON_ASSERT(char_traits<char_type>::to_char_type(current) == literal_text[0]);
+ for (std::size_t i = 1; i < length; ++i)
+ {
+ if (JSON_HEDLEY_UNLIKELY(char_traits<char_type>::to_char_type(get()) != literal_text[i]))
+ {
+ error_message = "invalid literal";
+ return token_type::parse_error;
+ }
+ }
+ return return_type;
+ }
+
+ /////////////////////
+ // input management
+ /////////////////////
+
+ /// reset token_buffer; current character is beginning of token
+ void reset() noexcept
+ {
+ token_buffer.clear();
+ token_string.clear();
+ token_string.push_back(char_traits<char_type>::to_char_type(current));
+ }
+
+ /*
+ @brief get next character from the input
+
+ This function provides the interface to the used input adapter. It does
+ not throw in case the input reached EOF, but returns a
+ `char_traits<char>::eof()` in that case. Stores the scanned characters
+ for use in error messages.
+
+ @return character read from the input
+ */
+ char_int_type get()
+ {
+ ++position.chars_read_total;
+ ++position.chars_read_current_line;
+
+ if (next_unget)
+ {
+ // just reset the next_unget variable and work with current
+ next_unget = false;
+ }
+ else
+ {
+ current = ia.get_character();
+ }
+
+ if (JSON_HEDLEY_LIKELY(current != char_traits<char_type>::eof()))
+ {
+ token_string.push_back(char_traits<char_type>::to_char_type(current));
+ }
+
+ if (current == '\n')
+ {
+ ++position.lines_read;
+ position.chars_read_current_line = 0;
+ }
+
+ return current;
+ }
+
+ /*!
+ @brief unget current character (read it again on next get)
+
+ We implement unget by setting variable next_unget to true. The input is not
+ changed - we just simulate ungetting by modifying chars_read_total,
+ chars_read_current_line, and token_string. The next call to get() will
+ behave as if the unget character is read again.
+ */
+ void unget()
+ {
+ next_unget = true;
+
+ --position.chars_read_total;
+
+ // in case we "unget" a newline, we have to also decrement the lines_read
+ if (position.chars_read_current_line == 0)
+ {
+ if (position.lines_read > 0)
+ {
+ --position.lines_read;
+ }
+ }
+ else
+ {
+ --position.chars_read_current_line;
+ }
+
+ if (JSON_HEDLEY_LIKELY(current != char_traits<char_type>::eof()))
+ {
+ JSON_ASSERT(!token_string.empty());
+ token_string.pop_back();
+ }
+ }
+
+ /// add a character to token_buffer
+ void add(char_int_type c)
+ {
+ token_buffer.push_back(static_cast<typename string_t::value_type>(c));
+ }
+
+ public:
+ /////////////////////
+ // value getters
+ /////////////////////
+
+ /// return integer value
+ constexpr number_integer_t get_number_integer() const noexcept
+ {
+ return value_integer;
+ }
+
+ /// return unsigned integer value
+ constexpr number_unsigned_t get_number_unsigned() const noexcept
+ {
+ return value_unsigned;
+ }
+
+ /// return floating-point value
+ constexpr number_float_t get_number_float() const noexcept
+ {
+ return value_float;
+ }
+
+ /// return current string value (implicitly resets the token; useful only once)
+ string_t& get_string()
+ {
+ return token_buffer;
+ }
+
+ /////////////////////
+ // diagnostics
+ /////////////////////
+
+ /// return position of last read token
+ constexpr position_t get_position() const noexcept
+ {
+ return position;
+ }
+
+ /// return the last read token (for errors only). Will never contain EOF
+ /// (an arbitrary value that is not a valid char value, often -1), because
+ /// 255 may legitimately occur. May contain NUL, which should be escaped.
+ std::string get_token_string() const
+ {
+ // escape control characters
+ std::string result;
+ for (const auto c : token_string)
+ {
+ if (static_cast<unsigned char>(c) <= '\x1F')
+ {
+ // escape control characters
+ std::array<char, 9> cs{{}};
+ static_cast<void>((std::snprintf)(cs.data(), cs.size(), "<U+%.4X>", static_cast<unsigned char>(c))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+ result += cs.data();
+ }
+ else
+ {
+ // add character as is
+ result.push_back(static_cast<std::string::value_type>(c));
+ }
+ }
+
+ return result;
+ }
+
+ /// return syntax error message
+ JSON_HEDLEY_RETURNS_NON_NULL
+ constexpr const char* get_error_message() const noexcept
+ {
+ return error_message;
+ }
+
+ /////////////////////
+ // actual scanner
+ /////////////////////
+
+ /*!
+ @brief skip the UTF-8 byte order mark
+ @return true iff there is no BOM or the correct BOM has been skipped
+ */
+ bool skip_bom()
+ {
+ if (get() == 0xEF)
+ {
+ // check if we completely parse the BOM
+ return get() == 0xBB && get() == 0xBF;
+ }
+
+ // the first character is not the beginning of the BOM; unget it to
+ // process is later
+ unget();
+ return true;
+ }
+
+ void skip_whitespace()
+ {
+ do
+ {
+ get();
+ }
+ while (current == ' ' || current == '\t' || current == '\n' || current == '\r');
+ }
+
+ token_type scan()
+ {
+ // initially, skip the BOM
+ if (position.chars_read_total == 0 && !skip_bom())
+ {
+ error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given";
+ return token_type::parse_error;
+ }
+
+ // read next character and ignore whitespace
+ skip_whitespace();
+
+ // ignore comments
+ while (ignore_comments && current == '/')
+ {
+ if (!scan_comment())
+ {
+ return token_type::parse_error;
+ }
+
+ // skip following whitespace
+ skip_whitespace();
+ }
+
+ switch (current)
+ {
+ // structural characters
+ case '[':
+ return token_type::begin_array;
+ case ']':
+ return token_type::end_array;
+ case '{':
+ return token_type::begin_object;
+ case '}':
+ return token_type::end_object;
+ case ':':
+ return token_type::name_separator;
+ case ',':
+ return token_type::value_separator;
+
+ // literals
+ case 't':
+ {
+ std::array<char_type, 4> true_literal = {{static_cast<char_type>('t'), static_cast<char_type>('r'), static_cast<char_type>('u'), static_cast<char_type>('e')}};
+ return scan_literal(true_literal.data(), true_literal.size(), token_type::literal_true);
+ }
+ case 'f':
+ {
+ std::array<char_type, 5> false_literal = {{static_cast<char_type>('f'), static_cast<char_type>('a'), static_cast<char_type>('l'), static_cast<char_type>('s'), static_cast<char_type>('e')}};
+ return scan_literal(false_literal.data(), false_literal.size(), token_type::literal_false);
+ }
+ case 'n':
+ {
+ std::array<char_type, 4> null_literal = {{static_cast<char_type>('n'), static_cast<char_type>('u'), static_cast<char_type>('l'), static_cast<char_type>('l')}};
+ return scan_literal(null_literal.data(), null_literal.size(), token_type::literal_null);
+ }
+
+ // string
+ case '\"':
+ return scan_string();
+
+ // number
+ case '-':
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ return scan_number();
+
+ // end of input (the null byte is needed when parsing from
+ // string literals)
+ case '\0':
+ case char_traits<char_type>::eof():
+ return token_type::end_of_input;
+
+ // error
+ default:
+ error_message = "invalid literal";
+ return token_type::parse_error;
+ }
+ }
+
+ private:
+ /// input adapter
+ InputAdapterType ia;
+
+ /// whether comments should be ignored (true) or signaled as errors (false)
+ const bool ignore_comments = false;
+
+ /// the current character
+ char_int_type current = char_traits<char_type>::eof();
+
+ /// whether the next get() call should just return current
+ bool next_unget = false;
+
+ /// the start position of the current token
+ position_t position {};
+
+ /// raw input token string (for error messages)
+ std::vector<char_type> token_string {};
+
+ /// buffer for variable-length tokens (numbers, strings)
+ string_t token_buffer {};
+
+ /// a description of occurred lexer errors
+ const char* error_message = "";
+
+ // number values
+ number_integer_t value_integer = 0;
+ number_unsigned_t value_unsigned = 0;
+ number_float_t value_float = 0;
+
+ /// the decimal point
+ const char_int_type decimal_point_char = '.';
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/is_sax.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <cstdint> // size_t
+#include <utility> // declval
+#include <string> // string
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+// #include <nlohmann/detail/meta/detected.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+template<typename T>
+using null_function_t = decltype(std::declval<T&>().null());
+
+template<typename T>
+using boolean_function_t =
+ decltype(std::declval<T&>().boolean(std::declval<bool>()));
+
+template<typename T, typename Integer>
+using number_integer_function_t =
+ decltype(std::declval<T&>().number_integer(std::declval<Integer>()));
+
+template<typename T, typename Unsigned>
+using number_unsigned_function_t =
+ decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>()));
+
+template<typename T, typename Float, typename String>
+using number_float_function_t = decltype(std::declval<T&>().number_float(
+ std::declval<Float>(), std::declval<const String&>()));
+
+template<typename T, typename String>
+using string_function_t =
+ decltype(std::declval<T&>().string(std::declval<String&>()));
+
+template<typename T, typename Binary>
+using binary_function_t =
+ decltype(std::declval<T&>().binary(std::declval<Binary&>()));
+
+template<typename T>
+using start_object_function_t =
+ decltype(std::declval<T&>().start_object(std::declval<std::size_t>()));
+
+template<typename T, typename String>
+using key_function_t =
+ decltype(std::declval<T&>().key(std::declval<String&>()));
+
+template<typename T>
+using end_object_function_t = decltype(std::declval<T&>().end_object());
+
+template<typename T>
+using start_array_function_t =
+ decltype(std::declval<T&>().start_array(std::declval<std::size_t>()));
+
+template<typename T>
+using end_array_function_t = decltype(std::declval<T&>().end_array());
+
+template<typename T, typename Exception>
+using parse_error_function_t = decltype(std::declval<T&>().parse_error(
+ std::declval<std::size_t>(), std::declval<const std::string&>(),
+ std::declval<const Exception&>()));
+
+template<typename SAX, typename BasicJsonType>
+struct is_sax
+{
+ private:
+ static_assert(is_basic_json<BasicJsonType>::value,
+ "BasicJsonType must be of type basic_json<...>");
+
+ using number_integer_t = typename BasicJsonType::number_integer_t;
+ using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+ using string_t = typename BasicJsonType::string_t;
+ using binary_t = typename BasicJsonType::binary_t;
+ using exception_t = typename BasicJsonType::exception;
+
+ public:
+ static constexpr bool value =
+ is_detected_exact<bool, null_function_t, SAX>::value &&
+ is_detected_exact<bool, boolean_function_t, SAX>::value &&
+ is_detected_exact<bool, number_integer_function_t, SAX, number_integer_t>::value &&
+ is_detected_exact<bool, number_unsigned_function_t, SAX, number_unsigned_t>::value &&
+ is_detected_exact<bool, number_float_function_t, SAX, number_float_t, string_t>::value &&
+ is_detected_exact<bool, string_function_t, SAX, string_t>::value &&
+ is_detected_exact<bool, binary_function_t, SAX, binary_t>::value &&
+ is_detected_exact<bool, start_object_function_t, SAX>::value &&
+ is_detected_exact<bool, key_function_t, SAX, string_t>::value &&
+ is_detected_exact<bool, end_object_function_t, SAX>::value &&
+ is_detected_exact<bool, start_array_function_t, SAX>::value &&
+ is_detected_exact<bool, end_array_function_t, SAX>::value &&
+ is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value;
+};
+
+template<typename SAX, typename BasicJsonType>
+struct is_sax_static_asserts
+{
+ private:
+ static_assert(is_basic_json<BasicJsonType>::value,
+ "BasicJsonType must be of type basic_json<...>");
+
+ using number_integer_t = typename BasicJsonType::number_integer_t;
+ using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+ using string_t = typename BasicJsonType::string_t;
+ using binary_t = typename BasicJsonType::binary_t;
+ using exception_t = typename BasicJsonType::exception;
+
+ public:
+ static_assert(is_detected_exact<bool, null_function_t, SAX>::value,
+ "Missing/invalid function: bool null()");
+ static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
+ "Missing/invalid function: bool boolean(bool)");
+ static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
+ "Missing/invalid function: bool boolean(bool)");
+ static_assert(
+ is_detected_exact<bool, number_integer_function_t, SAX,
+ number_integer_t>::value,
+ "Missing/invalid function: bool number_integer(number_integer_t)");
+ static_assert(
+ is_detected_exact<bool, number_unsigned_function_t, SAX,
+ number_unsigned_t>::value,
+ "Missing/invalid function: bool number_unsigned(number_unsigned_t)");
+ static_assert(is_detected_exact<bool, number_float_function_t, SAX,
+ number_float_t, string_t>::value,
+ "Missing/invalid function: bool number_float(number_float_t, const string_t&)");
+ static_assert(
+ is_detected_exact<bool, string_function_t, SAX, string_t>::value,
+ "Missing/invalid function: bool string(string_t&)");
+ static_assert(
+ is_detected_exact<bool, binary_function_t, SAX, binary_t>::value,
+ "Missing/invalid function: bool binary(binary_t&)");
+ static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value,
+ "Missing/invalid function: bool start_object(std::size_t)");
+ static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value,
+ "Missing/invalid function: bool key(string_t&)");
+ static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value,
+ "Missing/invalid function: bool end_object()");
+ static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value,
+ "Missing/invalid function: bool start_array(std::size_t)");
+ static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value,
+ "Missing/invalid function: bool end_array()");
+ static_assert(
+ is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value,
+ "Missing/invalid function: bool parse_error(std::size_t, const "
+ "std::string&, const exception&)");
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+// #include <nlohmann/detail/string_concat.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+/// how to treat CBOR tags
+enum class cbor_tag_handler_t
+{
+ error, ///< throw a parse_error exception in case of a tag
+ ignore, ///< ignore tags
+ store ///< store tags as binary type
+};
+
+/*!
+@brief determine system byte order
+
+@return true if and only if system's byte order is little endian
+
+@note from https://stackoverflow.com/a/1001328/266378
+*/
+static inline bool little_endianness(int num = 1) noexcept
+{
+ return *reinterpret_cast<char*>(&num) == 1;
+}
+
+///////////////////
+// binary reader //
+///////////////////
+
+/*!
+@brief deserialization of CBOR, MessagePack, and UBJSON values
+*/
+template<typename BasicJsonType, typename InputAdapterType, typename SAX = json_sax_dom_parser<BasicJsonType>>
+class binary_reader
+{
+ using number_integer_t = typename BasicJsonType::number_integer_t;
+ using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+ using string_t = typename BasicJsonType::string_t;
+ using binary_t = typename BasicJsonType::binary_t;
+ using json_sax_t = SAX;
+ using char_type = typename InputAdapterType::char_type;
+ using char_int_type = typename char_traits<char_type>::int_type;
+
+ public:
+ /*!
+ @brief create a binary reader
+
+ @param[in] adapter input adapter to read from
+ */
+ explicit binary_reader(InputAdapterType&& adapter, const input_format_t format = input_format_t::json) noexcept : ia(std::move(adapter)), input_format(format)
+ {
+ (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
+ }
+
+ // make class move-only
+ binary_reader(const binary_reader&) = delete;
+ binary_reader(binary_reader&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)
+ binary_reader& operator=(const binary_reader&) = delete;
+ binary_reader& operator=(binary_reader&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)
+ ~binary_reader() = default;
+
+ /*!
+ @param[in] format the binary format to parse
+ @param[in] sax_ a SAX event processor
+ @param[in] strict whether to expect the input to be consumed completed
+ @param[in] tag_handler how to treat CBOR tags
+
+ @return whether parsing was successful
+ */
+ JSON_HEDLEY_NON_NULL(3)
+ bool sax_parse(const input_format_t format,
+ json_sax_t* sax_,
+ const bool strict = true,
+ const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)
+ {
+ sax = sax_;
+ bool result = false;
+
+ switch (format)
+ {
+ case input_format_t::bson:
+ result = parse_bson_internal();
+ break;
+
+ case input_format_t::cbor:
+ result = parse_cbor_internal(true, tag_handler);
+ break;
+
+ case input_format_t::msgpack:
+ result = parse_msgpack_internal();
+ break;
+
+ case input_format_t::ubjson:
+ case input_format_t::bjdata:
+ result = parse_ubjson_internal();
+ break;
+
+ case input_format_t::json: // LCOV_EXCL_LINE
+ default: // LCOV_EXCL_LINE
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
+ }
+
+ // strict mode: next byte must be EOF
+ if (result && strict)
+ {
+ if (input_format == input_format_t::ubjson || input_format == input_format_t::bjdata)
+ {
+ get_ignore_noop();
+ }
+ else
+ {
+ get();
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(current != char_traits<char_type>::eof()))
+ {
+ return sax->parse_error(chars_read, get_token_string(), parse_error::create(110, chars_read,
+ exception_message(input_format, concat("expected end of input; last byte: 0x", get_token_string()), "value"), nullptr));
+ }
+ }
+
+ return result;
+ }
+
+ private:
+ //////////
+ // BSON //
+ //////////
+
+ /*!
+ @brief Reads in a BSON-object and passes it to the SAX-parser.
+ @return whether a valid BSON-value was passed to the SAX parser
+ */
+ bool parse_bson_internal()
+ {
+ std::int32_t document_size{};
+ get_number<std::int32_t, true>(input_format_t::bson, document_size);
+
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast<std::size_t>(-1))))
+ {
+ return false;
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false)))
+ {
+ return false;
+ }
+
+ return sax->end_object();
+ }
+
+ /*!
+ @brief Parses a C-style string from the BSON input.
+ @param[in,out] result A reference to the string variable where the read
+ string is to be stored.
+ @return `true` if the \x00-byte indicating the end of the string was
+ encountered before the EOF; false` indicates an unexpected EOF.
+ */
+ bool get_bson_cstr(string_t& result)
+ {
+ auto out = std::back_inserter(result);
+ while (true)
+ {
+ get();
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "cstring")))
+ {
+ return false;
+ }
+ if (current == 0x00)
+ {
+ return true;
+ }
+ *out++ = static_cast<typename string_t::value_type>(current);
+ }
+ }
+
+ /*!
+ @brief Parses a zero-terminated string of length @a len from the BSON
+ input.
+ @param[in] len The length (including the zero-byte at the end) of the
+ string to be read.
+ @param[in,out] result A reference to the string variable where the read
+ string is to be stored.
+ @tparam NumberType The type of the length @a len
+ @pre len >= 1
+ @return `true` if the string was successfully parsed
+ */
+ template<typename NumberType>
+ bool get_bson_string(const NumberType len, string_t& result)
+ {
+ if (JSON_HEDLEY_UNLIKELY(len < 1))
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+ exception_message(input_format_t::bson, concat("string length must be at least 1, is ", std::to_string(len)), "string"), nullptr));
+ }
+
+ return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) && get() != char_traits<char_type>::eof();
+ }
+
+ /*!
+ @brief Parses a byte array input of length @a len from the BSON input.
+ @param[in] len The length of the byte array to be read.
+ @param[in,out] result A reference to the binary variable where the read
+ array is to be stored.
+ @tparam NumberType The type of the length @a len
+ @pre len >= 0
+ @return `true` if the byte array was successfully parsed
+ */
+ template<typename NumberType>
+ bool get_bson_binary(const NumberType len, binary_t& result)
+ {
+ if (JSON_HEDLEY_UNLIKELY(len < 0))
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+ exception_message(input_format_t::bson, concat("byte array length cannot be negative, is ", std::to_string(len)), "binary"), nullptr));
+ }
+
+ // All BSON binary values have a subtype
+ std::uint8_t subtype{};
+ get_number<std::uint8_t>(input_format_t::bson, subtype);
+ result.set_subtype(subtype);
+
+ return get_binary(input_format_t::bson, len, result);
+ }
+
+ /*!
+ @brief Read a BSON document element of the given @a element_type.
+ @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html
+ @param[in] element_type_parse_position The position in the input stream,
+ where the `element_type` was read.
+ @warning Not all BSON element types are supported yet. An unsupported
+ @a element_type will give rise to a parse_error.114:
+ Unsupported BSON record type 0x...
+ @return whether a valid BSON-object/array was passed to the SAX parser
+ */
+ bool parse_bson_element_internal(const char_int_type element_type,
+ const std::size_t element_type_parse_position)
+ {
+ switch (element_type)
+ {
+ case 0x01: // double
+ {
+ double number{};
+ return get_number<double, true>(input_format_t::bson, number) && sax->number_float(static_cast<number_float_t>(number), "");
+ }
+
+ case 0x02: // string
+ {
+ std::int32_t len{};
+ string_t value;
+ return get_number<std::int32_t, true>(input_format_t::bson, len) && get_bson_string(len, value) && sax->string(value);
+ }
+
+ case 0x03: // object
+ {
+ return parse_bson_internal();
+ }
+
+ case 0x04: // array
+ {
+ return parse_bson_array();
+ }
+
+ case 0x05: // binary
+ {
+ std::int32_t len{};
+ binary_t value;
+ return get_number<std::int32_t, true>(input_format_t::bson, len) && get_bson_binary(len, value) && sax->binary(value);
+ }
+
+ case 0x08: // boolean
+ {
+ return sax->boolean(get() != 0);
+ }
+
+ case 0x0A: // null
+ {
+ return sax->null();
+ }
+
+ case 0x10: // int32
+ {
+ std::int32_t value{};
+ return get_number<std::int32_t, true>(input_format_t::bson, value) && sax->number_integer(value);
+ }
+
+ case 0x12: // int64
+ {
+ std::int64_t value{};
+ return get_number<std::int64_t, true>(input_format_t::bson, value) && sax->number_integer(value);
+ }
+
+ default: // anything else not supported (yet)
+ {
+ std::array<char, 3> cr{{}};
+ static_cast<void>((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast<unsigned char>(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+ const std::string cr_str{cr.data()};
+ return sax->parse_error(element_type_parse_position, cr_str,
+ parse_error::create(114, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr));
+ }
+ }
+ }
+
+ /*!
+ @brief Read a BSON element list (as specified in the BSON-spec)
+
+ The same binary layout is used for objects and arrays, hence it must be
+ indicated with the argument @a is_array which one is expected
+ (true --> array, false --> object).
+
+ @param[in] is_array Determines if the element list being read is to be
+ treated as an object (@a is_array == false), or as an
+ array (@a is_array == true).
+ @return whether a valid BSON-object/array was passed to the SAX parser
+ */
+ bool parse_bson_element_list(const bool is_array)
+ {
+ string_t key;
+
+ while (auto element_type = get())
+ {
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list")))
+ {
+ return false;
+ }
+
+ const std::size_t element_type_parse_position = chars_read;
+ if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
+ {
+ return false;
+ }
+
+ if (!is_array && !sax->key(key))
+ {
+ return false;
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position)))
+ {
+ return false;
+ }
+
+ // get_bson_cstr only appends
+ key.clear();
+ }
+
+ return true;
+ }
+
+ /*!
+ @brief Reads an array from the BSON input and passes it to the SAX-parser.
+ @return whether a valid BSON-array was passed to the SAX parser
+ */
+ bool parse_bson_array()
+ {
+ std::int32_t document_size{};
+ get_number<std::int32_t, true>(input_format_t::bson, document_size);
+
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast<std::size_t>(-1))))
+ {
+ return false;
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true)))
+ {
+ return false;
+ }
+
+ return sax->end_array();
+ }
+
+ //////////
+ // CBOR //
+ //////////
+
+ /*!
+ @param[in] get_char whether a new character should be retrieved from the
+ input (true) or whether the last read character should
+ be considered instead (false)
+ @param[in] tag_handler how CBOR tags should be treated
+
+ @return whether a valid CBOR value was passed to the SAX parser
+ */
+ bool parse_cbor_internal(const bool get_char,
+ const cbor_tag_handler_t tag_handler)
+ {
+ switch (get_char ? get() : current)
+ {
+ // EOF
+ case char_traits<char_type>::eof():
+ return unexpect_eof(input_format_t::cbor, "value");
+
+ // Integer 0x00..0x17 (0..23)
+ case 0x00:
+ case 0x01:
+ case 0x02:
+ case 0x03:
+ case 0x04:
+ case 0x05:
+ case 0x06:
+ case 0x07:
+ case 0x08:
+ case 0x09:
+ case 0x0A:
+ case 0x0B:
+ case 0x0C:
+ case 0x0D:
+ case 0x0E:
+ case 0x0F:
+ case 0x10:
+ case 0x11:
+ case 0x12:
+ case 0x13:
+ case 0x14:
+ case 0x15:
+ case 0x16:
+ case 0x17:
+ return sax->number_unsigned(static_cast<number_unsigned_t>(current));
+
+ case 0x18: // Unsigned integer (one-byte uint8_t follows)
+ {
+ std::uint8_t number{};
+ return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);
+ }
+
+ case 0x19: // Unsigned integer (two-byte uint16_t follows)
+ {
+ std::uint16_t number{};
+ return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);
+ }
+
+ case 0x1A: // Unsigned integer (four-byte uint32_t follows)
+ {
+ std::uint32_t number{};
+ return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);
+ }
+
+ case 0x1B: // Unsigned integer (eight-byte uint64_t follows)
+ {
+ std::uint64_t number{};
+ return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);
+ }
+
+ // Negative integer -1-0x00..-1-0x17 (-1..-24)
+ case 0x20:
+ case 0x21:
+ case 0x22:
+ case 0x23:
+ case 0x24:
+ case 0x25:
+ case 0x26:
+ case 0x27:
+ case 0x28:
+ case 0x29:
+ case 0x2A:
+ case 0x2B:
+ case 0x2C:
+ case 0x2D:
+ case 0x2E:
+ case 0x2F:
+ case 0x30:
+ case 0x31:
+ case 0x32:
+ case 0x33:
+ case 0x34:
+ case 0x35:
+ case 0x36:
+ case 0x37:
+ return sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - current));
+
+ case 0x38: // Negative integer (one-byte uint8_t follows)
+ {
+ std::uint8_t number{};
+ return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1) - number);
+ }
+
+ case 0x39: // Negative integer -1-n (two-byte uint16_t follows)
+ {
+ std::uint16_t number{};
+ return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1) - number);
+ }
+
+ case 0x3A: // Negative integer -1-n (four-byte uint32_t follows)
+ {
+ std::uint32_t number{};
+ return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1) - number);
+ }
+
+ case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows)
+ {
+ std::uint64_t number{};
+ return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1)
+ - static_cast<number_integer_t>(number));
+ }
+
+ // Binary data (0x00..0x17 bytes follow)
+ case 0x40:
+ case 0x41:
+ case 0x42:
+ case 0x43:
+ case 0x44:
+ case 0x45:
+ case 0x46:
+ case 0x47:
+ case 0x48:
+ case 0x49:
+ case 0x4A:
+ case 0x4B:
+ case 0x4C:
+ case 0x4D:
+ case 0x4E:
+ case 0x4F:
+ case 0x50:
+ case 0x51:
+ case 0x52:
+ case 0x53:
+ case 0x54:
+ case 0x55:
+ case 0x56:
+ case 0x57:
+ case 0x58: // Binary data (one-byte uint8_t for n follows)
+ case 0x59: // Binary data (two-byte uint16_t for n follow)
+ case 0x5A: // Binary data (four-byte uint32_t for n follow)
+ case 0x5B: // Binary data (eight-byte uint64_t for n follow)
+ case 0x5F: // Binary data (indefinite length)
+ {
+ binary_t b;
+ return get_cbor_binary(b) && sax->binary(b);
+ }
+
+ // UTF-8 string (0x00..0x17 bytes follow)
+ case 0x60:
+ case 0x61:
+ case 0x62:
+ case 0x63:
+ case 0x64:
+ case 0x65:
+ case 0x66:
+ case 0x67:
+ case 0x68:
+ case 0x69:
+ case 0x6A:
+ case 0x6B:
+ case 0x6C:
+ case 0x6D:
+ case 0x6E:
+ case 0x6F:
+ case 0x70:
+ case 0x71:
+ case 0x72:
+ case 0x73:
+ case 0x74:
+ case 0x75:
+ case 0x76:
+ case 0x77:
+ case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+ case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+ case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
+ case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
+ case 0x7F: // UTF-8 string (indefinite length)
+ {
+ string_t s;
+ return get_cbor_string(s) && sax->string(s);
+ }
+
+ // array (0x00..0x17 data items follow)
+ case 0x80:
+ case 0x81:
+ case 0x82:
+ case 0x83:
+ case 0x84:
+ case 0x85:
+ case 0x86:
+ case 0x87:
+ case 0x88:
+ case 0x89:
+ case 0x8A:
+ case 0x8B:
+ case 0x8C:
+ case 0x8D:
+ case 0x8E:
+ case 0x8F:
+ case 0x90:
+ case 0x91:
+ case 0x92:
+ case 0x93:
+ case 0x94:
+ case 0x95:
+ case 0x96:
+ case 0x97:
+ return get_cbor_array(
+ conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu), tag_handler);
+
+ case 0x98: // array (one-byte uint8_t for n follows)
+ {
+ std::uint8_t len{};
+ return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast<std::size_t>(len), tag_handler);
+ }
+
+ case 0x99: // array (two-byte uint16_t for n follow)
+ {
+ std::uint16_t len{};
+ return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast<std::size_t>(len), tag_handler);
+ }
+
+ case 0x9A: // array (four-byte uint32_t for n follow)
+ {
+ std::uint32_t len{};
+ return get_number(input_format_t::cbor, len) && get_cbor_array(conditional_static_cast<std::size_t>(len), tag_handler);
+ }
+
+ case 0x9B: // array (eight-byte uint64_t for n follow)
+ {
+ std::uint64_t len{};
+ return get_number(input_format_t::cbor, len) && get_cbor_array(conditional_static_cast<std::size_t>(len), tag_handler);
+ }
+
+ case 0x9F: // array (indefinite length)
+ return get_cbor_array(static_cast<std::size_t>(-1), tag_handler);
+
+ // map (0x00..0x17 pairs of data items follow)
+ case 0xA0:
+ case 0xA1:
+ case 0xA2:
+ case 0xA3:
+ case 0xA4:
+ case 0xA5:
+ case 0xA6:
+ case 0xA7:
+ case 0xA8:
+ case 0xA9:
+ case 0xAA:
+ case 0xAB:
+ case 0xAC:
+ case 0xAD:
+ case 0xAE:
+ case 0xAF:
+ case 0xB0:
+ case 0xB1:
+ case 0xB2:
+ case 0xB3:
+ case 0xB4:
+ case 0xB5:
+ case 0xB6:
+ case 0xB7:
+ return get_cbor_object(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu), tag_handler);
+
+ case 0xB8: // map (one-byte uint8_t for n follows)
+ {
+ std::uint8_t len{};
+ return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast<std::size_t>(len), tag_handler);
+ }
+
+ case 0xB9: // map (two-byte uint16_t for n follow)
+ {
+ std::uint16_t len{};
+ return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast<std::size_t>(len), tag_handler);
+ }
+
+ case 0xBA: // map (four-byte uint32_t for n follow)
+ {
+ std::uint32_t len{};
+ return get_number(input_format_t::cbor, len) && get_cbor_object(conditional_static_cast<std::size_t>(len), tag_handler);
+ }
+
+ case 0xBB: // map (eight-byte uint64_t for n follow)
+ {
+ std::uint64_t len{};
+ return get_number(input_format_t::cbor, len) && get_cbor_object(conditional_static_cast<std::size_t>(len), tag_handler);
+ }
+
+ case 0xBF: // map (indefinite length)
+ return get_cbor_object(static_cast<std::size_t>(-1), tag_handler);
+
+ case 0xC6: // tagged item
+ case 0xC7:
+ case 0xC8:
+ case 0xC9:
+ case 0xCA:
+ case 0xCB:
+ case 0xCC:
+ case 0xCD:
+ case 0xCE:
+ case 0xCF:
+ case 0xD0:
+ case 0xD1:
+ case 0xD2:
+ case 0xD3:
+ case 0xD4:
+ case 0xD8: // tagged item (1 bytes follow)
+ case 0xD9: // tagged item (2 bytes follow)
+ case 0xDA: // tagged item (4 bytes follow)
+ case 0xDB: // tagged item (8 bytes follow)
+ {
+ switch (tag_handler)
+ {
+ case cbor_tag_handler_t::error:
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+ exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr));
+ }
+
+ case cbor_tag_handler_t::ignore:
+ {
+ // ignore binary subtype
+ switch (current)
+ {
+ case 0xD8:
+ {
+ std::uint8_t subtype_to_ignore{};
+ get_number(input_format_t::cbor, subtype_to_ignore);
+ break;
+ }
+ case 0xD9:
+ {
+ std::uint16_t subtype_to_ignore{};
+ get_number(input_format_t::cbor, subtype_to_ignore);
+ break;
+ }
+ case 0xDA:
+ {
+ std::uint32_t subtype_to_ignore{};
+ get_number(input_format_t::cbor, subtype_to_ignore);
+ break;
+ }
+ case 0xDB:
+ {
+ std::uint64_t subtype_to_ignore{};
+ get_number(input_format_t::cbor, subtype_to_ignore);
+ break;
+ }
+ default:
+ break;
+ }
+ return parse_cbor_internal(true, tag_handler);
+ }
+
+ case cbor_tag_handler_t::store:
+ {
+ binary_t b;
+ // use binary subtype and store in binary container
+ switch (current)
+ {
+ case 0xD8:
+ {
+ std::uint8_t subtype{};
+ get_number(input_format_t::cbor, subtype);
+ b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));
+ break;
+ }
+ case 0xD9:
+ {
+ std::uint16_t subtype{};
+ get_number(input_format_t::cbor, subtype);
+ b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));
+ break;
+ }
+ case 0xDA:
+ {
+ std::uint32_t subtype{};
+ get_number(input_format_t::cbor, subtype);
+ b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));
+ break;
+ }
+ case 0xDB:
+ {
+ std::uint64_t subtype{};
+ get_number(input_format_t::cbor, subtype);
+ b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));
+ break;
+ }
+ default:
+ return parse_cbor_internal(true, tag_handler);
+ }
+ get();
+ return get_cbor_binary(b) && sax->binary(b);
+ }
+
+ default: // LCOV_EXCL_LINE
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
+ return false; // LCOV_EXCL_LINE
+ }
+ }
+
+ case 0xF4: // false
+ return sax->boolean(false);
+
+ case 0xF5: // true
+ return sax->boolean(true);
+
+ case 0xF6: // null
+ return sax->null();
+
+ case 0xF9: // Half-Precision Float (two-byte IEEE 754)
+ {
+ const auto byte1_raw = get();
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number")))
+ {
+ return false;
+ }
+ const auto byte2_raw = get();
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number")))
+ {
+ return false;
+ }
+
+ const auto byte1 = static_cast<unsigned char>(byte1_raw);
+ const auto byte2 = static_cast<unsigned char>(byte2_raw);
+
+ // code from RFC 7049, Appendix D, Figure 3:
+ // As half-precision floating-point numbers were only added
+ // to IEEE 754 in 2008, today's programming platforms often
+ // still only have limited support for them. It is very
+ // easy to include at least decoding support for them even
+ // without such support. An example of a small decoder for
+ // half-precision floating-point numbers in the C language
+ // is shown in Fig. 3.
+ const auto half = static_cast<unsigned int>((byte1 << 8u) + byte2);
+ const double val = [&half]
+ {
+ const int exp = (half >> 10u) & 0x1Fu;
+ const unsigned int mant = half & 0x3FFu;
+ JSON_ASSERT(0 <= exp&& exp <= 32);
+ JSON_ASSERT(mant <= 1024);
+ switch (exp)
+ {
+ case 0:
+ return std::ldexp(mant, -24);
+ case 31:
+ return (mant == 0)
+ ? std::numeric_limits<double>::infinity()
+ : std::numeric_limits<double>::quiet_NaN();
+ default:
+ return std::ldexp(mant + 1024, exp - 25);
+ }
+ }();
+ return sax->number_float((half & 0x8000u) != 0
+ ? static_cast<number_float_t>(-val)
+ : static_cast<number_float_t>(val), "");
+ }
+
+ case 0xFA: // Single-Precision Float (four-byte IEEE 754)
+ {
+ float number{};
+ return get_number(input_format_t::cbor, number) && sax->number_float(static_cast<number_float_t>(number), "");
+ }
+
+ case 0xFB: // Double-Precision Float (eight-byte IEEE 754)
+ {
+ double number{};
+ return get_number(input_format_t::cbor, number) && sax->number_float(static_cast<number_float_t>(number), "");
+ }
+
+ default: // anything else (0xFF is handled inside the other types)
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+ exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr));
+ }
+ }
+ }
+
+ /*!
+ @brief reads a CBOR string
+
+ This function first reads starting bytes to determine the expected
+ string length and then copies this number of bytes into a string.
+ Additionally, CBOR's strings with indefinite lengths are supported.
+
+ @param[out] result created string
+
+ @return whether string creation completed
+ */
+ bool get_cbor_string(string_t& result)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string")))
+ {
+ return false;
+ }
+
+ switch (current)
+ {
+ // UTF-8 string (0x00..0x17 bytes follow)
+ case 0x60:
+ case 0x61:
+ case 0x62:
+ case 0x63:
+ case 0x64:
+ case 0x65:
+ case 0x66:
+ case 0x67:
+ case 0x68:
+ case 0x69:
+ case 0x6A:
+ case 0x6B:
+ case 0x6C:
+ case 0x6D:
+ case 0x6E:
+ case 0x6F:
+ case 0x70:
+ case 0x71:
+ case 0x72:
+ case 0x73:
+ case 0x74:
+ case 0x75:
+ case 0x76:
+ case 0x77:
+ {
+ return get_string(input_format_t::cbor, static_cast<unsigned int>(current) & 0x1Fu, result);
+ }
+
+ case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+ {
+ std::uint8_t len{};
+ return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result);
+ }
+
+ case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+ {
+ std::uint16_t len{};
+ return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result);
+ }
+
+ case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
+ {
+ std::uint32_t len{};
+ return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result);
+ }
+
+ case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
+ {
+ std::uint64_t len{};
+ return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result);
+ }
+
+ case 0x7F: // UTF-8 string (indefinite length)
+ {
+ while (get() != 0xFF)
+ {
+ string_t chunk;
+ if (!get_cbor_string(chunk))
+ {
+ return false;
+ }
+ result.append(chunk);
+ }
+ return true;
+ }
+
+ default:
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read,
+ exception_message(input_format_t::cbor, concat("expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x", last_token), "string"), nullptr));
+ }
+ }
+ }
+
+ /*!
+ @brief reads a CBOR byte array
+
+ This function first reads starting bytes to determine the expected
+ byte array length and then copies this number of bytes into the byte array.
+ Additionally, CBOR's byte arrays with indefinite lengths are supported.
+
+ @param[out] result created byte array
+
+ @return whether byte array creation completed
+ */
+ bool get_cbor_binary(binary_t& result)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary")))
+ {
+ return false;
+ }
+
+ switch (current)
+ {
+ // Binary data (0x00..0x17 bytes follow)
+ case 0x40:
+ case 0x41:
+ case 0x42:
+ case 0x43:
+ case 0x44:
+ case 0x45:
+ case 0x46:
+ case 0x47:
+ case 0x48:
+ case 0x49:
+ case 0x4A:
+ case 0x4B:
+ case 0x4C:
+ case 0x4D:
+ case 0x4E:
+ case 0x4F:
+ case 0x50:
+ case 0x51:
+ case 0x52:
+ case 0x53:
+ case 0x54:
+ case 0x55:
+ case 0x56:
+ case 0x57:
+ {
+ return get_binary(input_format_t::cbor, static_cast<unsigned int>(current) & 0x1Fu, result);
+ }
+
+ case 0x58: // Binary data (one-byte uint8_t for n follows)
+ {
+ std::uint8_t len{};
+ return get_number(input_format_t::cbor, len) &&
+ get_binary(input_format_t::cbor, len, result);
+ }
+
+ case 0x59: // Binary data (two-byte uint16_t for n follow)
+ {
+ std::uint16_t len{};
+ return get_number(input_format_t::cbor, len) &&
+ get_binary(input_format_t::cbor, len, result);
+ }
+
+ case 0x5A: // Binary data (four-byte uint32_t for n follow)
+ {
+ std::uint32_t len{};
+ return get_number(input_format_t::cbor, len) &&
+ get_binary(input_format_t::cbor, len, result);
+ }
+
+ case 0x5B: // Binary data (eight-byte uint64_t for n follow)
+ {
+ std::uint64_t len{};
+ return get_number(input_format_t::cbor, len) &&
+ get_binary(input_format_t::cbor, len, result);
+ }
+
+ case 0x5F: // Binary data (indefinite length)
+ {
+ while (get() != 0xFF)
+ {
+ binary_t chunk;
+ if (!get_cbor_binary(chunk))
+ {
+ return false;
+ }
+ result.insert(result.end(), chunk.begin(), chunk.end());
+ }
+ return true;
+ }
+
+ default:
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read,
+ exception_message(input_format_t::cbor, concat("expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x", last_token), "binary"), nullptr));
+ }
+ }
+ }
+
+ /*!
+ @param[in] len the length of the array or static_cast<std::size_t>(-1) for an
+ array of indefinite size
+ @param[in] tag_handler how CBOR tags should be treated
+ @return whether array creation completed
+ */
+ bool get_cbor_array(const std::size_t len,
+ const cbor_tag_handler_t tag_handler)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len)))
+ {
+ return false;
+ }
+
+ if (len != static_cast<std::size_t>(-1))
+ {
+ for (std::size_t i = 0; i < len; ++i)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler)))
+ {
+ return false;
+ }
+ }
+ }
+ else
+ {
+ while (get() != 0xFF)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler)))
+ {
+ return false;
+ }
+ }
+ }
+
+ return sax->end_array();
+ }
+
+ /*!
+ @param[in] len the length of the object or static_cast<std::size_t>(-1) for an
+ object of indefinite size
+ @param[in] tag_handler how CBOR tags should be treated
+ @return whether object creation completed
+ */
+ bool get_cbor_object(const std::size_t len,
+ const cbor_tag_handler_t tag_handler)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len)))
+ {
+ return false;
+ }
+
+ if (len != 0)
+ {
+ string_t key;
+ if (len != static_cast<std::size_t>(-1))
+ {
+ for (std::size_t i = 0; i < len; ++i)
+ {
+ get();
+ if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key)))
+ {
+ return false;
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler)))
+ {
+ return false;
+ }
+ key.clear();
+ }
+ }
+ else
+ {
+ while (get() != 0xFF)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key)))
+ {
+ return false;
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler)))
+ {
+ return false;
+ }
+ key.clear();
+ }
+ }
+ }
+
+ return sax->end_object();
+ }
+
+ /////////////
+ // MsgPack //
+ /////////////
+
+ /*!
+ @return whether a valid MessagePack value was passed to the SAX parser
+ */
+ bool parse_msgpack_internal()
+ {
+ switch (get())
+ {
+ // EOF
+ case char_traits<char_type>::eof():
+ return unexpect_eof(input_format_t::msgpack, "value");
+
+ // positive fixint
+ case 0x00:
+ case 0x01:
+ case 0x02:
+ case 0x03:
+ case 0x04:
+ case 0x05:
+ case 0x06:
+ case 0x07:
+ case 0x08:
+ case 0x09:
+ case 0x0A:
+ case 0x0B:
+ case 0x0C:
+ case 0x0D:
+ case 0x0E:
+ case 0x0F:
+ case 0x10:
+ case 0x11:
+ case 0x12:
+ case 0x13:
+ case 0x14:
+ case 0x15:
+ case 0x16:
+ case 0x17:
+ case 0x18:
+ case 0x19:
+ case 0x1A:
+ case 0x1B:
+ case 0x1C:
+ case 0x1D:
+ case 0x1E:
+ case 0x1F:
+ case 0x20:
+ case 0x21:
+ case 0x22:
+ case 0x23:
+ case 0x24:
+ case 0x25:
+ case 0x26:
+ case 0x27:
+ case 0x28:
+ case 0x29:
+ case 0x2A:
+ case 0x2B:
+ case 0x2C:
+ case 0x2D:
+ case 0x2E:
+ case 0x2F:
+ case 0x30:
+ case 0x31:
+ case 0x32:
+ case 0x33:
+ case 0x34:
+ case 0x35:
+ case 0x36:
+ case 0x37:
+ case 0x38:
+ case 0x39:
+ case 0x3A:
+ case 0x3B:
+ case 0x3C:
+ case 0x3D:
+ case 0x3E:
+ case 0x3F:
+ case 0x40:
+ case 0x41:
+ case 0x42:
+ case 0x43:
+ case 0x44:
+ case 0x45:
+ case 0x46:
+ case 0x47:
+ case 0x48:
+ case 0x49:
+ case 0x4A:
+ case 0x4B:
+ case 0x4C:
+ case 0x4D:
+ case 0x4E:
+ case 0x4F:
+ case 0x50:
+ case 0x51:
+ case 0x52:
+ case 0x53:
+ case 0x54:
+ case 0x55:
+ case 0x56:
+ case 0x57:
+ case 0x58:
+ case 0x59:
+ case 0x5A:
+ case 0x5B:
+ case 0x5C:
+ case 0x5D:
+ case 0x5E:
+ case 0x5F:
+ case 0x60:
+ case 0x61:
+ case 0x62:
+ case 0x63:
+ case 0x64:
+ case 0x65:
+ case 0x66:
+ case 0x67:
+ case 0x68:
+ case 0x69:
+ case 0x6A:
+ case 0x6B:
+ case 0x6C:
+ case 0x6D:
+ case 0x6E:
+ case 0x6F:
+ case 0x70:
+ case 0x71:
+ case 0x72:
+ case 0x73:
+ case 0x74:
+ case 0x75:
+ case 0x76:
+ case 0x77:
+ case 0x78:
+ case 0x79:
+ case 0x7A:
+ case 0x7B:
+ case 0x7C:
+ case 0x7D:
+ case 0x7E:
+ case 0x7F:
+ return sax->number_unsigned(static_cast<number_unsigned_t>(current));
+
+ // fixmap
+ case 0x80:
+ case 0x81:
+ case 0x82:
+ case 0x83:
+ case 0x84:
+ case 0x85:
+ case 0x86:
+ case 0x87:
+ case 0x88:
+ case 0x89:
+ case 0x8A:
+ case 0x8B:
+ case 0x8C:
+ case 0x8D:
+ case 0x8E:
+ case 0x8F:
+ return get_msgpack_object(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x0Fu));
+
+ // fixarray
+ case 0x90:
+ case 0x91:
+ case 0x92:
+ case 0x93:
+ case 0x94:
+ case 0x95:
+ case 0x96:
+ case 0x97:
+ case 0x98:
+ case 0x99:
+ case 0x9A:
+ case 0x9B:
+ case 0x9C:
+ case 0x9D:
+ case 0x9E:
+ case 0x9F:
+ return get_msgpack_array(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x0Fu));
+
+ // fixstr
+ case 0xA0:
+ case 0xA1:
+ case 0xA2:
+ case 0xA3:
+ case 0xA4:
+ case 0xA5:
+ case 0xA6:
+ case 0xA7:
+ case 0xA8:
+ case 0xA9:
+ case 0xAA:
+ case 0xAB:
+ case 0xAC:
+ case 0xAD:
+ case 0xAE:
+ case 0xAF:
+ case 0xB0:
+ case 0xB1:
+ case 0xB2:
+ case 0xB3:
+ case 0xB4:
+ case 0xB5:
+ case 0xB6:
+ case 0xB7:
+ case 0xB8:
+ case 0xB9:
+ case 0xBA:
+ case 0xBB:
+ case 0xBC:
+ case 0xBD:
+ case 0xBE:
+ case 0xBF:
+ case 0xD9: // str 8
+ case 0xDA: // str 16
+ case 0xDB: // str 32
+ {
+ string_t s;
+ return get_msgpack_string(s) && sax->string(s);
+ }
+
+ case 0xC0: // nil
+ return sax->null();
+
+ case 0xC2: // false
+ return sax->boolean(false);
+
+ case 0xC3: // true
+ return sax->boolean(true);
+
+ case 0xC4: // bin 8
+ case 0xC5: // bin 16
+ case 0xC6: // bin 32
+ case 0xC7: // ext 8
+ case 0xC8: // ext 16
+ case 0xC9: // ext 32
+ case 0xD4: // fixext 1
+ case 0xD5: // fixext 2
+ case 0xD6: // fixext 4
+ case 0xD7: // fixext 8
+ case 0xD8: // fixext 16
+ {
+ binary_t b;
+ return get_msgpack_binary(b) && sax->binary(b);
+ }
+
+ case 0xCA: // float 32
+ {
+ float number{};
+ return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast<number_float_t>(number), "");
+ }
+
+ case 0xCB: // float 64
+ {
+ double number{};
+ return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast<number_float_t>(number), "");
+ }
+
+ case 0xCC: // uint 8
+ {
+ std::uint8_t number{};
+ return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);
+ }
+
+ case 0xCD: // uint 16
+ {
+ std::uint16_t number{};
+ return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);
+ }
+
+ case 0xCE: // uint 32
+ {
+ std::uint32_t number{};
+ return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);
+ }
+
+ case 0xCF: // uint 64
+ {
+ std::uint64_t number{};
+ return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);
+ }
+
+ case 0xD0: // int 8
+ {
+ std::int8_t number{};
+ return get_number(input_format_t::msgpack, number) && sax->number_integer(number);
+ }
+
+ case 0xD1: // int 16
+ {
+ std::int16_t number{};
+ return get_number(input_format_t::msgpack, number) && sax->number_integer(number);
+ }
+
+ case 0xD2: // int 32
+ {
+ std::int32_t number{};
+ return get_number(input_format_t::msgpack, number) && sax->number_integer(number);
+ }
+
+ case 0xD3: // int 64
+ {
+ std::int64_t number{};
+ return get_number(input_format_t::msgpack, number) && sax->number_integer(number);
+ }
+
+ case 0xDC: // array 16
+ {
+ std::uint16_t len{};
+ return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast<std::size_t>(len));
+ }
+
+ case 0xDD: // array 32
+ {
+ std::uint32_t len{};
+ return get_number(input_format_t::msgpack, len) && get_msgpack_array(conditional_static_cast<std::size_t>(len));
+ }
+
+ case 0xDE: // map 16
+ {
+ std::uint16_t len{};
+ return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast<std::size_t>(len));
+ }
+
+ case 0xDF: // map 32
+ {
+ std::uint32_t len{};
+ return get_number(input_format_t::msgpack, len) && get_msgpack_object(conditional_static_cast<std::size_t>(len));
+ }
+
+ // negative fixint
+ case 0xE0:
+ case 0xE1:
+ case 0xE2:
+ case 0xE3:
+ case 0xE4:
+ case 0xE5:
+ case 0xE6:
+ case 0xE7:
+ case 0xE8:
+ case 0xE9:
+ case 0xEA:
+ case 0xEB:
+ case 0xEC:
+ case 0xED:
+ case 0xEE:
+ case 0xEF:
+ case 0xF0:
+ case 0xF1:
+ case 0xF2:
+ case 0xF3:
+ case 0xF4:
+ case 0xF5:
+ case 0xF6:
+ case 0xF7:
+ case 0xF8:
+ case 0xF9:
+ case 0xFA:
+ case 0xFB:
+ case 0xFC:
+ case 0xFD:
+ case 0xFE:
+ case 0xFF:
+ return sax->number_integer(static_cast<std::int8_t>(current));
+
+ default: // anything else
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+ exception_message(input_format_t::msgpack, concat("invalid byte: 0x", last_token), "value"), nullptr));
+ }
+ }
+ }
+
+ /*!
+ @brief reads a MessagePack string
+
+ This function first reads starting bytes to determine the expected
+ string length and then copies this number of bytes into a string.
+
+ @param[out] result created string
+
+ @return whether string creation completed
+ */
+ bool get_msgpack_string(string_t& result)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::msgpack, "string")))
+ {
+ return false;
+ }
+
+ switch (current)
+ {
+ // fixstr
+ case 0xA0:
+ case 0xA1:
+ case 0xA2:
+ case 0xA3:
+ case 0xA4:
+ case 0xA5:
+ case 0xA6:
+ case 0xA7:
+ case 0xA8:
+ case 0xA9:
+ case 0xAA:
+ case 0xAB:
+ case 0xAC:
+ case 0xAD:
+ case 0xAE:
+ case 0xAF:
+ case 0xB0:
+ case 0xB1:
+ case 0xB2:
+ case 0xB3:
+ case 0xB4:
+ case 0xB5:
+ case 0xB6:
+ case 0xB7:
+ case 0xB8:
+ case 0xB9:
+ case 0xBA:
+ case 0xBB:
+ case 0xBC:
+ case 0xBD:
+ case 0xBE:
+ case 0xBF:
+ {
+ return get_string(input_format_t::msgpack, static_cast<unsigned int>(current) & 0x1Fu, result);
+ }
+
+ case 0xD9: // str 8
+ {
+ std::uint8_t len{};
+ return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result);
+ }
+
+ case 0xDA: // str 16
+ {
+ std::uint16_t len{};
+ return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result);
+ }
+
+ case 0xDB: // str 32
+ {
+ std::uint32_t len{};
+ return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result);
+ }
+
+ default:
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read,
+ exception_message(input_format_t::msgpack, concat("expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x", last_token), "string"), nullptr));
+ }
+ }
+ }
+
+ /*!
+ @brief reads a MessagePack byte array
+
+ This function first reads starting bytes to determine the expected
+ byte array length and then copies this number of bytes into a byte array.
+
+ @param[out] result created byte array
+
+ @return whether byte array creation completed
+ */
+ bool get_msgpack_binary(binary_t& result)
+ {
+ // helper function to set the subtype
+ auto assign_and_return_true = [&result](std::int8_t subtype)
+ {
+ result.set_subtype(static_cast<std::uint8_t>(subtype));
+ return true;
+ };
+
+ switch (current)
+ {
+ case 0xC4: // bin 8
+ {
+ std::uint8_t len{};
+ return get_number(input_format_t::msgpack, len) &&
+ get_binary(input_format_t::msgpack, len, result);
+ }
+
+ case 0xC5: // bin 16
+ {
+ std::uint16_t len{};
+ return get_number(input_format_t::msgpack, len) &&
+ get_binary(input_format_t::msgpack, len, result);
+ }
+
+ case 0xC6: // bin 32
+ {
+ std::uint32_t len{};
+ return get_number(input_format_t::msgpack, len) &&
+ get_binary(input_format_t::msgpack, len, result);
+ }
+
+ case 0xC7: // ext 8
+ {
+ std::uint8_t len{};
+ std::int8_t subtype{};
+ return get_number(input_format_t::msgpack, len) &&
+ get_number(input_format_t::msgpack, subtype) &&
+ get_binary(input_format_t::msgpack, len, result) &&
+ assign_and_return_true(subtype);
+ }
+
+ case 0xC8: // ext 16
+ {
+ std::uint16_t len{};
+ std::int8_t subtype{};
+ return get_number(input_format_t::msgpack, len) &&
+ get_number(input_format_t::msgpack, subtype) &&
+ get_binary(input_format_t::msgpack, len, result) &&
+ assign_and_return_true(subtype);
+ }
+
+ case 0xC9: // ext 32
+ {
+ std::uint32_t len{};
+ std::int8_t subtype{};
+ return get_number(input_format_t::msgpack, len) &&
+ get_number(input_format_t::msgpack, subtype) &&
+ get_binary(input_format_t::msgpack, len, result) &&
+ assign_and_return_true(subtype);
+ }
+
+ case 0xD4: // fixext 1
+ {
+ std::int8_t subtype{};
+ return get_number(input_format_t::msgpack, subtype) &&
+ get_binary(input_format_t::msgpack, 1, result) &&
+ assign_and_return_true(subtype);
+ }
+
+ case 0xD5: // fixext 2
+ {
+ std::int8_t subtype{};
+ return get_number(input_format_t::msgpack, subtype) &&
+ get_binary(input_format_t::msgpack, 2, result) &&
+ assign_and_return_true(subtype);
+ }
+
+ case 0xD6: // fixext 4
+ {
+ std::int8_t subtype{};
+ return get_number(input_format_t::msgpack, subtype) &&
+ get_binary(input_format_t::msgpack, 4, result) &&
+ assign_and_return_true(subtype);
+ }
+
+ case 0xD7: // fixext 8
+ {
+ std::int8_t subtype{};
+ return get_number(input_format_t::msgpack, subtype) &&
+ get_binary(input_format_t::msgpack, 8, result) &&
+ assign_and_return_true(subtype);
+ }
+
+ case 0xD8: // fixext 16
+ {
+ std::int8_t subtype{};
+ return get_number(input_format_t::msgpack, subtype) &&
+ get_binary(input_format_t::msgpack, 16, result) &&
+ assign_and_return_true(subtype);
+ }
+
+ default: // LCOV_EXCL_LINE
+ return false; // LCOV_EXCL_LINE
+ }
+ }
+
+ /*!
+ @param[in] len the length of the array
+ @return whether array creation completed
+ */
+ bool get_msgpack_array(const std::size_t len)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len)))
+ {
+ return false;
+ }
+
+ for (std::size_t i = 0; i < len; ++i)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal()))
+ {
+ return false;
+ }
+ }
+
+ return sax->end_array();
+ }
+
+ /*!
+ @param[in] len the length of the object
+ @return whether object creation completed
+ */
+ bool get_msgpack_object(const std::size_t len)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len)))
+ {
+ return false;
+ }
+
+ string_t key;
+ for (std::size_t i = 0; i < len; ++i)
+ {
+ get();
+ if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key)))
+ {
+ return false;
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal()))
+ {
+ return false;
+ }
+ key.clear();
+ }
+
+ return sax->end_object();
+ }
+
+ ////////////
+ // UBJSON //
+ ////////////
+
+ /*!
+ @param[in] get_char whether a new character should be retrieved from the
+ input (true, default) or whether the last read
+ character should be considered instead
+
+ @return whether a valid UBJSON value was passed to the SAX parser
+ */
+ bool parse_ubjson_internal(const bool get_char = true)
+ {
+ return get_ubjson_value(get_char ? get_ignore_noop() : current);
+ }
+
+ /*!
+ @brief reads a UBJSON string
+
+ This function is either called after reading the 'S' byte explicitly
+ indicating a string, or in case of an object key where the 'S' byte can be
+ left out.
+
+ @param[out] result created string
+ @param[in] get_char whether a new character should be retrieved from the
+ input (true, default) or whether the last read
+ character should be considered instead
+
+ @return whether string creation completed
+ */
+ bool get_ubjson_string(string_t& result, const bool get_char = true)
+ {
+ if (get_char)
+ {
+ get(); // TODO(niels): may we ignore N here?
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value")))
+ {
+ return false;
+ }
+
+ switch (current)
+ {
+ case 'U':
+ {
+ std::uint8_t len{};
+ return get_number(input_format, len) && get_string(input_format, len, result);
+ }
+
+ case 'i':
+ {
+ std::int8_t len{};
+ return get_number(input_format, len) && get_string(input_format, len, result);
+ }
+
+ case 'I':
+ {
+ std::int16_t len{};
+ return get_number(input_format, len) && get_string(input_format, len, result);
+ }
+
+ case 'l':
+ {
+ std::int32_t len{};
+ return get_number(input_format, len) && get_string(input_format, len, result);
+ }
+
+ case 'L':
+ {
+ std::int64_t len{};
+ return get_number(input_format, len) && get_string(input_format, len, result);
+ }
+
+ case 'u':
+ {
+ if (input_format != input_format_t::bjdata)
+ {
+ break;
+ }
+ std::uint16_t len{};
+ return get_number(input_format, len) && get_string(input_format, len, result);
+ }
+
+ case 'm':
+ {
+ if (input_format != input_format_t::bjdata)
+ {
+ break;
+ }
+ std::uint32_t len{};
+ return get_number(input_format, len) && get_string(input_format, len, result);
+ }
+
+ case 'M':
+ {
+ if (input_format != input_format_t::bjdata)
+ {
+ break;
+ }
+ std::uint64_t len{};
+ return get_number(input_format, len) && get_string(input_format, len, result);
+ }
+
+ default:
+ break;
+ }
+ auto last_token = get_token_string();
+ std::string message;
+
+ if (input_format != input_format_t::bjdata)
+ {
+ message = "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token;
+ }
+ else
+ {
+ message = "expected length type specification (U, i, u, I, m, l, M, L); last byte: 0x" + last_token;
+ }
+ return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "string"), nullptr));
+ }
+
+ /*!
+ @param[out] dim an integer vector storing the ND array dimensions
+ @return whether reading ND array size vector is successful
+ */
+ bool get_ubjson_ndarray_size(std::vector<size_t>& dim)
+ {
+ std::pair<std::size_t, char_int_type> size_and_type;
+ size_t dimlen = 0;
+ bool no_ndarray = true;
+
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type, no_ndarray)))
+ {
+ return false;
+ }
+
+ if (size_and_type.first != npos)
+ {
+ if (size_and_type.second != 0)
+ {
+ if (size_and_type.second != 'N')
+ {
+ for (std::size_t i = 0; i < size_and_type.first; ++i)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second)))
+ {
+ return false;
+ }
+ dim.push_back(dimlen);
+ }
+ }
+ }
+ else
+ {
+ for (std::size_t i = 0; i < size_and_type.first; ++i)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray)))
+ {
+ return false;
+ }
+ dim.push_back(dimlen);
+ }
+ }
+ }
+ else
+ {
+ while (current != ']')
+ {
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, current)))
+ {
+ return false;
+ }
+ dim.push_back(dimlen);
+ get_ignore_noop();
+ }
+ }
+ return true;
+ }
+
+ /*!
+ @param[out] result determined size
+ @param[in,out] is_ndarray for input, `true` means already inside an ndarray vector
+ or ndarray dimension is not allowed; `false` means ndarray
+ is allowed; for output, `true` means an ndarray is found;
+ is_ndarray can only return `true` when its initial value
+ is `false`
+ @param[in] prefix type marker if already read, otherwise set to 0
+
+ @return whether size determination completed
+ */
+ bool get_ubjson_size_value(std::size_t& result, bool& is_ndarray, char_int_type prefix = 0)
+ {
+ if (prefix == 0)
+ {
+ prefix = get_ignore_noop();
+ }
+
+ switch (prefix)
+ {
+ case 'U':
+ {
+ std::uint8_t number{};
+ if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))
+ {
+ return false;
+ }
+ result = static_cast<std::size_t>(number);
+ return true;
+ }
+
+ case 'i':
+ {
+ std::int8_t number{};
+ if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))
+ {
+ return false;
+ }
+ if (number < 0)
+ {
+ return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read,
+ exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr));
+ }
+ result = static_cast<std::size_t>(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char
+ return true;
+ }
+
+ case 'I':
+ {
+ std::int16_t number{};
+ if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))
+ {
+ return false;
+ }
+ if (number < 0)
+ {
+ return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read,
+ exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr));
+ }
+ result = static_cast<std::size_t>(number);
+ return true;
+ }
+
+ case 'l':
+ {
+ std::int32_t number{};
+ if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))
+ {
+ return false;
+ }
+ if (number < 0)
+ {
+ return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read,
+ exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr));
+ }
+ result = static_cast<std::size_t>(number);
+ return true;
+ }
+
+ case 'L':
+ {
+ std::int64_t number{};
+ if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))
+ {
+ return false;
+ }
+ if (number < 0)
+ {
+ return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read,
+ exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr));
+ }
+ if (!value_in_range_of<std::size_t>(number))
+ {
+ return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408,
+ exception_message(input_format, "integer value overflow", "size"), nullptr));
+ }
+ result = static_cast<std::size_t>(number);
+ return true;
+ }
+
+ case 'u':
+ {
+ if (input_format != input_format_t::bjdata)
+ {
+ break;
+ }
+ std::uint16_t number{};
+ if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))
+ {
+ return false;
+ }
+ result = static_cast<std::size_t>(number);
+ return true;
+ }
+
+ case 'm':
+ {
+ if (input_format != input_format_t::bjdata)
+ {
+ break;
+ }
+ std::uint32_t number{};
+ if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))
+ {
+ return false;
+ }
+ result = conditional_static_cast<std::size_t>(number);
+ return true;
+ }
+
+ case 'M':
+ {
+ if (input_format != input_format_t::bjdata)
+ {
+ break;
+ }
+ std::uint64_t number{};
+ if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))
+ {
+ return false;
+ }
+ if (!value_in_range_of<std::size_t>(number))
+ {
+ return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408,
+ exception_message(input_format, "integer value overflow", "size"), nullptr));
+ }
+ result = detail::conditional_static_cast<std::size_t>(number);
+ return true;
+ }
+
+ case '[':
+ {
+ if (input_format != input_format_t::bjdata)
+ {
+ break;
+ }
+ if (is_ndarray) // ndarray dimensional vector can only contain integers, and can not embed another array
+ {
+ return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, "ndarray dimensional vector is not allowed", "size"), nullptr));
+ }
+ std::vector<size_t> dim;
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_ndarray_size(dim)))
+ {
+ return false;
+ }
+ if (dim.size() == 1 || (dim.size() == 2 && dim.at(0) == 1)) // return normal array size if 1D row vector
+ {
+ result = dim.at(dim.size() - 1);
+ return true;
+ }
+ if (!dim.empty()) // if ndarray, convert to an object in JData annotated array format
+ {
+ for (auto i : dim) // test if any dimension in an ndarray is 0, if so, return a 1D empty container
+ {
+ if ( i == 0 )
+ {
+ result = 0;
+ return true;
+ }
+ }
+
+ string_t key = "_ArraySize_";
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3) || !sax->key(key) || !sax->start_array(dim.size())))
+ {
+ return false;
+ }
+ result = 1;
+ for (auto i : dim)
+ {
+ result *= i;
+ if (result == 0 || result == npos) // because dim elements shall not have zeros, result = 0 means overflow happened; it also can't be npos as it is used to initialize size in get_ubjson_size_type()
+ {
+ return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr));
+ }
+ if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(static_cast<number_unsigned_t>(i))))
+ {
+ return false;
+ }
+ }
+ is_ndarray = true;
+ return sax->end_array();
+ }
+ result = 0;
+ return true;
+ }
+
+ default:
+ break;
+ }
+ auto last_token = get_token_string();
+ std::string message;
+
+ if (input_format != input_format_t::bjdata)
+ {
+ message = "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token;
+ }
+ else
+ {
+ message = "expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x" + last_token;
+ }
+ return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "size"), nullptr));
+ }
+
+ /*!
+ @brief determine the type and size for a container
+
+ In the optimized UBJSON format, a type and a size can be provided to allow
+ for a more compact representation.
+
+ @param[out] result pair of the size and the type
+ @param[in] inside_ndarray whether the parser is parsing an ND array dimensional vector
+
+ @return whether pair creation completed
+ */
+ bool get_ubjson_size_type(std::pair<std::size_t, char_int_type>& result, bool inside_ndarray = false)
+ {
+ result.first = npos; // size
+ result.second = 0; // type
+ bool is_ndarray = false;
+
+ get_ignore_noop();
+
+ if (current == '$')
+ {
+ result.second = get(); // must not ignore 'N', because 'N' maybe the type
+ if (input_format == input_format_t::bjdata
+ && JSON_HEDLEY_UNLIKELY(std::binary_search(bjd_optimized_type_markers.begin(), bjd_optimized_type_markers.end(), result.second)))
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+ exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr));
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "type")))
+ {
+ return false;
+ }
+
+ get_ignore_noop();
+ if (JSON_HEDLEY_UNLIKELY(current != '#'))
+ {
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value")))
+ {
+ return false;
+ }
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+ exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr));
+ }
+
+ const bool is_error = get_ubjson_size_value(result.first, is_ndarray);
+ if (input_format == input_format_t::bjdata && is_ndarray)
+ {
+ if (inside_ndarray)
+ {
+ return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read,
+ exception_message(input_format, "ndarray can not be recursive", "size"), nullptr));
+ }
+ result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters
+ }
+ return is_error;
+ }
+
+ if (current == '#')
+ {
+ const bool is_error = get_ubjson_size_value(result.first, is_ndarray);
+ if (input_format == input_format_t::bjdata && is_ndarray)
+ {
+ return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read,
+ exception_message(input_format, "ndarray requires both type and size", "size"), nullptr));
+ }
+ return is_error;
+ }
+
+ return true;
+ }
+
+ /*!
+ @param prefix the previously read or set type prefix
+ @return whether value creation completed
+ */
+ bool get_ubjson_value(const char_int_type prefix)
+ {
+ switch (prefix)
+ {
+ case char_traits<char_type>::eof(): // EOF
+ return unexpect_eof(input_format, "value");
+
+ case 'T': // true
+ return sax->boolean(true);
+ case 'F': // false
+ return sax->boolean(false);
+
+ case 'Z': // null
+ return sax->null();
+
+ case 'U':
+ {
+ std::uint8_t number{};
+ return get_number(input_format, number) && sax->number_unsigned(number);
+ }
+
+ case 'i':
+ {
+ std::int8_t number{};
+ return get_number(input_format, number) && sax->number_integer(number);
+ }
+
+ case 'I':
+ {
+ std::int16_t number{};
+ return get_number(input_format, number) && sax->number_integer(number);
+ }
+
+ case 'l':
+ {
+ std::int32_t number{};
+ return get_number(input_format, number) && sax->number_integer(number);
+ }
+
+ case 'L':
+ {
+ std::int64_t number{};
+ return get_number(input_format, number) && sax->number_integer(number);
+ }
+
+ case 'u':
+ {
+ if (input_format != input_format_t::bjdata)
+ {
+ break;
+ }
+ std::uint16_t number{};
+ return get_number(input_format, number) && sax->number_unsigned(number);
+ }
+
+ case 'm':
+ {
+ if (input_format != input_format_t::bjdata)
+ {
+ break;
+ }
+ std::uint32_t number{};
+ return get_number(input_format, number) && sax->number_unsigned(number);
+ }
+
+ case 'M':
+ {
+ if (input_format != input_format_t::bjdata)
+ {
+ break;
+ }
+ std::uint64_t number{};
+ return get_number(input_format, number) && sax->number_unsigned(number);
+ }
+
+ case 'h':
+ {
+ if (input_format != input_format_t::bjdata)
+ {
+ break;
+ }
+ const auto byte1_raw = get();
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number")))
+ {
+ return false;
+ }
+ const auto byte2_raw = get();
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number")))
+ {
+ return false;
+ }
+
+ const auto byte1 = static_cast<unsigned char>(byte1_raw);
+ const auto byte2 = static_cast<unsigned char>(byte2_raw);
+
+ // code from RFC 7049, Appendix D, Figure 3:
+ // As half-precision floating-point numbers were only added
+ // to IEEE 754 in 2008, today's programming platforms often
+ // still only have limited support for them. It is very
+ // easy to include at least decoding support for them even
+ // without such support. An example of a small decoder for
+ // half-precision floating-point numbers in the C language
+ // is shown in Fig. 3.
+ const auto half = static_cast<unsigned int>((byte2 << 8u) + byte1);
+ const double val = [&half]
+ {
+ const int exp = (half >> 10u) & 0x1Fu;
+ const unsigned int mant = half & 0x3FFu;
+ JSON_ASSERT(0 <= exp&& exp <= 32);
+ JSON_ASSERT(mant <= 1024);
+ switch (exp)
+ {
+ case 0:
+ return std::ldexp(mant, -24);
+ case 31:
+ return (mant == 0)
+ ? std::numeric_limits<double>::infinity()
+ : std::numeric_limits<double>::quiet_NaN();
+ default:
+ return std::ldexp(mant + 1024, exp - 25);
+ }
+ }();
+ return sax->number_float((half & 0x8000u) != 0
+ ? static_cast<number_float_t>(-val)
+ : static_cast<number_float_t>(val), "");
+ }
+
+ case 'd':
+ {
+ float number{};
+ return get_number(input_format, number) && sax->number_float(static_cast<number_float_t>(number), "");
+ }
+
+ case 'D':
+ {
+ double number{};
+ return get_number(input_format, number) && sax->number_float(static_cast<number_float_t>(number), "");
+ }
+
+ case 'H':
+ {
+ return get_ubjson_high_precision_number();
+ }
+
+ case 'C': // char
+ {
+ get();
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "char")))
+ {
+ return false;
+ }
+ if (JSON_HEDLEY_UNLIKELY(current > 127))
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read,
+ exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr));
+ }
+ string_t s(1, static_cast<typename string_t::value_type>(current));
+ return sax->string(s);
+ }
+
+ case 'S': // string
+ {
+ string_t s;
+ return get_ubjson_string(s) && sax->string(s);
+ }
+
+ case '[': // array
+ return get_ubjson_array();
+
+ case '{': // object
+ return get_ubjson_object();
+
+ default: // anything else
+ break;
+ }
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, "invalid byte: 0x" + last_token, "value"), nullptr));
+ }
+
+ /*!
+ @return whether array creation completed
+ */
+ bool get_ubjson_array()
+ {
+ std::pair<std::size_t, char_int_type> size_and_type;
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type)))
+ {
+ return false;
+ }
+
+ // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata):
+ // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]}
+
+ if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
+ {
+ size_and_type.second &= ~(static_cast<char_int_type>(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker
+ auto it = std::lower_bound(bjd_types_map.begin(), bjd_types_map.end(), size_and_type.second, [](const bjd_type & p, char_int_type t)
+ {
+ return p.first < t;
+ });
+ string_t key = "_ArrayType_";
+ if (JSON_HEDLEY_UNLIKELY(it == bjd_types_map.end() || it->first != size_and_type.second))
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+ exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr));
+ }
+
+ string_t type = it->second; // sax->string() takes a reference
+ if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(type)))
+ {
+ return false;
+ }
+
+ if (size_and_type.second == 'C')
+ {
+ size_and_type.second = 'U';
+ }
+
+ key = "_ArrayData_";
+ if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(size_and_type.first) ))
+ {
+ return false;
+ }
+
+ for (std::size_t i = 0; i < size_and_type.first; ++i)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))
+ {
+ return false;
+ }
+ }
+
+ return (sax->end_array() && sax->end_object());
+ }
+
+ if (size_and_type.first != npos)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first)))
+ {
+ return false;
+ }
+
+ if (size_and_type.second != 0)
+ {
+ if (size_and_type.second != 'N')
+ {
+ for (std::size_t i = 0; i < size_and_type.first; ++i)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))
+ {
+ return false;
+ }
+ }
+ }
+ }
+ else
+ {
+ for (std::size_t i = 0; i < size_and_type.first; ++i)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))
+ {
+ return false;
+ }
+ }
+ }
+ }
+ else
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast<std::size_t>(-1))))
+ {
+ return false;
+ }
+
+ while (current != ']')
+ {
+ if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false)))
+ {
+ return false;
+ }
+ get_ignore_noop();
+ }
+ }
+
+ return sax->end_array();
+ }
+
+ /*!
+ @return whether object creation completed
+ */
+ bool get_ubjson_object()
+ {
+ std::pair<std::size_t, char_int_type> size_and_type;
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type)))
+ {
+ return false;
+ }
+
+ // do not accept ND-array size in objects in BJData
+ if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
+ {
+ auto last_token = get_token_string();
+ return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
+ exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr));
+ }
+
+ string_t key;
+ if (size_and_type.first != npos)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first)))
+ {
+ return false;
+ }
+
+ if (size_and_type.second != 0)
+ {
+ for (std::size_t i = 0; i < size_and_type.first; ++i)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))
+ {
+ return false;
+ }
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))
+ {
+ return false;
+ }
+ key.clear();
+ }
+ }
+ else
+ {
+ for (std::size_t i = 0; i < size_and_type.first; ++i)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))
+ {
+ return false;
+ }
+ if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))
+ {
+ return false;
+ }
+ key.clear();
+ }
+ }
+ }
+ else
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast<std::size_t>(-1))))
+ {
+ return false;
+ }
+
+ while (current != '}')
+ {
+ if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key)))
+ {
+ return false;
+ }
+ if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))
+ {
+ return false;
+ }
+ get_ignore_noop();
+ key.clear();
+ }
+ }
+
+ return sax->end_object();
+ }
+
+ // Note, no reader for UBJSON binary types is implemented because they do
+ // not exist
+
+ bool get_ubjson_high_precision_number()
+ {
+ // get size of following number string
+ std::size_t size{};
+ bool no_ndarray = true;
+ auto res = get_ubjson_size_value(size, no_ndarray);
+ if (JSON_HEDLEY_UNLIKELY(!res))
+ {
+ return res;
+ }
+
+ // get number string
+ std::vector<char> number_vector;
+ for (std::size_t i = 0; i < size; ++i)
+ {
+ get();
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number")))
+ {
+ return false;
+ }
+ number_vector.push_back(static_cast<char>(current));
+ }
+
+ // parse number string
+ using ia_type = decltype(detail::input_adapter(number_vector));
+ auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false);
+ const auto result_number = number_lexer.scan();
+ const auto number_string = number_lexer.get_token_string();
+ const auto result_remainder = number_lexer.scan();
+
+ using token_type = typename detail::lexer_base<BasicJsonType>::token_type;
+
+ if (JSON_HEDLEY_UNLIKELY(result_remainder != token_type::end_of_input))
+ {
+ return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read,
+ exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr));
+ }
+
+ switch (result_number)
+ {
+ case token_type::value_integer:
+ return sax->number_integer(number_lexer.get_number_integer());
+ case token_type::value_unsigned:
+ return sax->number_unsigned(number_lexer.get_number_unsigned());
+ case token_type::value_float:
+ return sax->number_float(number_lexer.get_number_float(), std::move(number_string));
+ case token_type::uninitialized:
+ case token_type::literal_true:
+ case token_type::literal_false:
+ case token_type::literal_null:
+ case token_type::value_string:
+ case token_type::begin_array:
+ case token_type::begin_object:
+ case token_type::end_array:
+ case token_type::end_object:
+ case token_type::name_separator:
+ case token_type::value_separator:
+ case token_type::parse_error:
+ case token_type::end_of_input:
+ case token_type::literal_or_value:
+ default:
+ return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read,
+ exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr));
+ }
+ }
+
+ ///////////////////////
+ // Utility functions //
+ ///////////////////////
+
+ /*!
+ @brief get next character from the input
+
+ This function provides the interface to the used input adapter. It does
+ not throw in case the input reached EOF, but returns a -'ve valued
+ `char_traits<char_type>::eof()` in that case.
+
+ @return character read from the input
+ */
+ char_int_type get()
+ {
+ ++chars_read;
+ return current = ia.get_character();
+ }
+
+ /*!
+ @return character read from the input after ignoring all 'N' entries
+ */
+ char_int_type get_ignore_noop()
+ {
+ do
+ {
+ get();
+ }
+ while (current == 'N');
+
+ return current;
+ }
+
+ /*
+ @brief read a number from the input
+
+ @tparam NumberType the type of the number
+ @param[in] format the current format (for diagnostics)
+ @param[out] result number of type @a NumberType
+
+ @return whether conversion completed
+
+ @note This function needs to respect the system's endianness, because
+ bytes in CBOR, MessagePack, and UBJSON are stored in network order
+ (big endian) and therefore need reordering on little endian systems.
+ On the other hand, BSON and BJData use little endian and should reorder
+ on big endian systems.
+ */
+ template<typename NumberType, bool InputIsLittleEndian = false>
+ bool get_number(const input_format_t format, NumberType& result)
+ {
+ // step 1: read input into array with system's byte order
+ std::array<std::uint8_t, sizeof(NumberType)> vec{};
+ for (std::size_t i = 0; i < sizeof(NumberType); ++i)
+ {
+ get();
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "number")))
+ {
+ return false;
+ }
+
+ // reverse byte order prior to conversion if necessary
+ if (is_little_endian != (InputIsLittleEndian || format == input_format_t::bjdata))
+ {
+ vec[sizeof(NumberType) - i - 1] = static_cast<std::uint8_t>(current);
+ }
+ else
+ {
+ vec[i] = static_cast<std::uint8_t>(current); // LCOV_EXCL_LINE
+ }
+ }
+
+ // step 2: convert array into number of type T and return
+ std::memcpy(&result, vec.data(), sizeof(NumberType));
+ return true;
+ }
+
+ /*!
+ @brief create a string by reading characters from the input
+
+ @tparam NumberType the type of the number
+ @param[in] format the current format (for diagnostics)
+ @param[in] len number of characters to read
+ @param[out] result string created by reading @a len bytes
+
+ @return whether string creation completed
+
+ @note We can not reserve @a len bytes for the result, because @a len
+ may be too large. Usually, @ref unexpect_eof() detects the end of
+ the input before we run out of string memory.
+ */
+ template<typename NumberType>
+ bool get_string(const input_format_t format,
+ const NumberType len,
+ string_t& result)
+ {
+ bool success = true;
+ for (NumberType i = 0; i < len; i++)
+ {
+ get();
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string")))
+ {
+ success = false;
+ break;
+ }
+ result.push_back(static_cast<typename string_t::value_type>(current));
+ }
+ return success;
+ }
+
+ /*!
+ @brief create a byte array by reading bytes from the input
+
+ @tparam NumberType the type of the number
+ @param[in] format the current format (for diagnostics)
+ @param[in] len number of bytes to read
+ @param[out] result byte array created by reading @a len bytes
+
+ @return whether byte array creation completed
+
+ @note We can not reserve @a len bytes for the result, because @a len
+ may be too large. Usually, @ref unexpect_eof() detects the end of
+ the input before we run out of memory.
+ */
+ template<typename NumberType>
+ bool get_binary(const input_format_t format,
+ const NumberType len,
+ binary_t& result)
+ {
+ bool success = true;
+ for (NumberType i = 0; i < len; i++)
+ {
+ get();
+ if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary")))
+ {
+ success = false;
+ break;
+ }
+ result.push_back(static_cast<std::uint8_t>(current));
+ }
+ return success;
+ }
+
+ /*!
+ @param[in] format the current format (for diagnostics)
+ @param[in] context further context information (for diagnostics)
+ @return whether the last read character is not EOF
+ */
+ JSON_HEDLEY_NON_NULL(3)
+ bool unexpect_eof(const input_format_t format, const char* context) const
+ {
+ if (JSON_HEDLEY_UNLIKELY(current == char_traits<char_type>::eof()))
+ {
+ return sax->parse_error(chars_read, "<end of file>",
+ parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr));
+ }
+ return true;
+ }
+
+ /*!
+ @return a string representation of the last read byte
+ */
+ std::string get_token_string() const
+ {
+ std::array<char, 3> cr{{}};
+ static_cast<void>((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast<unsigned char>(current))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+ return std::string{cr.data()};
+ }
+
+ /*!
+ @param[in] format the current format
+ @param[in] detail a detailed error message
+ @param[in] context further context information
+ @return a message string to use in the parse_error exceptions
+ */
+ std::string exception_message(const input_format_t format,
+ const std::string& detail,
+ const std::string& context) const
+ {
+ std::string error_msg = "syntax error while parsing ";
+
+ switch (format)
+ {
+ case input_format_t::cbor:
+ error_msg += "CBOR";
+ break;
+
+ case input_format_t::msgpack:
+ error_msg += "MessagePack";
+ break;
+
+ case input_format_t::ubjson:
+ error_msg += "UBJSON";
+ break;
+
+ case input_format_t::bson:
+ error_msg += "BSON";
+ break;
+
+ case input_format_t::bjdata:
+ error_msg += "BJData";
+ break;
+
+ case input_format_t::json: // LCOV_EXCL_LINE
+ default: // LCOV_EXCL_LINE
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
+ }
+
+ return concat(error_msg, ' ', context, ": ", detail);
+ }
+
+ private:
+ static JSON_INLINE_VARIABLE constexpr std::size_t npos = static_cast<std::size_t>(-1);
+
+ /// input adapter
+ InputAdapterType ia;
+
+ /// the current character
+ char_int_type current = char_traits<char_type>::eof();
+
+ /// the number of characters read
+ std::size_t chars_read = 0;
+
+ /// whether we can assume little endianness
+ const bool is_little_endian = little_endianness();
+
+ /// input format
+ const input_format_t input_format = input_format_t::json;
+
+ /// the SAX parser
+ json_sax_t* sax = nullptr;
+
+ // excluded markers in bjdata optimized type
+#define JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_ \
+ make_array<char_int_type>('F', 'H', 'N', 'S', 'T', 'Z', '[', '{')
+
+#define JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_ \
+ make_array<bjd_type>( \
+ bjd_type{'C', "char"}, \
+ bjd_type{'D', "double"}, \
+ bjd_type{'I', "int16"}, \
+ bjd_type{'L', "int64"}, \
+ bjd_type{'M', "uint64"}, \
+ bjd_type{'U', "uint8"}, \
+ bjd_type{'d', "single"}, \
+ bjd_type{'i', "int8"}, \
+ bjd_type{'l', "int32"}, \
+ bjd_type{'m', "uint32"}, \
+ bjd_type{'u', "uint16"})
+
+ JSON_PRIVATE_UNLESS_TESTED:
+ // lookup tables
+ // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes)
+ const decltype(JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_) bjd_optimized_type_markers =
+ JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_;
+
+ using bjd_type = std::pair<char_int_type, string_t>;
+ // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes)
+ const decltype(JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_) bjd_types_map =
+ JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_;
+
+#undef JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_
+#undef JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_
+};
+
+#ifndef JSON_HAS_CPP_17
+ template<typename BasicJsonType, typename InputAdapterType, typename SAX>
+ constexpr std::size_t binary_reader<BasicJsonType, InputAdapterType, SAX>::npos;
+#endif
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+// #include <nlohmann/detail/input/lexer.hpp>
+
+// #include <nlohmann/detail/input/parser.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <cmath> // isfinite
+#include <cstdint> // uint8_t
+#include <functional> // function
+#include <string> // string
+#include <utility> // move
+#include <vector> // vector
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+// #include <nlohmann/detail/input/json_sax.hpp>
+
+// #include <nlohmann/detail/input/lexer.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/is_sax.hpp>
+
+// #include <nlohmann/detail/string_concat.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+////////////
+// parser //
+////////////
+
+enum class parse_event_t : std::uint8_t
+{
+ /// the parser read `{` and started to process a JSON object
+ object_start,
+ /// the parser read `}` and finished processing a JSON object
+ object_end,
+ /// the parser read `[` and started to process a JSON array
+ array_start,
+ /// the parser read `]` and finished processing a JSON array
+ array_end,
+ /// the parser read a key of a value in an object
+ key,
+ /// the parser finished reading a JSON value
+ value
+};
+
+template<typename BasicJsonType>
+using parser_callback_t =
+ std::function<bool(int /*depth*/, parse_event_t /*event*/, BasicJsonType& /*parsed*/)>;
+
+/*!
+@brief syntax analysis
+
+This class implements a recursive descent parser.
+*/
+template<typename BasicJsonType, typename InputAdapterType>
+class parser
+{
+ using number_integer_t = typename BasicJsonType::number_integer_t;
+ using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+ using string_t = typename BasicJsonType::string_t;
+ using lexer_t = lexer<BasicJsonType, InputAdapterType>;
+ using token_type = typename lexer_t::token_type;
+
+ public:
+ /// a parser reading from an input adapter
+ explicit parser(InputAdapterType&& adapter,
+ const parser_callback_t<BasicJsonType> cb = nullptr,
+ const bool allow_exceptions_ = true,
+ const bool skip_comments = false)
+ : callback(cb)
+ , m_lexer(std::move(adapter), skip_comments)
+ , allow_exceptions(allow_exceptions_)
+ {
+ // read first token
+ get_token();
+ }
+
+ /*!
+ @brief public parser interface
+
+ @param[in] strict whether to expect the last token to be EOF
+ @param[in,out] result parsed JSON value
+
+ @throw parse_error.101 in case of an unexpected token
+ @throw parse_error.102 if to_unicode fails or surrogate error
+ @throw parse_error.103 if to_unicode fails
+ */
+ void parse(const bool strict, BasicJsonType& result)
+ {
+ if (callback)
+ {
+ json_sax_dom_callback_parser<BasicJsonType> sdp(result, callback, allow_exceptions);
+ sax_parse_internal(&sdp);
+
+ // in strict mode, input must be completely read
+ if (strict && (get_token() != token_type::end_of_input))
+ {
+ sdp.parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(),
+ exception_message(token_type::end_of_input, "value"), nullptr));
+ }
+
+ // in case of an error, return discarded value
+ if (sdp.is_errored())
+ {
+ result = value_t::discarded;
+ return;
+ }
+
+ // set top-level value to null if it was discarded by the callback
+ // function
+ if (result.is_discarded())
+ {
+ result = nullptr;
+ }
+ }
+ else
+ {
+ json_sax_dom_parser<BasicJsonType> sdp(result, allow_exceptions);
+ sax_parse_internal(&sdp);
+
+ // in strict mode, input must be completely read
+ if (strict && (get_token() != token_type::end_of_input))
+ {
+ sdp.parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr));
+ }
+
+ // in case of an error, return discarded value
+ if (sdp.is_errored())
+ {
+ result = value_t::discarded;
+ return;
+ }
+ }
+
+ result.assert_invariant();
+ }
+
+ /*!
+ @brief public accept interface
+
+ @param[in] strict whether to expect the last token to be EOF
+ @return whether the input is a proper JSON text
+ */
+ bool accept(const bool strict = true)
+ {
+ json_sax_acceptor<BasicJsonType> sax_acceptor;
+ return sax_parse(&sax_acceptor, strict);
+ }
+
+ template<typename SAX>
+ JSON_HEDLEY_NON_NULL(2)
+ bool sax_parse(SAX* sax, const bool strict = true)
+ {
+ (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
+ const bool result = sax_parse_internal(sax);
+
+ // strict mode: next byte must be EOF
+ if (result && strict && (get_token() != token_type::end_of_input))
+ {
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr));
+ }
+
+ return result;
+ }
+
+ private:
+ template<typename SAX>
+ JSON_HEDLEY_NON_NULL(2)
+ bool sax_parse_internal(SAX* sax)
+ {
+ // stack to remember the hierarchy of structured values we are parsing
+ // true = array; false = object
+ std::vector<bool> states;
+ // value to avoid a goto (see comment where set to true)
+ bool skip_to_state_evaluation = false;
+
+ while (true)
+ {
+ if (!skip_to_state_evaluation)
+ {
+ // invariant: get_token() was called before each iteration
+ switch (last_token)
+ {
+ case token_type::begin_object:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast<std::size_t>(-1))))
+ {
+ return false;
+ }
+
+ // closing } -> we are done
+ if (get_token() == token_type::end_object)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->end_object()))
+ {
+ return false;
+ }
+ break;
+ }
+
+ // parse key
+ if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string))
+ {
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr));
+ }
+ if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
+ {
+ return false;
+ }
+
+ // parse separator (:)
+ if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
+ {
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr));
+ }
+
+ // remember we are now inside an object
+ states.push_back(false);
+
+ // parse values
+ get_token();
+ continue;
+ }
+
+ case token_type::begin_array:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast<std::size_t>(-1))))
+ {
+ return false;
+ }
+
+ // closing ] -> we are done
+ if (get_token() == token_type::end_array)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->end_array()))
+ {
+ return false;
+ }
+ break;
+ }
+
+ // remember we are now inside an array
+ states.push_back(true);
+
+ // parse values (no need to call get_token)
+ continue;
+ }
+
+ case token_type::value_float:
+ {
+ const auto res = m_lexer.get_number_float();
+
+ if (JSON_HEDLEY_UNLIKELY(!std::isfinite(res)))
+ {
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr));
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string())))
+ {
+ return false;
+ }
+
+ break;
+ }
+
+ case token_type::literal_false:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->boolean(false)))
+ {
+ return false;
+ }
+ break;
+ }
+
+ case token_type::literal_null:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->null()))
+ {
+ return false;
+ }
+ break;
+ }
+
+ case token_type::literal_true:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->boolean(true)))
+ {
+ return false;
+ }
+ break;
+ }
+
+ case token_type::value_integer:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->number_integer(m_lexer.get_number_integer())))
+ {
+ return false;
+ }
+ break;
+ }
+
+ case token_type::value_string:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string())))
+ {
+ return false;
+ }
+ break;
+ }
+
+ case token_type::value_unsigned:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(m_lexer.get_number_unsigned())))
+ {
+ return false;
+ }
+ break;
+ }
+
+ case token_type::parse_error:
+ {
+ // using "uninitialized" to avoid "expected" message
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr));
+ }
+ case token_type::end_of_input:
+ {
+ if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1))
+ {
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(),
+ "attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr));
+ }
+
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr));
+ }
+ case token_type::uninitialized:
+ case token_type::end_array:
+ case token_type::end_object:
+ case token_type::name_separator:
+ case token_type::value_separator:
+ case token_type::literal_or_value:
+ default: // the last token was unexpected
+ {
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr));
+ }
+ }
+ }
+ else
+ {
+ skip_to_state_evaluation = false;
+ }
+
+ // we reached this line after we successfully parsed a value
+ if (states.empty())
+ {
+ // empty stack: we reached the end of the hierarchy: done
+ return true;
+ }
+
+ if (states.back()) // array
+ {
+ // comma -> next value
+ if (get_token() == token_type::value_separator)
+ {
+ // parse a new value
+ get_token();
+ continue;
+ }
+
+ // closing ]
+ if (JSON_HEDLEY_LIKELY(last_token == token_type::end_array))
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->end_array()))
+ {
+ return false;
+ }
+
+ // We are done with this array. Before we can parse a
+ // new value, we need to evaluate the new state first.
+ // By setting skip_to_state_evaluation to false, we
+ // are effectively jumping to the beginning of this if.
+ JSON_ASSERT(!states.empty());
+ states.pop_back();
+ skip_to_state_evaluation = true;
+ continue;
+ }
+
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), nullptr));
+ }
+
+ // states.back() is false -> object
+
+ // comma -> next value
+ if (get_token() == token_type::value_separator)
+ {
+ // parse key
+ if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::value_string))
+ {
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr));
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
+ {
+ return false;
+ }
+
+ // parse separator (:)
+ if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
+ {
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr));
+ }
+
+ // parse values
+ get_token();
+ continue;
+ }
+
+ // closing }
+ if (JSON_HEDLEY_LIKELY(last_token == token_type::end_object))
+ {
+ if (JSON_HEDLEY_UNLIKELY(!sax->end_object()))
+ {
+ return false;
+ }
+
+ // We are done with this object. Before we can parse a
+ // new value, we need to evaluate the new state first.
+ // By setting skip_to_state_evaluation to false, we
+ // are effectively jumping to the beginning of this if.
+ JSON_ASSERT(!states.empty());
+ states.pop_back();
+ skip_to_state_evaluation = true;
+ continue;
+ }
+
+ return sax->parse_error(m_lexer.get_position(),
+ m_lexer.get_token_string(),
+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), nullptr));
+ }
+ }
+
+ /// get next token from lexer
+ token_type get_token()
+ {
+ return last_token = m_lexer.scan();
+ }
+
+ std::string exception_message(const token_type expected, const std::string& context)
+ {
+ std::string error_msg = "syntax error ";
+
+ if (!context.empty())
+ {
+ error_msg += concat("while parsing ", context, ' ');
+ }
+
+ error_msg += "- ";
+
+ if (last_token == token_type::parse_error)
+ {
+ error_msg += concat(m_lexer.get_error_message(), "; last read: '",
+ m_lexer.get_token_string(), '\'');
+ }
+ else
+ {
+ error_msg += concat("unexpected ", lexer_t::token_type_name(last_token));
+ }
+
+ if (expected != token_type::uninitialized)
+ {
+ error_msg += concat("; expected ", lexer_t::token_type_name(expected));
+ }
+
+ return error_msg;
+ }
+
+ private:
+ /// callback function
+ const parser_callback_t<BasicJsonType> callback = nullptr;
+ /// the type of the last read token
+ token_type last_token = token_type::uninitialized;
+ /// the lexer
+ lexer_t m_lexer;
+ /// whether to throw exceptions in case of errors
+ const bool allow_exceptions = true;
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/iterators/internal_iterator.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <cstddef> // ptrdiff_t
+#include <limits> // numeric_limits
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+/*
+@brief an iterator for primitive JSON types
+
+This class models an iterator for primitive JSON types (boolean, number,
+string). It's only purpose is to allow the iterator/const_iterator classes
+to "iterate" over primitive values. Internally, the iterator is modeled by
+a `difference_type` variable. Value begin_value (`0`) models the begin,
+end_value (`1`) models past the end.
+*/
+class primitive_iterator_t
+{
+ private:
+ using difference_type = std::ptrdiff_t;
+ static constexpr difference_type begin_value = 0;
+ static constexpr difference_type end_value = begin_value + 1;
+
+ JSON_PRIVATE_UNLESS_TESTED:
+ /// iterator as signed integer type
+ difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)();
+
+ public:
+ constexpr difference_type get_value() const noexcept
+ {
+ return m_it;
+ }
+
+ /// set iterator to a defined beginning
+ void set_begin() noexcept
+ {
+ m_it = begin_value;
+ }
+
+ /// set iterator to a defined past the end
+ void set_end() noexcept
+ {
+ m_it = end_value;
+ }
+
+ /// return whether the iterator can be dereferenced
+ constexpr bool is_begin() const noexcept
+ {
+ return m_it == begin_value;
+ }
+
+ /// return whether the iterator is at end
+ constexpr bool is_end() const noexcept
+ {
+ return m_it == end_value;
+ }
+
+ friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+ {
+ return lhs.m_it == rhs.m_it;
+ }
+
+ friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+ {
+ return lhs.m_it < rhs.m_it;
+ }
+
+ primitive_iterator_t operator+(difference_type n) noexcept
+ {
+ auto result = *this;
+ result += n;
+ return result;
+ }
+
+ friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+ {
+ return lhs.m_it - rhs.m_it;
+ }
+
+ primitive_iterator_t& operator++() noexcept
+ {
+ ++m_it;
+ return *this;
+ }
+
+ primitive_iterator_t operator++(int)& noexcept // NOLINT(cert-dcl21-cpp)
+ {
+ auto result = *this;
+ ++m_it;
+ return result;
+ }
+
+ primitive_iterator_t& operator--() noexcept
+ {
+ --m_it;
+ return *this;
+ }
+
+ primitive_iterator_t operator--(int)& noexcept // NOLINT(cert-dcl21-cpp)
+ {
+ auto result = *this;
+ --m_it;
+ return result;
+ }
+
+ primitive_iterator_t& operator+=(difference_type n) noexcept
+ {
+ m_it += n;
+ return *this;
+ }
+
+ primitive_iterator_t& operator-=(difference_type n) noexcept
+ {
+ m_it -= n;
+ return *this;
+ }
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+/*!
+@brief an iterator value
+
+@note This structure could easily be a union, but MSVC currently does not allow
+unions members with complex constructors, see https://github.com/nlohmann/json/pull/105.
+*/
+template<typename BasicJsonType> struct internal_iterator
+{
+ /// iterator for JSON objects
+ typename BasicJsonType::object_t::iterator object_iterator {};
+ /// iterator for JSON arrays
+ typename BasicJsonType::array_t::iterator array_iterator {};
+ /// generic iterator for all other types
+ primitive_iterator_t primitive_iterator {};
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/iterators/iter_impl.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <iterator> // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next
+#include <type_traits> // conditional, is_const, remove_const
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/iterators/internal_iterator.hpp>
+
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+// forward declare, to be able to friend it later on
+template<typename IteratorType> class iteration_proxy;
+template<typename IteratorType> class iteration_proxy_value;
+
+/*!
+@brief a template for a bidirectional iterator for the @ref basic_json class
+This class implements a both iterators (iterator and const_iterator) for the
+@ref basic_json class.
+@note An iterator is called *initialized* when a pointer to a JSON value has
+ been set (e.g., by a constructor or a copy assignment). If the iterator is
+ default-constructed, it is *uninitialized* and most methods are undefined.
+ **The library uses assertions to detect calls on uninitialized iterators.**
+@requirement The class satisfies the following concept requirements:
+-
+[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):
+ The iterator that can be moved can be moved in both directions (i.e.
+ incremented and decremented).
+@since version 1.0.0, simplified in version 2.0.9, change to bidirectional
+ iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593)
+*/
+template<typename BasicJsonType>
+class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions)
+{
+ /// the iterator with BasicJsonType of different const-ness
+ using other_iter_impl = iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>;
+ /// allow basic_json to access private members
+ friend other_iter_impl;
+ friend BasicJsonType;
+ friend iteration_proxy<iter_impl>;
+ friend iteration_proxy_value<iter_impl>;
+
+ using object_t = typename BasicJsonType::object_t;
+ using array_t = typename BasicJsonType::array_t;
+ // make sure BasicJsonType is basic_json or const basic_json
+ static_assert(is_basic_json<typename std::remove_const<BasicJsonType>::type>::value,
+ "iter_impl only accepts (const) basic_json");
+ // superficial check for the LegacyBidirectionalIterator named requirement
+ static_assert(std::is_base_of<std::bidirectional_iterator_tag, std::bidirectional_iterator_tag>::value
+ && std::is_base_of<std::bidirectional_iterator_tag, typename std::iterator_traits<typename array_t::iterator>::iterator_category>::value,
+ "basic_json iterator assumes array and object type iterators satisfy the LegacyBidirectionalIterator named requirement.");
+
+ public:
+ /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17.
+ /// The C++ Standard has never required user-defined iterators to derive from std::iterator.
+ /// A user-defined iterator should provide publicly accessible typedefs named
+ /// iterator_category, value_type, difference_type, pointer, and reference.
+ /// Note that value_type is required to be non-const, even for constant iterators.
+ using iterator_category = std::bidirectional_iterator_tag;
+
+ /// the type of the values when the iterator is dereferenced
+ using value_type = typename BasicJsonType::value_type;
+ /// a type to represent differences between iterators
+ using difference_type = typename BasicJsonType::difference_type;
+ /// defines a pointer to the type iterated over (value_type)
+ using pointer = typename std::conditional<std::is_const<BasicJsonType>::value,
+ typename BasicJsonType::const_pointer,
+ typename BasicJsonType::pointer>::type;
+ /// defines a reference to the type iterated over (value_type)
+ using reference =
+ typename std::conditional<std::is_const<BasicJsonType>::value,
+ typename BasicJsonType::const_reference,
+ typename BasicJsonType::reference>::type;
+
+ iter_impl() = default;
+ ~iter_impl() = default;
+ iter_impl(iter_impl&&) noexcept = default;
+ iter_impl& operator=(iter_impl&&) noexcept = default;
+
+ /*!
+ @brief constructor for a given JSON instance
+ @param[in] object pointer to a JSON object for this iterator
+ @pre object != nullptr
+ @post The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ explicit iter_impl(pointer object) noexcept : m_object(object)
+ {
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ {
+ m_it.object_iterator = typename object_t::iterator();
+ break;
+ }
+
+ case value_t::array:
+ {
+ m_it.array_iterator = typename array_t::iterator();
+ break;
+ }
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ m_it.primitive_iterator = primitive_iterator_t();
+ break;
+ }
+ }
+ }
+
+ /*!
+ @note The conventional copy constructor and copy assignment are implicitly
+ defined. Combined with the following converting constructor and
+ assignment, they support: (1) copy from iterator to iterator, (2)
+ copy from const iterator to const iterator, and (3) conversion from
+ iterator to const iterator. However conversion from const iterator
+ to iterator is not defined.
+ */
+
+ /*!
+ @brief const copy constructor
+ @param[in] other const iterator to copy from
+ @note This copy constructor had to be defined explicitly to circumvent a bug
+ occurring on msvc v19.0 compiler (VS 2015) debug build. For more
+ information refer to: https://github.com/nlohmann/json/issues/1608
+ */
+ iter_impl(const iter_impl<const BasicJsonType>& other) noexcept
+ : m_object(other.m_object), m_it(other.m_it)
+ {}
+
+ /*!
+ @brief converting assignment
+ @param[in] other const iterator to copy from
+ @return const/non-const iterator
+ @note It is not checked whether @a other is initialized.
+ */
+ iter_impl& operator=(const iter_impl<const BasicJsonType>& other) noexcept
+ {
+ if (&other != this)
+ {
+ m_object = other.m_object;
+ m_it = other.m_it;
+ }
+ return *this;
+ }
+
+ /*!
+ @brief converting constructor
+ @param[in] other non-const iterator to copy from
+ @note It is not checked whether @a other is initialized.
+ */
+ iter_impl(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept
+ : m_object(other.m_object), m_it(other.m_it)
+ {}
+
+ /*!
+ @brief converting assignment
+ @param[in] other non-const iterator to copy from
+ @return const/non-const iterator
+ @note It is not checked whether @a other is initialized.
+ */
+ iter_impl& operator=(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept // NOLINT(cert-oop54-cpp)
+ {
+ m_object = other.m_object;
+ m_it = other.m_it;
+ return *this;
+ }
+
+ JSON_PRIVATE_UNLESS_TESTED:
+ /*!
+ @brief set the iterator to the first value
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ void set_begin() noexcept
+ {
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ {
+ m_it.object_iterator = m_object->m_data.m_value.object->begin();
+ break;
+ }
+
+ case value_t::array:
+ {
+ m_it.array_iterator = m_object->m_data.m_value.array->begin();
+ break;
+ }
+
+ case value_t::null:
+ {
+ // set to end so begin()==end() is true: null is empty
+ m_it.primitive_iterator.set_end();
+ break;
+ }
+
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ m_it.primitive_iterator.set_begin();
+ break;
+ }
+ }
+ }
+
+ /*!
+ @brief set the iterator past the last value
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ void set_end() noexcept
+ {
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ {
+ m_it.object_iterator = m_object->m_data.m_value.object->end();
+ break;
+ }
+
+ case value_t::array:
+ {
+ m_it.array_iterator = m_object->m_data.m_value.array->end();
+ break;
+ }
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ m_it.primitive_iterator.set_end();
+ break;
+ }
+ }
+ }
+
+ public:
+ /*!
+ @brief return a reference to the value pointed to by the iterator
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ reference operator*() const
+ {
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ {
+ JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end());
+ return m_it.object_iterator->second;
+ }
+
+ case value_t::array:
+ {
+ JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end());
+ return *m_it.array_iterator;
+ }
+
+ case value_t::null:
+ JSON_THROW(invalid_iterator::create(214, "cannot get value", m_object));
+
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin()))
+ {
+ return *m_object;
+ }
+
+ JSON_THROW(invalid_iterator::create(214, "cannot get value", m_object));
+ }
+ }
+ }
+
+ /*!
+ @brief dereference the iterator
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ pointer operator->() const
+ {
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ {
+ JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end());
+ return &(m_it.object_iterator->second);
+ }
+
+ case value_t::array:
+ {
+ JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end());
+ return &*m_it.array_iterator;
+ }
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin()))
+ {
+ return m_object;
+ }
+
+ JSON_THROW(invalid_iterator::create(214, "cannot get value", m_object));
+ }
+ }
+ }
+
+ /*!
+ @brief post-increment (it++)
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ iter_impl operator++(int)& // NOLINT(cert-dcl21-cpp)
+ {
+ auto result = *this;
+ ++(*this);
+ return result;
+ }
+
+ /*!
+ @brief pre-increment (++it)
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ iter_impl& operator++()
+ {
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ {
+ std::advance(m_it.object_iterator, 1);
+ break;
+ }
+
+ case value_t::array:
+ {
+ std::advance(m_it.array_iterator, 1);
+ break;
+ }
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ ++m_it.primitive_iterator;
+ break;
+ }
+ }
+
+ return *this;
+ }
+
+ /*!
+ @brief post-decrement (it--)
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ iter_impl operator--(int)& // NOLINT(cert-dcl21-cpp)
+ {
+ auto result = *this;
+ --(*this);
+ return result;
+ }
+
+ /*!
+ @brief pre-decrement (--it)
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ iter_impl& operator--()
+ {
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ {
+ std::advance(m_it.object_iterator, -1);
+ break;
+ }
+
+ case value_t::array:
+ {
+ std::advance(m_it.array_iterator, -1);
+ break;
+ }
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ --m_it.primitive_iterator;
+ break;
+ }
+ }
+
+ return *this;
+ }
+
+ /*!
+ @brief comparison: equal
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ template < typename IterImpl, detail::enable_if_t < (std::is_same<IterImpl, iter_impl>::value || std::is_same<IterImpl, other_iter_impl>::value), std::nullptr_t > = nullptr >
+ bool operator==(const IterImpl& other) const
+ {
+ // if objects are not the same, the comparison is undefined
+ if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object))
+ {
+ JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers", m_object));
+ }
+
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ return (m_it.object_iterator == other.m_it.object_iterator);
+
+ case value_t::array:
+ return (m_it.array_iterator == other.m_it.array_iterator);
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ return (m_it.primitive_iterator == other.m_it.primitive_iterator);
+ }
+ }
+
+ /*!
+ @brief comparison: not equal
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ template < typename IterImpl, detail::enable_if_t < (std::is_same<IterImpl, iter_impl>::value || std::is_same<IterImpl, other_iter_impl>::value), std::nullptr_t > = nullptr >
+ bool operator!=(const IterImpl& other) const
+ {
+ return !operator==(other);
+ }
+
+ /*!
+ @brief comparison: smaller
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ bool operator<(const iter_impl& other) const
+ {
+ // if objects are not the same, the comparison is undefined
+ if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object))
+ {
+ JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers", m_object));
+ }
+
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators", m_object));
+
+ case value_t::array:
+ return (m_it.array_iterator < other.m_it.array_iterator);
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ return (m_it.primitive_iterator < other.m_it.primitive_iterator);
+ }
+ }
+
+ /*!
+ @brief comparison: less than or equal
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ bool operator<=(const iter_impl& other) const
+ {
+ return !other.operator < (*this);
+ }
+
+ /*!
+ @brief comparison: greater than
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ bool operator>(const iter_impl& other) const
+ {
+ return !operator<=(other);
+ }
+
+ /*!
+ @brief comparison: greater than or equal
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ bool operator>=(const iter_impl& other) const
+ {
+ return !operator<(other);
+ }
+
+ /*!
+ @brief add to iterator
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ iter_impl& operator+=(difference_type i)
+ {
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", m_object));
+
+ case value_t::array:
+ {
+ std::advance(m_it.array_iterator, i);
+ break;
+ }
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ m_it.primitive_iterator += i;
+ break;
+ }
+ }
+
+ return *this;
+ }
+
+ /*!
+ @brief subtract from iterator
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ iter_impl& operator-=(difference_type i)
+ {
+ return operator+=(-i);
+ }
+
+ /*!
+ @brief add to iterator
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ iter_impl operator+(difference_type i) const
+ {
+ auto result = *this;
+ result += i;
+ return result;
+ }
+
+ /*!
+ @brief addition of distance and iterator
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ friend iter_impl operator+(difference_type i, const iter_impl& it)
+ {
+ auto result = it;
+ result += i;
+ return result;
+ }
+
+ /*!
+ @brief subtract from iterator
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ iter_impl operator-(difference_type i) const
+ {
+ auto result = *this;
+ result -= i;
+ return result;
+ }
+
+ /*!
+ @brief return difference
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ difference_type operator-(const iter_impl& other) const
+ {
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", m_object));
+
+ case value_t::array:
+ return m_it.array_iterator - other.m_it.array_iterator;
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ return m_it.primitive_iterator - other.m_it.primitive_iterator;
+ }
+ }
+
+ /*!
+ @brief access to successor
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ reference operator[](difference_type n) const
+ {
+ JSON_ASSERT(m_object != nullptr);
+
+ switch (m_object->m_data.m_type)
+ {
+ case value_t::object:
+ JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators", m_object));
+
+ case value_t::array:
+ return *std::next(m_it.array_iterator, n);
+
+ case value_t::null:
+ JSON_THROW(invalid_iterator::create(214, "cannot get value", m_object));
+
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.get_value() == -n))
+ {
+ return *m_object;
+ }
+
+ JSON_THROW(invalid_iterator::create(214, "cannot get value", m_object));
+ }
+ }
+ }
+
+ /*!
+ @brief return the key of an object iterator
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ const typename object_t::key_type& key() const
+ {
+ JSON_ASSERT(m_object != nullptr);
+
+ if (JSON_HEDLEY_LIKELY(m_object->is_object()))
+ {
+ return m_it.object_iterator->first;
+ }
+
+ JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators", m_object));
+ }
+
+ /*!
+ @brief return the value of an iterator
+ @pre The iterator is initialized; i.e. `m_object != nullptr`.
+ */
+ reference value() const
+ {
+ return operator*();
+ }
+
+ JSON_PRIVATE_UNLESS_TESTED:
+ /// associated JSON instance
+ pointer m_object = nullptr;
+ /// the actual iterator of the associated instance
+ internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it {};
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/iterators/iteration_proxy.hpp>
+
+// #include <nlohmann/detail/iterators/json_reverse_iterator.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <cstddef> // ptrdiff_t
+#include <iterator> // reverse_iterator
+#include <utility> // declval
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+//////////////////////
+// reverse_iterator //
+//////////////////////
+
+/*!
+@brief a template for a reverse iterator class
+
+@tparam Base the base iterator type to reverse. Valid types are @ref
+iterator (to create @ref reverse_iterator) and @ref const_iterator (to
+create @ref const_reverse_iterator).
+
+@requirement The class satisfies the following concept requirements:
+-
+[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):
+ The iterator that can be moved can be moved in both directions (i.e.
+ incremented and decremented).
+- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator):
+ It is possible to write to the pointed-to element (only if @a Base is
+ @ref iterator).
+
+@since version 1.0.0
+*/
+template<typename Base>
+class json_reverse_iterator : public std::reverse_iterator<Base>
+{
+ public:
+ using difference_type = std::ptrdiff_t;
+ /// shortcut to the reverse iterator adapter
+ using base_iterator = std::reverse_iterator<Base>;
+ /// the reference type for the pointed-to element
+ using reference = typename Base::reference;
+
+ /// create reverse iterator from iterator
+ explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept
+ : base_iterator(it) {}
+
+ /// create reverse iterator from base class
+ explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {}
+
+ /// post-increment (it++)
+ json_reverse_iterator operator++(int)& // NOLINT(cert-dcl21-cpp)
+ {
+ return static_cast<json_reverse_iterator>(base_iterator::operator++(1));
+ }
+
+ /// pre-increment (++it)
+ json_reverse_iterator& operator++()
+ {
+ return static_cast<json_reverse_iterator&>(base_iterator::operator++());
+ }
+
+ /// post-decrement (it--)
+ json_reverse_iterator operator--(int)& // NOLINT(cert-dcl21-cpp)
+ {
+ return static_cast<json_reverse_iterator>(base_iterator::operator--(1));
+ }
+
+ /// pre-decrement (--it)
+ json_reverse_iterator& operator--()
+ {
+ return static_cast<json_reverse_iterator&>(base_iterator::operator--());
+ }
+
+ /// add to iterator
+ json_reverse_iterator& operator+=(difference_type i)
+ {
+ return static_cast<json_reverse_iterator&>(base_iterator::operator+=(i));
+ }
+
+ /// add to iterator
+ json_reverse_iterator operator+(difference_type i) const
+ {
+ return static_cast<json_reverse_iterator>(base_iterator::operator+(i));
+ }
+
+ /// subtract from iterator
+ json_reverse_iterator operator-(difference_type i) const
+ {
+ return static_cast<json_reverse_iterator>(base_iterator::operator-(i));
+ }
+
+ /// return difference
+ difference_type operator-(const json_reverse_iterator& other) const
+ {
+ return base_iterator(*this) - base_iterator(other);
+ }
+
+ /// access to successor
+ reference operator[](difference_type n) const
+ {
+ return *(this->operator+(n));
+ }
+
+ /// return the key of an object iterator
+ auto key() const -> decltype(std::declval<Base>().key())
+ {
+ auto it = --this->base();
+ return it.key();
+ }
+
+ /// return the value of an iterator
+ reference value() const
+ {
+ auto it = --this->base();
+ return it.operator * ();
+ }
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
+
+// #include <nlohmann/detail/json_custom_base_class.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <type_traits> // conditional, is_same
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+/*!
+@brief Default base class of the @ref basic_json class.
+
+So that the correct implementations of the copy / move ctors / assign operators
+of @ref basic_json do not require complex case distinctions
+(no base class / custom base class used as customization point),
+@ref basic_json always has a base class.
+By default, this class is used because it is empty and thus has no effect
+on the behavior of @ref basic_json.
+*/
+struct json_default_base {};
+
+template<class T>
+using json_base_class = typename std::conditional <
+ std::is_same<T, void>::value,
+ json_default_base,
+ T
+ >::type;
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/json_pointer.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <algorithm> // all_of
+#include <cctype> // isdigit
+#include <cerrno> // errno, ERANGE
+#include <cstdlib> // strtoull
+#ifndef JSON_NO_IO
+ #include <iosfwd> // ostream
+#endif // JSON_NO_IO
+#include <limits> // max
+#include <numeric> // accumulate
+#include <string> // string
+#include <utility> // move
+#include <vector> // vector
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/string_concat.hpp>
+
+// #include <nlohmann/detail/string_escape.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+
+/// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document
+/// @sa https://json.nlohmann.me/api/json_pointer/
+template<typename RefStringType>
+class json_pointer
+{
+ // allow basic_json to access private members
+ NLOHMANN_BASIC_JSON_TPL_DECLARATION
+ friend class basic_json;
+
+ template<typename>
+ friend class json_pointer;
+
+ template<typename T>
+ struct string_t_helper
+ {
+ using type = T;
+ };
+
+ NLOHMANN_BASIC_JSON_TPL_DECLARATION
+ struct string_t_helper<NLOHMANN_BASIC_JSON_TPL>
+ {
+ using type = StringType;
+ };
+
+ public:
+ // for backwards compatibility accept BasicJsonType
+ using string_t = typename string_t_helper<RefStringType>::type;
+
+ /// @brief create JSON pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/json_pointer/
+ explicit json_pointer(const string_t& s = "")
+ : reference_tokens(split(s))
+ {}
+
+ /// @brief return a string representation of the JSON pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/to_string/
+ string_t to_string() const
+ {
+ return std::accumulate(reference_tokens.begin(), reference_tokens.end(),
+ string_t{},
+ [](const string_t& a, const string_t& b)
+ {
+ return detail::concat(a, '/', detail::escape(b));
+ });
+ }
+
+ /// @brief return a string representation of the JSON pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_string/
+ JSON_HEDLEY_DEPRECATED_FOR(3.11.0, to_string())
+ operator string_t() const
+ {
+ return to_string();
+ }
+
+#ifndef JSON_NO_IO
+ /// @brief write string representation of the JSON pointer to stream
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/
+ friend std::ostream& operator<<(std::ostream& o, const json_pointer& ptr)
+ {
+ o << ptr.to_string();
+ return o;
+ }
+#endif
+
+ /// @brief append another JSON pointer at the end of this JSON pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/
+ json_pointer& operator/=(const json_pointer& ptr)
+ {
+ reference_tokens.insert(reference_tokens.end(),
+ ptr.reference_tokens.begin(),
+ ptr.reference_tokens.end());
+ return *this;
+ }
+
+ /// @brief append an unescaped reference token at the end of this JSON pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/
+ json_pointer& operator/=(string_t token)
+ {
+ push_back(std::move(token));
+ return *this;
+ }
+
+ /// @brief append an array index at the end of this JSON pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/
+ json_pointer& operator/=(std::size_t array_idx)
+ {
+ return *this /= std::to_string(array_idx);
+ }
+
+ /// @brief create a new JSON pointer by appending the right JSON pointer at the end of the left JSON pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/
+ friend json_pointer operator/(const json_pointer& lhs,
+ const json_pointer& rhs)
+ {
+ return json_pointer(lhs) /= rhs;
+ }
+
+ /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/
+ friend json_pointer operator/(const json_pointer& lhs, string_t token) // NOLINT(performance-unnecessary-value-param)
+ {
+ return json_pointer(lhs) /= std::move(token);
+ }
+
+ /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/
+ friend json_pointer operator/(const json_pointer& lhs, std::size_t array_idx)
+ {
+ return json_pointer(lhs) /= array_idx;
+ }
+
+ /// @brief returns the parent of this JSON pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/parent_pointer/
+ json_pointer parent_pointer() const
+ {
+ if (empty())
+ {
+ return *this;
+ }
+
+ json_pointer res = *this;
+ res.pop_back();
+ return res;
+ }
+
+ /// @brief remove last reference token
+ /// @sa https://json.nlohmann.me/api/json_pointer/pop_back/
+ void pop_back()
+ {
+ if (JSON_HEDLEY_UNLIKELY(empty()))
+ {
+ JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", nullptr));
+ }
+
+ reference_tokens.pop_back();
+ }
+
+ /// @brief return last reference token
+ /// @sa https://json.nlohmann.me/api/json_pointer/back/
+ const string_t& back() const
+ {
+ if (JSON_HEDLEY_UNLIKELY(empty()))
+ {
+ JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", nullptr));
+ }
+
+ return reference_tokens.back();
+ }
+
+ /// @brief append an unescaped token at the end of the reference pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/push_back/
+ void push_back(const string_t& token)
+ {
+ reference_tokens.push_back(token);
+ }
+
+ /// @brief append an unescaped token at the end of the reference pointer
+ /// @sa https://json.nlohmann.me/api/json_pointer/push_back/
+ void push_back(string_t&& token)
+ {
+ reference_tokens.push_back(std::move(token));
+ }
+
+ /// @brief return whether pointer points to the root document
+ /// @sa https://json.nlohmann.me/api/json_pointer/empty/
+ bool empty() const noexcept
+ {
+ return reference_tokens.empty();
+ }
+
+ private:
+ /*!
+ @param[in] s reference token to be converted into an array index
+
+ @return integer representation of @a s
+
+ @throw parse_error.106 if an array index begins with '0'
+ @throw parse_error.109 if an array index begins not with a digit
+ @throw out_of_range.404 if string @a s could not be converted to an integer
+ @throw out_of_range.410 if an array index exceeds size_type
+ */
+ template<typename BasicJsonType>
+ static typename BasicJsonType::size_type array_index(const string_t& s)
+ {
+ using size_type = typename BasicJsonType::size_type;
+
+ // error condition (cf. RFC 6901, Sect. 4)
+ if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0'))
+ {
+ JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", s, "' must not begin with '0'"), nullptr));
+ }
+
+ // error condition (cf. RFC 6901, Sect. 4)
+ if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9')))
+ {
+ JSON_THROW(detail::parse_error::create(109, 0, detail::concat("array index '", s, "' is not a number"), nullptr));
+ }
+
+ const char* p = s.c_str();
+ char* p_end = nullptr;
+ errno = 0; // strtoull doesn't reset errno
+ const unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int)
+ if (p == p_end // invalid input or empty string
+ || errno == ERANGE // out of range
+ || JSON_HEDLEY_UNLIKELY(static_cast<std::size_t>(p_end - p) != s.size())) // incomplete read
+ {
+ JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", s, "'"), nullptr));
+ }
+
+ // only triggered on special platforms (like 32bit), see also
+ // https://github.com/nlohmann/json/pull/2203
+ if (res >= static_cast<unsigned long long>((std::numeric_limits<size_type>::max)())) // NOLINT(runtime/int)
+ {
+ JSON_THROW(detail::out_of_range::create(410, detail::concat("array index ", s, " exceeds size_type"), nullptr)); // LCOV_EXCL_LINE
+ }
+
+ return static_cast<size_type>(res);
+ }
+
+ JSON_PRIVATE_UNLESS_TESTED:
+ json_pointer top() const
+ {
+ if (JSON_HEDLEY_UNLIKELY(empty()))
+ {
+ JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", nullptr));
+ }
+
+ json_pointer result = *this;
+ result.reference_tokens = {reference_tokens[0]};
+ return result;
+ }
+
+ private:
+ /*!
+ @brief create and return a reference to the pointed to value
+
+ @complexity Linear in the number of reference tokens.
+
+ @throw parse_error.109 if array index is not a number
+ @throw type_error.313 if value cannot be unflattened
+ */
+ template<typename BasicJsonType>
+ BasicJsonType& get_and_create(BasicJsonType& j) const
+ {
+ auto* result = &j;
+
+ // in case no reference tokens exist, return a reference to the JSON value
+ // j which will be overwritten by a primitive value
+ for (const auto& reference_token : reference_tokens)
+ {
+ switch (result->type())
+ {
+ case detail::value_t::null:
+ {
+ if (reference_token == "0")
+ {
+ // start a new array if reference token is 0
+ result = &result->operator[](0);
+ }
+ else
+ {
+ // start a new object otherwise
+ result = &result->operator[](reference_token);
+ }
+ break;
+ }
+
+ case detail::value_t::object:
+ {
+ // create an entry in the object
+ result = &result->operator[](reference_token);
+ break;
+ }
+
+ case detail::value_t::array:
+ {
+ // create an entry in the array
+ result = &result->operator[](array_index<BasicJsonType>(reference_token));
+ break;
+ }
+
+ /*
+ The following code is only reached if there exists a reference
+ token _and_ the current value is primitive. In this case, we have
+ an error situation, because primitive values may only occur as
+ single value; that is, with an empty list of reference tokens.
+ */
+ case detail::value_t::string:
+ case detail::value_t::boolean:
+ case detail::value_t::number_integer:
+ case detail::value_t::number_unsigned:
+ case detail::value_t::number_float:
+ case detail::value_t::binary:
+ case detail::value_t::discarded:
+ default:
+ JSON_THROW(detail::type_error::create(313, "invalid value to unflatten", &j));
+ }
+ }
+
+ return *result;
+ }
+
+ /*!
+ @brief return a reference to the pointed to value
+
+ @note This version does not throw if a value is not present, but tries to
+ create nested values instead. For instance, calling this function
+ with pointer `"/this/that"` on a null value is equivalent to calling
+ `operator[]("this").operator[]("that")` on that value, effectively
+ changing the null value to an object.
+
+ @param[in] ptr a JSON value
+
+ @return reference to the JSON value pointed to by the JSON pointer
+
+ @complexity Linear in the length of the JSON pointer.
+
+ @throw parse_error.106 if an array index begins with '0'
+ @throw parse_error.109 if an array index was not a number
+ @throw out_of_range.404 if the JSON pointer can not be resolved
+ */
+ template<typename BasicJsonType>
+ BasicJsonType& get_unchecked(BasicJsonType* ptr) const
+ {
+ for (const auto& reference_token : reference_tokens)
+ {
+ // convert null values to arrays or objects before continuing
+ if (ptr->is_null())
+ {
+ // check if reference token is a number
+ const bool nums =
+ std::all_of(reference_token.begin(), reference_token.end(),
+ [](const unsigned char x)
+ {
+ return std::isdigit(x);
+ });
+
+ // change value to array for numbers or "-" or to object otherwise
+ *ptr = (nums || reference_token == "-")
+ ? detail::value_t::array
+ : detail::value_t::object;
+ }
+
+ switch (ptr->type())
+ {
+ case detail::value_t::object:
+ {
+ // use unchecked object access
+ ptr = &ptr->operator[](reference_token);
+ break;
+ }
+
+ case detail::value_t::array:
+ {
+ if (reference_token == "-")
+ {
+ // explicitly treat "-" as index beyond the end
+ ptr = &ptr->operator[](ptr->m_data.m_value.array->size());
+ }
+ else
+ {
+ // convert array index to number; unchecked access
+ ptr = &ptr->operator[](array_index<BasicJsonType>(reference_token));
+ }
+ break;
+ }
+
+ case detail::value_t::null:
+ case detail::value_t::string:
+ case detail::value_t::boolean:
+ case detail::value_t::number_integer:
+ case detail::value_t::number_unsigned:
+ case detail::value_t::number_float:
+ case detail::value_t::binary:
+ case detail::value_t::discarded:
+ default:
+ JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", reference_token, "'"), ptr));
+ }
+ }
+
+ return *ptr;
+ }
+
+ /*!
+ @throw parse_error.106 if an array index begins with '0'
+ @throw parse_error.109 if an array index was not a number
+ @throw out_of_range.402 if the array index '-' is used
+ @throw out_of_range.404 if the JSON pointer can not be resolved
+ */
+ template<typename BasicJsonType>
+ BasicJsonType& get_checked(BasicJsonType* ptr) const
+ {
+ for (const auto& reference_token : reference_tokens)
+ {
+ switch (ptr->type())
+ {
+ case detail::value_t::object:
+ {
+ // note: at performs range check
+ ptr = &ptr->at(reference_token);
+ break;
+ }
+
+ case detail::value_t::array:
+ {
+ if (JSON_HEDLEY_UNLIKELY(reference_token == "-"))
+ {
+ // "-" always fails the range check
+ JSON_THROW(detail::out_of_range::create(402, detail::concat(
+ "array index '-' (", std::to_string(ptr->m_data.m_value.array->size()),
+ ") is out of range"), ptr));
+ }
+
+ // note: at performs range check
+ ptr = &ptr->at(array_index<BasicJsonType>(reference_token));
+ break;
+ }
+
+ case detail::value_t::null:
+ case detail::value_t::string:
+ case detail::value_t::boolean:
+ case detail::value_t::number_integer:
+ case detail::value_t::number_unsigned:
+ case detail::value_t::number_float:
+ case detail::value_t::binary:
+ case detail::value_t::discarded:
+ default:
+ JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", reference_token, "'"), ptr));
+ }
+ }
+
+ return *ptr;
+ }
+
+ /*!
+ @brief return a const reference to the pointed to value
+
+ @param[in] ptr a JSON value
+
+ @return const reference to the JSON value pointed to by the JSON
+ pointer
+
+ @throw parse_error.106 if an array index begins with '0'
+ @throw parse_error.109 if an array index was not a number
+ @throw out_of_range.402 if the array index '-' is used
+ @throw out_of_range.404 if the JSON pointer can not be resolved
+ */
+ template<typename BasicJsonType>
+ const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const
+ {
+ for (const auto& reference_token : reference_tokens)
+ {
+ switch (ptr->type())
+ {
+ case detail::value_t::object:
+ {
+ // use unchecked object access
+ ptr = &ptr->operator[](reference_token);
+ break;
+ }
+
+ case detail::value_t::array:
+ {
+ if (JSON_HEDLEY_UNLIKELY(reference_token == "-"))
+ {
+ // "-" cannot be used for const access
+ JSON_THROW(detail::out_of_range::create(402, detail::concat("array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr));
+ }
+
+ // use unchecked array access
+ ptr = &ptr->operator[](array_index<BasicJsonType>(reference_token));
+ break;
+ }
+
+ case detail::value_t::null:
+ case detail::value_t::string:
+ case detail::value_t::boolean:
+ case detail::value_t::number_integer:
+ case detail::value_t::number_unsigned:
+ case detail::value_t::number_float:
+ case detail::value_t::binary:
+ case detail::value_t::discarded:
+ default:
+ JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", reference_token, "'"), ptr));
+ }
+ }
+
+ return *ptr;
+ }
+
+ /*!
+ @throw parse_error.106 if an array index begins with '0'
+ @throw parse_error.109 if an array index was not a number
+ @throw out_of_range.402 if the array index '-' is used
+ @throw out_of_range.404 if the JSON pointer can not be resolved
+ */
+ template<typename BasicJsonType>
+ const BasicJsonType& get_checked(const BasicJsonType* ptr) const
+ {
+ for (const auto& reference_token : reference_tokens)
+ {
+ switch (ptr->type())
+ {
+ case detail::value_t::object:
+ {
+ // note: at performs range check
+ ptr = &ptr->at(reference_token);
+ break;
+ }
+
+ case detail::value_t::array:
+ {
+ if (JSON_HEDLEY_UNLIKELY(reference_token == "-"))
+ {
+ // "-" always fails the range check
+ JSON_THROW(detail::out_of_range::create(402, detail::concat(
+ "array index '-' (", std::to_string(ptr->m_data.m_value.array->size()),
+ ") is out of range"), ptr));
+ }
+
+ // note: at performs range check
+ ptr = &ptr->at(array_index<BasicJsonType>(reference_token));
+ break;
+ }
+
+ case detail::value_t::null:
+ case detail::value_t::string:
+ case detail::value_t::boolean:
+ case detail::value_t::number_integer:
+ case detail::value_t::number_unsigned:
+ case detail::value_t::number_float:
+ case detail::value_t::binary:
+ case detail::value_t::discarded:
+ default:
+ JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", reference_token, "'"), ptr));
+ }
+ }
+
+ return *ptr;
+ }
+
+ /*!
+ @throw parse_error.106 if an array index begins with '0'
+ @throw parse_error.109 if an array index was not a number
+ */
+ template<typename BasicJsonType>
+ bool contains(const BasicJsonType* ptr) const
+ {
+ for (const auto& reference_token : reference_tokens)
+ {
+ switch (ptr->type())
+ {
+ case detail::value_t::object:
+ {
+ if (!ptr->contains(reference_token))
+ {
+ // we did not find the key in the object
+ return false;
+ }
+
+ ptr = &ptr->operator[](reference_token);
+ break;
+ }
+
+ case detail::value_t::array:
+ {
+ if (JSON_HEDLEY_UNLIKELY(reference_token == "-"))
+ {
+ // "-" always fails the range check
+ return false;
+ }
+ if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !("0" <= reference_token && reference_token <= "9")))
+ {
+ // invalid char
+ return false;
+ }
+ if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1))
+ {
+ if (JSON_HEDLEY_UNLIKELY(!('1' <= reference_token[0] && reference_token[0] <= '9')))
+ {
+ // first char should be between '1' and '9'
+ return false;
+ }
+ for (std::size_t i = 1; i < reference_token.size(); i++)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!('0' <= reference_token[i] && reference_token[i] <= '9')))
+ {
+ // other char should be between '0' and '9'
+ return false;
+ }
+ }
+ }
+
+ const auto idx = array_index<BasicJsonType>(reference_token);
+ if (idx >= ptr->size())
+ {
+ // index out of range
+ return false;
+ }
+
+ ptr = &ptr->operator[](idx);
+ break;
+ }
+
+ case detail::value_t::null:
+ case detail::value_t::string:
+ case detail::value_t::boolean:
+ case detail::value_t::number_integer:
+ case detail::value_t::number_unsigned:
+ case detail::value_t::number_float:
+ case detail::value_t::binary:
+ case detail::value_t::discarded:
+ default:
+ {
+ // we do not expect primitive values if there is still a
+ // reference token to process
+ return false;
+ }
+ }
+ }
+
+ // no reference token left means we found a primitive value
+ return true;
+ }
+
+ /*!
+ @brief split the string input to reference tokens
+
+ @note This function is only called by the json_pointer constructor.
+ All exceptions below are documented there.
+
+ @throw parse_error.107 if the pointer is not empty or begins with '/'
+ @throw parse_error.108 if character '~' is not followed by '0' or '1'
+ */
+ static std::vector<string_t> split(const string_t& reference_string)
+ {
+ std::vector<string_t> result;
+
+ // special case: empty reference string -> no reference tokens
+ if (reference_string.empty())
+ {
+ return result;
+ }
+
+ // check if nonempty reference string begins with slash
+ if (JSON_HEDLEY_UNLIKELY(reference_string[0] != '/'))
+ {
+ JSON_THROW(detail::parse_error::create(107, 1, detail::concat("JSON pointer must be empty or begin with '/' - was: '", reference_string, "'"), nullptr));
+ }
+
+ // extract the reference tokens:
+ // - slash: position of the last read slash (or end of string)
+ // - start: position after the previous slash
+ for (
+ // search for the first slash after the first character
+ std::size_t slash = reference_string.find_first_of('/', 1),
+ // set the beginning of the first reference token
+ start = 1;
+ // we can stop if start == 0 (if slash == string_t::npos)
+ start != 0;
+ // set the beginning of the next reference token
+ // (will eventually be 0 if slash == string_t::npos)
+ start = (slash == string_t::npos) ? 0 : slash + 1,
+ // find next slash
+ slash = reference_string.find_first_of('/', start))
+ {
+ // use the text between the beginning of the reference token
+ // (start) and the last slash (slash).
+ auto reference_token = reference_string.substr(start, slash - start);
+
+ // check reference tokens are properly escaped
+ for (std::size_t pos = reference_token.find_first_of('~');
+ pos != string_t::npos;
+ pos = reference_token.find_first_of('~', pos + 1))
+ {
+ JSON_ASSERT(reference_token[pos] == '~');
+
+ // ~ must be followed by 0 or 1
+ if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 ||
+ (reference_token[pos + 1] != '0' &&
+ reference_token[pos + 1] != '1')))
+ {
+ JSON_THROW(detail::parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'", nullptr));
+ }
+ }
+
+ // finally, store the reference token
+ detail::unescape(reference_token);
+ result.push_back(reference_token);
+ }
+
+ return result;
+ }
+
+ private:
+ /*!
+ @param[in] reference_string the reference string to the current value
+ @param[in] value the value to consider
+ @param[in,out] result the result object to insert values to
+
+ @note Empty objects or arrays are flattened to `null`.
+ */
+ template<typename BasicJsonType>
+ static void flatten(const string_t& reference_string,
+ const BasicJsonType& value,
+ BasicJsonType& result)
+ {
+ switch (value.type())
+ {
+ case detail::value_t::array:
+ {
+ if (value.m_data.m_value.array->empty())
+ {
+ // flatten empty array as null
+ result[reference_string] = nullptr;
+ }
+ else
+ {
+ // iterate array and use index as reference string
+ for (std::size_t i = 0; i < value.m_data.m_value.array->size(); ++i)
+ {
+ flatten(detail::concat(reference_string, '/', std::to_string(i)),
+ value.m_data.m_value.array->operator[](i), result);
+ }
+ }
+ break;
+ }
+
+ case detail::value_t::object:
+ {
+ if (value.m_data.m_value.object->empty())
+ {
+ // flatten empty object as null
+ result[reference_string] = nullptr;
+ }
+ else
+ {
+ // iterate object and use keys as reference string
+ for (const auto& element : *value.m_data.m_value.object)
+ {
+ flatten(detail::concat(reference_string, '/', detail::escape(element.first)), element.second, result);
+ }
+ }
+ break;
+ }
+
+ case detail::value_t::null:
+ case detail::value_t::string:
+ case detail::value_t::boolean:
+ case detail::value_t::number_integer:
+ case detail::value_t::number_unsigned:
+ case detail::value_t::number_float:
+ case detail::value_t::binary:
+ case detail::value_t::discarded:
+ default:
+ {
+ // add primitive value with its reference string
+ result[reference_string] = value;
+ break;
+ }
+ }
+ }
+
+ /*!
+ @param[in] value flattened JSON
+
+ @return unflattened JSON
+
+ @throw parse_error.109 if array index is not a number
+ @throw type_error.314 if value is not an object
+ @throw type_error.315 if object values are not primitive
+ @throw type_error.313 if value cannot be unflattened
+ */
+ template<typename BasicJsonType>
+ static BasicJsonType
+ unflatten(const BasicJsonType& value)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!value.is_object()))
+ {
+ JSON_THROW(detail::type_error::create(314, "only objects can be unflattened", &value));
+ }
+
+ BasicJsonType result;
+
+ // iterate the JSON object values
+ for (const auto& element : *value.m_data.m_value.object)
+ {
+ if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive()))
+ {
+ JSON_THROW(detail::type_error::create(315, "values in object must be primitive", &element.second));
+ }
+
+ // assign value to reference pointed to by JSON pointer; Note that if
+ // the JSON pointer is "" (i.e., points to the whole value), function
+ // get_and_create returns a reference to result itself. An assignment
+ // will then create a primitive value.
+ json_pointer(element.first).get_and_create(result) = element.second;
+ }
+
+ return result;
+ }
+
+ // can't use conversion operator because of ambiguity
+ json_pointer<string_t> convert() const&
+ {
+ json_pointer<string_t> result;
+ result.reference_tokens = reference_tokens;
+ return result;
+ }
+
+ json_pointer<string_t> convert()&&
+ {
+ json_pointer<string_t> result;
+ result.reference_tokens = std::move(reference_tokens);
+ return result;
+ }
+
+ public:
+#if JSON_HAS_THREE_WAY_COMPARISON
+ /// @brief compares two JSON pointers for equality
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_eq/
+ template<typename RefStringTypeRhs>
+ bool operator==(const json_pointer<RefStringTypeRhs>& rhs) const noexcept
+ {
+ return reference_tokens == rhs.reference_tokens;
+ }
+
+ /// @brief compares JSON pointer and string for equality
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_eq/
+ JSON_HEDLEY_DEPRECATED_FOR(3.11.2, operator==(json_pointer))
+ bool operator==(const string_t& rhs) const
+ {
+ return *this == json_pointer(rhs);
+ }
+
+ /// @brief 3-way compares two JSON pointers
+ template<typename RefStringTypeRhs>
+ std::strong_ordering operator<=>(const json_pointer<RefStringTypeRhs>& rhs) const noexcept // *NOPAD*
+ {
+ return reference_tokens <=> rhs.reference_tokens; // *NOPAD*
+ }
+#else
+ /// @brief compares two JSON pointers for equality
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_eq/
+ template<typename RefStringTypeLhs, typename RefStringTypeRhs>
+ // NOLINTNEXTLINE(readability-redundant-declaration)
+ friend bool operator==(const json_pointer<RefStringTypeLhs>& lhs,
+ const json_pointer<RefStringTypeRhs>& rhs) noexcept;
+
+ /// @brief compares JSON pointer and string for equality
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_eq/
+ template<typename RefStringTypeLhs, typename StringType>
+ // NOLINTNEXTLINE(readability-redundant-declaration)
+ friend bool operator==(const json_pointer<RefStringTypeLhs>& lhs,
+ const StringType& rhs);
+
+ /// @brief compares string and JSON pointer for equality
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_eq/
+ template<typename RefStringTypeRhs, typename StringType>
+ // NOLINTNEXTLINE(readability-redundant-declaration)
+ friend bool operator==(const StringType& lhs,
+ const json_pointer<RefStringTypeRhs>& rhs);
+
+ /// @brief compares two JSON pointers for inequality
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_ne/
+ template<typename RefStringTypeLhs, typename RefStringTypeRhs>
+ // NOLINTNEXTLINE(readability-redundant-declaration)
+ friend bool operator!=(const json_pointer<RefStringTypeLhs>& lhs,
+ const json_pointer<RefStringTypeRhs>& rhs) noexcept;
+
+ /// @brief compares JSON pointer and string for inequality
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_ne/
+ template<typename RefStringTypeLhs, typename StringType>
+ // NOLINTNEXTLINE(readability-redundant-declaration)
+ friend bool operator!=(const json_pointer<RefStringTypeLhs>& lhs,
+ const StringType& rhs);
+
+ /// @brief compares string and JSON pointer for inequality
+ /// @sa https://json.nlohmann.me/api/json_pointer/operator_ne/
+ template<typename RefStringTypeRhs, typename StringType>
+ // NOLINTNEXTLINE(readability-redundant-declaration)
+ friend bool operator!=(const StringType& lhs,
+ const json_pointer<RefStringTypeRhs>& rhs);
+
+ /// @brief compares two JSON pointer for less-than
+ template<typename RefStringTypeLhs, typename RefStringTypeRhs>
+ // NOLINTNEXTLINE(readability-redundant-declaration)
+ friend bool operator<(const json_pointer<RefStringTypeLhs>& lhs,
+ const json_pointer<RefStringTypeRhs>& rhs) noexcept;
+#endif
+
+ private:
+ /// the reference tokens
+ std::vector<string_t> reference_tokens;
+};
+
+#if !JSON_HAS_THREE_WAY_COMPARISON
+// functions cannot be defined inside class due to ODR violations
+template<typename RefStringTypeLhs, typename RefStringTypeRhs>
+inline bool operator==(const json_pointer<RefStringTypeLhs>& lhs,
+ const json_pointer<RefStringTypeRhs>& rhs) noexcept
+{
+ return lhs.reference_tokens == rhs.reference_tokens;
+}
+
+template<typename RefStringTypeLhs,
+ typename StringType = typename json_pointer<RefStringTypeLhs>::string_t>
+JSON_HEDLEY_DEPRECATED_FOR(3.11.2, operator==(json_pointer, json_pointer))
+inline bool operator==(const json_pointer<RefStringTypeLhs>& lhs,
+ const StringType& rhs)
+{
+ return lhs == json_pointer<RefStringTypeLhs>(rhs);
+}
+
+template<typename RefStringTypeRhs,
+ typename StringType = typename json_pointer<RefStringTypeRhs>::string_t>
+JSON_HEDLEY_DEPRECATED_FOR(3.11.2, operator==(json_pointer, json_pointer))
+inline bool operator==(const StringType& lhs,
+ const json_pointer<RefStringTypeRhs>& rhs)
+{
+ return json_pointer<RefStringTypeRhs>(lhs) == rhs;
+}
+
+template<typename RefStringTypeLhs, typename RefStringTypeRhs>
+inline bool operator!=(const json_pointer<RefStringTypeLhs>& lhs,
+ const json_pointer<RefStringTypeRhs>& rhs) noexcept
+{
+ return !(lhs == rhs);
+}
+
+template<typename RefStringTypeLhs,
+ typename StringType = typename json_pointer<RefStringTypeLhs>::string_t>
+JSON_HEDLEY_DEPRECATED_FOR(3.11.2, operator!=(json_pointer, json_pointer))
+inline bool operator!=(const json_pointer<RefStringTypeLhs>& lhs,
+ const StringType& rhs)
+{
+ return !(lhs == rhs);
+}
+
+template<typename RefStringTypeRhs,
+ typename StringType = typename json_pointer<RefStringTypeRhs>::string_t>
+JSON_HEDLEY_DEPRECATED_FOR(3.11.2, operator!=(json_pointer, json_pointer))
+inline bool operator!=(const StringType& lhs,
+ const json_pointer<RefStringTypeRhs>& rhs)
+{
+ return !(lhs == rhs);
+}
+
+template<typename RefStringTypeLhs, typename RefStringTypeRhs>
+inline bool operator<(const json_pointer<RefStringTypeLhs>& lhs,
+ const json_pointer<RefStringTypeRhs>& rhs) noexcept
+{
+ return lhs.reference_tokens < rhs.reference_tokens;
+}
+#endif
+
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/json_ref.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <initializer_list>
+#include <utility>
+
+// #include <nlohmann/detail/abi_macros.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+template<typename BasicJsonType>
+class json_ref
+{
+ public:
+ using value_type = BasicJsonType;
+
+ json_ref(value_type&& value)
+ : owned_value(std::move(value))
+ {}
+
+ json_ref(const value_type& value)
+ : value_ref(&value)
+ {}
+
+ json_ref(std::initializer_list<json_ref> init)
+ : owned_value(init)
+ {}
+
+ template <
+ class... Args,
+ enable_if_t<std::is_constructible<value_type, Args...>::value, int> = 0 >
+ json_ref(Args && ... args)
+ : owned_value(std::forward<Args>(args)...)
+ {}
+
+ // class should be movable only
+ json_ref(json_ref&&) noexcept = default;
+ json_ref(const json_ref&) = delete;
+ json_ref& operator=(const json_ref&) = delete;
+ json_ref& operator=(json_ref&&) = delete;
+ ~json_ref() = default;
+
+ value_type moved_or_copied() const
+ {
+ if (value_ref == nullptr)
+ {
+ return std::move(owned_value);
+ }
+ return *value_ref;
+ }
+
+ value_type const& operator*() const
+ {
+ return value_ref ? *value_ref : owned_value;
+ }
+
+ value_type const* operator->() const
+ {
+ return &** this;
+ }
+
+ private:
+ mutable value_type owned_value = nullptr;
+ value_type const* value_ref = nullptr;
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/string_concat.hpp>
+
+// #include <nlohmann/detail/string_escape.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+// #include <nlohmann/detail/output/binary_writer.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <algorithm> // reverse
+#include <array> // array
+#include <map> // map
+#include <cmath> // isnan, isinf
+#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
+#include <cstring> // memcpy
+#include <limits> // numeric_limits
+#include <string> // string
+#include <utility> // move
+#include <vector> // vector
+
+// #include <nlohmann/detail/input/binary_reader.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/output/output_adapters.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <algorithm> // copy
+#include <cstddef> // size_t
+#include <iterator> // back_inserter
+#include <memory> // shared_ptr, make_shared
+#include <string> // basic_string
+#include <vector> // vector
+
+#ifndef JSON_NO_IO
+ #include <ios> // streamsize
+ #include <ostream> // basic_ostream
+#endif // JSON_NO_IO
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+/// abstract output adapter interface
+template<typename CharType> struct output_adapter_protocol
+{
+ virtual void write_character(CharType c) = 0;
+ virtual void write_characters(const CharType* s, std::size_t length) = 0;
+ virtual ~output_adapter_protocol() = default;
+
+ output_adapter_protocol() = default;
+ output_adapter_protocol(const output_adapter_protocol&) = default;
+ output_adapter_protocol(output_adapter_protocol&&) noexcept = default;
+ output_adapter_protocol& operator=(const output_adapter_protocol&) = default;
+ output_adapter_protocol& operator=(output_adapter_protocol&&) noexcept = default;
+};
+
+/// a type to simplify interfaces
+template<typename CharType>
+using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;
+
+/// output adapter for byte vectors
+template<typename CharType, typename AllocatorType = std::allocator<CharType>>
+class output_vector_adapter : public output_adapter_protocol<CharType>
+{
+ public:
+ explicit output_vector_adapter(std::vector<CharType, AllocatorType>& vec) noexcept
+ : v(vec)
+ {}
+
+ void write_character(CharType c) override
+ {
+ v.push_back(c);
+ }
+
+ JSON_HEDLEY_NON_NULL(2)
+ void write_characters(const CharType* s, std::size_t length) override
+ {
+ v.insert(v.end(), s, s + length);
+ }
+
+ private:
+ std::vector<CharType, AllocatorType>& v;
+};
+
+#ifndef JSON_NO_IO
+/// output adapter for output streams
+template<typename CharType>
+class output_stream_adapter : public output_adapter_protocol<CharType>
+{
+ public:
+ explicit output_stream_adapter(std::basic_ostream<CharType>& s) noexcept
+ : stream(s)
+ {}
+
+ void write_character(CharType c) override
+ {
+ stream.put(c);
+ }
+
+ JSON_HEDLEY_NON_NULL(2)
+ void write_characters(const CharType* s, std::size_t length) override
+ {
+ stream.write(s, static_cast<std::streamsize>(length));
+ }
+
+ private:
+ std::basic_ostream<CharType>& stream;
+};
+#endif // JSON_NO_IO
+
+/// output adapter for basic_string
+template<typename CharType, typename StringType = std::basic_string<CharType>>
+class output_string_adapter : public output_adapter_protocol<CharType>
+{
+ public:
+ explicit output_string_adapter(StringType& s) noexcept
+ : str(s)
+ {}
+
+ void write_character(CharType c) override
+ {
+ str.push_back(c);
+ }
+
+ JSON_HEDLEY_NON_NULL(2)
+ void write_characters(const CharType* s, std::size_t length) override
+ {
+ str.append(s, length);
+ }
+
+ private:
+ StringType& str;
+};
+
+template<typename CharType, typename StringType = std::basic_string<CharType>>
+class output_adapter
+{
+ public:
+ template<typename AllocatorType = std::allocator<CharType>>
+ output_adapter(std::vector<CharType, AllocatorType>& vec)
+ : oa(std::make_shared<output_vector_adapter<CharType, AllocatorType>>(vec)) {}
+
+#ifndef JSON_NO_IO
+ output_adapter(std::basic_ostream<CharType>& s)
+ : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {}
+#endif // JSON_NO_IO
+
+ output_adapter(StringType& s)
+ : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {}
+
+ operator output_adapter_t<CharType>()
+ {
+ return oa;
+ }
+
+ private:
+ output_adapter_t<CharType> oa = nullptr;
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/string_concat.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+///////////////////
+// binary writer //
+///////////////////
+
+/*!
+@brief serialization to CBOR and MessagePack values
+*/
+template<typename BasicJsonType, typename CharType>
+class binary_writer
+{
+ using string_t = typename BasicJsonType::string_t;
+ using binary_t = typename BasicJsonType::binary_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+
+ public:
+ /*!
+ @brief create a binary writer
+
+ @param[in] adapter output adapter to write to
+ */
+ explicit binary_writer(output_adapter_t<CharType> adapter) : oa(std::move(adapter))
+ {
+ JSON_ASSERT(oa);
+ }
+
+ /*!
+ @param[in] j JSON value to serialize
+ @pre j.type() == value_t::object
+ */
+ void write_bson(const BasicJsonType& j)
+ {
+ switch (j.type())
+ {
+ case value_t::object:
+ {
+ write_bson_object(*j.m_data.m_value.object);
+ break;
+ }
+
+ case value_t::null:
+ case value_t::array:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ JSON_THROW(type_error::create(317, concat("to serialize to BSON, top-level type must be object, but is ", j.type_name()), &j));
+ }
+ }
+ }
+
+ /*!
+ @param[in] j JSON value to serialize
+ */
+ void write_cbor(const BasicJsonType& j)
+ {
+ switch (j.type())
+ {
+ case value_t::null:
+ {
+ oa->write_character(to_char_type(0xF6));
+ break;
+ }
+
+ case value_t::boolean:
+ {
+ oa->write_character(j.m_data.m_value.boolean
+ ? to_char_type(0xF5)
+ : to_char_type(0xF4));
+ break;
+ }
+
+ case value_t::number_integer:
+ {
+ if (j.m_data.m_value.number_integer >= 0)
+ {
+ // CBOR does not differentiate between positive signed
+ // integers and unsigned integers. Therefore, we used the
+ // code from the value_t::number_unsigned case here.
+ if (j.m_data.m_value.number_integer <= 0x17)
+ {
+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ oa->write_character(to_char_type(0x18));
+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ oa->write_character(to_char_type(0x19));
+ write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ oa->write_character(to_char_type(0x1A));
+ write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_integer));
+ }
+ else
+ {
+ oa->write_character(to_char_type(0x1B));
+ write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));
+ }
+ }
+ else
+ {
+ // The conversions below encode the sign in the first
+ // byte, and the value is converted to a positive number.
+ const auto positive_number = -1 - j.m_data.m_value.number_integer;
+ if (j.m_data.m_value.number_integer >= -24)
+ {
+ write_number(static_cast<std::uint8_t>(0x20 + positive_number));
+ }
+ else if (positive_number <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ oa->write_character(to_char_type(0x38));
+ write_number(static_cast<std::uint8_t>(positive_number));
+ }
+ else if (positive_number <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ oa->write_character(to_char_type(0x39));
+ write_number(static_cast<std::uint16_t>(positive_number));
+ }
+ else if (positive_number <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ oa->write_character(to_char_type(0x3A));
+ write_number(static_cast<std::uint32_t>(positive_number));
+ }
+ else
+ {
+ oa->write_character(to_char_type(0x3B));
+ write_number(static_cast<std::uint64_t>(positive_number));
+ }
+ }
+ break;
+ }
+
+ case value_t::number_unsigned:
+ {
+ if (j.m_data.m_value.number_unsigned <= 0x17)
+ {
+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_unsigned));
+ }
+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ oa->write_character(to_char_type(0x18));
+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_unsigned));
+ }
+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ oa->write_character(to_char_type(0x19));
+ write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_unsigned));
+ }
+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ oa->write_character(to_char_type(0x1A));
+ write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_unsigned));
+ }
+ else
+ {
+ oa->write_character(to_char_type(0x1B));
+ write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_unsigned));
+ }
+ break;
+ }
+
+ case value_t::number_float:
+ {
+ if (std::isnan(j.m_data.m_value.number_float))
+ {
+ // NaN is 0xf97e00 in CBOR
+ oa->write_character(to_char_type(0xF9));
+ oa->write_character(to_char_type(0x7E));
+ oa->write_character(to_char_type(0x00));
+ }
+ else if (std::isinf(j.m_data.m_value.number_float))
+ {
+ // Infinity is 0xf97c00, -Infinity is 0xf9fc00
+ oa->write_character(to_char_type(0xf9));
+ oa->write_character(j.m_data.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC));
+ oa->write_character(to_char_type(0x00));
+ }
+ else
+ {
+ write_compact_float(j.m_data.m_value.number_float, detail::input_format_t::cbor);
+ }
+ break;
+ }
+
+ case value_t::string:
+ {
+ // step 1: write control byte and the string length
+ const auto N = j.m_data.m_value.string->size();
+ if (N <= 0x17)
+ {
+ write_number(static_cast<std::uint8_t>(0x60 + N));
+ }
+ else if (N <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ oa->write_character(to_char_type(0x78));
+ write_number(static_cast<std::uint8_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ oa->write_character(to_char_type(0x79));
+ write_number(static_cast<std::uint16_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ oa->write_character(to_char_type(0x7A));
+ write_number(static_cast<std::uint32_t>(N));
+ }
+ // LCOV_EXCL_START
+ else if (N <= (std::numeric_limits<std::uint64_t>::max)())
+ {
+ oa->write_character(to_char_type(0x7B));
+ write_number(static_cast<std::uint64_t>(N));
+ }
+ // LCOV_EXCL_STOP
+
+ // step 2: write the string
+ oa->write_characters(
+ reinterpret_cast<const CharType*>(j.m_data.m_value.string->c_str()),
+ j.m_data.m_value.string->size());
+ break;
+ }
+
+ case value_t::array:
+ {
+ // step 1: write control byte and the array size
+ const auto N = j.m_data.m_value.array->size();
+ if (N <= 0x17)
+ {
+ write_number(static_cast<std::uint8_t>(0x80 + N));
+ }
+ else if (N <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ oa->write_character(to_char_type(0x98));
+ write_number(static_cast<std::uint8_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ oa->write_character(to_char_type(0x99));
+ write_number(static_cast<std::uint16_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ oa->write_character(to_char_type(0x9A));
+ write_number(static_cast<std::uint32_t>(N));
+ }
+ // LCOV_EXCL_START
+ else if (N <= (std::numeric_limits<std::uint64_t>::max)())
+ {
+ oa->write_character(to_char_type(0x9B));
+ write_number(static_cast<std::uint64_t>(N));
+ }
+ // LCOV_EXCL_STOP
+
+ // step 2: write each element
+ for (const auto& el : *j.m_data.m_value.array)
+ {
+ write_cbor(el);
+ }
+ break;
+ }
+
+ case value_t::binary:
+ {
+ if (j.m_data.m_value.binary->has_subtype())
+ {
+ if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ write_number(static_cast<std::uint8_t>(0xd8));
+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.binary->subtype()));
+ }
+ else if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ write_number(static_cast<std::uint8_t>(0xd9));
+ write_number(static_cast<std::uint16_t>(j.m_data.m_value.binary->subtype()));
+ }
+ else if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ write_number(static_cast<std::uint8_t>(0xda));
+ write_number(static_cast<std::uint32_t>(j.m_data.m_value.binary->subtype()));
+ }
+ else if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits<std::uint64_t>::max)())
+ {
+ write_number(static_cast<std::uint8_t>(0xdb));
+ write_number(static_cast<std::uint64_t>(j.m_data.m_value.binary->subtype()));
+ }
+ }
+
+ // step 1: write control byte and the binary array size
+ const auto N = j.m_data.m_value.binary->size();
+ if (N <= 0x17)
+ {
+ write_number(static_cast<std::uint8_t>(0x40 + N));
+ }
+ else if (N <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ oa->write_character(to_char_type(0x58));
+ write_number(static_cast<std::uint8_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ oa->write_character(to_char_type(0x59));
+ write_number(static_cast<std::uint16_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ oa->write_character(to_char_type(0x5A));
+ write_number(static_cast<std::uint32_t>(N));
+ }
+ // LCOV_EXCL_START
+ else if (N <= (std::numeric_limits<std::uint64_t>::max)())
+ {
+ oa->write_character(to_char_type(0x5B));
+ write_number(static_cast<std::uint64_t>(N));
+ }
+ // LCOV_EXCL_STOP
+
+ // step 2: write each element
+ oa->write_characters(
+ reinterpret_cast<const CharType*>(j.m_data.m_value.binary->data()),
+ N);
+
+ break;
+ }
+
+ case value_t::object:
+ {
+ // step 1: write control byte and the object size
+ const auto N = j.m_data.m_value.object->size();
+ if (N <= 0x17)
+ {
+ write_number(static_cast<std::uint8_t>(0xA0 + N));
+ }
+ else if (N <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ oa->write_character(to_char_type(0xB8));
+ write_number(static_cast<std::uint8_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ oa->write_character(to_char_type(0xB9));
+ write_number(static_cast<std::uint16_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ oa->write_character(to_char_type(0xBA));
+ write_number(static_cast<std::uint32_t>(N));
+ }
+ // LCOV_EXCL_START
+ else if (N <= (std::numeric_limits<std::uint64_t>::max)())
+ {
+ oa->write_character(to_char_type(0xBB));
+ write_number(static_cast<std::uint64_t>(N));
+ }
+ // LCOV_EXCL_STOP
+
+ // step 2: write each element
+ for (const auto& el : *j.m_data.m_value.object)
+ {
+ write_cbor(el.first);
+ write_cbor(el.second);
+ }
+ break;
+ }
+
+ case value_t::discarded:
+ default:
+ break;
+ }
+ }
+
+ /*!
+ @param[in] j JSON value to serialize
+ */
+ void write_msgpack(const BasicJsonType& j)
+ {
+ switch (j.type())
+ {
+ case value_t::null: // nil
+ {
+ oa->write_character(to_char_type(0xC0));
+ break;
+ }
+
+ case value_t::boolean: // true and false
+ {
+ oa->write_character(j.m_data.m_value.boolean
+ ? to_char_type(0xC3)
+ : to_char_type(0xC2));
+ break;
+ }
+
+ case value_t::number_integer:
+ {
+ if (j.m_data.m_value.number_integer >= 0)
+ {
+ // MessagePack does not differentiate between positive
+ // signed integers and unsigned integers. Therefore, we used
+ // the code from the value_t::number_unsigned case here.
+ if (j.m_data.m_value.number_unsigned < 128)
+ {
+ // positive fixnum
+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ // uint 8
+ oa->write_character(to_char_type(0xCC));
+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ // uint 16
+ oa->write_character(to_char_type(0xCD));
+ write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ // uint 32
+ oa->write_character(to_char_type(0xCE));
+ write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)())
+ {
+ // uint 64
+ oa->write_character(to_char_type(0xCF));
+ write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));
+ }
+ }
+ else
+ {
+ if (j.m_data.m_value.number_integer >= -32)
+ {
+ // negative fixnum
+ write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&
+ j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())
+ {
+ // int 8
+ oa->write_character(to_char_type(0xD0));
+ write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&
+ j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())
+ {
+ // int 16
+ oa->write_character(to_char_type(0xD1));
+ write_number(static_cast<std::int16_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&
+ j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())
+ {
+ // int 32
+ oa->write_character(to_char_type(0xD2));
+ write_number(static_cast<std::int32_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int64_t>::min)() &&
+ j.m_data.m_value.number_integer <= (std::numeric_limits<std::int64_t>::max)())
+ {
+ // int 64
+ oa->write_character(to_char_type(0xD3));
+ write_number(static_cast<std::int64_t>(j.m_data.m_value.number_integer));
+ }
+ }
+ break;
+ }
+
+ case value_t::number_unsigned:
+ {
+ if (j.m_data.m_value.number_unsigned < 128)
+ {
+ // positive fixnum
+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ // uint 8
+ oa->write_character(to_char_type(0xCC));
+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ // uint 16
+ oa->write_character(to_char_type(0xCD));
+ write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ // uint 32
+ oa->write_character(to_char_type(0xCE));
+ write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_integer));
+ }
+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)())
+ {
+ // uint 64
+ oa->write_character(to_char_type(0xCF));
+ write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));
+ }
+ break;
+ }
+
+ case value_t::number_float:
+ {
+ write_compact_float(j.m_data.m_value.number_float, detail::input_format_t::msgpack);
+ break;
+ }
+
+ case value_t::string:
+ {
+ // step 1: write control byte and the string length
+ const auto N = j.m_data.m_value.string->size();
+ if (N <= 31)
+ {
+ // fixstr
+ write_number(static_cast<std::uint8_t>(0xA0 | N));
+ }
+ else if (N <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ // str 8
+ oa->write_character(to_char_type(0xD9));
+ write_number(static_cast<std::uint8_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ // str 16
+ oa->write_character(to_char_type(0xDA));
+ write_number(static_cast<std::uint16_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ // str 32
+ oa->write_character(to_char_type(0xDB));
+ write_number(static_cast<std::uint32_t>(N));
+ }
+
+ // step 2: write the string
+ oa->write_characters(
+ reinterpret_cast<const CharType*>(j.m_data.m_value.string->c_str()),
+ j.m_data.m_value.string->size());
+ break;
+ }
+
+ case value_t::array:
+ {
+ // step 1: write control byte and the array size
+ const auto N = j.m_data.m_value.array->size();
+ if (N <= 15)
+ {
+ // fixarray
+ write_number(static_cast<std::uint8_t>(0x90 | N));
+ }
+ else if (N <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ // array 16
+ oa->write_character(to_char_type(0xDC));
+ write_number(static_cast<std::uint16_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ // array 32
+ oa->write_character(to_char_type(0xDD));
+ write_number(static_cast<std::uint32_t>(N));
+ }
+
+ // step 2: write each element
+ for (const auto& el : *j.m_data.m_value.array)
+ {
+ write_msgpack(el);
+ }
+ break;
+ }
+
+ case value_t::binary:
+ {
+ // step 0: determine if the binary type has a set subtype to
+ // determine whether or not to use the ext or fixext types
+ const bool use_ext = j.m_data.m_value.binary->has_subtype();
+
+ // step 1: write control byte and the byte string length
+ const auto N = j.m_data.m_value.binary->size();
+ if (N <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ std::uint8_t output_type{};
+ bool fixed = true;
+ if (use_ext)
+ {
+ switch (N)
+ {
+ case 1:
+ output_type = 0xD4; // fixext 1
+ break;
+ case 2:
+ output_type = 0xD5; // fixext 2
+ break;
+ case 4:
+ output_type = 0xD6; // fixext 4
+ break;
+ case 8:
+ output_type = 0xD7; // fixext 8
+ break;
+ case 16:
+ output_type = 0xD8; // fixext 16
+ break;
+ default:
+ output_type = 0xC7; // ext 8
+ fixed = false;
+ break;
+ }
+
+ }
+ else
+ {
+ output_type = 0xC4; // bin 8
+ fixed = false;
+ }
+
+ oa->write_character(to_char_type(output_type));
+ if (!fixed)
+ {
+ write_number(static_cast<std::uint8_t>(N));
+ }
+ }
+ else if (N <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ const std::uint8_t output_type = use_ext
+ ? 0xC8 // ext 16
+ : 0xC5; // bin 16
+
+ oa->write_character(to_char_type(output_type));
+ write_number(static_cast<std::uint16_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ const std::uint8_t output_type = use_ext
+ ? 0xC9 // ext 32
+ : 0xC6; // bin 32
+
+ oa->write_character(to_char_type(output_type));
+ write_number(static_cast<std::uint32_t>(N));
+ }
+
+ // step 1.5: if this is an ext type, write the subtype
+ if (use_ext)
+ {
+ write_number(static_cast<std::int8_t>(j.m_data.m_value.binary->subtype()));
+ }
+
+ // step 2: write the byte string
+ oa->write_characters(
+ reinterpret_cast<const CharType*>(j.m_data.m_value.binary->data()),
+ N);
+
+ break;
+ }
+
+ case value_t::object:
+ {
+ // step 1: write control byte and the object size
+ const auto N = j.m_data.m_value.object->size();
+ if (N <= 15)
+ {
+ // fixmap
+ write_number(static_cast<std::uint8_t>(0x80 | (N & 0xF)));
+ }
+ else if (N <= (std::numeric_limits<std::uint16_t>::max)())
+ {
+ // map 16
+ oa->write_character(to_char_type(0xDE));
+ write_number(static_cast<std::uint16_t>(N));
+ }
+ else if (N <= (std::numeric_limits<std::uint32_t>::max)())
+ {
+ // map 32
+ oa->write_character(to_char_type(0xDF));
+ write_number(static_cast<std::uint32_t>(N));
+ }
+
+ // step 2: write each element
+ for (const auto& el : *j.m_data.m_value.object)
+ {
+ write_msgpack(el.first);
+ write_msgpack(el.second);
+ }
+ break;
+ }
+
+ case value_t::discarded:
+ default:
+ break;
+ }
+ }
+
+ /*!
+ @param[in] j JSON value to serialize
+ @param[in] use_count whether to use '#' prefixes (optimized format)
+ @param[in] use_type whether to use '$' prefixes (optimized format)
+ @param[in] add_prefix whether prefixes need to be used for this value
+ @param[in] use_bjdata whether write in BJData format, default is false
+ */
+ void write_ubjson(const BasicJsonType& j, const bool use_count,
+ const bool use_type, const bool add_prefix = true,
+ const bool use_bjdata = false)
+ {
+ switch (j.type())
+ {
+ case value_t::null:
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('Z'));
+ }
+ break;
+ }
+
+ case value_t::boolean:
+ {
+ if (add_prefix)
+ {
+ oa->write_character(j.m_data.m_value.boolean
+ ? to_char_type('T')
+ : to_char_type('F'));
+ }
+ break;
+ }
+
+ case value_t::number_integer:
+ {
+ write_number_with_ubjson_prefix(j.m_data.m_value.number_integer, add_prefix, use_bjdata);
+ break;
+ }
+
+ case value_t::number_unsigned:
+ {
+ write_number_with_ubjson_prefix(j.m_data.m_value.number_unsigned, add_prefix, use_bjdata);
+ break;
+ }
+
+ case value_t::number_float:
+ {
+ write_number_with_ubjson_prefix(j.m_data.m_value.number_float, add_prefix, use_bjdata);
+ break;
+ }
+
+ case value_t::string:
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('S'));
+ }
+ write_number_with_ubjson_prefix(j.m_data.m_value.string->size(), true, use_bjdata);
+ oa->write_characters(
+ reinterpret_cast<const CharType*>(j.m_data.m_value.string->c_str()),
+ j.m_data.m_value.string->size());
+ break;
+ }
+
+ case value_t::array:
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('['));
+ }
+
+ bool prefix_required = true;
+ if (use_type && !j.m_data.m_value.array->empty())
+ {
+ JSON_ASSERT(use_count);
+ const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata);
+ const bool same_prefix = std::all_of(j.begin() + 1, j.end(),
+ [this, first_prefix, use_bjdata](const BasicJsonType & v)
+ {
+ return ubjson_prefix(v, use_bjdata) == first_prefix;
+ });
+
+ std::vector<CharType> bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type
+
+ if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end()))
+ {
+ prefix_required = false;
+ oa->write_character(to_char_type('$'));
+ oa->write_character(first_prefix);
+ }
+ }
+
+ if (use_count)
+ {
+ oa->write_character(to_char_type('#'));
+ write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata);
+ }
+
+ for (const auto& el : *j.m_data.m_value.array)
+ {
+ write_ubjson(el, use_count, use_type, prefix_required, use_bjdata);
+ }
+
+ if (!use_count)
+ {
+ oa->write_character(to_char_type(']'));
+ }
+
+ break;
+ }
+
+ case value_t::binary:
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('['));
+ }
+
+ if (use_type && !j.m_data.m_value.binary->empty())
+ {
+ JSON_ASSERT(use_count);
+ oa->write_character(to_char_type('$'));
+ oa->write_character('U');
+ }
+
+ if (use_count)
+ {
+ oa->write_character(to_char_type('#'));
+ write_number_with_ubjson_prefix(j.m_data.m_value.binary->size(), true, use_bjdata);
+ }
+
+ if (use_type)
+ {
+ oa->write_characters(
+ reinterpret_cast<const CharType*>(j.m_data.m_value.binary->data()),
+ j.m_data.m_value.binary->size());
+ }
+ else
+ {
+ for (size_t i = 0; i < j.m_data.m_value.binary->size(); ++i)
+ {
+ oa->write_character(to_char_type('U'));
+ oa->write_character(j.m_data.m_value.binary->data()[i]);
+ }
+ }
+
+ if (!use_count)
+ {
+ oa->write_character(to_char_type(']'));
+ }
+
+ break;
+ }
+
+ case value_t::object:
+ {
+ if (use_bjdata && j.m_data.m_value.object->size() == 3 && j.m_data.m_value.object->find("_ArrayType_") != j.m_data.m_value.object->end() && j.m_data.m_value.object->find("_ArraySize_") != j.m_data.m_value.object->end() && j.m_data.m_value.object->find("_ArrayData_") != j.m_data.m_value.object->end())
+ {
+ if (!write_bjdata_ndarray(*j.m_data.m_value.object, use_count, use_type)) // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata)
+ {
+ break;
+ }
+ }
+
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('{'));
+ }
+
+ bool prefix_required = true;
+ if (use_type && !j.m_data.m_value.object->empty())
+ {
+ JSON_ASSERT(use_count);
+ const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata);
+ const bool same_prefix = std::all_of(j.begin(), j.end(),
+ [this, first_prefix, use_bjdata](const BasicJsonType & v)
+ {
+ return ubjson_prefix(v, use_bjdata) == first_prefix;
+ });
+
+ std::vector<CharType> bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type
+
+ if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end()))
+ {
+ prefix_required = false;
+ oa->write_character(to_char_type('$'));
+ oa->write_character(first_prefix);
+ }
+ }
+
+ if (use_count)
+ {
+ oa->write_character(to_char_type('#'));
+ write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata);
+ }
+
+ for (const auto& el : *j.m_data.m_value.object)
+ {
+ write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata);
+ oa->write_characters(
+ reinterpret_cast<const CharType*>(el.first.c_str()),
+ el.first.size());
+ write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata);
+ }
+
+ if (!use_count)
+ {
+ oa->write_character(to_char_type('}'));
+ }
+
+ break;
+ }
+
+ case value_t::discarded:
+ default:
+ break;
+ }
+ }
+
+ private:
+ //////////
+ // BSON //
+ //////////
+
+ /*!
+ @return The size of a BSON document entry header, including the id marker
+ and the entry name size (and its null-terminator).
+ */
+ static std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j)
+ {
+ const auto it = name.find(static_cast<typename string_t::value_type>(0));
+ if (JSON_HEDLEY_UNLIKELY(it != BasicJsonType::string_t::npos))
+ {
+ JSON_THROW(out_of_range::create(409, concat("BSON key cannot contain code point U+0000 (at byte ", std::to_string(it), ")"), &j));
+ static_cast<void>(j);
+ }
+
+ return /*id*/ 1ul + name.size() + /*zero-terminator*/1u;
+ }
+
+ /*!
+ @brief Writes the given @a element_type and @a name to the output adapter
+ */
+ void write_bson_entry_header(const string_t& name,
+ const std::uint8_t element_type)
+ {
+ oa->write_character(to_char_type(element_type)); // boolean
+ oa->write_characters(
+ reinterpret_cast<const CharType*>(name.c_str()),
+ name.size() + 1u);
+ }
+
+ /*!
+ @brief Writes a BSON element with key @a name and boolean value @a value
+ */
+ void write_bson_boolean(const string_t& name,
+ const bool value)
+ {
+ write_bson_entry_header(name, 0x08);
+ oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00));
+ }
+
+ /*!
+ @brief Writes a BSON element with key @a name and double value @a value
+ */
+ void write_bson_double(const string_t& name,
+ const double value)
+ {
+ write_bson_entry_header(name, 0x01);
+ write_number<double>(value, true);
+ }
+
+ /*!
+ @return The size of the BSON-encoded string in @a value
+ */
+ static std::size_t calc_bson_string_size(const string_t& value)
+ {
+ return sizeof(std::int32_t) + value.size() + 1ul;
+ }
+
+ /*!
+ @brief Writes a BSON element with key @a name and string value @a value
+ */
+ void write_bson_string(const string_t& name,
+ const string_t& value)
+ {
+ write_bson_entry_header(name, 0x02);
+
+ write_number<std::int32_t>(static_cast<std::int32_t>(value.size() + 1ul), true);
+ oa->write_characters(
+ reinterpret_cast<const CharType*>(value.c_str()),
+ value.size() + 1);
+ }
+
+ /*!
+ @brief Writes a BSON element with key @a name and null value
+ */
+ void write_bson_null(const string_t& name)
+ {
+ write_bson_entry_header(name, 0x0A);
+ }
+
+ /*!
+ @return The size of the BSON-encoded integer @a value
+ */
+ static std::size_t calc_bson_integer_size(const std::int64_t value)
+ {
+ return (std::numeric_limits<std::int32_t>::min)() <= value && value <= (std::numeric_limits<std::int32_t>::max)()
+ ? sizeof(std::int32_t)
+ : sizeof(std::int64_t);
+ }
+
+ /*!
+ @brief Writes a BSON element with key @a name and integer @a value
+ */
+ void write_bson_integer(const string_t& name,
+ const std::int64_t value)
+ {
+ if ((std::numeric_limits<std::int32_t>::min)() <= value && value <= (std::numeric_limits<std::int32_t>::max)())
+ {
+ write_bson_entry_header(name, 0x10); // int32
+ write_number<std::int32_t>(static_cast<std::int32_t>(value), true);
+ }
+ else
+ {
+ write_bson_entry_header(name, 0x12); // int64
+ write_number<std::int64_t>(static_cast<std::int64_t>(value), true);
+ }
+ }
+
+ /*!
+ @return The size of the BSON-encoded unsigned integer in @a j
+ */
+ static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept
+ {
+ return (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
+ ? sizeof(std::int32_t)
+ : sizeof(std::int64_t);
+ }
+
+ /*!
+ @brief Writes a BSON element with key @a name and unsigned @a value
+ */
+ void write_bson_unsigned(const string_t& name,
+ const BasicJsonType& j)
+ {
+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
+ {
+ write_bson_entry_header(name, 0x10 /* int32 */);
+ write_number<std::int32_t>(static_cast<std::int32_t>(j.m_data.m_value.number_unsigned), true);
+ }
+ else if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
+ {
+ write_bson_entry_header(name, 0x12 /* int64 */);
+ write_number<std::int64_t>(static_cast<std::int64_t>(j.m_data.m_value.number_unsigned), true);
+ }
+ else
+ {
+ JSON_THROW(out_of_range::create(407, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BSON as it does not fit int64"), &j));
+ }
+ }
+
+ /*!
+ @brief Writes a BSON element with key @a name and object @a value
+ */
+ void write_bson_object_entry(const string_t& name,
+ const typename BasicJsonType::object_t& value)
+ {
+ write_bson_entry_header(name, 0x03); // object
+ write_bson_object(value);
+ }
+
+ /*!
+ @return The size of the BSON-encoded array @a value
+ */
+ static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value)
+ {
+ std::size_t array_index = 0ul;
+
+ const std::size_t embedded_document_size = std::accumulate(std::begin(value), std::end(value), static_cast<std::size_t>(0), [&array_index](std::size_t result, const typename BasicJsonType::array_t::value_type & el)
+ {
+ return result + calc_bson_element_size(std::to_string(array_index++), el);
+ });
+
+ return sizeof(std::int32_t) + embedded_document_size + 1ul;
+ }
+
+ /*!
+ @return The size of the BSON-encoded binary array @a value
+ */
+ static std::size_t calc_bson_binary_size(const typename BasicJsonType::binary_t& value)
+ {
+ return sizeof(std::int32_t) + value.size() + 1ul;
+ }
+
+ /*!
+ @brief Writes a BSON element with key @a name and array @a value
+ */
+ void write_bson_array(const string_t& name,
+ const typename BasicJsonType::array_t& value)
+ {
+ write_bson_entry_header(name, 0x04); // array
+ write_number<std::int32_t>(static_cast<std::int32_t>(calc_bson_array_size(value)), true);
+
+ std::size_t array_index = 0ul;
+
+ for (const auto& el : value)
+ {
+ write_bson_element(std::to_string(array_index++), el);
+ }
+
+ oa->write_character(to_char_type(0x00));
+ }
+
+ /*!
+ @brief Writes a BSON element with key @a name and binary value @a value
+ */
+ void write_bson_binary(const string_t& name,
+ const binary_t& value)
+ {
+ write_bson_entry_header(name, 0x05);
+
+ write_number<std::int32_t>(static_cast<std::int32_t>(value.size()), true);
+ write_number(value.has_subtype() ? static_cast<std::uint8_t>(value.subtype()) : static_cast<std::uint8_t>(0x00));
+
+ oa->write_characters(reinterpret_cast<const CharType*>(value.data()), value.size());
+ }
+
+ /*!
+ @brief Calculates the size necessary to serialize the JSON value @a j with its @a name
+ @return The calculated size for the BSON document entry for @a j with the given @a name.
+ */
+ static std::size_t calc_bson_element_size(const string_t& name,
+ const BasicJsonType& j)
+ {
+ const auto header_size = calc_bson_entry_header_size(name, j);
+ switch (j.type())
+ {
+ case value_t::object:
+ return header_size + calc_bson_object_size(*j.m_data.m_value.object);
+
+ case value_t::array:
+ return header_size + calc_bson_array_size(*j.m_data.m_value.array);
+
+ case value_t::binary:
+ return header_size + calc_bson_binary_size(*j.m_data.m_value.binary);
+
+ case value_t::boolean:
+ return header_size + 1ul;
+
+ case value_t::number_float:
+ return header_size + 8ul;
+
+ case value_t::number_integer:
+ return header_size + calc_bson_integer_size(j.m_data.m_value.number_integer);
+
+ case value_t::number_unsigned:
+ return header_size + calc_bson_unsigned_size(j.m_data.m_value.number_unsigned);
+
+ case value_t::string:
+ return header_size + calc_bson_string_size(*j.m_data.m_value.string);
+
+ case value_t::null:
+ return header_size + 0ul;
+
+ // LCOV_EXCL_START
+ case value_t::discarded:
+ default:
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert)
+ return 0ul;
+ // LCOV_EXCL_STOP
+ }
+ }
+
+ /*!
+ @brief Serializes the JSON value @a j to BSON and associates it with the
+ key @a name.
+ @param name The name to associate with the JSON entity @a j within the
+ current BSON document
+ */
+ void write_bson_element(const string_t& name,
+ const BasicJsonType& j)
+ {
+ switch (j.type())
+ {
+ case value_t::object:
+ return write_bson_object_entry(name, *j.m_data.m_value.object);
+
+ case value_t::array:
+ return write_bson_array(name, *j.m_data.m_value.array);
+
+ case value_t::binary:
+ return write_bson_binary(name, *j.m_data.m_value.binary);
+
+ case value_t::boolean:
+ return write_bson_boolean(name, j.m_data.m_value.boolean);
+
+ case value_t::number_float:
+ return write_bson_double(name, j.m_data.m_value.number_float);
+
+ case value_t::number_integer:
+ return write_bson_integer(name, j.m_data.m_value.number_integer);
+
+ case value_t::number_unsigned:
+ return write_bson_unsigned(name, j);
+
+ case value_t::string:
+ return write_bson_string(name, *j.m_data.m_value.string);
+
+ case value_t::null:
+ return write_bson_null(name);
+
+ // LCOV_EXCL_START
+ case value_t::discarded:
+ default:
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert)
+ return;
+ // LCOV_EXCL_STOP
+ }
+ }
+
+ /*!
+ @brief Calculates the size of the BSON serialization of the given
+ JSON-object @a j.
+ @param[in] value JSON value to serialize
+ @pre value.type() == value_t::object
+ */
+ static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value)
+ {
+ const std::size_t document_size = std::accumulate(value.begin(), value.end(), static_cast<std::size_t>(0),
+ [](size_t result, const typename BasicJsonType::object_t::value_type & el)
+ {
+ return result += calc_bson_element_size(el.first, el.second);
+ });
+
+ return sizeof(std::int32_t) + document_size + 1ul;
+ }
+
+ /*!
+ @param[in] value JSON value to serialize
+ @pre value.type() == value_t::object
+ */
+ void write_bson_object(const typename BasicJsonType::object_t& value)
+ {
+ write_number<std::int32_t>(static_cast<std::int32_t>(calc_bson_object_size(value)), true);
+
+ for (const auto& el : value)
+ {
+ write_bson_element(el.first, el.second);
+ }
+
+ oa->write_character(to_char_type(0x00));
+ }
+
+ //////////
+ // CBOR //
+ //////////
+
+ static constexpr CharType get_cbor_float_prefix(float /*unused*/)
+ {
+ return to_char_type(0xFA); // Single-Precision Float
+ }
+
+ static constexpr CharType get_cbor_float_prefix(double /*unused*/)
+ {
+ return to_char_type(0xFB); // Double-Precision Float
+ }
+
+ /////////////
+ // MsgPack //
+ /////////////
+
+ static constexpr CharType get_msgpack_float_prefix(float /*unused*/)
+ {
+ return to_char_type(0xCA); // float 32
+ }
+
+ static constexpr CharType get_msgpack_float_prefix(double /*unused*/)
+ {
+ return to_char_type(0xCB); // float 64
+ }
+
+ ////////////
+ // UBJSON //
+ ////////////
+
+ // UBJSON: write number (floating point)
+ template<typename NumberType, typename std::enable_if<
+ std::is_floating_point<NumberType>::value, int>::type = 0>
+ void write_number_with_ubjson_prefix(const NumberType n,
+ const bool add_prefix,
+ const bool use_bjdata)
+ {
+ if (add_prefix)
+ {
+ oa->write_character(get_ubjson_float_prefix(n));
+ }
+ write_number(n, use_bjdata);
+ }
+
+ // UBJSON: write number (unsigned integer)
+ template<typename NumberType, typename std::enable_if<
+ std::is_unsigned<NumberType>::value, int>::type = 0>
+ void write_number_with_ubjson_prefix(const NumberType n,
+ const bool add_prefix,
+ const bool use_bjdata)
+ {
+ if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int8_t>::max)()))
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('i')); // int8
+ }
+ write_number(static_cast<std::uint8_t>(n), use_bjdata);
+ }
+ else if (n <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('U')); // uint8
+ }
+ write_number(static_cast<std::uint8_t>(n), use_bjdata);
+ }
+ else if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int16_t>::max)()))
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('I')); // int16
+ }
+ write_number(static_cast<std::int16_t>(n), use_bjdata);
+ }
+ else if (use_bjdata && n <= static_cast<uint64_t>((std::numeric_limits<uint16_t>::max)()))
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('u')); // uint16 - bjdata only
+ }
+ write_number(static_cast<std::uint16_t>(n), use_bjdata);
+ }
+ else if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('l')); // int32
+ }
+ write_number(static_cast<std::int32_t>(n), use_bjdata);
+ }
+ else if (use_bjdata && n <= static_cast<uint64_t>((std::numeric_limits<uint32_t>::max)()))
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('m')); // uint32 - bjdata only
+ }
+ write_number(static_cast<std::uint32_t>(n), use_bjdata);
+ }
+ else if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('L')); // int64
+ }
+ write_number(static_cast<std::int64_t>(n), use_bjdata);
+ }
+ else if (use_bjdata && n <= (std::numeric_limits<uint64_t>::max)())
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('M')); // uint64 - bjdata only
+ }
+ write_number(static_cast<std::uint64_t>(n), use_bjdata);
+ }
+ else
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('H')); // high-precision number
+ }
+
+ const auto number = BasicJsonType(n).dump();
+ write_number_with_ubjson_prefix(number.size(), true, use_bjdata);
+ for (std::size_t i = 0; i < number.size(); ++i)
+ {
+ oa->write_character(to_char_type(static_cast<std::uint8_t>(number[i])));
+ }
+ }
+ }
+
+ // UBJSON: write number (signed integer)
+ template < typename NumberType, typename std::enable_if <
+ std::is_signed<NumberType>::value&&
+ !std::is_floating_point<NumberType>::value, int >::type = 0 >
+ void write_number_with_ubjson_prefix(const NumberType n,
+ const bool add_prefix,
+ const bool use_bjdata)
+ {
+ if ((std::numeric_limits<std::int8_t>::min)() <= n && n <= (std::numeric_limits<std::int8_t>::max)())
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('i')); // int8
+ }
+ write_number(static_cast<std::int8_t>(n), use_bjdata);
+ }
+ else if (static_cast<std::int64_t>((std::numeric_limits<std::uint8_t>::min)()) <= n && n <= static_cast<std::int64_t>((std::numeric_limits<std::uint8_t>::max)()))
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('U')); // uint8
+ }
+ write_number(static_cast<std::uint8_t>(n), use_bjdata);
+ }
+ else if ((std::numeric_limits<std::int16_t>::min)() <= n && n <= (std::numeric_limits<std::int16_t>::max)())
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('I')); // int16
+ }
+ write_number(static_cast<std::int16_t>(n), use_bjdata);
+ }
+ else if (use_bjdata && (static_cast<std::int64_t>((std::numeric_limits<std::uint16_t>::min)()) <= n && n <= static_cast<std::int64_t>((std::numeric_limits<std::uint16_t>::max)())))
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('u')); // uint16 - bjdata only
+ }
+ write_number(static_cast<uint16_t>(n), use_bjdata);
+ }
+ else if ((std::numeric_limits<std::int32_t>::min)() <= n && n <= (std::numeric_limits<std::int32_t>::max)())
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('l')); // int32
+ }
+ write_number(static_cast<std::int32_t>(n), use_bjdata);
+ }
+ else if (use_bjdata && (static_cast<std::int64_t>((std::numeric_limits<std::uint32_t>::min)()) <= n && n <= static_cast<std::int64_t>((std::numeric_limits<std::uint32_t>::max)())))
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('m')); // uint32 - bjdata only
+ }
+ write_number(static_cast<uint32_t>(n), use_bjdata);
+ }
+ else if ((std::numeric_limits<std::int64_t>::min)() <= n && n <= (std::numeric_limits<std::int64_t>::max)())
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('L')); // int64
+ }
+ write_number(static_cast<std::int64_t>(n), use_bjdata);
+ }
+ // LCOV_EXCL_START
+ else
+ {
+ if (add_prefix)
+ {
+ oa->write_character(to_char_type('H')); // high-precision number
+ }
+
+ const auto number = BasicJsonType(n).dump();
+ write_number_with_ubjson_prefix(number.size(), true, use_bjdata);
+ for (std::size_t i = 0; i < number.size(); ++i)
+ {
+ oa->write_character(to_char_type(static_cast<std::uint8_t>(number[i])));
+ }
+ }
+ // LCOV_EXCL_STOP
+ }
+
+ /*!
+ @brief determine the type prefix of container values
+ */
+ CharType ubjson_prefix(const BasicJsonType& j, const bool use_bjdata) const noexcept
+ {
+ switch (j.type())
+ {
+ case value_t::null:
+ return 'Z';
+
+ case value_t::boolean:
+ return j.m_data.m_value.boolean ? 'T' : 'F';
+
+ case value_t::number_integer:
+ {
+ if ((std::numeric_limits<std::int8_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())
+ {
+ return 'i';
+ }
+ if ((std::numeric_limits<std::uint8_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint8_t>::max)())
+ {
+ return 'U';
+ }
+ if ((std::numeric_limits<std::int16_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())
+ {
+ return 'I';
+ }
+ if (use_bjdata && ((std::numeric_limits<std::uint16_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint16_t>::max)()))
+ {
+ return 'u';
+ }
+ if ((std::numeric_limits<std::int32_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())
+ {
+ return 'l';
+ }
+ if (use_bjdata && ((std::numeric_limits<std::uint32_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint32_t>::max)()))
+ {
+ return 'm';
+ }
+ if ((std::numeric_limits<std::int64_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::int64_t>::max)())
+ {
+ return 'L';
+ }
+ // anything else is treated as high-precision number
+ return 'H'; // LCOV_EXCL_LINE
+ }
+
+ case value_t::number_unsigned:
+ {
+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int8_t>::max)()))
+ {
+ return 'i';
+ }
+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint8_t>::max)()))
+ {
+ return 'U';
+ }
+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int16_t>::max)()))
+ {
+ return 'I';
+ }
+ if (use_bjdata && j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint16_t>::max)()))
+ {
+ return 'u';
+ }
+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
+ {
+ return 'l';
+ }
+ if (use_bjdata && j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint32_t>::max)()))
+ {
+ return 'm';
+ }
+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
+ {
+ return 'L';
+ }
+ if (use_bjdata && j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)())
+ {
+ return 'M';
+ }
+ // anything else is treated as high-precision number
+ return 'H'; // LCOV_EXCL_LINE
+ }
+
+ case value_t::number_float:
+ return get_ubjson_float_prefix(j.m_data.m_value.number_float);
+
+ case value_t::string:
+ return 'S';
+
+ case value_t::array: // fallthrough
+ case value_t::binary:
+ return '[';
+
+ case value_t::object:
+ return '{';
+
+ case value_t::discarded:
+ default: // discarded values
+ return 'N';
+ }
+ }
+
+ static constexpr CharType get_ubjson_float_prefix(float /*unused*/)
+ {
+ return 'd'; // float 32
+ }
+
+ static constexpr CharType get_ubjson_float_prefix(double /*unused*/)
+ {
+ return 'D'; // float 64
+ }
+
+ /*!
+ @return false if the object is successfully converted to a bjdata ndarray, true if the type or size is invalid
+ */
+ bool write_bjdata_ndarray(const typename BasicJsonType::object_t& value, const bool use_count, const bool use_type)
+ {
+ std::map<string_t, CharType> bjdtype = {{"uint8", 'U'}, {"int8", 'i'}, {"uint16", 'u'}, {"int16", 'I'},
+ {"uint32", 'm'}, {"int32", 'l'}, {"uint64", 'M'}, {"int64", 'L'}, {"single", 'd'}, {"double", 'D'}, {"char", 'C'}
+ };
+
+ string_t key = "_ArrayType_";
+ auto it = bjdtype.find(static_cast<string_t>(value.at(key)));
+ if (it == bjdtype.end())
+ {
+ return true;
+ }
+ CharType dtype = it->second;
+
+ key = "_ArraySize_";
+ std::size_t len = (value.at(key).empty() ? 0 : 1);
+ for (const auto& el : value.at(key))
+ {
+ len *= static_cast<std::size_t>(el.m_data.m_value.number_unsigned);
+ }
+
+ key = "_ArrayData_";
+ if (value.at(key).size() != len)
+ {
+ return true;
+ }
+
+ oa->write_character('[');
+ oa->write_character('$');
+ oa->write_character(dtype);
+ oa->write_character('#');
+
+ key = "_ArraySize_";
+ write_ubjson(value.at(key), use_count, use_type, true, true);
+
+ key = "_ArrayData_";
+ if (dtype == 'U' || dtype == 'C')
+ {
+ for (const auto& el : value.at(key))
+ {
+ write_number(static_cast<std::uint8_t>(el.m_data.m_value.number_unsigned), true);
+ }
+ }
+ else if (dtype == 'i')
+ {
+ for (const auto& el : value.at(key))
+ {
+ write_number(static_cast<std::int8_t>(el.m_data.m_value.number_integer), true);
+ }
+ }
+ else if (dtype == 'u')
+ {
+ for (const auto& el : value.at(key))
+ {
+ write_number(static_cast<std::uint16_t>(el.m_data.m_value.number_unsigned), true);
+ }
+ }
+ else if (dtype == 'I')
+ {
+ for (const auto& el : value.at(key))
+ {
+ write_number(static_cast<std::int16_t>(el.m_data.m_value.number_integer), true);
+ }
+ }
+ else if (dtype == 'm')
+ {
+ for (const auto& el : value.at(key))
+ {
+ write_number(static_cast<std::uint32_t>(el.m_data.m_value.number_unsigned), true);
+ }
+ }
+ else if (dtype == 'l')
+ {
+ for (const auto& el : value.at(key))
+ {
+ write_number(static_cast<std::int32_t>(el.m_data.m_value.number_integer), true);
+ }
+ }
+ else if (dtype == 'M')
+ {
+ for (const auto& el : value.at(key))
+ {
+ write_number(static_cast<std::uint64_t>(el.m_data.m_value.number_unsigned), true);
+ }
+ }
+ else if (dtype == 'L')
+ {
+ for (const auto& el : value.at(key))
+ {
+ write_number(static_cast<std::int64_t>(el.m_data.m_value.number_integer), true);
+ }
+ }
+ else if (dtype == 'd')
+ {
+ for (const auto& el : value.at(key))
+ {
+ write_number(static_cast<float>(el.m_data.m_value.number_float), true);
+ }
+ }
+ else if (dtype == 'D')
+ {
+ for (const auto& el : value.at(key))
+ {
+ write_number(static_cast<double>(el.m_data.m_value.number_float), true);
+ }
+ }
+ return false;
+ }
+
+ ///////////////////////
+ // Utility functions //
+ ///////////////////////
+
+ /*
+ @brief write a number to output input
+ @param[in] n number of type @a NumberType
+ @param[in] OutputIsLittleEndian Set to true if output data is
+ required to be little endian
+ @tparam NumberType the type of the number
+
+ @note This function needs to respect the system's endianness, because bytes
+ in CBOR, MessagePack, and UBJSON are stored in network order (big
+ endian) and therefore need reordering on little endian systems.
+ On the other hand, BSON and BJData use little endian and should reorder
+ on big endian systems.
+ */
+ template<typename NumberType>
+ void write_number(const NumberType n, const bool OutputIsLittleEndian = false)
+ {
+ // step 1: write number to array of length NumberType
+ std::array<CharType, sizeof(NumberType)> vec{};
+ std::memcpy(vec.data(), &n, sizeof(NumberType));
+
+ // step 2: write array to output (with possible reordering)
+ if (is_little_endian != OutputIsLittleEndian)
+ {
+ // reverse byte order prior to conversion if necessary
+ std::reverse(vec.begin(), vec.end());
+ }
+
+ oa->write_characters(vec.data(), sizeof(NumberType));
+ }
+
+ void write_compact_float(const number_float_t n, detail::input_format_t format)
+ {
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wfloat-equal"
+#endif
+ if (static_cast<double>(n) >= static_cast<double>(std::numeric_limits<float>::lowest()) &&
+ static_cast<double>(n) <= static_cast<double>((std::numeric_limits<float>::max)()) &&
+ static_cast<double>(static_cast<float>(n)) == static_cast<double>(n))
+ {
+ oa->write_character(format == detail::input_format_t::cbor
+ ? get_cbor_float_prefix(static_cast<float>(n))
+ : get_msgpack_float_prefix(static_cast<float>(n)));
+ write_number(static_cast<float>(n));
+ }
+ else
+ {
+ oa->write_character(format == detail::input_format_t::cbor
+ ? get_cbor_float_prefix(n)
+ : get_msgpack_float_prefix(n));
+ write_number(n);
+ }
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+ }
+
+ public:
+ // The following to_char_type functions are implement the conversion
+ // between uint8_t and CharType. In case CharType is not unsigned,
+ // such a conversion is required to allow values greater than 128.
+ // See <https://github.com/nlohmann/json/issues/1286> for a discussion.
+ template < typename C = CharType,
+ enable_if_t < std::is_signed<C>::value && std::is_signed<char>::value > * = nullptr >
+ static constexpr CharType to_char_type(std::uint8_t x) noexcept
+ {
+ return *reinterpret_cast<char*>(&x);
+ }
+
+ template < typename C = CharType,
+ enable_if_t < std::is_signed<C>::value && std::is_unsigned<char>::value > * = nullptr >
+ static CharType to_char_type(std::uint8_t x) noexcept
+ {
+ static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t");
+ static_assert(std::is_trivial<CharType>::value, "CharType must be trivial");
+ CharType result;
+ std::memcpy(&result, &x, sizeof(x));
+ return result;
+ }
+
+ template<typename C = CharType,
+ enable_if_t<std::is_unsigned<C>::value>* = nullptr>
+ static constexpr CharType to_char_type(std::uint8_t x) noexcept
+ {
+ return x;
+ }
+
+ template < typename InputCharType, typename C = CharType,
+ enable_if_t <
+ std::is_signed<C>::value &&
+ std::is_signed<char>::value &&
+ std::is_same<char, typename std::remove_cv<InputCharType>::type>::value
+ > * = nullptr >
+ static constexpr CharType to_char_type(InputCharType x) noexcept
+ {
+ return x;
+ }
+
+ private:
+ /// whether we can assume little endianness
+ const bool is_little_endian = little_endianness();
+
+ /// the output
+ output_adapter_t<CharType> oa = nullptr;
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/output/output_adapters.hpp>
+
+// #include <nlohmann/detail/output/serializer.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2008-2009 Björn Hoehrmann <bjoern@hoehrmann.de>
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <algorithm> // reverse, remove, fill, find, none_of
+#include <array> // array
+#include <clocale> // localeconv, lconv
+#include <cmath> // labs, isfinite, isnan, signbit
+#include <cstddef> // size_t, ptrdiff_t
+#include <cstdint> // uint8_t
+#include <cstdio> // snprintf
+#include <limits> // numeric_limits
+#include <string> // string, char_traits
+#include <iomanip> // setfill, setw
+#include <type_traits> // is_same
+#include <utility> // move
+
+// #include <nlohmann/detail/conversions/to_chars.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2009 Florian Loitsch <https://florian.loitsch.com/>
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <array> // array
+#include <cmath> // signbit, isfinite
+#include <cstdint> // intN_t, uintN_t
+#include <cstring> // memcpy, memmove
+#include <limits> // numeric_limits
+#include <type_traits> // conditional
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+/*!
+@brief implements the Grisu2 algorithm for binary to decimal floating-point
+conversion.
+
+This implementation is a slightly modified version of the reference
+implementation which may be obtained from
+http://florian.loitsch.com/publications (bench.tar.gz).
+
+The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch.
+
+For a detailed description of the algorithm see:
+
+[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with
+ Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming
+ Language Design and Implementation, PLDI 2010
+[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately",
+ Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language
+ Design and Implementation, PLDI 1996
+*/
+namespace dtoa_impl
+{
+
+template<typename Target, typename Source>
+Target reinterpret_bits(const Source source)
+{
+ static_assert(sizeof(Target) == sizeof(Source), "size mismatch");
+
+ Target target;
+ std::memcpy(&target, &source, sizeof(Source));
+ return target;
+}
+
+struct diyfp // f * 2^e
+{
+ static constexpr int kPrecision = 64; // = q
+
+ std::uint64_t f = 0;
+ int e = 0;
+
+ constexpr diyfp(std::uint64_t f_, int e_) noexcept : f(f_), e(e_) {}
+
+ /*!
+ @brief returns x - y
+ @pre x.e == y.e and x.f >= y.f
+ */
+ static diyfp sub(const diyfp& x, const diyfp& y) noexcept
+ {
+ JSON_ASSERT(x.e == y.e);
+ JSON_ASSERT(x.f >= y.f);
+
+ return {x.f - y.f, x.e};
+ }
+
+ /*!
+ @brief returns x * y
+ @note The result is rounded. (Only the upper q bits are returned.)
+ */
+ static diyfp mul(const diyfp& x, const diyfp& y) noexcept
+ {
+ static_assert(kPrecision == 64, "internal error");
+
+ // Computes:
+ // f = round((x.f * y.f) / 2^q)
+ // e = x.e + y.e + q
+
+ // Emulate the 64-bit * 64-bit multiplication:
+ //
+ // p = u * v
+ // = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi)
+ // = (u_lo v_lo ) + 2^32 ((u_lo v_hi ) + (u_hi v_lo )) + 2^64 (u_hi v_hi )
+ // = (p0 ) + 2^32 ((p1 ) + (p2 )) + 2^64 (p3 )
+ // = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3 )
+ // = (p0_lo ) + 2^32 (p0_hi + p1_lo + p2_lo ) + 2^64 (p1_hi + p2_hi + p3)
+ // = (p0_lo ) + 2^32 (Q ) + 2^64 (H )
+ // = (p0_lo ) + 2^32 (Q_lo + 2^32 Q_hi ) + 2^64 (H )
+ //
+ // (Since Q might be larger than 2^32 - 1)
+ //
+ // = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H)
+ //
+ // (Q_hi + H does not overflow a 64-bit int)
+ //
+ // = p_lo + 2^64 p_hi
+
+ const std::uint64_t u_lo = x.f & 0xFFFFFFFFu;
+ const std::uint64_t u_hi = x.f >> 32u;
+ const std::uint64_t v_lo = y.f & 0xFFFFFFFFu;
+ const std::uint64_t v_hi = y.f >> 32u;
+
+ const std::uint64_t p0 = u_lo * v_lo;
+ const std::uint64_t p1 = u_lo * v_hi;
+ const std::uint64_t p2 = u_hi * v_lo;
+ const std::uint64_t p3 = u_hi * v_hi;
+
+ const std::uint64_t p0_hi = p0 >> 32u;
+ const std::uint64_t p1_lo = p1 & 0xFFFFFFFFu;
+ const std::uint64_t p1_hi = p1 >> 32u;
+ const std::uint64_t p2_lo = p2 & 0xFFFFFFFFu;
+ const std::uint64_t p2_hi = p2 >> 32u;
+
+ std::uint64_t Q = p0_hi + p1_lo + p2_lo;
+
+ // The full product might now be computed as
+ //
+ // p_hi = p3 + p2_hi + p1_hi + (Q >> 32)
+ // p_lo = p0_lo + (Q << 32)
+ //
+ // But in this particular case here, the full p_lo is not required.
+ // Effectively we only need to add the highest bit in p_lo to p_hi (and
+ // Q_hi + 1 does not overflow).
+
+ Q += std::uint64_t{1} << (64u - 32u - 1u); // round, ties up
+
+ const std::uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32u);
+
+ return {h, x.e + y.e + 64};
+ }
+
+ /*!
+ @brief normalize x such that the significand is >= 2^(q-1)
+ @pre x.f != 0
+ */
+ static diyfp normalize(diyfp x) noexcept
+ {
+ JSON_ASSERT(x.f != 0);
+
+ while ((x.f >> 63u) == 0)
+ {
+ x.f <<= 1u;
+ x.e--;
+ }
+
+ return x;
+ }
+
+ /*!
+ @brief normalize x such that the result has the exponent E
+ @pre e >= x.e and the upper e - x.e bits of x.f must be zero.
+ */
+ static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept
+ {
+ const int delta = x.e - target_exponent;
+
+ JSON_ASSERT(delta >= 0);
+ JSON_ASSERT(((x.f << delta) >> delta) == x.f);
+
+ return {x.f << delta, target_exponent};
+ }
+};
+
+struct boundaries
+{
+ diyfp w;
+ diyfp minus;
+ diyfp plus;
+};
+
+/*!
+Compute the (normalized) diyfp representing the input number 'value' and its
+boundaries.
+
+@pre value must be finite and positive
+*/
+template<typename FloatType>
+boundaries compute_boundaries(FloatType value)
+{
+ JSON_ASSERT(std::isfinite(value));
+ JSON_ASSERT(value > 0);
+
+ // Convert the IEEE representation into a diyfp.
+ //
+ // If v is denormal:
+ // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1))
+ // If v is normalized:
+ // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1))
+
+ static_assert(std::numeric_limits<FloatType>::is_iec559,
+ "internal error: dtoa_short requires an IEEE-754 floating-point implementation");
+
+ constexpr int kPrecision = std::numeric_limits<FloatType>::digits; // = p (includes the hidden bit)
+ constexpr int kBias = std::numeric_limits<FloatType>::max_exponent - 1 + (kPrecision - 1);
+ constexpr int kMinExp = 1 - kBias;
+ constexpr std::uint64_t kHiddenBit = std::uint64_t{1} << (kPrecision - 1); // = 2^(p-1)
+
+ using bits_type = typename std::conditional<kPrecision == 24, std::uint32_t, std::uint64_t >::type;
+
+ const auto bits = static_cast<std::uint64_t>(reinterpret_bits<bits_type>(value));
+ const std::uint64_t E = bits >> (kPrecision - 1);
+ const std::uint64_t F = bits & (kHiddenBit - 1);
+
+ const bool is_denormal = E == 0;
+ const diyfp v = is_denormal
+ ? diyfp(F, kMinExp)
+ : diyfp(F + kHiddenBit, static_cast<int>(E) - kBias);
+
+ // Compute the boundaries m- and m+ of the floating-point value
+ // v = f * 2^e.
+ //
+ // Determine v- and v+, the floating-point predecessor and successor if v,
+ // respectively.
+ //
+ // v- = v - 2^e if f != 2^(p-1) or e == e_min (A)
+ // = v - 2^(e-1) if f == 2^(p-1) and e > e_min (B)
+ //
+ // v+ = v + 2^e
+ //
+ // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_
+ // between m- and m+ round to v, regardless of how the input rounding
+ // algorithm breaks ties.
+ //
+ // ---+-------------+-------------+-------------+-------------+--- (A)
+ // v- m- v m+ v+
+ //
+ // -----------------+------+------+-------------+-------------+--- (B)
+ // v- m- v m+ v+
+
+ const bool lower_boundary_is_closer = F == 0 && E > 1;
+ const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1);
+ const diyfp m_minus = lower_boundary_is_closer
+ ? diyfp(4 * v.f - 1, v.e - 2) // (B)
+ : diyfp(2 * v.f - 1, v.e - 1); // (A)
+
+ // Determine the normalized w+ = m+.
+ const diyfp w_plus = diyfp::normalize(m_plus);
+
+ // Determine w- = m- such that e_(w-) = e_(w+).
+ const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e);
+
+ return {diyfp::normalize(v), w_minus, w_plus};
+}
+
+// Given normalized diyfp w, Grisu needs to find a (normalized) cached
+// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies
+// within a certain range [alpha, gamma] (Definition 3.2 from [1])
+//
+// alpha <= e = e_c + e_w + q <= gamma
+//
+// or
+//
+// f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q
+// <= f_c * f_w * 2^gamma
+//
+// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies
+//
+// 2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma
+//
+// or
+//
+// 2^(q - 2 + alpha) <= c * w < 2^(q + gamma)
+//
+// The choice of (alpha,gamma) determines the size of the table and the form of
+// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well
+// in practice:
+//
+// The idea is to cut the number c * w = f * 2^e into two parts, which can be
+// processed independently: An integral part p1, and a fractional part p2:
+//
+// f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e
+// = (f div 2^-e) + (f mod 2^-e) * 2^e
+// = p1 + p2 * 2^e
+//
+// The conversion of p1 into decimal form requires a series of divisions and
+// modulos by (a power of) 10. These operations are faster for 32-bit than for
+// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be
+// achieved by choosing
+//
+// -e >= 32 or e <= -32 := gamma
+//
+// In order to convert the fractional part
+//
+// p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ...
+//
+// into decimal form, the fraction is repeatedly multiplied by 10 and the digits
+// d[-i] are extracted in order:
+//
+// (10 * p2) div 2^-e = d[-1]
+// (10 * p2) mod 2^-e = d[-2] / 10^1 + ...
+//
+// The multiplication by 10 must not overflow. It is sufficient to choose
+//
+// 10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64.
+//
+// Since p2 = f mod 2^-e < 2^-e,
+//
+// -e <= 60 or e >= -60 := alpha
+
+constexpr int kAlpha = -60;
+constexpr int kGamma = -32;
+
+struct cached_power // c = f * 2^e ~= 10^k
+{
+ std::uint64_t f;
+ int e;
+ int k;
+};
+
+/*!
+For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached
+power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c
+satisfies (Definition 3.2 from [1])
+
+ alpha <= e_c + e + q <= gamma.
+*/
+inline cached_power get_cached_power_for_binary_exponent(int e)
+{
+ // Now
+ //
+ // alpha <= e_c + e + q <= gamma (1)
+ // ==> f_c * 2^alpha <= c * 2^e * 2^q
+ //
+ // and since the c's are normalized, 2^(q-1) <= f_c,
+ //
+ // ==> 2^(q - 1 + alpha) <= c * 2^(e + q)
+ // ==> 2^(alpha - e - 1) <= c
+ //
+ // If c were an exact power of ten, i.e. c = 10^k, one may determine k as
+ //
+ // k = ceil( log_10( 2^(alpha - e - 1) ) )
+ // = ceil( (alpha - e - 1) * log_10(2) )
+ //
+ // From the paper:
+ // "In theory the result of the procedure could be wrong since c is rounded,
+ // and the computation itself is approximated [...]. In practice, however,
+ // this simple function is sufficient."
+ //
+ // For IEEE double precision floating-point numbers converted into
+ // normalized diyfp's w = f * 2^e, with q = 64,
+ //
+ // e >= -1022 (min IEEE exponent)
+ // -52 (p - 1)
+ // -52 (p - 1, possibly normalize denormal IEEE numbers)
+ // -11 (normalize the diyfp)
+ // = -1137
+ //
+ // and
+ //
+ // e <= +1023 (max IEEE exponent)
+ // -52 (p - 1)
+ // -11 (normalize the diyfp)
+ // = 960
+ //
+ // This binary exponent range [-1137,960] results in a decimal exponent
+ // range [-307,324]. One does not need to store a cached power for each
+ // k in this range. For each such k it suffices to find a cached power
+ // such that the exponent of the product lies in [alpha,gamma].
+ // This implies that the difference of the decimal exponents of adjacent
+ // table entries must be less than or equal to
+ //
+ // floor( (gamma - alpha) * log_10(2) ) = 8.
+ //
+ // (A smaller distance gamma-alpha would require a larger table.)
+
+ // NB:
+ // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34.
+
+ constexpr int kCachedPowersMinDecExp = -300;
+ constexpr int kCachedPowersDecStep = 8;
+
+ static constexpr std::array<cached_power, 79> kCachedPowers =
+ {
+ {
+ { 0xAB70FE17C79AC6CA, -1060, -300 },
+ { 0xFF77B1FCBEBCDC4F, -1034, -292 },
+ { 0xBE5691EF416BD60C, -1007, -284 },
+ { 0x8DD01FAD907FFC3C, -980, -276 },
+ { 0xD3515C2831559A83, -954, -268 },
+ { 0x9D71AC8FADA6C9B5, -927, -260 },
+ { 0xEA9C227723EE8BCB, -901, -252 },
+ { 0xAECC49914078536D, -874, -244 },
+ { 0x823C12795DB6CE57, -847, -236 },
+ { 0xC21094364DFB5637, -821, -228 },
+ { 0x9096EA6F3848984F, -794, -220 },
+ { 0xD77485CB25823AC7, -768, -212 },
+ { 0xA086CFCD97BF97F4, -741, -204 },
+ { 0xEF340A98172AACE5, -715, -196 },
+ { 0xB23867FB2A35B28E, -688, -188 },
+ { 0x84C8D4DFD2C63F3B, -661, -180 },
+ { 0xC5DD44271AD3CDBA, -635, -172 },
+ { 0x936B9FCEBB25C996, -608, -164 },
+ { 0xDBAC6C247D62A584, -582, -156 },
+ { 0xA3AB66580D5FDAF6, -555, -148 },
+ { 0xF3E2F893DEC3F126, -529, -140 },
+ { 0xB5B5ADA8AAFF80B8, -502, -132 },
+ { 0x87625F056C7C4A8B, -475, -124 },
+ { 0xC9BCFF6034C13053, -449, -116 },
+ { 0x964E858C91BA2655, -422, -108 },
+ { 0xDFF9772470297EBD, -396, -100 },
+ { 0xA6DFBD9FB8E5B88F, -369, -92 },
+ { 0xF8A95FCF88747D94, -343, -84 },
+ { 0xB94470938FA89BCF, -316, -76 },
+ { 0x8A08F0F8BF0F156B, -289, -68 },
+ { 0xCDB02555653131B6, -263, -60 },
+ { 0x993FE2C6D07B7FAC, -236, -52 },
+ { 0xE45C10C42A2B3B06, -210, -44 },
+ { 0xAA242499697392D3, -183, -36 },
+ { 0xFD87B5F28300CA0E, -157, -28 },
+ { 0xBCE5086492111AEB, -130, -20 },
+ { 0x8CBCCC096F5088CC, -103, -12 },
+ { 0xD1B71758E219652C, -77, -4 },
+ { 0x9C40000000000000, -50, 4 },
+ { 0xE8D4A51000000000, -24, 12 },
+ { 0xAD78EBC5AC620000, 3, 20 },
+ { 0x813F3978F8940984, 30, 28 },
+ { 0xC097CE7BC90715B3, 56, 36 },
+ { 0x8F7E32CE7BEA5C70, 83, 44 },
+ { 0xD5D238A4ABE98068, 109, 52 },
+ { 0x9F4F2726179A2245, 136, 60 },
+ { 0xED63A231D4C4FB27, 162, 68 },
+ { 0xB0DE65388CC8ADA8, 189, 76 },
+ { 0x83C7088E1AAB65DB, 216, 84 },
+ { 0xC45D1DF942711D9A, 242, 92 },
+ { 0x924D692CA61BE758, 269, 100 },
+ { 0xDA01EE641A708DEA, 295, 108 },
+ { 0xA26DA3999AEF774A, 322, 116 },
+ { 0xF209787BB47D6B85, 348, 124 },
+ { 0xB454E4A179DD1877, 375, 132 },
+ { 0x865B86925B9BC5C2, 402, 140 },
+ { 0xC83553C5C8965D3D, 428, 148 },
+ { 0x952AB45CFA97A0B3, 455, 156 },
+ { 0xDE469FBD99A05FE3, 481, 164 },
+ { 0xA59BC234DB398C25, 508, 172 },
+ { 0xF6C69A72A3989F5C, 534, 180 },
+ { 0xB7DCBF5354E9BECE, 561, 188 },
+ { 0x88FCF317F22241E2, 588, 196 },
+ { 0xCC20CE9BD35C78A5, 614, 204 },
+ { 0x98165AF37B2153DF, 641, 212 },
+ { 0xE2A0B5DC971F303A, 667, 220 },
+ { 0xA8D9D1535CE3B396, 694, 228 },
+ { 0xFB9B7CD9A4A7443C, 720, 236 },
+ { 0xBB764C4CA7A44410, 747, 244 },
+ { 0x8BAB8EEFB6409C1A, 774, 252 },
+ { 0xD01FEF10A657842C, 800, 260 },
+ { 0x9B10A4E5E9913129, 827, 268 },
+ { 0xE7109BFBA19C0C9D, 853, 276 },
+ { 0xAC2820D9623BF429, 880, 284 },
+ { 0x80444B5E7AA7CF85, 907, 292 },
+ { 0xBF21E44003ACDD2D, 933, 300 },
+ { 0x8E679C2F5E44FF8F, 960, 308 },
+ { 0xD433179D9C8CB841, 986, 316 },
+ { 0x9E19DB92B4E31BA9, 1013, 324 },
+ }
+ };
+
+ // This computation gives exactly the same results for k as
+ // k = ceil((kAlpha - e - 1) * 0.30102999566398114)
+ // for |e| <= 1500, but doesn't require floating-point operations.
+ // NB: log_10(2) ~= 78913 / 2^18
+ JSON_ASSERT(e >= -1500);
+ JSON_ASSERT(e <= 1500);
+ const int f = kAlpha - e - 1;
+ const int k = (f * 78913) / (1 << 18) + static_cast<int>(f > 0);
+
+ const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep;
+ JSON_ASSERT(index >= 0);
+ JSON_ASSERT(static_cast<std::size_t>(index) < kCachedPowers.size());
+
+ const cached_power cached = kCachedPowers[static_cast<std::size_t>(index)];
+ JSON_ASSERT(kAlpha <= cached.e + e + 64);
+ JSON_ASSERT(kGamma >= cached.e + e + 64);
+
+ return cached;
+}
+
+/*!
+For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k.
+For n == 0, returns 1 and sets pow10 := 1.
+*/
+inline int find_largest_pow10(const std::uint32_t n, std::uint32_t& pow10)
+{
+ // LCOV_EXCL_START
+ if (n >= 1000000000)
+ {
+ pow10 = 1000000000;
+ return 10;
+ }
+ // LCOV_EXCL_STOP
+ if (n >= 100000000)
+ {
+ pow10 = 100000000;
+ return 9;
+ }
+ if (n >= 10000000)
+ {
+ pow10 = 10000000;
+ return 8;
+ }
+ if (n >= 1000000)
+ {
+ pow10 = 1000000;
+ return 7;
+ }
+ if (n >= 100000)
+ {
+ pow10 = 100000;
+ return 6;
+ }
+ if (n >= 10000)
+ {
+ pow10 = 10000;
+ return 5;
+ }
+ if (n >= 1000)
+ {
+ pow10 = 1000;
+ return 4;
+ }
+ if (n >= 100)
+ {
+ pow10 = 100;
+ return 3;
+ }
+ if (n >= 10)
+ {
+ pow10 = 10;
+ return 2;
+ }
+
+ pow10 = 1;
+ return 1;
+}
+
+inline void grisu2_round(char* buf, int len, std::uint64_t dist, std::uint64_t delta,
+ std::uint64_t rest, std::uint64_t ten_k)
+{
+ JSON_ASSERT(len >= 1);
+ JSON_ASSERT(dist <= delta);
+ JSON_ASSERT(rest <= delta);
+ JSON_ASSERT(ten_k > 0);
+
+ // <--------------------------- delta ---->
+ // <---- dist --------->
+ // --------------[------------------+-------------------]--------------
+ // M- w M+
+ //
+ // ten_k
+ // <------>
+ // <---- rest ---->
+ // --------------[------------------+----+--------------]--------------
+ // w V
+ // = buf * 10^k
+ //
+ // ten_k represents a unit-in-the-last-place in the decimal representation
+ // stored in buf.
+ // Decrement buf by ten_k while this takes buf closer to w.
+
+ // The tests are written in this order to avoid overflow in unsigned
+ // integer arithmetic.
+
+ while (rest < dist
+ && delta - rest >= ten_k
+ && (rest + ten_k < dist || dist - rest > rest + ten_k - dist))
+ {
+ JSON_ASSERT(buf[len - 1] != '0');
+ buf[len - 1]--;
+ rest += ten_k;
+ }
+}
+
+/*!
+Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+.
+M- and M+ must be normalized and share the same exponent -60 <= e <= -32.
+*/
+inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent,
+ diyfp M_minus, diyfp w, diyfp M_plus)
+{
+ static_assert(kAlpha >= -60, "internal error");
+ static_assert(kGamma <= -32, "internal error");
+
+ // Generates the digits (and the exponent) of a decimal floating-point
+ // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's
+ // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma.
+ //
+ // <--------------------------- delta ---->
+ // <---- dist --------->
+ // --------------[------------------+-------------------]--------------
+ // M- w M+
+ //
+ // Grisu2 generates the digits of M+ from left to right and stops as soon as
+ // V is in [M-,M+].
+
+ JSON_ASSERT(M_plus.e >= kAlpha);
+ JSON_ASSERT(M_plus.e <= kGamma);
+
+ std::uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e)
+ std::uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e)
+
+ // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0):
+ //
+ // M+ = f * 2^e
+ // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e
+ // = ((p1 ) * 2^-e + (p2 )) * 2^e
+ // = p1 + p2 * 2^e
+
+ const diyfp one(std::uint64_t{1} << -M_plus.e, M_plus.e);
+
+ auto p1 = static_cast<std::uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.)
+ std::uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e
+
+ // 1)
+ //
+ // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0]
+
+ JSON_ASSERT(p1 > 0);
+
+ std::uint32_t pow10{};
+ const int k = find_largest_pow10(p1, pow10);
+
+ // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1)
+ //
+ // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1))
+ // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1))
+ //
+ // M+ = p1 + p2 * 2^e
+ // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e
+ // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e
+ // = d[k-1] * 10^(k-1) + ( rest) * 2^e
+ //
+ // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0)
+ //
+ // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0]
+ //
+ // but stop as soon as
+ //
+ // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e
+
+ int n = k;
+ while (n > 0)
+ {
+ // Invariants:
+ // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k)
+ // pow10 = 10^(n-1) <= p1 < 10^n
+ //
+ const std::uint32_t d = p1 / pow10; // d = p1 div 10^(n-1)
+ const std::uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1)
+ //
+ // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e
+ // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e)
+ //
+ JSON_ASSERT(d <= 9);
+ buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d
+ //
+ // M+ = buffer * 10^(n-1) + (r + p2 * 2^e)
+ //
+ p1 = r;
+ n--;
+ //
+ // M+ = buffer * 10^n + (p1 + p2 * 2^e)
+ // pow10 = 10^n
+ //
+
+ // Now check if enough digits have been generated.
+ // Compute
+ //
+ // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e
+ //
+ // Note:
+ // Since rest and delta share the same exponent e, it suffices to
+ // compare the significands.
+ const std::uint64_t rest = (std::uint64_t{p1} << -one.e) + p2;
+ if (rest <= delta)
+ {
+ // V = buffer * 10^n, with M- <= V <= M+.
+
+ decimal_exponent += n;
+
+ // We may now just stop. But instead look if the buffer could be
+ // decremented to bring V closer to w.
+ //
+ // pow10 = 10^n is now 1 ulp in the decimal representation V.
+ // The rounding procedure works with diyfp's with an implicit
+ // exponent of e.
+ //
+ // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e
+ //
+ const std::uint64_t ten_n = std::uint64_t{pow10} << -one.e;
+ grisu2_round(buffer, length, dist, delta, rest, ten_n);
+
+ return;
+ }
+
+ pow10 /= 10;
+ //
+ // pow10 = 10^(n-1) <= p1 < 10^n
+ // Invariants restored.
+ }
+
+ // 2)
+ //
+ // The digits of the integral part have been generated:
+ //
+ // M+ = d[k-1]...d[1]d[0] + p2 * 2^e
+ // = buffer + p2 * 2^e
+ //
+ // Now generate the digits of the fractional part p2 * 2^e.
+ //
+ // Note:
+ // No decimal point is generated: the exponent is adjusted instead.
+ //
+ // p2 actually represents the fraction
+ //
+ // p2 * 2^e
+ // = p2 / 2^-e
+ // = d[-1] / 10^1 + d[-2] / 10^2 + ...
+ //
+ // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...)
+ //
+ // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m
+ // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...)
+ //
+ // using
+ //
+ // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e)
+ // = ( d) * 2^-e + ( r)
+ //
+ // or
+ // 10^m * p2 * 2^e = d + r * 2^e
+ //
+ // i.e.
+ //
+ // M+ = buffer + p2 * 2^e
+ // = buffer + 10^-m * (d + r * 2^e)
+ // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e
+ //
+ // and stop as soon as 10^-m * r * 2^e <= delta * 2^e
+
+ JSON_ASSERT(p2 > delta);
+
+ int m = 0;
+ for (;;)
+ {
+ // Invariant:
+ // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e
+ // = buffer * 10^-m + 10^-m * (p2 ) * 2^e
+ // = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e
+ // = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e
+ //
+ JSON_ASSERT(p2 <= (std::numeric_limits<std::uint64_t>::max)() / 10);
+ p2 *= 10;
+ const std::uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e
+ const std::uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e
+ //
+ // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e
+ // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e))
+ // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e
+ //
+ JSON_ASSERT(d <= 9);
+ buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d
+ //
+ // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e
+ //
+ p2 = r;
+ m++;
+ //
+ // M+ = buffer * 10^-m + 10^-m * p2 * 2^e
+ // Invariant restored.
+
+ // Check if enough digits have been generated.
+ //
+ // 10^-m * p2 * 2^e <= delta * 2^e
+ // p2 * 2^e <= 10^m * delta * 2^e
+ // p2 <= 10^m * delta
+ delta *= 10;
+ dist *= 10;
+ if (p2 <= delta)
+ {
+ break;
+ }
+ }
+
+ // V = buffer * 10^-m, with M- <= V <= M+.
+
+ decimal_exponent -= m;
+
+ // 1 ulp in the decimal representation is now 10^-m.
+ // Since delta and dist are now scaled by 10^m, we need to do the
+ // same with ulp in order to keep the units in sync.
+ //
+ // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e
+ //
+ const std::uint64_t ten_m = one.f;
+ grisu2_round(buffer, length, dist, delta, p2, ten_m);
+
+ // By construction this algorithm generates the shortest possible decimal
+ // number (Loitsch, Theorem 6.2) which rounds back to w.
+ // For an input number of precision p, at least
+ //
+ // N = 1 + ceil(p * log_10(2))
+ //
+ // decimal digits are sufficient to identify all binary floating-point
+ // numbers (Matula, "In-and-Out conversions").
+ // This implies that the algorithm does not produce more than N decimal
+ // digits.
+ //
+ // N = 17 for p = 53 (IEEE double precision)
+ // N = 9 for p = 24 (IEEE single precision)
+}
+
+/*!
+v = buf * 10^decimal_exponent
+len is the length of the buffer (number of decimal digits)
+The buffer must be large enough, i.e. >= max_digits10.
+*/
+JSON_HEDLEY_NON_NULL(1)
+inline void grisu2(char* buf, int& len, int& decimal_exponent,
+ diyfp m_minus, diyfp v, diyfp m_plus)
+{
+ JSON_ASSERT(m_plus.e == m_minus.e);
+ JSON_ASSERT(m_plus.e == v.e);
+
+ // --------(-----------------------+-----------------------)-------- (A)
+ // m- v m+
+ //
+ // --------------------(-----------+-----------------------)-------- (B)
+ // m- v m+
+ //
+ // First scale v (and m- and m+) such that the exponent is in the range
+ // [alpha, gamma].
+
+ const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e);
+
+ const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k
+
+ // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma]
+ const diyfp w = diyfp::mul(v, c_minus_k);
+ const diyfp w_minus = diyfp::mul(m_minus, c_minus_k);
+ const diyfp w_plus = diyfp::mul(m_plus, c_minus_k);
+
+ // ----(---+---)---------------(---+---)---------------(---+---)----
+ // w- w w+
+ // = c*m- = c*v = c*m+
+ //
+ // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and
+ // w+ are now off by a small amount.
+ // In fact:
+ //
+ // w - v * 10^k < 1 ulp
+ //
+ // To account for this inaccuracy, add resp. subtract 1 ulp.
+ //
+ // --------+---[---------------(---+---)---------------]---+--------
+ // w- M- w M+ w+
+ //
+ // Now any number in [M-, M+] (bounds included) will round to w when input,
+ // regardless of how the input rounding algorithm breaks ties.
+ //
+ // And digit_gen generates the shortest possible such number in [M-, M+].
+ // Note that this does not mean that Grisu2 always generates the shortest
+ // possible number in the interval (m-, m+).
+ const diyfp M_minus(w_minus.f + 1, w_minus.e);
+ const diyfp M_plus (w_plus.f - 1, w_plus.e );
+
+ decimal_exponent = -cached.k; // = -(-k) = k
+
+ grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus);
+}
+
+/*!
+v = buf * 10^decimal_exponent
+len is the length of the buffer (number of decimal digits)
+The buffer must be large enough, i.e. >= max_digits10.
+*/
+template<typename FloatType>
+JSON_HEDLEY_NON_NULL(1)
+void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value)
+{
+ static_assert(diyfp::kPrecision >= std::numeric_limits<FloatType>::digits + 3,
+ "internal error: not enough precision");
+
+ JSON_ASSERT(std::isfinite(value));
+ JSON_ASSERT(value > 0);
+
+ // If the neighbors (and boundaries) of 'value' are always computed for double-precision
+ // numbers, all float's can be recovered using strtod (and strtof). However, the resulting
+ // decimal representations are not exactly "short".
+ //
+ // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars)
+ // says "value is converted to a string as if by std::sprintf in the default ("C") locale"
+ // and since sprintf promotes floats to doubles, I think this is exactly what 'std::to_chars'
+ // does.
+ // On the other hand, the documentation for 'std::to_chars' requires that "parsing the
+ // representation using the corresponding std::from_chars function recovers value exactly". That
+ // indicates that single precision floating-point numbers should be recovered using
+ // 'std::strtof'.
+ //
+ // NB: If the neighbors are computed for single-precision numbers, there is a single float
+ // (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision
+ // value is off by 1 ulp.
+#if 0 // NOLINT(readability-avoid-unconditional-preprocessor-if)
+ const boundaries w = compute_boundaries(static_cast<double>(value));
+#else
+ const boundaries w = compute_boundaries(value);
+#endif
+
+ grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus);
+}
+
+/*!
+@brief appends a decimal representation of e to buf
+@return a pointer to the element following the exponent.
+@pre -1000 < e < 1000
+*/
+JSON_HEDLEY_NON_NULL(1)
+JSON_HEDLEY_RETURNS_NON_NULL
+inline char* append_exponent(char* buf, int e)
+{
+ JSON_ASSERT(e > -1000);
+ JSON_ASSERT(e < 1000);
+
+ if (e < 0)
+ {
+ e = -e;
+ *buf++ = '-';
+ }
+ else
+ {
+ *buf++ = '+';
+ }
+
+ auto k = static_cast<std::uint32_t>(e);
+ if (k < 10)
+ {
+ // Always print at least two digits in the exponent.
+ // This is for compatibility with printf("%g").
+ *buf++ = '0';
+ *buf++ = static_cast<char>('0' + k);
+ }
+ else if (k < 100)
+ {
+ *buf++ = static_cast<char>('0' + k / 10);
+ k %= 10;
+ *buf++ = static_cast<char>('0' + k);
+ }
+ else
+ {
+ *buf++ = static_cast<char>('0' + k / 100);
+ k %= 100;
+ *buf++ = static_cast<char>('0' + k / 10);
+ k %= 10;
+ *buf++ = static_cast<char>('0' + k);
+ }
+
+ return buf;
+}
+
+/*!
+@brief prettify v = buf * 10^decimal_exponent
+
+If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point
+notation. Otherwise it will be printed in exponential notation.
+
+@pre min_exp < 0
+@pre max_exp > 0
+*/
+JSON_HEDLEY_NON_NULL(1)
+JSON_HEDLEY_RETURNS_NON_NULL
+inline char* format_buffer(char* buf, int len, int decimal_exponent,
+ int min_exp, int max_exp)
+{
+ JSON_ASSERT(min_exp < 0);
+ JSON_ASSERT(max_exp > 0);
+
+ const int k = len;
+ const int n = len + decimal_exponent;
+
+ // v = buf * 10^(n-k)
+ // k is the length of the buffer (number of decimal digits)
+ // n is the position of the decimal point relative to the start of the buffer.
+
+ if (k <= n && n <= max_exp)
+ {
+ // digits[000]
+ // len <= max_exp + 2
+
+ std::memset(buf + k, '0', static_cast<size_t>(n) - static_cast<size_t>(k));
+ // Make it look like a floating-point number (#362, #378)
+ buf[n + 0] = '.';
+ buf[n + 1] = '0';
+ return buf + (static_cast<size_t>(n) + 2);
+ }
+
+ if (0 < n && n <= max_exp)
+ {
+ // dig.its
+ // len <= max_digits10 + 1
+
+ JSON_ASSERT(k > n);
+
+ std::memmove(buf + (static_cast<size_t>(n) + 1), buf + n, static_cast<size_t>(k) - static_cast<size_t>(n));
+ buf[n] = '.';
+ return buf + (static_cast<size_t>(k) + 1U);
+ }
+
+ if (min_exp < n && n <= 0)
+ {
+ // 0.[000]digits
+ // len <= 2 + (-min_exp - 1) + max_digits10
+
+ std::memmove(buf + (2 + static_cast<size_t>(-n)), buf, static_cast<size_t>(k));
+ buf[0] = '0';
+ buf[1] = '.';
+ std::memset(buf + 2, '0', static_cast<size_t>(-n));
+ return buf + (2U + static_cast<size_t>(-n) + static_cast<size_t>(k));
+ }
+
+ if (k == 1)
+ {
+ // dE+123
+ // len <= 1 + 5
+
+ buf += 1;
+ }
+ else
+ {
+ // d.igitsE+123
+ // len <= max_digits10 + 1 + 5
+
+ std::memmove(buf + 2, buf + 1, static_cast<size_t>(k) - 1);
+ buf[1] = '.';
+ buf += 1 + static_cast<size_t>(k);
+ }
+
+ *buf++ = 'e';
+ return append_exponent(buf, n - 1);
+}
+
+} // namespace dtoa_impl
+
+/*!
+@brief generates a decimal representation of the floating-point number value in [first, last).
+
+The format of the resulting decimal representation is similar to printf's %g
+format. Returns an iterator pointing past-the-end of the decimal representation.
+
+@note The input number must be finite, i.e. NaN's and Inf's are not supported.
+@note The buffer must be large enough.
+@note The result is NOT null-terminated.
+*/
+template<typename FloatType>
+JSON_HEDLEY_NON_NULL(1, 2)
+JSON_HEDLEY_RETURNS_NON_NULL
+char* to_chars(char* first, const char* last, FloatType value)
+{
+ static_cast<void>(last); // maybe unused - fix warning
+ JSON_ASSERT(std::isfinite(value));
+
+ // Use signbit(value) instead of (value < 0) since signbit works for -0.
+ if (std::signbit(value))
+ {
+ value = -value;
+ *first++ = '-';
+ }
+
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wfloat-equal"
+#endif
+ if (value == 0) // +-0
+ {
+ *first++ = '0';
+ // Make it look like a floating-point number (#362, #378)
+ *first++ = '.';
+ *first++ = '0';
+ return first;
+ }
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+
+ JSON_ASSERT(last - first >= std::numeric_limits<FloatType>::max_digits10);
+
+ // Compute v = buffer * 10^decimal_exponent.
+ // The decimal digits are stored in the buffer, which needs to be interpreted
+ // as an unsigned decimal integer.
+ // len is the length of the buffer, i.e. the number of decimal digits.
+ int len = 0;
+ int decimal_exponent = 0;
+ dtoa_impl::grisu2(first, len, decimal_exponent, value);
+
+ JSON_ASSERT(len <= std::numeric_limits<FloatType>::max_digits10);
+
+ // Format the buffer like printf("%.*g", prec, value)
+ constexpr int kMinExp = -4;
+ // Use digits10 here to increase compatibility with version 2.
+ constexpr int kMaxExp = std::numeric_limits<FloatType>::digits10;
+
+ JSON_ASSERT(last - first >= kMaxExp + 2);
+ JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits<FloatType>::max_digits10);
+ JSON_ASSERT(last - first >= std::numeric_limits<FloatType>::max_digits10 + 6);
+
+ return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp);
+}
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/output/binary_writer.hpp>
+
+// #include <nlohmann/detail/output/output_adapters.hpp>
+
+// #include <nlohmann/detail/string_concat.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+///////////////////
+// serialization //
+///////////////////
+
+/// how to treat decoding errors
+enum class error_handler_t
+{
+ strict, ///< throw a type_error exception in case of invalid UTF-8
+ replace, ///< replace invalid UTF-8 sequences with U+FFFD
+ ignore ///< ignore invalid UTF-8 sequences
+};
+
+template<typename BasicJsonType>
+class serializer
+{
+ using string_t = typename BasicJsonType::string_t;
+ using number_float_t = typename BasicJsonType::number_float_t;
+ using number_integer_t = typename BasicJsonType::number_integer_t;
+ using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using binary_char_t = typename BasicJsonType::binary_t::value_type;
+ static constexpr std::uint8_t UTF8_ACCEPT = 0;
+ static constexpr std::uint8_t UTF8_REJECT = 1;
+
+ public:
+ /*!
+ @param[in] s output stream to serialize to
+ @param[in] ichar indentation character to use
+ @param[in] error_handler_ how to react on decoding errors
+ */
+ serializer(output_adapter_t<char> s, const char ichar,
+ error_handler_t error_handler_ = error_handler_t::strict)
+ : o(std::move(s))
+ , loc(std::localeconv())
+ , thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits<char>::to_char_type(* (loc->thousands_sep)))
+ , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits<char>::to_char_type(* (loc->decimal_point)))
+ , indent_char(ichar)
+ , indent_string(512, indent_char)
+ , error_handler(error_handler_)
+ {}
+
+ // delete because of pointer members
+ serializer(const serializer&) = delete;
+ serializer& operator=(const serializer&) = delete;
+ serializer(serializer&&) = delete;
+ serializer& operator=(serializer&&) = delete;
+ ~serializer() = default;
+
+ /*!
+ @brief internal implementation of the serialization function
+
+ This function is called by the public member function dump and organizes
+ the serialization internally. The indentation level is propagated as
+ additional parameter. In case of arrays and objects, the function is
+ called recursively.
+
+ - strings and object keys are escaped using `escape_string()`
+ - integer numbers are converted implicitly via `operator<<`
+ - floating-point numbers are converted to a string using `"%g"` format
+ - binary values are serialized as objects containing the subtype and the
+ byte array
+
+ @param[in] val value to serialize
+ @param[in] pretty_print whether the output shall be pretty-printed
+ @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters
+ in the output are escaped with `\uXXXX` sequences, and the result consists
+ of ASCII characters only.
+ @param[in] indent_step the indent level
+ @param[in] current_indent the current indent level (only used internally)
+ */
+ void dump(const BasicJsonType& val,
+ const bool pretty_print,
+ const bool ensure_ascii,
+ const unsigned int indent_step,
+ const unsigned int current_indent = 0)
+ {
+ switch (val.m_data.m_type)
+ {
+ case value_t::object:
+ {
+ if (val.m_data.m_value.object->empty())
+ {
+ o->write_characters("{}", 2);
+ return;
+ }
+
+ if (pretty_print)
+ {
+ o->write_characters("{\n", 2);
+
+ // variable to hold indentation for recursive calls
+ const auto new_indent = current_indent + indent_step;
+ if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
+ {
+ indent_string.resize(indent_string.size() * 2, ' ');
+ }
+
+ // first n-1 elements
+ auto i = val.m_data.m_value.object->cbegin();
+ for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i)
+ {
+ o->write_characters(indent_string.c_str(), new_indent);
+ o->write_character('\"');
+ dump_escaped(i->first, ensure_ascii);
+ o->write_characters("\": ", 3);
+ dump(i->second, true, ensure_ascii, indent_step, new_indent);
+ o->write_characters(",\n", 2);
+ }
+
+ // last element
+ JSON_ASSERT(i != val.m_data.m_value.object->cend());
+ JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend());
+ o->write_characters(indent_string.c_str(), new_indent);
+ o->write_character('\"');
+ dump_escaped(i->first, ensure_ascii);
+ o->write_characters("\": ", 3);
+ dump(i->second, true, ensure_ascii, indent_step, new_indent);
+
+ o->write_character('\n');
+ o->write_characters(indent_string.c_str(), current_indent);
+ o->write_character('}');
+ }
+ else
+ {
+ o->write_character('{');
+
+ // first n-1 elements
+ auto i = val.m_data.m_value.object->cbegin();
+ for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i)
+ {
+ o->write_character('\"');
+ dump_escaped(i->first, ensure_ascii);
+ o->write_characters("\":", 2);
+ dump(i->second, false, ensure_ascii, indent_step, current_indent);
+ o->write_character(',');
+ }
+
+ // last element
+ JSON_ASSERT(i != val.m_data.m_value.object->cend());
+ JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend());
+ o->write_character('\"');
+ dump_escaped(i->first, ensure_ascii);
+ o->write_characters("\":", 2);
+ dump(i->second, false, ensure_ascii, indent_step, current_indent);
+
+ o->write_character('}');
+ }
+
+ return;
+ }
+
+ case value_t::array:
+ {
+ if (val.m_data.m_value.array->empty())
+ {
+ o->write_characters("[]", 2);
+ return;
+ }
+
+ if (pretty_print)
+ {
+ o->write_characters("[\n", 2);
+
+ // variable to hold indentation for recursive calls
+ const auto new_indent = current_indent + indent_step;
+ if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
+ {
+ indent_string.resize(indent_string.size() * 2, ' ');
+ }
+
+ // first n-1 elements
+ for (auto i = val.m_data.m_value.array->cbegin();
+ i != val.m_data.m_value.array->cend() - 1; ++i)
+ {
+ o->write_characters(indent_string.c_str(), new_indent);
+ dump(*i, true, ensure_ascii, indent_step, new_indent);
+ o->write_characters(",\n", 2);
+ }
+
+ // last element
+ JSON_ASSERT(!val.m_data.m_value.array->empty());
+ o->write_characters(indent_string.c_str(), new_indent);
+ dump(val.m_data.m_value.array->back(), true, ensure_ascii, indent_step, new_indent);
+
+ o->write_character('\n');
+ o->write_characters(indent_string.c_str(), current_indent);
+ o->write_character(']');
+ }
+ else
+ {
+ o->write_character('[');
+
+ // first n-1 elements
+ for (auto i = val.m_data.m_value.array->cbegin();
+ i != val.m_data.m_value.array->cend() - 1; ++i)
+ {
+ dump(*i, false, ensure_ascii, indent_step, current_indent);
+ o->write_character(',');
+ }
+
+ // last element
+ JSON_ASSERT(!val.m_data.m_value.array->empty());
+ dump(val.m_data.m_value.array->back(), false, ensure_ascii, indent_step, current_indent);
+
+ o->write_character(']');
+ }
+
+ return;
+ }
+
+ case value_t::string:
+ {
+ o->write_character('\"');
+ dump_escaped(*val.m_data.m_value.string, ensure_ascii);
+ o->write_character('\"');
+ return;
+ }
+
+ case value_t::binary:
+ {
+ if (pretty_print)
+ {
+ o->write_characters("{\n", 2);
+
+ // variable to hold indentation for recursive calls
+ const auto new_indent = current_indent + indent_step;
+ if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
+ {
+ indent_string.resize(indent_string.size() * 2, ' ');
+ }
+
+ o->write_characters(indent_string.c_str(), new_indent);
+
+ o->write_characters("\"bytes\": [", 10);
+
+ if (!val.m_data.m_value.binary->empty())
+ {
+ for (auto i = val.m_data.m_value.binary->cbegin();
+ i != val.m_data.m_value.binary->cend() - 1; ++i)
+ {
+ dump_integer(*i);
+ o->write_characters(", ", 2);
+ }
+ dump_integer(val.m_data.m_value.binary->back());
+ }
+
+ o->write_characters("],\n", 3);
+ o->write_characters(indent_string.c_str(), new_indent);
+
+ o->write_characters("\"subtype\": ", 11);
+ if (val.m_data.m_value.binary->has_subtype())
+ {
+ dump_integer(val.m_data.m_value.binary->subtype());
+ }
+ else
+ {
+ o->write_characters("null", 4);
+ }
+ o->write_character('\n');
+ o->write_characters(indent_string.c_str(), current_indent);
+ o->write_character('}');
+ }
+ else
+ {
+ o->write_characters("{\"bytes\":[", 10);
+
+ if (!val.m_data.m_value.binary->empty())
+ {
+ for (auto i = val.m_data.m_value.binary->cbegin();
+ i != val.m_data.m_value.binary->cend() - 1; ++i)
+ {
+ dump_integer(*i);
+ o->write_character(',');
+ }
+ dump_integer(val.m_data.m_value.binary->back());
+ }
+
+ o->write_characters("],\"subtype\":", 12);
+ if (val.m_data.m_value.binary->has_subtype())
+ {
+ dump_integer(val.m_data.m_value.binary->subtype());
+ o->write_character('}');
+ }
+ else
+ {
+ o->write_characters("null}", 5);
+ }
+ }
+ return;
+ }
+
+ case value_t::boolean:
+ {
+ if (val.m_data.m_value.boolean)
+ {
+ o->write_characters("true", 4);
+ }
+ else
+ {
+ o->write_characters("false", 5);
+ }
+ return;
+ }
+
+ case value_t::number_integer:
+ {
+ dump_integer(val.m_data.m_value.number_integer);
+ return;
+ }
+
+ case value_t::number_unsigned:
+ {
+ dump_integer(val.m_data.m_value.number_unsigned);
+ return;
+ }
+
+ case value_t::number_float:
+ {
+ dump_float(val.m_data.m_value.number_float);
+ return;
+ }
+
+ case value_t::discarded:
+ {
+ o->write_characters("<discarded>", 11);
+ return;
+ }
+
+ case value_t::null:
+ {
+ o->write_characters("null", 4);
+ return;
+ }
+
+ default: // LCOV_EXCL_LINE
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
+ }
+ }
+
+ JSON_PRIVATE_UNLESS_TESTED:
+ /*!
+ @brief dump escaped string
+
+ Escape a string by replacing certain special characters by a sequence of an
+ escape character (backslash) and another character and other control
+ characters by a sequence of "\u" followed by a four-digit hex
+ representation. The escaped string is written to output stream @a o.
+
+ @param[in] s the string to escape
+ @param[in] ensure_ascii whether to escape non-ASCII characters with
+ \uXXXX sequences
+
+ @complexity Linear in the length of string @a s.
+ */
+ void dump_escaped(const string_t& s, const bool ensure_ascii)
+ {
+ std::uint32_t codepoint{};
+ std::uint8_t state = UTF8_ACCEPT;
+ std::size_t bytes = 0; // number of bytes written to string_buffer
+
+ // number of bytes written at the point of the last valid byte
+ std::size_t bytes_after_last_accept = 0;
+ std::size_t undumped_chars = 0;
+
+ for (std::size_t i = 0; i < s.size(); ++i)
+ {
+ const auto byte = static_cast<std::uint8_t>(s[i]);
+
+ switch (decode(state, codepoint, byte))
+ {
+ case UTF8_ACCEPT: // decode found a new code point
+ {
+ switch (codepoint)
+ {
+ case 0x08: // backspace
+ {
+ string_buffer[bytes++] = '\\';
+ string_buffer[bytes++] = 'b';
+ break;
+ }
+
+ case 0x09: // horizontal tab
+ {
+ string_buffer[bytes++] = '\\';
+ string_buffer[bytes++] = 't';
+ break;
+ }
+
+ case 0x0A: // newline
+ {
+ string_buffer[bytes++] = '\\';
+ string_buffer[bytes++] = 'n';
+ break;
+ }
+
+ case 0x0C: // formfeed
+ {
+ string_buffer[bytes++] = '\\';
+ string_buffer[bytes++] = 'f';
+ break;
+ }
+
+ case 0x0D: // carriage return
+ {
+ string_buffer[bytes++] = '\\';
+ string_buffer[bytes++] = 'r';
+ break;
+ }
+
+ case 0x22: // quotation mark
+ {
+ string_buffer[bytes++] = '\\';
+ string_buffer[bytes++] = '\"';
+ break;
+ }
+
+ case 0x5C: // reverse solidus
+ {
+ string_buffer[bytes++] = '\\';
+ string_buffer[bytes++] = '\\';
+ break;
+ }
+
+ default:
+ {
+ // escape control characters (0x00..0x1F) or, if
+ // ensure_ascii parameter is used, non-ASCII characters
+ if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F)))
+ {
+ if (codepoint <= 0xFFFF)
+ {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+ static_cast<void>((std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x",
+ static_cast<std::uint16_t>(codepoint)));
+ bytes += 6;
+ }
+ else
+ {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+ static_cast<void>((std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x",
+ static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)),
+ static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu))));
+ bytes += 12;
+ }
+ }
+ else
+ {
+ // copy byte to buffer (all previous bytes
+ // been copied have in default case above)
+ string_buffer[bytes++] = s[i];
+ }
+ break;
+ }
+ }
+
+ // write buffer and reset index; there must be 13 bytes
+ // left, as this is the maximal number of bytes to be
+ // written ("\uxxxx\uxxxx\0") for one code point
+ if (string_buffer.size() - bytes < 13)
+ {
+ o->write_characters(string_buffer.data(), bytes);
+ bytes = 0;
+ }
+
+ // remember the byte position of this accept
+ bytes_after_last_accept = bytes;
+ undumped_chars = 0;
+ break;
+ }
+
+ case UTF8_REJECT: // decode found invalid UTF-8 byte
+ {
+ switch (error_handler)
+ {
+ case error_handler_t::strict:
+ {
+ JSON_THROW(type_error::create(316, concat("invalid UTF-8 byte at index ", std::to_string(i), ": 0x", hex_bytes(byte | 0)), nullptr));
+ }
+
+ case error_handler_t::ignore:
+ case error_handler_t::replace:
+ {
+ // in case we saw this character the first time, we
+ // would like to read it again, because the byte
+ // may be OK for itself, but just not OK for the
+ // previous sequence
+ if (undumped_chars > 0)
+ {
+ --i;
+ }
+
+ // reset length buffer to the last accepted index;
+ // thus removing/ignoring the invalid characters
+ bytes = bytes_after_last_accept;
+
+ if (error_handler == error_handler_t::replace)
+ {
+ // add a replacement character
+ if (ensure_ascii)
+ {
+ string_buffer[bytes++] = '\\';
+ string_buffer[bytes++] = 'u';
+ string_buffer[bytes++] = 'f';
+ string_buffer[bytes++] = 'f';
+ string_buffer[bytes++] = 'f';
+ string_buffer[bytes++] = 'd';
+ }
+ else
+ {
+ string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xEF');
+ string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF');
+ string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD');
+ }
+
+ // write buffer and reset index; there must be 13 bytes
+ // left, as this is the maximal number of bytes to be
+ // written ("\uxxxx\uxxxx\0") for one code point
+ if (string_buffer.size() - bytes < 13)
+ {
+ o->write_characters(string_buffer.data(), bytes);
+ bytes = 0;
+ }
+
+ bytes_after_last_accept = bytes;
+ }
+
+ undumped_chars = 0;
+
+ // continue processing the string
+ state = UTF8_ACCEPT;
+ break;
+ }
+
+ default: // LCOV_EXCL_LINE
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
+ }
+ break;
+ }
+
+ default: // decode found yet incomplete multi-byte code point
+ {
+ if (!ensure_ascii)
+ {
+ // code point will not be escaped - copy byte to buffer
+ string_buffer[bytes++] = s[i];
+ }
+ ++undumped_chars;
+ break;
+ }
+ }
+ }
+
+ // we finished processing the string
+ if (JSON_HEDLEY_LIKELY(state == UTF8_ACCEPT))
+ {
+ // write buffer
+ if (bytes > 0)
+ {
+ o->write_characters(string_buffer.data(), bytes);
+ }
+ }
+ else
+ {
+ // we finish reading, but do not accept: string was incomplete
+ switch (error_handler)
+ {
+ case error_handler_t::strict:
+ {
+ JSON_THROW(type_error::create(316, concat("incomplete UTF-8 string; last byte: 0x", hex_bytes(static_cast<std::uint8_t>(s.back() | 0))), nullptr));
+ }
+
+ case error_handler_t::ignore:
+ {
+ // write all accepted bytes
+ o->write_characters(string_buffer.data(), bytes_after_last_accept);
+ break;
+ }
+
+ case error_handler_t::replace:
+ {
+ // write all accepted bytes
+ o->write_characters(string_buffer.data(), bytes_after_last_accept);
+ // add a replacement character
+ if (ensure_ascii)
+ {
+ o->write_characters("\\ufffd", 6);
+ }
+ else
+ {
+ o->write_characters("\xEF\xBF\xBD", 3);
+ }
+ break;
+ }
+
+ default: // LCOV_EXCL_LINE
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
+ }
+ }
+ }
+
+ private:
+ /*!
+ @brief count digits
+
+ Count the number of decimal (base 10) digits for an input unsigned integer.
+
+ @param[in] x unsigned integer number to count its digits
+ @return number of decimal digits
+ */
+ inline unsigned int count_digits(number_unsigned_t x) noexcept
+ {
+ unsigned int n_digits = 1;
+ for (;;)
+ {
+ if (x < 10)
+ {
+ return n_digits;
+ }
+ if (x < 100)
+ {
+ return n_digits + 1;
+ }
+ if (x < 1000)
+ {
+ return n_digits + 2;
+ }
+ if (x < 10000)
+ {
+ return n_digits + 3;
+ }
+ x = x / 10000u;
+ n_digits += 4;
+ }
+ }
+
+ /*!
+ * @brief convert a byte to a uppercase hex representation
+ * @param[in] byte byte to represent
+ * @return representation ("00".."FF")
+ */
+ static std::string hex_bytes(std::uint8_t byte)
+ {
+ std::string result = "FF";
+ constexpr const char* nibble_to_hex = "0123456789ABCDEF";
+ result[0] = nibble_to_hex[byte / 16];
+ result[1] = nibble_to_hex[byte % 16];
+ return result;
+ }
+
+ // templates to avoid warnings about useless casts
+ template <typename NumberType, enable_if_t<std::is_signed<NumberType>::value, int> = 0>
+ bool is_negative_number(NumberType x)
+ {
+ return x < 0;
+ }
+
+ template < typename NumberType, enable_if_t <std::is_unsigned<NumberType>::value, int > = 0 >
+ bool is_negative_number(NumberType /*unused*/)
+ {
+ return false;
+ }
+
+ /*!
+ @brief dump an integer
+
+ Dump a given integer to output stream @a o. Works internally with
+ @a number_buffer.
+
+ @param[in] x integer number (signed or unsigned) to dump
+ @tparam NumberType either @a number_integer_t or @a number_unsigned_t
+ */
+ template < typename NumberType, detail::enable_if_t <
+ std::is_integral<NumberType>::value ||
+ std::is_same<NumberType, number_unsigned_t>::value ||
+ std::is_same<NumberType, number_integer_t>::value ||
+ std::is_same<NumberType, binary_char_t>::value,
+ int > = 0 >
+ void dump_integer(NumberType x)
+ {
+ static constexpr std::array<std::array<char, 2>, 100> digits_to_99
+ {
+ {
+ {{'0', '0'}}, {{'0', '1'}}, {{'0', '2'}}, {{'0', '3'}}, {{'0', '4'}}, {{'0', '5'}}, {{'0', '6'}}, {{'0', '7'}}, {{'0', '8'}}, {{'0', '9'}},
+ {{'1', '0'}}, {{'1', '1'}}, {{'1', '2'}}, {{'1', '3'}}, {{'1', '4'}}, {{'1', '5'}}, {{'1', '6'}}, {{'1', '7'}}, {{'1', '8'}}, {{'1', '9'}},
+ {{'2', '0'}}, {{'2', '1'}}, {{'2', '2'}}, {{'2', '3'}}, {{'2', '4'}}, {{'2', '5'}}, {{'2', '6'}}, {{'2', '7'}}, {{'2', '8'}}, {{'2', '9'}},
+ {{'3', '0'}}, {{'3', '1'}}, {{'3', '2'}}, {{'3', '3'}}, {{'3', '4'}}, {{'3', '5'}}, {{'3', '6'}}, {{'3', '7'}}, {{'3', '8'}}, {{'3', '9'}},
+ {{'4', '0'}}, {{'4', '1'}}, {{'4', '2'}}, {{'4', '3'}}, {{'4', '4'}}, {{'4', '5'}}, {{'4', '6'}}, {{'4', '7'}}, {{'4', '8'}}, {{'4', '9'}},
+ {{'5', '0'}}, {{'5', '1'}}, {{'5', '2'}}, {{'5', '3'}}, {{'5', '4'}}, {{'5', '5'}}, {{'5', '6'}}, {{'5', '7'}}, {{'5', '8'}}, {{'5', '9'}},
+ {{'6', '0'}}, {{'6', '1'}}, {{'6', '2'}}, {{'6', '3'}}, {{'6', '4'}}, {{'6', '5'}}, {{'6', '6'}}, {{'6', '7'}}, {{'6', '8'}}, {{'6', '9'}},
+ {{'7', '0'}}, {{'7', '1'}}, {{'7', '2'}}, {{'7', '3'}}, {{'7', '4'}}, {{'7', '5'}}, {{'7', '6'}}, {{'7', '7'}}, {{'7', '8'}}, {{'7', '9'}},
+ {{'8', '0'}}, {{'8', '1'}}, {{'8', '2'}}, {{'8', '3'}}, {{'8', '4'}}, {{'8', '5'}}, {{'8', '6'}}, {{'8', '7'}}, {{'8', '8'}}, {{'8', '9'}},
+ {{'9', '0'}}, {{'9', '1'}}, {{'9', '2'}}, {{'9', '3'}}, {{'9', '4'}}, {{'9', '5'}}, {{'9', '6'}}, {{'9', '7'}}, {{'9', '8'}}, {{'9', '9'}},
+ }
+ };
+
+ // special case for "0"
+ if (x == 0)
+ {
+ o->write_character('0');
+ return;
+ }
+
+ // use a pointer to fill the buffer
+ auto buffer_ptr = number_buffer.begin(); // NOLINT(llvm-qualified-auto,readability-qualified-auto,cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+
+ number_unsigned_t abs_value;
+
+ unsigned int n_chars{};
+
+ if (is_negative_number(x))
+ {
+ *buffer_ptr = '-';
+ abs_value = remove_sign(static_cast<number_integer_t>(x));
+
+ // account one more byte for the minus sign
+ n_chars = 1 + count_digits(abs_value);
+ }
+ else
+ {
+ abs_value = static_cast<number_unsigned_t>(x);
+ n_chars = count_digits(abs_value);
+ }
+
+ // spare 1 byte for '\0'
+ JSON_ASSERT(n_chars < number_buffer.size() - 1);
+
+ // jump to the end to generate the string from backward,
+ // so we later avoid reversing the result
+ buffer_ptr += n_chars;
+
+ // Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu
+ // See: https://www.youtube.com/watch?v=o4-CwDo2zpg
+ while (abs_value >= 100)
+ {
+ const auto digits_index = static_cast<unsigned>((abs_value % 100));
+ abs_value /= 100;
+ *(--buffer_ptr) = digits_to_99[digits_index][1];
+ *(--buffer_ptr) = digits_to_99[digits_index][0];
+ }
+
+ if (abs_value >= 10)
+ {
+ const auto digits_index = static_cast<unsigned>(abs_value);
+ *(--buffer_ptr) = digits_to_99[digits_index][1];
+ *(--buffer_ptr) = digits_to_99[digits_index][0];
+ }
+ else
+ {
+ *(--buffer_ptr) = static_cast<char>('0' + abs_value);
+ }
+
+ o->write_characters(number_buffer.data(), n_chars);
+ }
+
+ /*!
+ @brief dump a floating-point number
+
+ Dump a given floating-point number to output stream @a o. Works internally
+ with @a number_buffer.
+
+ @param[in] x floating-point number to dump
+ */
+ void dump_float(number_float_t x)
+ {
+ // NaN / inf
+ if (!std::isfinite(x))
+ {
+ o->write_characters("null", 4);
+ return;
+ }
+
+ // If number_float_t is an IEEE-754 single or double precision number,
+ // use the Grisu2 algorithm to produce short numbers which are
+ // guaranteed to round-trip, using strtof and strtod, resp.
+ //
+ // NB: The test below works if <long double> == <double>.
+ static constexpr bool is_ieee_single_or_double
+ = (std::numeric_limits<number_float_t>::is_iec559 && std::numeric_limits<number_float_t>::digits == 24 && std::numeric_limits<number_float_t>::max_exponent == 128) ||
+ (std::numeric_limits<number_float_t>::is_iec559 && std::numeric_limits<number_float_t>::digits == 53 && std::numeric_limits<number_float_t>::max_exponent == 1024);
+
+ dump_float(x, std::integral_constant<bool, is_ieee_single_or_double>());
+ }
+
+ void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/)
+ {
+ auto* begin = number_buffer.data();
+ auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x);
+
+ o->write_characters(begin, static_cast<size_t>(end - begin));
+ }
+
+ void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/)
+ {
+ // get number of digits for a float -> text -> float round-trip
+ static constexpr auto d = std::numeric_limits<number_float_t>::max_digits10;
+
+ // the actual conversion
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+ std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x);
+
+ // negative value indicates an error
+ JSON_ASSERT(len > 0);
+ // check if buffer was large enough
+ JSON_ASSERT(static_cast<std::size_t>(len) < number_buffer.size());
+
+ // erase thousands separator
+ if (thousands_sep != '\0')
+ {
+ // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081
+ const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep);
+ std::fill(end, number_buffer.end(), '\0');
+ JSON_ASSERT((end - number_buffer.begin()) <= len);
+ len = (end - number_buffer.begin());
+ }
+
+ // convert decimal point to '.'
+ if (decimal_point != '\0' && decimal_point != '.')
+ {
+ // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081
+ const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);
+ if (dec_pos != number_buffer.end())
+ {
+ *dec_pos = '.';
+ }
+ }
+
+ o->write_characters(number_buffer.data(), static_cast<std::size_t>(len));
+
+ // determine if we need to append ".0"
+ const bool value_is_int_like =
+ std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1,
+ [](char c)
+ {
+ return c == '.' || c == 'e';
+ });
+
+ if (value_is_int_like)
+ {
+ o->write_characters(".0", 2);
+ }
+ }
+
+ /*!
+ @brief check whether a string is UTF-8 encoded
+
+ The function checks each byte of a string whether it is UTF-8 encoded. The
+ result of the check is stored in the @a state parameter. The function must
+ be called initially with state 0 (accept). State 1 means the string must
+ be rejected, because the current byte is not allowed. If the string is
+ completely processed, but the state is non-zero, the string ended
+ prematurely; that is, the last byte indicated more bytes should have
+ followed.
+
+ @param[in,out] state the state of the decoding
+ @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT)
+ @param[in] byte next byte to decode
+ @return new state
+
+ @note The function has been edited: a std::array is used.
+
+ @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+ @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
+ */
+ static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept
+ {
+ static const std::array<std::uint8_t, 400> utf8d =
+ {
+ {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF
+ 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF
+ 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF
+ 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF
+ 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2
+ 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4
+ 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6
+ 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8
+ }
+ };
+
+ JSON_ASSERT(byte < utf8d.size());
+ const std::uint8_t type = utf8d[byte];
+
+ codep = (state != UTF8_ACCEPT)
+ ? (byte & 0x3fu) | (codep << 6u)
+ : (0xFFu >> type) & (byte);
+
+ const std::size_t index = 256u + static_cast<size_t>(state) * 16u + static_cast<size_t>(type);
+ JSON_ASSERT(index < utf8d.size());
+ state = utf8d[index];
+ return state;
+ }
+
+ /*
+ * Overload to make the compiler happy while it is instantiating
+ * dump_integer for number_unsigned_t.
+ * Must never be called.
+ */
+ number_unsigned_t remove_sign(number_unsigned_t x)
+ {
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
+ return x; // LCOV_EXCL_LINE
+ }
+
+ /*
+ * Helper function for dump_integer
+ *
+ * This function takes a negative signed integer and returns its absolute
+ * value as unsigned integer. The plus/minus shuffling is necessary as we can
+ * not directly remove the sign of an arbitrary signed integer as the
+ * absolute values of INT_MIN and INT_MAX are usually not the same. See
+ * #1708 for details.
+ */
+ inline number_unsigned_t remove_sign(number_integer_t x) noexcept
+ {
+ JSON_ASSERT(x < 0 && x < (std::numeric_limits<number_integer_t>::max)()); // NOLINT(misc-redundant-expression)
+ return static_cast<number_unsigned_t>(-(x + 1)) + 1;
+ }
+
+ private:
+ /// the output of the serializer
+ output_adapter_t<char> o = nullptr;
+
+ /// a (hopefully) large enough character buffer
+ std::array<char, 64> number_buffer{{}};
+
+ /// the locale
+ const std::lconv* loc = nullptr;
+ /// the locale's thousand separator character
+ const char thousands_sep = '\0';
+ /// the locale's decimal point character
+ const char decimal_point = '\0';
+
+ /// string buffer
+ std::array<char, 512> string_buffer{{}};
+
+ /// the indentation character
+ const char indent_char;
+ /// the indentation string
+ string_t indent_string;
+
+ /// error_handler how to react on decoding errors
+ const error_handler_t error_handler;
+};
+
+} // namespace detail
+NLOHMANN_JSON_NAMESPACE_END
+
+// #include <nlohmann/detail/value_t.hpp>
+
+// #include <nlohmann/json_fwd.hpp>
+
+// #include <nlohmann/ordered_map.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#include <functional> // equal_to, less
+#include <initializer_list> // initializer_list
+#include <iterator> // input_iterator_tag, iterator_traits
+#include <memory> // allocator
+#include <stdexcept> // for out_of_range
+#include <type_traits> // enable_if, is_convertible
+#include <utility> // pair
+#include <vector> // vector
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+
+/// ordered_map: a minimal map-like container that preserves insertion order
+/// for use within nlohmann::basic_json<ordered_map>
+template <class Key, class T, class IgnoredLess = std::less<Key>,
+ class Allocator = std::allocator<std::pair<const Key, T>>>
+ struct ordered_map : std::vector<std::pair<const Key, T>, Allocator>
+{
+ using key_type = Key;
+ using mapped_type = T;
+ using Container = std::vector<std::pair<const Key, T>, Allocator>;
+ using iterator = typename Container::iterator;
+ using const_iterator = typename Container::const_iterator;
+ using size_type = typename Container::size_type;
+ using value_type = typename Container::value_type;
+#ifdef JSON_HAS_CPP_14
+ using key_compare = std::equal_to<>;
+#else
+ using key_compare = std::equal_to<Key>;
+#endif
+
+ // Explicit constructors instead of `using Container::Container`
+ // otherwise older compilers choke on it (GCC <= 5.5, xcode <= 9.4)
+ ordered_map() noexcept(noexcept(Container())) : Container{} {}
+ explicit ordered_map(const Allocator& alloc) noexcept(noexcept(Container(alloc))) : Container{alloc} {}
+ template <class It>
+ ordered_map(It first, It last, const Allocator& alloc = Allocator())
+ : Container{first, last, alloc} {}
+ ordered_map(std::initializer_list<value_type> init, const Allocator& alloc = Allocator() )
+ : Container{init, alloc} {}
+
+ std::pair<iterator, bool> emplace(const key_type& key, T&& t)
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ return {it, false};
+ }
+ }
+ Container::emplace_back(key, std::forward<T>(t));
+ return {std::prev(this->end()), true};
+ }
+
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
+ std::pair<iterator, bool> emplace(KeyType && key, T && t)
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ return {it, false};
+ }
+ }
+ Container::emplace_back(std::forward<KeyType>(key), std::forward<T>(t));
+ return {std::prev(this->end()), true};
+ }
+
+ T& operator[](const key_type& key)
+ {
+ return emplace(key, T{}).first->second;
+ }
+
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
+ T & operator[](KeyType && key)
+ {
+ return emplace(std::forward<KeyType>(key), T{}).first->second;
+ }
+
+ const T& operator[](const key_type& key) const
+ {
+ return at(key);
+ }
+
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
+ const T & operator[](KeyType && key) const
+ {
+ return at(std::forward<KeyType>(key));
+ }
+
+ T& at(const key_type& key)
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ return it->second;
+ }
+ }
+
+ JSON_THROW(std::out_of_range("key not found"));
+ }
+
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
+ T & at(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward)
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ return it->second;
+ }
+ }
+
+ JSON_THROW(std::out_of_range("key not found"));
+ }
+
+ const T& at(const key_type& key) const
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ return it->second;
+ }
+ }
+
+ JSON_THROW(std::out_of_range("key not found"));
+ }
+
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
+ const T & at(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward)
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ return it->second;
+ }
+ }
+
+ JSON_THROW(std::out_of_range("key not found"));
+ }
+
+ size_type erase(const key_type& key)
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ // Since we cannot move const Keys, re-construct them in place
+ for (auto next = it; ++next != this->end(); ++it)
+ {
+ it->~value_type(); // Destroy but keep allocation
+ new (&*it) value_type{std::move(*next)};
+ }
+ Container::pop_back();
+ return 1;
+ }
+ }
+ return 0;
+ }
+
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
+ size_type erase(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward)
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ // Since we cannot move const Keys, re-construct them in place
+ for (auto next = it; ++next != this->end(); ++it)
+ {
+ it->~value_type(); // Destroy but keep allocation
+ new (&*it) value_type{std::move(*next)};
+ }
+ Container::pop_back();
+ return 1;
+ }
+ }
+ return 0;
+ }
+
+ iterator erase(iterator pos)
+ {
+ return erase(pos, std::next(pos));
+ }
+
+ iterator erase(iterator first, iterator last)
+ {
+ if (first == last)
+ {
+ return first;
+ }
+
+ const auto elements_affected = std::distance(first, last);
+ const auto offset = std::distance(Container::begin(), first);
+
+ // This is the start situation. We need to delete elements_affected
+ // elements (3 in this example: e, f, g), and need to return an
+ // iterator past the last deleted element (h in this example).
+ // Note that offset is the distance from the start of the vector
+ // to first. We will need this later.
+
+ // [ a, b, c, d, e, f, g, h, i, j ]
+ // ^ ^
+ // first last
+
+ // Since we cannot move const Keys, we re-construct them in place.
+ // We start at first and re-construct (viz. copy) the elements from
+ // the back of the vector. Example for first iteration:
+
+ // ,--------.
+ // v | destroy e and re-construct with h
+ // [ a, b, c, d, e, f, g, h, i, j ]
+ // ^ ^
+ // it it + elements_affected
+
+ for (auto it = first; std::next(it, elements_affected) != Container::end(); ++it)
+ {
+ it->~value_type(); // destroy but keep allocation
+ new (&*it) value_type{std::move(*std::next(it, elements_affected))}; // "move" next element to it
+ }
+
+ // [ a, b, c, d, h, i, j, h, i, j ]
+ // ^ ^
+ // first last
+
+ // remove the unneeded elements at the end of the vector
+ Container::resize(this->size() - static_cast<size_type>(elements_affected));
+
+ // [ a, b, c, d, h, i, j ]
+ // ^ ^
+ // first last
+
+ // first is now pointing past the last deleted element, but we cannot
+ // use this iterator, because it may have been invalidated by the
+ // resize call. Instead, we can return begin() + offset.
+ return Container::begin() + offset;
+ }
+
+ size_type count(const key_type& key) const
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ return 1;
+ }
+ }
+ return 0;
+ }
+
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
+ size_type count(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward)
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ return 1;
+ }
+ }
+ return 0;
+ }
+
+ iterator find(const key_type& key)
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ return it;
+ }
+ }
+ return Container::end();
+ }
+
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
+ iterator find(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward)
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ return it;
+ }
+ }
+ return Container::end();
+ }
+
+ const_iterator find(const key_type& key) const
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, key))
+ {
+ return it;
+ }
+ }
+ return Container::end();
+ }
+
+ std::pair<iterator, bool> insert( value_type&& value )
+ {
+ return emplace(value.first, std::move(value.second));
+ }
+
+ std::pair<iterator, bool> insert( const value_type& value )
+ {
+ for (auto it = this->begin(); it != this->end(); ++it)
+ {
+ if (m_compare(it->first, value.first))
+ {
+ return {it, false};
+ }
+ }
+ Container::push_back(value);
+ return {--this->end(), true};
+ }
+
+ template<typename InputIt>
+ using require_input_iter = typename std::enable_if<std::is_convertible<typename std::iterator_traits<InputIt>::iterator_category,
+ std::input_iterator_tag>::value>::type;
+
+ template<typename InputIt, typename = require_input_iter<InputIt>>
+ void insert(InputIt first, InputIt last)
+ {
+ for (auto it = first; it != last; ++it)
+ {
+ insert(*it);
+ }
+ }
+
+private:
+ JSON_NO_UNIQUE_ADDRESS key_compare m_compare = key_compare();
+};
+
+NLOHMANN_JSON_NAMESPACE_END
+
+
+#if defined(JSON_HAS_CPP_17)
+ #if JSON_HAS_STATIC_RTTI
+ #include <any>
+ #endif
+ #include <string_view>
+#endif
+
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+NLOHMANN_JSON_NAMESPACE_BEGIN
+
+/*!
+@brief a class to store JSON values
+
+@internal
+@invariant The member variables @a m_value and @a m_type have the following
+relationship:
+- If `m_type == value_t::object`, then `m_value.object != nullptr`.
+- If `m_type == value_t::array`, then `m_value.array != nullptr`.
+- If `m_type == value_t::string`, then `m_value.string != nullptr`.
+The invariants are checked by member function assert_invariant().
+
+@note ObjectType trick from https://stackoverflow.com/a/9860911
+@endinternal
+
+@since version 1.0.0
+
+@nosubgrouping
+*/
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions)
+ : public ::nlohmann::detail::json_base_class<CustomBaseClass>
+{
+ private:
+ template<detail::value_t> friend struct detail::external_constructor;
+
+ template<typename>
+ friend class ::nlohmann::json_pointer;
+ // can be restored when json_pointer backwards compatibility is removed
+ // friend ::nlohmann::json_pointer<StringType>;
+
+ template<typename BasicJsonType, typename InputType>
+ friend class ::nlohmann::detail::parser;
+ friend ::nlohmann::detail::serializer<basic_json>;
+ template<typename BasicJsonType>
+ friend class ::nlohmann::detail::iter_impl;
+ template<typename BasicJsonType, typename CharType>
+ friend class ::nlohmann::detail::binary_writer;
+ template<typename BasicJsonType, typename InputType, typename SAX>
+ friend class ::nlohmann::detail::binary_reader;
+ template<typename BasicJsonType>
+ friend class ::nlohmann::detail::json_sax_dom_parser;
+ template<typename BasicJsonType>
+ friend class ::nlohmann::detail::json_sax_dom_callback_parser;
+ friend class ::nlohmann::detail::exception;
+
+ /// workaround type for MSVC
+ using basic_json_t = NLOHMANN_BASIC_JSON_TPL;
+ using json_base_class_t = ::nlohmann::detail::json_base_class<CustomBaseClass>;
+
+ JSON_PRIVATE_UNLESS_TESTED:
+ // convenience aliases for types residing in namespace detail;
+ using lexer = ::nlohmann::detail::lexer_base<basic_json>;
+
+ template<typename InputAdapterType>
+ static ::nlohmann::detail::parser<basic_json, InputAdapterType> parser(
+ InputAdapterType adapter,
+ detail::parser_callback_t<basic_json>cb = nullptr,
+ const bool allow_exceptions = true,
+ const bool ignore_comments = false
+ )
+ {
+ return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),
+ std::move(cb), allow_exceptions, ignore_comments);
+ }
+
+ private:
+ using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t;
+ template<typename BasicJsonType>
+ using internal_iterator = ::nlohmann::detail::internal_iterator<BasicJsonType>;
+ template<typename BasicJsonType>
+ using iter_impl = ::nlohmann::detail::iter_impl<BasicJsonType>;
+ template<typename Iterator>
+ using iteration_proxy = ::nlohmann::detail::iteration_proxy<Iterator>;
+ template<typename Base> using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator<Base>;
+
+ template<typename CharType>
+ using output_adapter_t = ::nlohmann::detail::output_adapter_t<CharType>;
+
+ template<typename InputType>
+ using binary_reader = ::nlohmann::detail::binary_reader<basic_json, InputType>;
+ template<typename CharType> using binary_writer = ::nlohmann::detail::binary_writer<basic_json, CharType>;
+
+ JSON_PRIVATE_UNLESS_TESTED:
+ using serializer = ::nlohmann::detail::serializer<basic_json>;
+
+ public:
+ using value_t = detail::value_t;
+ /// JSON Pointer, see @ref nlohmann::json_pointer
+ using json_pointer = ::nlohmann::json_pointer<StringType>;
+ template<typename T, typename SFINAE>
+ using json_serializer = JSONSerializer<T, SFINAE>;
+ /// how to treat decoding errors
+ using error_handler_t = detail::error_handler_t;
+ /// how to treat CBOR tags
+ using cbor_tag_handler_t = detail::cbor_tag_handler_t;
+ /// helper type for initializer lists of basic_json values
+ using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>;
+
+ using input_format_t = detail::input_format_t;
+ /// SAX interface type, see @ref nlohmann::json_sax
+ using json_sax_t = json_sax<basic_json>;
+
+ ////////////////
+ // exceptions //
+ ////////////////
+
+ /// @name exceptions
+ /// Classes to implement user-defined exceptions.
+ /// @{
+
+ using exception = detail::exception;
+ using parse_error = detail::parse_error;
+ using invalid_iterator = detail::invalid_iterator;
+ using type_error = detail::type_error;
+ using out_of_range = detail::out_of_range;
+ using other_error = detail::other_error;
+
+ /// @}
+
+ /////////////////////
+ // container types //
+ /////////////////////
+
+ /// @name container types
+ /// The canonic container types to use @ref basic_json like any other STL
+ /// container.
+ /// @{
+
+ /// the type of elements in a basic_json container
+ using value_type = basic_json;
+
+ /// the type of an element reference
+ using reference = value_type&;
+ /// the type of an element const reference
+ using const_reference = const value_type&;
+
+ /// a type to represent differences between iterators
+ using difference_type = std::ptrdiff_t;
+ /// a type to represent container sizes
+ using size_type = std::size_t;
+
+ /// the allocator type
+ using allocator_type = AllocatorType<basic_json>;
+
+ /// the type of an element pointer
+ using pointer = typename std::allocator_traits<allocator_type>::pointer;
+ /// the type of an element const pointer
+ using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer;
+
+ /// an iterator for a basic_json container
+ using iterator = iter_impl<basic_json>;
+ /// a const iterator for a basic_json container
+ using const_iterator = iter_impl<const basic_json>;
+ /// a reverse iterator for a basic_json container
+ using reverse_iterator = json_reverse_iterator<typename basic_json::iterator>;
+ /// a const reverse iterator for a basic_json container
+ using const_reverse_iterator = json_reverse_iterator<typename basic_json::const_iterator>;
+
+ /// @}
+
+ /// @brief returns the allocator associated with the container
+ /// @sa https://json.nlohmann.me/api/basic_json/get_allocator/
+ static allocator_type get_allocator()
+ {
+ return allocator_type();
+ }
+
+ /// @brief returns version information on the library
+ /// @sa https://json.nlohmann.me/api/basic_json/meta/
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json meta()
+ {
+ basic_json result;
+
+ result["copyright"] = "(C) 2013-2023 Niels Lohmann";
+ result["name"] = "JSON for Modern C++";
+ result["url"] = "https://github.com/nlohmann/json";
+ result["version"]["string"] =
+ detail::concat(std::to_string(NLOHMANN_JSON_VERSION_MAJOR), '.',
+ std::to_string(NLOHMANN_JSON_VERSION_MINOR), '.',
+ std::to_string(NLOHMANN_JSON_VERSION_PATCH));
+ result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR;
+ result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR;
+ result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH;
+
+#ifdef _WIN32
+ result["platform"] = "win32";
+#elif defined __linux__
+ result["platform"] = "linux";
+#elif defined __APPLE__
+ result["platform"] = "apple";
+#elif defined __unix__
+ result["platform"] = "unix";
+#else
+ result["platform"] = "unknown";
+#endif
+
+#if defined(__ICC) || defined(__INTEL_COMPILER)
+ result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}};
+#elif defined(__clang__)
+ result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}};
+#elif defined(__GNUC__) || defined(__GNUG__)
+ result["compiler"] = {{"family", "gcc"}, {"version", detail::concat(
+ std::to_string(__GNUC__), '.',
+ std::to_string(__GNUC_MINOR__), '.',
+ std::to_string(__GNUC_PATCHLEVEL__))
+ }
+ };
+#elif defined(__HP_cc) || defined(__HP_aCC)
+ result["compiler"] = "hp"
+#elif defined(__IBMCPP__)
+ result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}};
+#elif defined(_MSC_VER)
+ result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}};
+#elif defined(__PGI)
+ result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}};
+#elif defined(__SUNPRO_CC)
+ result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}};
+#else
+ result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}};
+#endif
+
+#if defined(_MSVC_LANG)
+ result["compiler"]["c++"] = std::to_string(_MSVC_LANG);
+#elif defined(__cplusplus)
+ result["compiler"]["c++"] = std::to_string(__cplusplus);
+#else
+ result["compiler"]["c++"] = "unknown";
+#endif
+ return result;
+ }
+
+ ///////////////////////////
+ // JSON value data types //
+ ///////////////////////////
+
+ /// @name JSON value data types
+ /// The data types to store a JSON value. These types are derived from
+ /// the template arguments passed to class @ref basic_json.
+ /// @{
+
+ /// @brief default object key comparator type
+ /// The actual object key comparator type (@ref object_comparator_t) may be
+ /// different.
+ /// @sa https://json.nlohmann.me/api/basic_json/default_object_comparator_t/
+#if defined(JSON_HAS_CPP_14)
+ // use of transparent comparator avoids unnecessary repeated construction of temporaries
+ // in functions involving lookup by key with types other than object_t::key_type (aka. StringType)
+ using default_object_comparator_t = std::less<>;
+#else
+ using default_object_comparator_t = std::less<StringType>;
+#endif
+
+ /// @brief a type for an object
+ /// @sa https://json.nlohmann.me/api/basic_json/object_t/
+ using object_t = ObjectType<StringType,
+ basic_json,
+ default_object_comparator_t,
+ AllocatorType<std::pair<const StringType,
+ basic_json>>>;
+
+ /// @brief a type for an array
+ /// @sa https://json.nlohmann.me/api/basic_json/array_t/
+ using array_t = ArrayType<basic_json, AllocatorType<basic_json>>;
+
+ /// @brief a type for a string
+ /// @sa https://json.nlohmann.me/api/basic_json/string_t/
+ using string_t = StringType;
+
+ /// @brief a type for a boolean
+ /// @sa https://json.nlohmann.me/api/basic_json/boolean_t/
+ using boolean_t = BooleanType;
+
+ /// @brief a type for a number (integer)
+ /// @sa https://json.nlohmann.me/api/basic_json/number_integer_t/
+ using number_integer_t = NumberIntegerType;
+
+ /// @brief a type for a number (unsigned)
+ /// @sa https://json.nlohmann.me/api/basic_json/number_unsigned_t/
+ using number_unsigned_t = NumberUnsignedType;
+
+ /// @brief a type for a number (floating-point)
+ /// @sa https://json.nlohmann.me/api/basic_json/number_float_t/
+ using number_float_t = NumberFloatType;
+
+ /// @brief a type for a packed binary type
+ /// @sa https://json.nlohmann.me/api/basic_json/binary_t/
+ using binary_t = nlohmann::byte_container_with_subtype<BinaryType>;
+
+ /// @brief object key comparator type
+ /// @sa https://json.nlohmann.me/api/basic_json/object_comparator_t/
+ using object_comparator_t = detail::actual_object_comparator_t<basic_json>;
+
+ /// @}
+
+ private:
+
+ /// helper for exception-safe object creation
+ template<typename T, typename... Args>
+ JSON_HEDLEY_RETURNS_NON_NULL
+ static T* create(Args&& ... args)
+ {
+ AllocatorType<T> alloc;
+ using AllocatorTraits = std::allocator_traits<AllocatorType<T>>;
+
+ auto deleter = [&](T * obj)
+ {
+ AllocatorTraits::deallocate(alloc, obj, 1);
+ };
+ std::unique_ptr<T, decltype(deleter)> obj(AllocatorTraits::allocate(alloc, 1), deleter);
+ AllocatorTraits::construct(alloc, obj.get(), std::forward<Args>(args)...);
+ JSON_ASSERT(obj != nullptr);
+ return obj.release();
+ }
+
+ ////////////////////////
+ // JSON value storage //
+ ////////////////////////
+
+ JSON_PRIVATE_UNLESS_TESTED:
+ /*!
+ @brief a JSON value
+
+ The actual storage for a JSON value of the @ref basic_json class. This
+ union combines the different storage types for the JSON value types
+ defined in @ref value_t.
+
+ JSON type | value_t type | used type
+ --------- | --------------- | ------------------------
+ object | object | pointer to @ref object_t
+ array | array | pointer to @ref array_t
+ string | string | pointer to @ref string_t
+ boolean | boolean | @ref boolean_t
+ number | number_integer | @ref number_integer_t
+ number | number_unsigned | @ref number_unsigned_t
+ number | number_float | @ref number_float_t
+ binary | binary | pointer to @ref binary_t
+ null | null | *no value is stored*
+
+ @note Variable-length types (objects, arrays, and strings) are stored as
+ pointers. The size of the union should not exceed 64 bits if the default
+ value types are used.
+
+ @since version 1.0.0
+ */
+ union json_value
+ {
+ /// object (stored with pointer to save storage)
+ object_t* object;
+ /// array (stored with pointer to save storage)
+ array_t* array;
+ /// string (stored with pointer to save storage)
+ string_t* string;
+ /// binary (stored with pointer to save storage)
+ binary_t* binary;
+ /// boolean
+ boolean_t boolean;
+ /// number (integer)
+ number_integer_t number_integer;
+ /// number (unsigned integer)
+ number_unsigned_t number_unsigned;
+ /// number (floating-point)
+ number_float_t number_float;
+
+ /// default constructor (for null values)
+ json_value() = default;
+ /// constructor for booleans
+ json_value(boolean_t v) noexcept : boolean(v) {}
+ /// constructor for numbers (integer)
+ json_value(number_integer_t v) noexcept : number_integer(v) {}
+ /// constructor for numbers (unsigned)
+ json_value(number_unsigned_t v) noexcept : number_unsigned(v) {}
+ /// constructor for numbers (floating-point)
+ json_value(number_float_t v) noexcept : number_float(v) {}
+ /// constructor for empty values of a given type
+ json_value(value_t t)
+ {
+ switch (t)
+ {
+ case value_t::object:
+ {
+ object = create<object_t>();
+ break;
+ }
+
+ case value_t::array:
+ {
+ array = create<array_t>();
+ break;
+ }
+
+ case value_t::string:
+ {
+ string = create<string_t>("");
+ break;
+ }
+
+ case value_t::binary:
+ {
+ binary = create<binary_t>();
+ break;
+ }
+
+ case value_t::boolean:
+ {
+ boolean = static_cast<boolean_t>(false);
+ break;
+ }
+
+ case value_t::number_integer:
+ {
+ number_integer = static_cast<number_integer_t>(0);
+ break;
+ }
+
+ case value_t::number_unsigned:
+ {
+ number_unsigned = static_cast<number_unsigned_t>(0);
+ break;
+ }
+
+ case value_t::number_float:
+ {
+ number_float = static_cast<number_float_t>(0.0);
+ break;
+ }
+
+ case value_t::null:
+ {
+ object = nullptr; // silence warning, see #821
+ break;
+ }
+
+ case value_t::discarded:
+ default:
+ {
+ object = nullptr; // silence warning, see #821
+ if (JSON_HEDLEY_UNLIKELY(t == value_t::null))
+ {
+ JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.11.3", nullptr)); // LCOV_EXCL_LINE
+ }
+ break;
+ }
+ }
+ }
+
+ /// constructor for strings
+ json_value(const string_t& value) : string(create<string_t>(value)) {}
+
+ /// constructor for rvalue strings
+ json_value(string_t&& value) : string(create<string_t>(std::move(value))) {}
+
+ /// constructor for objects
+ json_value(const object_t& value) : object(create<object_t>(value)) {}
+
+ /// constructor for rvalue objects
+ json_value(object_t&& value) : object(create<object_t>(std::move(value))) {}
+
+ /// constructor for arrays
+ json_value(const array_t& value) : array(create<array_t>(value)) {}
+
+ /// constructor for rvalue arrays
+ json_value(array_t&& value) : array(create<array_t>(std::move(value))) {}
+
+ /// constructor for binary arrays
+ json_value(const typename binary_t::container_type& value) : binary(create<binary_t>(value)) {}
+
+ /// constructor for rvalue binary arrays
+ json_value(typename binary_t::container_type&& value) : binary(create<binary_t>(std::move(value))) {}
+
+ /// constructor for binary arrays (internal type)
+ json_value(const binary_t& value) : binary(create<binary_t>(value)) {}
+
+ /// constructor for rvalue binary arrays (internal type)
+ json_value(binary_t&& value) : binary(create<binary_t>(std::move(value))) {}
+
+ void destroy(value_t t)
+ {
+ if (
+ (t == value_t::object && object == nullptr) ||
+ (t == value_t::array && array == nullptr) ||
+ (t == value_t::string && string == nullptr) ||
+ (t == value_t::binary && binary == nullptr)
+ )
+ {
+ //not initialized (e.g. due to exception in the ctor)
+ return;
+ }
+ if (t == value_t::array || t == value_t::object)
+ {
+ // flatten the current json_value to a heap-allocated stack
+ std::vector<basic_json> stack;
+
+ // move the top-level items to stack
+ if (t == value_t::array)
+ {
+ stack.reserve(array->size());
+ std::move(array->begin(), array->end(), std::back_inserter(stack));
+ }
+ else
+ {
+ stack.reserve(object->size());
+ for (auto&& it : *object)
+ {
+ stack.push_back(std::move(it.second));
+ }
+ }
+
+ while (!stack.empty())
+ {
+ // move the last item to local variable to be processed
+ basic_json current_item(std::move(stack.back()));
+ stack.pop_back();
+
+ // if current_item is array/object, move
+ // its children to the stack to be processed later
+ if (current_item.is_array())
+ {
+ std::move(current_item.m_data.m_value.array->begin(), current_item.m_data.m_value.array->end(), std::back_inserter(stack));
+
+ current_item.m_data.m_value.array->clear();
+ }
+ else if (current_item.is_object())
+ {
+ for (auto&& it : *current_item.m_data.m_value.object)
+ {
+ stack.push_back(std::move(it.second));
+ }
+
+ current_item.m_data.m_value.object->clear();
+ }
+
+ // it's now safe that current_item get destructed
+ // since it doesn't have any children
+ }
+ }
+
+ switch (t)
+ {
+ case value_t::object:
+ {
+ AllocatorType<object_t> alloc;
+ std::allocator_traits<decltype(alloc)>::destroy(alloc, object);
+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, object, 1);
+ break;
+ }
+
+ case value_t::array:
+ {
+ AllocatorType<array_t> alloc;
+ std::allocator_traits<decltype(alloc)>::destroy(alloc, array);
+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, array, 1);
+ break;
+ }
+
+ case value_t::string:
+ {
+ AllocatorType<string_t> alloc;
+ std::allocator_traits<decltype(alloc)>::destroy(alloc, string);
+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, string, 1);
+ break;
+ }
+
+ case value_t::binary:
+ {
+ AllocatorType<binary_t> alloc;
+ std::allocator_traits<decltype(alloc)>::destroy(alloc, binary);
+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, binary, 1);
+ break;
+ }
+
+ case value_t::null:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::discarded:
+ default:
+ {
+ break;
+ }
+ }
+ }
+ };
+
+ private:
+ /*!
+ @brief checks the class invariants
+
+ This function asserts the class invariants. It needs to be called at the
+ end of every constructor to make sure that created objects respect the
+ invariant. Furthermore, it has to be called each time the type of a JSON
+ value is changed, because the invariant expresses a relationship between
+ @a m_type and @a m_value.
+
+ Furthermore, the parent relation is checked for arrays and objects: If
+ @a check_parents true and the value is an array or object, then the
+ container's elements must have the current value as parent.
+
+ @param[in] check_parents whether the parent relation should be checked.
+ The value is true by default and should only be set to false
+ during destruction of objects when the invariant does not
+ need to hold.
+ */
+ void assert_invariant(bool check_parents = true) const noexcept
+ {
+ JSON_ASSERT(m_data.m_type != value_t::object || m_data.m_value.object != nullptr);
+ JSON_ASSERT(m_data.m_type != value_t::array || m_data.m_value.array != nullptr);
+ JSON_ASSERT(m_data.m_type != value_t::string || m_data.m_value.string != nullptr);
+ JSON_ASSERT(m_data.m_type != value_t::binary || m_data.m_value.binary != nullptr);
+
+#if JSON_DIAGNOSTICS
+ JSON_TRY
+ {
+ // cppcheck-suppress assertWithSideEffect
+ JSON_ASSERT(!check_parents || !is_structured() || std::all_of(begin(), end(), [this](const basic_json & j)
+ {
+ return j.m_parent == this;
+ }));
+ }
+ JSON_CATCH(...) {} // LCOV_EXCL_LINE
+#endif
+ static_cast<void>(check_parents);
+ }
+
+ void set_parents()
+ {
+#if JSON_DIAGNOSTICS
+ switch (m_data.m_type)
+ {
+ case value_t::array:
+ {
+ for (auto& element : *m_data.m_value.array)
+ {
+ element.m_parent = this;
+ }
+ break;
+ }
+
+ case value_t::object:
+ {
+ for (auto& element : *m_data.m_value.object)
+ {
+ element.second.m_parent = this;
+ }
+ break;
+ }
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ break;
+ }
+#endif
+ }
+
+ iterator set_parents(iterator it, typename iterator::difference_type count_set_parents)
+ {
+#if JSON_DIAGNOSTICS
+ for (typename iterator::difference_type i = 0; i < count_set_parents; ++i)
+ {
+ (it + i)->m_parent = this;
+ }
+#else
+ static_cast<void>(count_set_parents);
+#endif
+ return it;
+ }
+
+ reference set_parent(reference j, std::size_t old_capacity = static_cast<std::size_t>(-1))
+ {
+#if JSON_DIAGNOSTICS
+ if (old_capacity != static_cast<std::size_t>(-1))
+ {
+ // see https://github.com/nlohmann/json/issues/2838
+ JSON_ASSERT(type() == value_t::array);
+ if (JSON_HEDLEY_UNLIKELY(m_data.m_value.array->capacity() != old_capacity))
+ {
+ // capacity has changed: update all parents
+ set_parents();
+ return j;
+ }
+ }
+
+ // ordered_json uses a vector internally, so pointers could have
+ // been invalidated; see https://github.com/nlohmann/json/issues/2962
+#ifdef JSON_HEDLEY_MSVC_VERSION
+#pragma warning(push )
+#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr
+#endif
+ if (detail::is_ordered_map<object_t>::value)
+ {
+ set_parents();
+ return j;
+ }
+#ifdef JSON_HEDLEY_MSVC_VERSION
+#pragma warning( pop )
+#endif
+
+ j.m_parent = this;
+#else
+ static_cast<void>(j);
+ static_cast<void>(old_capacity);
+#endif
+ return j;
+ }
+
+ public:
+ //////////////////////////
+ // JSON parser callback //
+ //////////////////////////
+
+ /// @brief parser event types
+ /// @sa https://json.nlohmann.me/api/basic_json/parse_event_t/
+ using parse_event_t = detail::parse_event_t;
+
+ /// @brief per-element parser callback type
+ /// @sa https://json.nlohmann.me/api/basic_json/parser_callback_t/
+ using parser_callback_t = detail::parser_callback_t<basic_json>;
+
+ //////////////////
+ // constructors //
+ //////////////////
+
+ /// @name constructors and destructors
+ /// Constructors of class @ref basic_json, copy/move constructor, copy
+ /// assignment, static functions creating objects, and the destructor.
+ /// @{
+
+ /// @brief create an empty value with a given type
+ /// @sa https://json.nlohmann.me/api/basic_json/basic_json/
+ basic_json(const value_t v)
+ : m_data(v)
+ {
+ assert_invariant();
+ }
+
+ /// @brief create a null object
+ /// @sa https://json.nlohmann.me/api/basic_json/basic_json/
+ basic_json(std::nullptr_t = nullptr) noexcept // NOLINT(bugprone-exception-escape)
+ : basic_json(value_t::null)
+ {
+ assert_invariant();
+ }
+
+ /// @brief create a JSON value from compatible types
+ /// @sa https://json.nlohmann.me/api/basic_json/basic_json/
+ template < typename CompatibleType,
+ typename U = detail::uncvref_t<CompatibleType>,
+ detail::enable_if_t <
+ !detail::is_basic_json<U>::value && detail::is_compatible_type<basic_json_t, U>::value, int > = 0 >
+ basic_json(CompatibleType && val) noexcept(noexcept( // NOLINT(bugprone-forwarding-reference-overload,bugprone-exception-escape)
+ JSONSerializer<U>::to_json(std::declval<basic_json_t&>(),
+ std::forward<CompatibleType>(val))))
+ {
+ JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val));
+ set_parents();
+ assert_invariant();
+ }
+
+ /// @brief create a JSON value from an existing one
+ /// @sa https://json.nlohmann.me/api/basic_json/basic_json/
+ template < typename BasicJsonType,
+ detail::enable_if_t <
+ detail::is_basic_json<BasicJsonType>::value&& !std::is_same<basic_json, BasicJsonType>::value, int > = 0 >
+ basic_json(const BasicJsonType& val)
+ {
+ using other_boolean_t = typename BasicJsonType::boolean_t;
+ using other_number_float_t = typename BasicJsonType::number_float_t;
+ using other_number_integer_t = typename BasicJsonType::number_integer_t;
+ using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+ using other_string_t = typename BasicJsonType::string_t;
+ using other_object_t = typename BasicJsonType::object_t;
+ using other_array_t = typename BasicJsonType::array_t;
+ using other_binary_t = typename BasicJsonType::binary_t;
+
+ switch (val.type())
+ {
+ case value_t::boolean:
+ JSONSerializer<other_boolean_t>::to_json(*this, val.template get<other_boolean_t>());
+ break;
+ case value_t::number_float:
+ JSONSerializer<other_number_float_t>::to_json(*this, val.template get<other_number_float_t>());
+ break;
+ case value_t::number_integer:
+ JSONSerializer<other_number_integer_t>::to_json(*this, val.template get<other_number_integer_t>());
+ break;
+ case value_t::number_unsigned:
+ JSONSerializer<other_number_unsigned_t>::to_json(*this, val.template get<other_number_unsigned_t>());
+ break;
+ case value_t::string:
+ JSONSerializer<other_string_t>::to_json(*this, val.template get_ref<const other_string_t&>());
+ break;
+ case value_t::object:
+ JSONSerializer<other_object_t>::to_json(*this, val.template get_ref<const other_object_t&>());
+ break;
+ case value_t::array:
+ JSONSerializer<other_array_t>::to_json(*this, val.template get_ref<const other_array_t&>());
+ break;
+ case value_t::binary:
+ JSONSerializer<other_binary_t>::to_json(*this, val.template get_ref<const other_binary_t&>());
+ break;
+ case value_t::null:
+ *this = nullptr;
+ break;
+ case value_t::discarded:
+ m_data.m_type = value_t::discarded;
+ break;
+ default: // LCOV_EXCL_LINE
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
+ }
+ JSON_ASSERT(m_data.m_type == val.type());
+ set_parents();
+ assert_invariant();
+ }
+
+ /// @brief create a container (array or object) from an initializer list
+ /// @sa https://json.nlohmann.me/api/basic_json/basic_json/
+ basic_json(initializer_list_t init,
+ bool type_deduction = true,
+ value_t manual_type = value_t::array)
+ {
+ // check if each element is an array with two elements whose first
+ // element is a string
+ bool is_an_object = std::all_of(init.begin(), init.end(),
+ [](const detail::json_ref<basic_json>& element_ref)
+ {
+ // The cast is to ensure op[size_type] is called, bearing in mind size_type may not be int;
+ // (many string types can be constructed from 0 via its null-pointer guise, so we get a
+ // broken call to op[key_type], the wrong semantics and a 4804 warning on Windows)
+ return element_ref->is_array() && element_ref->size() == 2 && (*element_ref)[static_cast<size_type>(0)].is_string();
+ });
+
+ // adjust type if type deduction is not wanted
+ if (!type_deduction)
+ {
+ // if array is wanted, do not create an object though possible
+ if (manual_type == value_t::array)
+ {
+ is_an_object = false;
+ }
+
+ // if object is wanted but impossible, throw an exception
+ if (JSON_HEDLEY_UNLIKELY(manual_type == value_t::object && !is_an_object))
+ {
+ JSON_THROW(type_error::create(301, "cannot create object from initializer list", nullptr));
+ }
+ }
+
+ if (is_an_object)
+ {
+ // the initializer list is a list of pairs -> create object
+ m_data.m_type = value_t::object;
+ m_data.m_value = value_t::object;
+
+ for (auto& element_ref : init)
+ {
+ auto element = element_ref.moved_or_copied();
+ m_data.m_value.object->emplace(
+ std::move(*((*element.m_data.m_value.array)[0].m_data.m_value.string)),
+ std::move((*element.m_data.m_value.array)[1]));
+ }
+ }
+ else
+ {
+ // the initializer list describes an array -> create array
+ m_data.m_type = value_t::array;
+ m_data.m_value.array = create<array_t>(init.begin(), init.end());
+ }
+
+ set_parents();
+ assert_invariant();
+ }
+
+ /// @brief explicitly create a binary array (without subtype)
+ /// @sa https://json.nlohmann.me/api/basic_json/binary/
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json binary(const typename binary_t::container_type& init)
+ {
+ auto res = basic_json();
+ res.m_data.m_type = value_t::binary;
+ res.m_data.m_value = init;
+ return res;
+ }
+
+ /// @brief explicitly create a binary array (with subtype)
+ /// @sa https://json.nlohmann.me/api/basic_json/binary/
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json binary(const typename binary_t::container_type& init, typename binary_t::subtype_type subtype)
+ {
+ auto res = basic_json();
+ res.m_data.m_type = value_t::binary;
+ res.m_data.m_value = binary_t(init, subtype);
+ return res;
+ }
+
+ /// @brief explicitly create a binary array
+ /// @sa https://json.nlohmann.me/api/basic_json/binary/
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json binary(typename binary_t::container_type&& init)
+ {
+ auto res = basic_json();
+ res.m_data.m_type = value_t::binary;
+ res.m_data.m_value = std::move(init);
+ return res;
+ }
+
+ /// @brief explicitly create a binary array (with subtype)
+ /// @sa https://json.nlohmann.me/api/basic_json/binary/
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json binary(typename binary_t::container_type&& init, typename binary_t::subtype_type subtype)
+ {
+ auto res = basic_json();
+ res.m_data.m_type = value_t::binary;
+ res.m_data.m_value = binary_t(std::move(init), subtype);
+ return res;
+ }
+
+ /// @brief explicitly create an array from an initializer list
+ /// @sa https://json.nlohmann.me/api/basic_json/array/
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json array(initializer_list_t init = {})
+ {
+ return basic_json(init, false, value_t::array);
+ }
+
+ /// @brief explicitly create an object from an initializer list
+ /// @sa https://json.nlohmann.me/api/basic_json/object/
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json object(initializer_list_t init = {})
+ {
+ return basic_json(init, false, value_t::object);
+ }
+
+ /// @brief construct an array with count copies of given value
+ /// @sa https://json.nlohmann.me/api/basic_json/basic_json/
+ basic_json(size_type cnt, const basic_json& val):
+ m_data{cnt, val}
+ {
+ set_parents();
+ assert_invariant();
+ }
+
+ /// @brief construct a JSON container given an iterator range
+ /// @sa https://json.nlohmann.me/api/basic_json/basic_json/
+ template < class InputIT, typename std::enable_if <
+ std::is_same<InputIT, typename basic_json_t::iterator>::value ||
+ std::is_same<InputIT, typename basic_json_t::const_iterator>::value, int >::type = 0 >
+ basic_json(InputIT first, InputIT last)
+ {
+ JSON_ASSERT(first.m_object != nullptr);
+ JSON_ASSERT(last.m_object != nullptr);
+
+ // make sure iterator fits the current value
+ if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object))
+ {
+ JSON_THROW(invalid_iterator::create(201, "iterators are not compatible", nullptr));
+ }
+
+ // copy type from first iterator
+ m_data.m_type = first.m_object->m_data.m_type;
+
+ // check if iterator range is complete for primitive values
+ switch (m_data.m_type)
+ {
+ case value_t::boolean:
+ case value_t::number_float:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::string:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin()
+ || !last.m_it.primitive_iterator.is_end()))
+ {
+ JSON_THROW(invalid_iterator::create(204, "iterators out of range", first.m_object));
+ }
+ break;
+ }
+
+ case value_t::null:
+ case value_t::object:
+ case value_t::array:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ break;
+ }
+
+ switch (m_data.m_type)
+ {
+ case value_t::number_integer:
+ {
+ m_data.m_value.number_integer = first.m_object->m_data.m_value.number_integer;
+ break;
+ }
+
+ case value_t::number_unsigned:
+ {
+ m_data.m_value.number_unsigned = first.m_object->m_data.m_value.number_unsigned;
+ break;
+ }
+
+ case value_t::number_float:
+ {
+ m_data.m_value.number_float = first.m_object->m_data.m_value.number_float;
+ break;
+ }
+
+ case value_t::boolean:
+ {
+ m_data.m_value.boolean = first.m_object->m_data.m_value.boolean;
+ break;
+ }
+
+ case value_t::string:
+ {
+ m_data.m_value = *first.m_object->m_data.m_value.string;
+ break;
+ }
+
+ case value_t::object:
+ {
+ m_data.m_value.object = create<object_t>(first.m_it.object_iterator,
+ last.m_it.object_iterator);
+ break;
+ }
+
+ case value_t::array:
+ {
+ m_data.m_value.array = create<array_t>(first.m_it.array_iterator,
+ last.m_it.array_iterator);
+ break;
+ }
+
+ case value_t::binary:
+ {
+ m_data.m_value = *first.m_object->m_data.m_value.binary;
+ break;
+ }
+
+ case value_t::null:
+ case value_t::discarded:
+ default:
+ JSON_THROW(invalid_iterator::create(206, detail::concat("cannot construct with iterators from ", first.m_object->type_name()), first.m_object));
+ }
+
+ set_parents();
+ assert_invariant();
+ }
+
+ ///////////////////////////////////////
+ // other constructors and destructor //
+ ///////////////////////////////////////
+
+ template<typename JsonRef,
+ detail::enable_if_t<detail::conjunction<detail::is_json_ref<JsonRef>,
+ std::is_same<typename JsonRef::value_type, basic_json>>::value, int> = 0 >
+ basic_json(const JsonRef& ref) : basic_json(ref.moved_or_copied()) {}
+
+ /// @brief copy constructor
+ /// @sa https://json.nlohmann.me/api/basic_json/basic_json/
+ basic_json(const basic_json& other)
+ : json_base_class_t(other)
+ {
+ m_data.m_type = other.m_data.m_type;
+ // check of passed value is valid
+ other.assert_invariant();
+
+ switch (m_data.m_type)
+ {
+ case value_t::object:
+ {
+ m_data.m_value = *other.m_data.m_value.object;
+ break;
+ }
+
+ case value_t::array:
+ {
+ m_data.m_value = *other.m_data.m_value.array;
+ break;
+ }
+
+ case value_t::string:
+ {
+ m_data.m_value = *other.m_data.m_value.string;
+ break;
+ }
+
+ case value_t::boolean:
+ {
+ m_data.m_value = other.m_data.m_value.boolean;
+ break;
+ }
+
+ case value_t::number_integer:
+ {
+ m_data.m_value = other.m_data.m_value.number_integer;
+ break;
+ }
+
+ case value_t::number_unsigned:
+ {
+ m_data.m_value = other.m_data.m_value.number_unsigned;
+ break;
+ }
+
+ case value_t::number_float:
+ {
+ m_data.m_value = other.m_data.m_value.number_float;
+ break;
+ }
+
+ case value_t::binary:
+ {
+ m_data.m_value = *other.m_data.m_value.binary;
+ break;
+ }
+
+ case value_t::null:
+ case value_t::discarded:
+ default:
+ break;
+ }
+
+ set_parents();
+ assert_invariant();
+ }
+
+ /// @brief move constructor
+ /// @sa https://json.nlohmann.me/api/basic_json/basic_json/
+ basic_json(basic_json&& other) noexcept
+ : json_base_class_t(std::forward<json_base_class_t>(other)),
+ m_data(std::move(other.m_data))
+ {
+ // check that passed value is valid
+ other.assert_invariant(false);
+
+ // invalidate payload
+ other.m_data.m_type = value_t::null;
+ other.m_data.m_value = {};
+
+ set_parents();
+ assert_invariant();
+ }
+
+ /// @brief copy assignment
+ /// @sa https://json.nlohmann.me/api/basic_json/operator=/
+ basic_json& operator=(basic_json other) noexcept (
+ std::is_nothrow_move_constructible<value_t>::value&&
+ std::is_nothrow_move_assignable<value_t>::value&&
+ std::is_nothrow_move_constructible<json_value>::value&&
+ std::is_nothrow_move_assignable<json_value>::value&&
+ std::is_nothrow_move_assignable<json_base_class_t>::value
+ )
+ {
+ // check that passed value is valid
+ other.assert_invariant();
+
+ using std::swap;
+ swap(m_data.m_type, other.m_data.m_type);
+ swap(m_data.m_value, other.m_data.m_value);
+ json_base_class_t::operator=(std::move(other));
+
+ set_parents();
+ assert_invariant();
+ return *this;
+ }
+
+ /// @brief destructor
+ /// @sa https://json.nlohmann.me/api/basic_json/~basic_json/
+ ~basic_json() noexcept
+ {
+ assert_invariant(false);
+ }
+
+ /// @}
+
+ public:
+ ///////////////////////
+ // object inspection //
+ ///////////////////////
+
+ /// @name object inspection
+ /// Functions to inspect the type of a JSON value.
+ /// @{
+
+ /// @brief serialization
+ /// @sa https://json.nlohmann.me/api/basic_json/dump/
+ string_t dump(const int indent = -1,
+ const char indent_char = ' ',
+ const bool ensure_ascii = false,
+ const error_handler_t error_handler = error_handler_t::strict) const
+ {
+ string_t result;
+ serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler);
+
+ if (indent >= 0)
+ {
+ s.dump(*this, true, ensure_ascii, static_cast<unsigned int>(indent));
+ }
+ else
+ {
+ s.dump(*this, false, ensure_ascii, 0);
+ }
+
+ return result;
+ }
+
+ /// @brief return the type of the JSON value (explicit)
+ /// @sa https://json.nlohmann.me/api/basic_json/type/
+ constexpr value_t type() const noexcept
+ {
+ return m_data.m_type;
+ }
+
+ /// @brief return whether type is primitive
+ /// @sa https://json.nlohmann.me/api/basic_json/is_primitive/
+ constexpr bool is_primitive() const noexcept
+ {
+ return is_null() || is_string() || is_boolean() || is_number() || is_binary();
+ }
+
+ /// @brief return whether type is structured
+ /// @sa https://json.nlohmann.me/api/basic_json/is_structured/
+ constexpr bool is_structured() const noexcept
+ {
+ return is_array() || is_object();
+ }
+
+ /// @brief return whether value is null
+ /// @sa https://json.nlohmann.me/api/basic_json/is_null/
+ constexpr bool is_null() const noexcept
+ {
+ return m_data.m_type == value_t::null;
+ }
+
+ /// @brief return whether value is a boolean
+ /// @sa https://json.nlohmann.me/api/basic_json/is_boolean/
+ constexpr bool is_boolean() const noexcept
+ {
+ return m_data.m_type == value_t::boolean;
+ }
+
+ /// @brief return whether value is a number
+ /// @sa https://json.nlohmann.me/api/basic_json/is_number/
+ constexpr bool is_number() const noexcept
+ {
+ return is_number_integer() || is_number_float();
+ }
+
+ /// @brief return whether value is an integer number
+ /// @sa https://json.nlohmann.me/api/basic_json/is_number_integer/
+ constexpr bool is_number_integer() const noexcept
+ {
+ return m_data.m_type == value_t::number_integer || m_data.m_type == value_t::number_unsigned;
+ }
+
+ /// @brief return whether value is an unsigned integer number
+ /// @sa https://json.nlohmann.me/api/basic_json/is_number_unsigned/
+ constexpr bool is_number_unsigned() const noexcept
+ {
+ return m_data.m_type == value_t::number_unsigned;
+ }
+
+ /// @brief return whether value is a floating-point number
+ /// @sa https://json.nlohmann.me/api/basic_json/is_number_float/
+ constexpr bool is_number_float() const noexcept
+ {
+ return m_data.m_type == value_t::number_float;
+ }
+
+ /// @brief return whether value is an object
+ /// @sa https://json.nlohmann.me/api/basic_json/is_object/
+ constexpr bool is_object() const noexcept
+ {
+ return m_data.m_type == value_t::object;
+ }
+
+ /// @brief return whether value is an array
+ /// @sa https://json.nlohmann.me/api/basic_json/is_array/
+ constexpr bool is_array() const noexcept
+ {
+ return m_data.m_type == value_t::array;
+ }
+
+ /// @brief return whether value is a string
+ /// @sa https://json.nlohmann.me/api/basic_json/is_string/
+ constexpr bool is_string() const noexcept
+ {
+ return m_data.m_type == value_t::string;
+ }
+
+ /// @brief return whether value is a binary array
+ /// @sa https://json.nlohmann.me/api/basic_json/is_binary/
+ constexpr bool is_binary() const noexcept
+ {
+ return m_data.m_type == value_t::binary;
+ }
+
+ /// @brief return whether value is discarded
+ /// @sa https://json.nlohmann.me/api/basic_json/is_discarded/
+ constexpr bool is_discarded() const noexcept
+ {
+ return m_data.m_type == value_t::discarded;
+ }
+
+ /// @brief return the type of the JSON value (implicit)
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_value_t/
+ constexpr operator value_t() const noexcept
+ {
+ return m_data.m_type;
+ }
+
+ /// @}
+
+ private:
+ //////////////////
+ // value access //
+ //////////////////
+
+ /// get a boolean (explicit)
+ boolean_t get_impl(boolean_t* /*unused*/) const
+ {
+ if (JSON_HEDLEY_LIKELY(is_boolean()))
+ {
+ return m_data.m_value.boolean;
+ }
+
+ JSON_THROW(type_error::create(302, detail::concat("type must be boolean, but is ", type_name()), this));
+ }
+
+ /// get a pointer to the value (object)
+ object_t* get_impl_ptr(object_t* /*unused*/) noexcept
+ {
+ return is_object() ? m_data.m_value.object : nullptr;
+ }
+
+ /// get a pointer to the value (object)
+ constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept
+ {
+ return is_object() ? m_data.m_value.object : nullptr;
+ }
+
+ /// get a pointer to the value (array)
+ array_t* get_impl_ptr(array_t* /*unused*/) noexcept
+ {
+ return is_array() ? m_data.m_value.array : nullptr;
+ }
+
+ /// get a pointer to the value (array)
+ constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept
+ {
+ return is_array() ? m_data.m_value.array : nullptr;
+ }
+
+ /// get a pointer to the value (string)
+ string_t* get_impl_ptr(string_t* /*unused*/) noexcept
+ {
+ return is_string() ? m_data.m_value.string : nullptr;
+ }
+
+ /// get a pointer to the value (string)
+ constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept
+ {
+ return is_string() ? m_data.m_value.string : nullptr;
+ }
+
+ /// get a pointer to the value (boolean)
+ boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept
+ {
+ return is_boolean() ? &m_data.m_value.boolean : nullptr;
+ }
+
+ /// get a pointer to the value (boolean)
+ constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept
+ {
+ return is_boolean() ? &m_data.m_value.boolean : nullptr;
+ }
+
+ /// get a pointer to the value (integer number)
+ number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept
+ {
+ return is_number_integer() ? &m_data.m_value.number_integer : nullptr;
+ }
+
+ /// get a pointer to the value (integer number)
+ constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept
+ {
+ return is_number_integer() ? &m_data.m_value.number_integer : nullptr;
+ }
+
+ /// get a pointer to the value (unsigned number)
+ number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept
+ {
+ return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr;
+ }
+
+ /// get a pointer to the value (unsigned number)
+ constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept
+ {
+ return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr;
+ }
+
+ /// get a pointer to the value (floating-point number)
+ number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept
+ {
+ return is_number_float() ? &m_data.m_value.number_float : nullptr;
+ }
+
+ /// get a pointer to the value (floating-point number)
+ constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept
+ {
+ return is_number_float() ? &m_data.m_value.number_float : nullptr;
+ }
+
+ /// get a pointer to the value (binary)
+ binary_t* get_impl_ptr(binary_t* /*unused*/) noexcept
+ {
+ return is_binary() ? m_data.m_value.binary : nullptr;
+ }
+
+ /// get a pointer to the value (binary)
+ constexpr const binary_t* get_impl_ptr(const binary_t* /*unused*/) const noexcept
+ {
+ return is_binary() ? m_data.m_value.binary : nullptr;
+ }
+
+ /*!
+ @brief helper function to implement get_ref()
+
+ This function helps to implement get_ref() without code duplication for
+ const and non-const overloads
+
+ @tparam ThisType will be deduced as `basic_json` or `const basic_json`
+
+ @throw type_error.303 if ReferenceType does not match underlying value
+ type of the current JSON
+ */
+ template<typename ReferenceType, typename ThisType>
+ static ReferenceType get_ref_impl(ThisType& obj)
+ {
+ // delegate the call to get_ptr<>()
+ auto* ptr = obj.template get_ptr<typename std::add_pointer<ReferenceType>::type>();
+
+ if (JSON_HEDLEY_LIKELY(ptr != nullptr))
+ {
+ return *ptr;
+ }
+
+ JSON_THROW(type_error::create(303, detail::concat("incompatible ReferenceType for get_ref, actual type is ", obj.type_name()), &obj));
+ }
+
+ public:
+ /// @name value access
+ /// Direct access to the stored value of a JSON value.
+ /// @{
+
+ /// @brief get a pointer value (implicit)
+ /// @sa https://json.nlohmann.me/api/basic_json/get_ptr/
+ template<typename PointerType, typename std::enable_if<
+ std::is_pointer<PointerType>::value, int>::type = 0>
+ auto get_ptr() noexcept -> decltype(std::declval<basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))
+ {
+ // delegate the call to get_impl_ptr<>()
+ return get_impl_ptr(static_cast<PointerType>(nullptr));
+ }
+
+ /// @brief get a pointer value (implicit)
+ /// @sa https://json.nlohmann.me/api/basic_json/get_ptr/
+ template < typename PointerType, typename std::enable_if <
+ std::is_pointer<PointerType>::value&&
+ std::is_const<typename std::remove_pointer<PointerType>::type>::value, int >::type = 0 >
+ constexpr auto get_ptr() const noexcept -> decltype(std::declval<const basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))
+ {
+ // delegate the call to get_impl_ptr<>() const
+ return get_impl_ptr(static_cast<PointerType>(nullptr));
+ }
+
+ private:
+ /*!
+ @brief get a value (explicit)
+
+ Explicit type conversion between the JSON value and a compatible value
+ which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)
+ and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).
+ The value is converted by calling the @ref json_serializer<ValueType>
+ `from_json()` method.
+
+ The function is equivalent to executing
+ @code {.cpp}
+ ValueType ret;
+ JSONSerializer<ValueType>::from_json(*this, ret);
+ return ret;
+ @endcode
+
+ This overloads is chosen if:
+ - @a ValueType is not @ref basic_json,
+ - @ref json_serializer<ValueType> has a `from_json()` method of the form
+ `void from_json(const basic_json&, ValueType&)`, and
+ - @ref json_serializer<ValueType> does not have a `from_json()` method of
+ the form `ValueType from_json(const basic_json&)`
+
+ @tparam ValueType the returned value type
+
+ @return copy of the JSON value, converted to @a ValueType
+
+ @throw what @ref json_serializer<ValueType> `from_json()` method throws
+
+ @liveexample{The example below shows several conversions from JSON values
+ to other types. There a few things to note: (1) Floating-point numbers can
+ be converted to integers\, (2) A JSON array can be converted to a standard
+ `std::vector<short>`\, (3) A JSON object can be converted to C++
+ associative containers such as `std::unordered_map<std::string\,
+ json>`.,get__ValueType_const}
+
+ @since version 2.1.0
+ */
+ template < typename ValueType,
+ detail::enable_if_t <
+ detail::is_default_constructible<ValueType>::value&&
+ detail::has_from_json<basic_json_t, ValueType>::value,
+ int > = 0 >
+ ValueType get_impl(detail::priority_tag<0> /*unused*/) const noexcept(noexcept(
+ JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), std::declval<ValueType&>())))
+ {
+ auto ret = ValueType();
+ JSONSerializer<ValueType>::from_json(*this, ret);
+ return ret;
+ }
+
+ /*!
+ @brief get a value (explicit); special case
+
+ Explicit type conversion between the JSON value and a compatible value
+ which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)
+ and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).
+ The value is converted by calling the @ref json_serializer<ValueType>
+ `from_json()` method.
+
+ The function is equivalent to executing
+ @code {.cpp}
+ return JSONSerializer<ValueType>::from_json(*this);
+ @endcode
+
+ This overloads is chosen if:
+ - @a ValueType is not @ref basic_json and
+ - @ref json_serializer<ValueType> has a `from_json()` method of the form
+ `ValueType from_json(const basic_json&)`
+
+ @note If @ref json_serializer<ValueType> has both overloads of
+ `from_json()`, this one is chosen.
+
+ @tparam ValueType the returned value type
+
+ @return copy of the JSON value, converted to @a ValueType
+
+ @throw what @ref json_serializer<ValueType> `from_json()` method throws
+
+ @since version 2.1.0
+ */
+ template < typename ValueType,
+ detail::enable_if_t <
+ detail::has_non_default_from_json<basic_json_t, ValueType>::value,
+ int > = 0 >
+ ValueType get_impl(detail::priority_tag<1> /*unused*/) const noexcept(noexcept(
+ JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>())))
+ {
+ return JSONSerializer<ValueType>::from_json(*this);
+ }
+
+ /*!
+ @brief get special-case overload
+
+ This overloads converts the current @ref basic_json in a different
+ @ref basic_json type
+
+ @tparam BasicJsonType == @ref basic_json
+
+ @return a copy of *this, converted into @a BasicJsonType
+
+ @complexity Depending on the implementation of the called `from_json()`
+ method.
+
+ @since version 3.2.0
+ */
+ template < typename BasicJsonType,
+ detail::enable_if_t <
+ detail::is_basic_json<BasicJsonType>::value,
+ int > = 0 >
+ BasicJsonType get_impl(detail::priority_tag<2> /*unused*/) const
+ {
+ return *this;
+ }
+
+ /*!
+ @brief get special-case overload
+
+ This overloads avoids a lot of template boilerplate, it can be seen as the
+ identity method
+
+ @tparam BasicJsonType == @ref basic_json
+
+ @return a copy of *this
+
+ @complexity Constant.
+
+ @since version 2.1.0
+ */
+ template<typename BasicJsonType,
+ detail::enable_if_t<
+ std::is_same<BasicJsonType, basic_json_t>::value,
+ int> = 0>
+ basic_json get_impl(detail::priority_tag<3> /*unused*/) const
+ {
+ return *this;
+ }
+
+ /*!
+ @brief get a pointer value (explicit)
+ @copydoc get()
+ */
+ template<typename PointerType,
+ detail::enable_if_t<
+ std::is_pointer<PointerType>::value,
+ int> = 0>
+ constexpr auto get_impl(detail::priority_tag<4> /*unused*/) const noexcept
+ -> decltype(std::declval<const basic_json_t&>().template get_ptr<PointerType>())
+ {
+ // delegate the call to get_ptr
+ return get_ptr<PointerType>();
+ }
+
+ public:
+ /*!
+ @brief get a (pointer) value (explicit)
+
+ Performs explicit type conversion between the JSON value and a compatible value if required.
+
+ - If the requested type is a pointer to the internally stored JSON value that pointer is returned.
+ No copies are made.
+
+ - If the requested type is the current @ref basic_json, or a different @ref basic_json convertible
+ from the current @ref basic_json.
+
+ - Otherwise the value is converted by calling the @ref json_serializer<ValueType> `from_json()`
+ method.
+
+ @tparam ValueTypeCV the provided value type
+ @tparam ValueType the returned value type
+
+ @return copy of the JSON value, converted to @tparam ValueType if necessary
+
+ @throw what @ref json_serializer<ValueType> `from_json()` method throws if conversion is required
+
+ @since version 2.1.0
+ */
+ template < typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>>
+#if defined(JSON_HAS_CPP_14)
+ constexpr
+#endif
+ auto get() const noexcept(
+ noexcept(std::declval<const basic_json_t&>().template get_impl<ValueType>(detail::priority_tag<4> {})))
+ -> decltype(std::declval<const basic_json_t&>().template get_impl<ValueType>(detail::priority_tag<4> {}))
+ {
+ // we cannot static_assert on ValueTypeCV being non-const, because
+ // there is support for get<const basic_json_t>(), which is why we
+ // still need the uncvref
+ static_assert(!std::is_reference<ValueTypeCV>::value,
+ "get() cannot be used with reference types, you might want to use get_ref()");
+ return get_impl<ValueType>(detail::priority_tag<4> {});
+ }
+
+ /*!
+ @brief get a pointer value (explicit)
+
+ Explicit pointer access to the internally stored JSON value. No copies are
+ made.
+
+ @warning The pointer becomes invalid if the underlying JSON object
+ changes.
+
+ @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref
+ object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,
+ @ref number_unsigned_t, or @ref number_float_t.
+
+ @return pointer to the internally stored JSON value if the requested
+ pointer type @a PointerType fits to the JSON value; `nullptr` otherwise
+
+ @complexity Constant.
+
+ @liveexample{The example below shows how pointers to internal values of a
+ JSON value can be requested. Note that no type conversions are made and a
+ `nullptr` is returned if the value and the requested pointer type does not
+ match.,get__PointerType}
+
+ @sa see @ref get_ptr() for explicit pointer-member access
+
+ @since version 1.0.0
+ */
+ template<typename PointerType, typename std::enable_if<
+ std::is_pointer<PointerType>::value, int>::type = 0>
+ auto get() noexcept -> decltype(std::declval<basic_json_t&>().template get_ptr<PointerType>())
+ {
+ // delegate the call to get_ptr
+ return get_ptr<PointerType>();
+ }
+
+ /// @brief get a value (explicit)
+ /// @sa https://json.nlohmann.me/api/basic_json/get_to/
+ template < typename ValueType,
+ detail::enable_if_t <
+ !detail::is_basic_json<ValueType>::value&&
+ detail::has_from_json<basic_json_t, ValueType>::value,
+ int > = 0 >
+ ValueType & get_to(ValueType& v) const noexcept(noexcept(
+ JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), v)))
+ {
+ JSONSerializer<ValueType>::from_json(*this, v);
+ return v;
+ }
+
+ // specialization to allow calling get_to with a basic_json value
+ // see https://github.com/nlohmann/json/issues/2175
+ template<typename ValueType,
+ detail::enable_if_t <
+ detail::is_basic_json<ValueType>::value,
+ int> = 0>
+ ValueType & get_to(ValueType& v) const
+ {
+ v = *this;
+ return v;
+ }
+
+ template <
+ typename T, std::size_t N,
+ typename Array = T (&)[N], // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+ detail::enable_if_t <
+ detail::has_from_json<basic_json_t, Array>::value, int > = 0 >
+ Array get_to(T (&v)[N]) const // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+ noexcept(noexcept(JSONSerializer<Array>::from_json(
+ std::declval<const basic_json_t&>(), v)))
+ {
+ JSONSerializer<Array>::from_json(*this, v);
+ return v;
+ }
+
+ /// @brief get a reference value (implicit)
+ /// @sa https://json.nlohmann.me/api/basic_json/get_ref/
+ template<typename ReferenceType, typename std::enable_if<
+ std::is_reference<ReferenceType>::value, int>::type = 0>
+ ReferenceType get_ref()
+ {
+ // delegate call to get_ref_impl
+ return get_ref_impl<ReferenceType>(*this);
+ }
+
+ /// @brief get a reference value (implicit)
+ /// @sa https://json.nlohmann.me/api/basic_json/get_ref/
+ template < typename ReferenceType, typename std::enable_if <
+ std::is_reference<ReferenceType>::value&&
+ std::is_const<typename std::remove_reference<ReferenceType>::type>::value, int >::type = 0 >
+ ReferenceType get_ref() const
+ {
+ // delegate call to get_ref_impl
+ return get_ref_impl<ReferenceType>(*this);
+ }
+
+ /*!
+ @brief get a value (implicit)
+
+ Implicit type conversion between the JSON value and a compatible value.
+ The call is realized by calling @ref get() const.
+
+ @tparam ValueType non-pointer type compatible to the JSON value, for
+ instance `int` for JSON integer numbers, `bool` for JSON booleans, or
+ `std::vector` types for JSON arrays. The character type of @ref string_t
+ as well as an initializer list of this type is excluded to avoid
+ ambiguities as these types implicitly convert to `std::string`.
+
+ @return copy of the JSON value, converted to type @a ValueType
+
+ @throw type_error.302 in case passed type @a ValueType is incompatible
+ to the JSON value type (e.g., the JSON value is of type boolean, but a
+ string is requested); see example below
+
+ @complexity Linear in the size of the JSON value.
+
+ @liveexample{The example below shows several conversions from JSON values
+ to other types. There a few things to note: (1) Floating-point numbers can
+ be converted to integers\, (2) A JSON array can be converted to a standard
+ `std::vector<short>`\, (3) A JSON object can be converted to C++
+ associative containers such as `std::unordered_map<std::string\,
+ json>`.,operator__ValueType}
+
+ @since version 1.0.0
+ */
+ template < typename ValueType, typename std::enable_if <
+ detail::conjunction <
+ detail::negation<std::is_pointer<ValueType>>,
+ detail::negation<std::is_same<ValueType, std::nullptr_t>>,
+ detail::negation<std::is_same<ValueType, detail::json_ref<basic_json>>>,
+ detail::negation<std::is_same<ValueType, typename string_t::value_type>>,
+ detail::negation<detail::is_basic_json<ValueType>>,
+ detail::negation<std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>>,
+#if defined(JSON_HAS_CPP_17) && (defined(__GNUC__) || (defined(_MSC_VER) && _MSC_VER >= 1910 && _MSC_VER <= 1914))
+ detail::negation<std::is_same<ValueType, std::string_view>>,
+#endif
+#if defined(JSON_HAS_CPP_17) && JSON_HAS_STATIC_RTTI
+ detail::negation<std::is_same<ValueType, std::any>>,
+#endif
+ detail::is_detected_lazy<detail::get_template_function, const basic_json_t&, ValueType>
+ >::value, int >::type = 0 >
+ JSON_EXPLICIT operator ValueType() const
+ {
+ // delegate the call to get<>() const
+ return get<ValueType>();
+ }
+
+ /// @brief get a binary value
+ /// @sa https://json.nlohmann.me/api/basic_json/get_binary/
+ binary_t& get_binary()
+ {
+ if (!is_binary())
+ {
+ JSON_THROW(type_error::create(302, detail::concat("type must be binary, but is ", type_name()), this));
+ }
+
+ return *get_ptr<binary_t*>();
+ }
+
+ /// @brief get a binary value
+ /// @sa https://json.nlohmann.me/api/basic_json/get_binary/
+ const binary_t& get_binary() const
+ {
+ if (!is_binary())
+ {
+ JSON_THROW(type_error::create(302, detail::concat("type must be binary, but is ", type_name()), this));
+ }
+
+ return *get_ptr<const binary_t*>();
+ }
+
+ /// @}
+
+ ////////////////////
+ // element access //
+ ////////////////////
+
+ /// @name element access
+ /// Access to the JSON value.
+ /// @{
+
+ /// @brief access specified array element with bounds checking
+ /// @sa https://json.nlohmann.me/api/basic_json/at/
+ reference at(size_type idx)
+ {
+ // at only works for arrays
+ if (JSON_HEDLEY_LIKELY(is_array()))
+ {
+ JSON_TRY
+ {
+ return set_parent(m_data.m_value.array->at(idx));
+ }
+ JSON_CATCH (std::out_of_range&)
+ {
+ // create better exception explanation
+ JSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), this));
+ }
+ }
+ else
+ {
+ JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this));
+ }
+ }
+
+ /// @brief access specified array element with bounds checking
+ /// @sa https://json.nlohmann.me/api/basic_json/at/
+ const_reference at(size_type idx) const
+ {
+ // at only works for arrays
+ if (JSON_HEDLEY_LIKELY(is_array()))
+ {
+ JSON_TRY
+ {
+ return m_data.m_value.array->at(idx);
+ }
+ JSON_CATCH (std::out_of_range&)
+ {
+ // create better exception explanation
+ JSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), this));
+ }
+ }
+ else
+ {
+ JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this));
+ }
+ }
+
+ /// @brief access specified object element with bounds checking
+ /// @sa https://json.nlohmann.me/api/basic_json/at/
+ reference at(const typename object_t::key_type& key)
+ {
+ // at only works for objects
+ if (JSON_HEDLEY_UNLIKELY(!is_object()))
+ {
+ JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this));
+ }
+
+ auto it = m_data.m_value.object->find(key);
+ if (it == m_data.m_value.object->end())
+ {
+ JSON_THROW(out_of_range::create(403, detail::concat("key '", key, "' not found"), this));
+ }
+ return set_parent(it->second);
+ }
+
+ /// @brief access specified object element with bounds checking
+ /// @sa https://json.nlohmann.me/api/basic_json/at/
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>
+ reference at(KeyType && key)
+ {
+ // at only works for objects
+ if (JSON_HEDLEY_UNLIKELY(!is_object()))
+ {
+ JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this));
+ }
+
+ auto it = m_data.m_value.object->find(std::forward<KeyType>(key));
+ if (it == m_data.m_value.object->end())
+ {
+ JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward<KeyType>(key)), "' not found"), this));
+ }
+ return set_parent(it->second);
+ }
+
+ /// @brief access specified object element with bounds checking
+ /// @sa https://json.nlohmann.me/api/basic_json/at/
+ const_reference at(const typename object_t::key_type& key) const
+ {
+ // at only works for objects
+ if (JSON_HEDLEY_UNLIKELY(!is_object()))
+ {
+ JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this));
+ }
+
+ auto it = m_data.m_value.object->find(key);
+ if (it == m_data.m_value.object->end())
+ {
+ JSON_THROW(out_of_range::create(403, detail::concat("key '", key, "' not found"), this));
+ }
+ return it->second;
+ }
+
+ /// @brief access specified object element with bounds checking
+ /// @sa https://json.nlohmann.me/api/basic_json/at/
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>
+ const_reference at(KeyType && key) const
+ {
+ // at only works for objects
+ if (JSON_HEDLEY_UNLIKELY(!is_object()))
+ {
+ JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this));
+ }
+
+ auto it = m_data.m_value.object->find(std::forward<KeyType>(key));
+ if (it == m_data.m_value.object->end())
+ {
+ JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward<KeyType>(key)), "' not found"), this));
+ }
+ return it->second;
+ }
+
+ /// @brief access specified array element
+ /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/
+ reference operator[](size_type idx)
+ {
+ // implicitly convert null value to an empty array
+ if (is_null())
+ {
+ m_data.m_type = value_t::array;
+ m_data.m_value.array = create<array_t>();
+ assert_invariant();
+ }
+
+ // operator[] only works for arrays
+ if (JSON_HEDLEY_LIKELY(is_array()))
+ {
+ // fill up array with null values if given idx is outside range
+ if (idx >= m_data.m_value.array->size())
+ {
+#if JSON_DIAGNOSTICS
+ // remember array size & capacity before resizing
+ const auto old_size = m_data.m_value.array->size();
+ const auto old_capacity = m_data.m_value.array->capacity();
+#endif
+ m_data.m_value.array->resize(idx + 1);
+
+#if JSON_DIAGNOSTICS
+ if (JSON_HEDLEY_UNLIKELY(m_data.m_value.array->capacity() != old_capacity))
+ {
+ // capacity has changed: update all parents
+ set_parents();
+ }
+ else
+ {
+ // set parent for values added above
+ set_parents(begin() + static_cast<typename iterator::difference_type>(old_size), static_cast<typename iterator::difference_type>(idx + 1 - old_size));
+ }
+#endif
+ assert_invariant();
+ }
+
+ return m_data.m_value.array->operator[](idx);
+ }
+
+ JSON_THROW(type_error::create(305, detail::concat("cannot use operator[] with a numeric argument with ", type_name()), this));
+ }
+
+ /// @brief access specified array element
+ /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/
+ const_reference operator[](size_type idx) const
+ {
+ // const operator[] only works for arrays
+ if (JSON_HEDLEY_LIKELY(is_array()))
+ {
+ return m_data.m_value.array->operator[](idx);
+ }
+
+ JSON_THROW(type_error::create(305, detail::concat("cannot use operator[] with a numeric argument with ", type_name()), this));
+ }
+
+ /// @brief access specified object element
+ /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/
+ reference operator[](typename object_t::key_type key)
+ {
+ // implicitly convert null value to an empty object
+ if (is_null())
+ {
+ m_data.m_type = value_t::object;
+ m_data.m_value.object = create<object_t>();
+ assert_invariant();
+ }
+
+ // operator[] only works for objects
+ if (JSON_HEDLEY_LIKELY(is_object()))
+ {
+ auto result = m_data.m_value.object->emplace(std::move(key), nullptr);
+ return set_parent(result.first->second);
+ }
+
+ JSON_THROW(type_error::create(305, detail::concat("cannot use operator[] with a string argument with ", type_name()), this));
+ }
+
+ /// @brief access specified object element
+ /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/
+ const_reference operator[](const typename object_t::key_type& key) const
+ {
+ // const operator[] only works for objects
+ if (JSON_HEDLEY_LIKELY(is_object()))
+ {
+ auto it = m_data.m_value.object->find(key);
+ JSON_ASSERT(it != m_data.m_value.object->end());
+ return it->second;
+ }
+
+ JSON_THROW(type_error::create(305, detail::concat("cannot use operator[] with a string argument with ", type_name()), this));
+ }
+
+ // these two functions resolve a (const) char * ambiguity affecting Clang and MSVC
+ // (they seemingly cannot be constrained to resolve the ambiguity)
+ template<typename T>
+ reference operator[](T* key)
+ {
+ return operator[](typename object_t::key_type(key));
+ }
+
+ template<typename T>
+ const_reference operator[](T* key) const
+ {
+ return operator[](typename object_t::key_type(key));
+ }
+
+ /// @brief access specified object element
+ /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int > = 0 >
+ reference operator[](KeyType && key)
+ {
+ // implicitly convert null value to an empty object
+ if (is_null())
+ {
+ m_data.m_type = value_t::object;
+ m_data.m_value.object = create<object_t>();
+ assert_invariant();
+ }
+
+ // operator[] only works for objects
+ if (JSON_HEDLEY_LIKELY(is_object()))
+ {
+ auto result = m_data.m_value.object->emplace(std::forward<KeyType>(key), nullptr);
+ return set_parent(result.first->second);
+ }
+
+ JSON_THROW(type_error::create(305, detail::concat("cannot use operator[] with a string argument with ", type_name()), this));
+ }
+
+ /// @brief access specified object element
+ /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int > = 0 >
+ const_reference operator[](KeyType && key) const
+ {
+ // const operator[] only works for objects
+ if (JSON_HEDLEY_LIKELY(is_object()))
+ {
+ auto it = m_data.m_value.object->find(std::forward<KeyType>(key));
+ JSON_ASSERT(it != m_data.m_value.object->end());
+ return it->second;
+ }
+
+ JSON_THROW(type_error::create(305, detail::concat("cannot use operator[] with a string argument with ", type_name()), this));
+ }
+
+ private:
+ template<typename KeyType>
+ using is_comparable_with_object_key = detail::is_comparable <
+ object_comparator_t, const typename object_t::key_type&, KeyType >;
+
+ template<typename ValueType>
+ using value_return_type = std::conditional <
+ detail::is_c_string_uncvref<ValueType>::value,
+ string_t, typename std::decay<ValueType>::type >;
+
+ public:
+ /// @brief access specified object element with default value
+ /// @sa https://json.nlohmann.me/api/basic_json/value/
+ template < class ValueType, detail::enable_if_t <
+ !detail::is_transparent<object_comparator_t>::value
+ && detail::is_getable<basic_json_t, ValueType>::value
+ && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >
+ ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const
+ {
+ // value only works for objects
+ if (JSON_HEDLEY_LIKELY(is_object()))
+ {
+ // if key is found, return value and given default value otherwise
+ const auto it = find(key);
+ if (it != end())
+ {
+ return it->template get<ValueType>();
+ }
+
+ return default_value;
+ }
+
+ JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this));
+ }
+
+ /// @brief access specified object element with default value
+ /// @sa https://json.nlohmann.me/api/basic_json/value/
+ template < class ValueType, class ReturnType = typename value_return_type<ValueType>::type,
+ detail::enable_if_t <
+ !detail::is_transparent<object_comparator_t>::value
+ && detail::is_getable<basic_json_t, ReturnType>::value
+ && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >
+ ReturnType value(const typename object_t::key_type& key, ValueType && default_value) const
+ {
+ // value only works for objects
+ if (JSON_HEDLEY_LIKELY(is_object()))
+ {
+ // if key is found, return value and given default value otherwise
+ const auto it = find(key);
+ if (it != end())
+ {
+ return it->template get<ReturnType>();
+ }
+
+ return std::forward<ValueType>(default_value);
+ }
+
+ JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this));
+ }
+
+ /// @brief access specified object element with default value
+ /// @sa https://json.nlohmann.me/api/basic_json/value/
+ template < class ValueType, class KeyType, detail::enable_if_t <
+ detail::is_transparent<object_comparator_t>::value
+ && !detail::is_json_pointer<KeyType>::value
+ && is_comparable_with_object_key<KeyType>::value
+ && detail::is_getable<basic_json_t, ValueType>::value
+ && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >
+ ValueType value(KeyType && key, const ValueType& default_value) const
+ {
+ // value only works for objects
+ if (JSON_HEDLEY_LIKELY(is_object()))
+ {
+ // if key is found, return value and given default value otherwise
+ const auto it = find(std::forward<KeyType>(key));
+ if (it != end())
+ {
+ return it->template get<ValueType>();
+ }
+
+ return default_value;
+ }
+
+ JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this));
+ }
+
+ /// @brief access specified object element via JSON Pointer with default value
+ /// @sa https://json.nlohmann.me/api/basic_json/value/
+ template < class ValueType, class KeyType, class ReturnType = typename value_return_type<ValueType>::type,
+ detail::enable_if_t <
+ detail::is_transparent<object_comparator_t>::value
+ && !detail::is_json_pointer<KeyType>::value
+ && is_comparable_with_object_key<KeyType>::value
+ && detail::is_getable<basic_json_t, ReturnType>::value
+ && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >
+ ReturnType value(KeyType && key, ValueType && default_value) const
+ {
+ // value only works for objects
+ if (JSON_HEDLEY_LIKELY(is_object()))
+ {
+ // if key is found, return value and given default value otherwise
+ const auto it = find(std::forward<KeyType>(key));
+ if (it != end())
+ {
+ return it->template get<ReturnType>();
+ }
+
+ return std::forward<ValueType>(default_value);
+ }
+
+ JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this));
+ }
+
+ /// @brief access specified object element via JSON Pointer with default value
+ /// @sa https://json.nlohmann.me/api/basic_json/value/
+ template < class ValueType, detail::enable_if_t <
+ detail::is_getable<basic_json_t, ValueType>::value
+ && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >
+ ValueType value(const json_pointer& ptr, const ValueType& default_value) const
+ {
+ // value only works for objects
+ if (JSON_HEDLEY_LIKELY(is_object()))
+ {
+ // if pointer resolves a value, return it or use default value
+ JSON_TRY
+ {
+ return ptr.get_checked(this).template get<ValueType>();
+ }
+ JSON_INTERNAL_CATCH (out_of_range&)
+ {
+ return default_value;
+ }
+ }
+
+ JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this));
+ }
+
+ /// @brief access specified object element via JSON Pointer with default value
+ /// @sa https://json.nlohmann.me/api/basic_json/value/
+ template < class ValueType, class ReturnType = typename value_return_type<ValueType>::type,
+ detail::enable_if_t <
+ detail::is_getable<basic_json_t, ReturnType>::value
+ && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >
+ ReturnType value(const json_pointer& ptr, ValueType && default_value) const
+ {
+ // value only works for objects
+ if (JSON_HEDLEY_LIKELY(is_object()))
+ {
+ // if pointer resolves a value, return it or use default value
+ JSON_TRY
+ {
+ return ptr.get_checked(this).template get<ReturnType>();
+ }
+ JSON_INTERNAL_CATCH (out_of_range&)
+ {
+ return std::forward<ValueType>(default_value);
+ }
+ }
+
+ JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this));
+ }
+
+ template < class ValueType, class BasicJsonType, detail::enable_if_t <
+ detail::is_basic_json<BasicJsonType>::value
+ && detail::is_getable<basic_json_t, ValueType>::value
+ && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >
+ JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)
+ ValueType value(const ::nlohmann::json_pointer<BasicJsonType>& ptr, const ValueType& default_value) const
+ {
+ return value(ptr.convert(), default_value);
+ }
+
+ template < class ValueType, class BasicJsonType, class ReturnType = typename value_return_type<ValueType>::type,
+ detail::enable_if_t <
+ detail::is_basic_json<BasicJsonType>::value
+ && detail::is_getable<basic_json_t, ReturnType>::value
+ && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >
+ JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)
+ ReturnType value(const ::nlohmann::json_pointer<BasicJsonType>& ptr, ValueType && default_value) const
+ {
+ return value(ptr.convert(), std::forward<ValueType>(default_value));
+ }
+
+ /// @brief access the first element
+ /// @sa https://json.nlohmann.me/api/basic_json/front/
+ reference front()
+ {
+ return *begin();
+ }
+
+ /// @brief access the first element
+ /// @sa https://json.nlohmann.me/api/basic_json/front/
+ const_reference front() const
+ {
+ return *cbegin();
+ }
+
+ /// @brief access the last element
+ /// @sa https://json.nlohmann.me/api/basic_json/back/
+ reference back()
+ {
+ auto tmp = end();
+ --tmp;
+ return *tmp;
+ }
+
+ /// @brief access the last element
+ /// @sa https://json.nlohmann.me/api/basic_json/back/
+ const_reference back() const
+ {
+ auto tmp = cend();
+ --tmp;
+ return *tmp;
+ }
+
+ /// @brief remove element given an iterator
+ /// @sa https://json.nlohmann.me/api/basic_json/erase/
+ template < class IteratorType, detail::enable_if_t <
+ std::is_same<IteratorType, typename basic_json_t::iterator>::value ||
+ std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int > = 0 >
+ IteratorType erase(IteratorType pos)
+ {
+ // make sure iterator fits the current value
+ if (JSON_HEDLEY_UNLIKELY(this != pos.m_object))
+ {
+ JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", this));
+ }
+
+ IteratorType result = end();
+
+ switch (m_data.m_type)
+ {
+ case value_t::boolean:
+ case value_t::number_float:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::string:
+ case value_t::binary:
+ {
+ if (JSON_HEDLEY_UNLIKELY(!pos.m_it.primitive_iterator.is_begin()))
+ {
+ JSON_THROW(invalid_iterator::create(205, "iterator out of range", this));
+ }
+
+ if (is_string())
+ {
+ AllocatorType<string_t> alloc;
+ std::allocator_traits<decltype(alloc)>::destroy(alloc, m_data.m_value.string);
+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_data.m_value.string, 1);
+ m_data.m_value.string = nullptr;
+ }
+ else if (is_binary())
+ {
+ AllocatorType<binary_t> alloc;
+ std::allocator_traits<decltype(alloc)>::destroy(alloc, m_data.m_value.binary);
+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_data.m_value.binary, 1);
+ m_data.m_value.binary = nullptr;
+ }
+
+ m_data.m_type = value_t::null;
+ assert_invariant();
+ break;
+ }
+
+ case value_t::object:
+ {
+ result.m_it.object_iterator = m_data.m_value.object->erase(pos.m_it.object_iterator);
+ break;
+ }
+
+ case value_t::array:
+ {
+ result.m_it.array_iterator = m_data.m_value.array->erase(pos.m_it.array_iterator);
+ break;
+ }
+
+ case value_t::null:
+ case value_t::discarded:
+ default:
+ JSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this));
+ }
+
+ return result;
+ }
+
+ /// @brief remove elements given an iterator range
+ /// @sa https://json.nlohmann.me/api/basic_json/erase/
+ template < class IteratorType, detail::enable_if_t <
+ std::is_same<IteratorType, typename basic_json_t::iterator>::value ||
+ std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int > = 0 >
+ IteratorType erase(IteratorType first, IteratorType last)
+ {
+ // make sure iterator fits the current value
+ if (JSON_HEDLEY_UNLIKELY(this != first.m_object || this != last.m_object))
+ {
+ JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value", this));
+ }
+
+ IteratorType result = end();
+
+ switch (m_data.m_type)
+ {
+ case value_t::boolean:
+ case value_t::number_float:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::string:
+ case value_t::binary:
+ {
+ if (JSON_HEDLEY_LIKELY(!first.m_it.primitive_iterator.is_begin()
+ || !last.m_it.primitive_iterator.is_end()))
+ {
+ JSON_THROW(invalid_iterator::create(204, "iterators out of range", this));
+ }
+
+ if (is_string())
+ {
+ AllocatorType<string_t> alloc;
+ std::allocator_traits<decltype(alloc)>::destroy(alloc, m_data.m_value.string);
+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_data.m_value.string, 1);
+ m_data.m_value.string = nullptr;
+ }
+ else if (is_binary())
+ {
+ AllocatorType<binary_t> alloc;
+ std::allocator_traits<decltype(alloc)>::destroy(alloc, m_data.m_value.binary);
+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_data.m_value.binary, 1);
+ m_data.m_value.binary = nullptr;
+ }
+
+ m_data.m_type = value_t::null;
+ assert_invariant();
+ break;
+ }
+
+ case value_t::object:
+ {
+ result.m_it.object_iterator = m_data.m_value.object->erase(first.m_it.object_iterator,
+ last.m_it.object_iterator);
+ break;
+ }
+
+ case value_t::array:
+ {
+ result.m_it.array_iterator = m_data.m_value.array->erase(first.m_it.array_iterator,
+ last.m_it.array_iterator);
+ break;
+ }
+
+ case value_t::null:
+ case value_t::discarded:
+ default:
+ JSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this));
+ }
+
+ return result;
+ }
+
+ private:
+ template < typename KeyType, detail::enable_if_t <
+ detail::has_erase_with_key_type<basic_json_t, KeyType>::value, int > = 0 >
+ size_type erase_internal(KeyType && key)
+ {
+ // this erase only works for objects
+ if (JSON_HEDLEY_UNLIKELY(!is_object()))
+ {
+ JSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this));
+ }
+
+ return m_data.m_value.object->erase(std::forward<KeyType>(key));
+ }
+
+ template < typename KeyType, detail::enable_if_t <
+ !detail::has_erase_with_key_type<basic_json_t, KeyType>::value, int > = 0 >
+ size_type erase_internal(KeyType && key)
+ {
+ // this erase only works for objects
+ if (JSON_HEDLEY_UNLIKELY(!is_object()))
+ {
+ JSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this));
+ }
+
+ const auto it = m_data.m_value.object->find(std::forward<KeyType>(key));
+ if (it != m_data.m_value.object->end())
+ {
+ m_data.m_value.object->erase(it);
+ return 1;
+ }
+ return 0;
+ }
+
+ public:
+
+ /// @brief remove element from a JSON object given a key
+ /// @sa https://json.nlohmann.me/api/basic_json/erase/
+ size_type erase(const typename object_t::key_type& key)
+ {
+ // the indirection via erase_internal() is added to avoid making this
+ // function a template and thus de-rank it during overload resolution
+ return erase_internal(key);
+ }
+
+ /// @brief remove element from a JSON object given a key
+ /// @sa https://json.nlohmann.me/api/basic_json/erase/
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>
+ size_type erase(KeyType && key)
+ {
+ return erase_internal(std::forward<KeyType>(key));
+ }
+
+ /// @brief remove element from a JSON array given an index
+ /// @sa https://json.nlohmann.me/api/basic_json/erase/
+ void erase(const size_type idx)
+ {
+ // this erase only works for arrays
+ if (JSON_HEDLEY_LIKELY(is_array()))
+ {
+ if (JSON_HEDLEY_UNLIKELY(idx >= size()))
+ {
+ JSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), this));
+ }
+
+ m_data.m_value.array->erase(m_data.m_value.array->begin() + static_cast<difference_type>(idx));
+ }
+ else
+ {
+ JSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this));
+ }
+ }
+
+ /// @}
+
+ ////////////
+ // lookup //
+ ////////////
+
+ /// @name lookup
+ /// @{
+
+ /// @brief find an element in a JSON object
+ /// @sa https://json.nlohmann.me/api/basic_json/find/
+ iterator find(const typename object_t::key_type& key)
+ {
+ auto result = end();
+
+ if (is_object())
+ {
+ result.m_it.object_iterator = m_data.m_value.object->find(key);
+ }
+
+ return result;
+ }
+
+ /// @brief find an element in a JSON object
+ /// @sa https://json.nlohmann.me/api/basic_json/find/
+ const_iterator find(const typename object_t::key_type& key) const
+ {
+ auto result = cend();
+
+ if (is_object())
+ {
+ result.m_it.object_iterator = m_data.m_value.object->find(key);
+ }
+
+ return result;
+ }
+
+ /// @brief find an element in a JSON object
+ /// @sa https://json.nlohmann.me/api/basic_json/find/
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>
+ iterator find(KeyType && key)
+ {
+ auto result = end();
+
+ if (is_object())
+ {
+ result.m_it.object_iterator = m_data.m_value.object->find(std::forward<KeyType>(key));
+ }
+
+ return result;
+ }
+
+ /// @brief find an element in a JSON object
+ /// @sa https://json.nlohmann.me/api/basic_json/find/
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>
+ const_iterator find(KeyType && key) const
+ {
+ auto result = cend();
+
+ if (is_object())
+ {
+ result.m_it.object_iterator = m_data.m_value.object->find(std::forward<KeyType>(key));
+ }
+
+ return result;
+ }
+
+ /// @brief returns the number of occurrences of a key in a JSON object
+ /// @sa https://json.nlohmann.me/api/basic_json/count/
+ size_type count(const typename object_t::key_type& key) const
+ {
+ // return 0 for all nonobject types
+ return is_object() ? m_data.m_value.object->count(key) : 0;
+ }
+
+ /// @brief returns the number of occurrences of a key in a JSON object
+ /// @sa https://json.nlohmann.me/api/basic_json/count/
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>
+ size_type count(KeyType && key) const
+ {
+ // return 0 for all nonobject types
+ return is_object() ? m_data.m_value.object->count(std::forward<KeyType>(key)) : 0;
+ }
+
+ /// @brief check the existence of an element in a JSON object
+ /// @sa https://json.nlohmann.me/api/basic_json/contains/
+ bool contains(const typename object_t::key_type& key) const
+ {
+ return is_object() && m_data.m_value.object->find(key) != m_data.m_value.object->end();
+ }
+
+ /// @brief check the existence of an element in a JSON object
+ /// @sa https://json.nlohmann.me/api/basic_json/contains/
+ template<class KeyType, detail::enable_if_t<
+ detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>
+ bool contains(KeyType && key) const
+ {
+ return is_object() && m_data.m_value.object->find(std::forward<KeyType>(key)) != m_data.m_value.object->end();
+ }
+
+ /// @brief check the existence of an element in a JSON object given a JSON pointer
+ /// @sa https://json.nlohmann.me/api/basic_json/contains/
+ bool contains(const json_pointer& ptr) const
+ {
+ return ptr.contains(this);
+ }
+
+ template<typename BasicJsonType, detail::enable_if_t<detail::is_basic_json<BasicJsonType>::value, int> = 0>
+ JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)
+ bool contains(const typename ::nlohmann::json_pointer<BasicJsonType>& ptr) const
+ {
+ return ptr.contains(this);
+ }
+
+ /// @}
+
+ ///////////////
+ // iterators //
+ ///////////////
+
+ /// @name iterators
+ /// @{
+
+ /// @brief returns an iterator to the first element
+ /// @sa https://json.nlohmann.me/api/basic_json/begin/
+ iterator begin() noexcept
+ {
+ iterator result(this);
+ result.set_begin();
+ return result;
+ }
+
+ /// @brief returns an iterator to the first element
+ /// @sa https://json.nlohmann.me/api/basic_json/begin/
+ const_iterator begin() const noexcept
+ {
+ return cbegin();
+ }
+
+ /// @brief returns a const iterator to the first element
+ /// @sa https://json.nlohmann.me/api/basic_json/cbegin/
+ const_iterator cbegin() const noexcept
+ {
+ const_iterator result(this);
+ result.set_begin();
+ return result;
+ }
+
+ /// @brief returns an iterator to one past the last element
+ /// @sa https://json.nlohmann.me/api/basic_json/end/
+ iterator end() noexcept
+ {
+ iterator result(this);
+ result.set_end();
+ return result;
+ }
+
+ /// @brief returns an iterator to one past the last element
+ /// @sa https://json.nlohmann.me/api/basic_json/end/
+ const_iterator end() const noexcept
+ {
+ return cend();
+ }
+
+ /// @brief returns an iterator to one past the last element
+ /// @sa https://json.nlohmann.me/api/basic_json/cend/
+ const_iterator cend() const noexcept
+ {
+ const_iterator result(this);
+ result.set_end();
+ return result;
+ }
+
+ /// @brief returns an iterator to the reverse-beginning
+ /// @sa https://json.nlohmann.me/api/basic_json/rbegin/
+ reverse_iterator rbegin() noexcept
+ {
+ return reverse_iterator(end());
+ }
+
+ /// @brief returns an iterator to the reverse-beginning
+ /// @sa https://json.nlohmann.me/api/basic_json/rbegin/
+ const_reverse_iterator rbegin() const noexcept
+ {
+ return crbegin();
+ }
+
+ /// @brief returns an iterator to the reverse-end
+ /// @sa https://json.nlohmann.me/api/basic_json/rend/
+ reverse_iterator rend() noexcept
+ {
+ return reverse_iterator(begin());
+ }
+
+ /// @brief returns an iterator to the reverse-end
+ /// @sa https://json.nlohmann.me/api/basic_json/rend/
+ const_reverse_iterator rend() const noexcept
+ {
+ return crend();
+ }
+
+ /// @brief returns a const reverse iterator to the last element
+ /// @sa https://json.nlohmann.me/api/basic_json/crbegin/
+ const_reverse_iterator crbegin() const noexcept
+ {
+ return const_reverse_iterator(cend());
+ }
+
+ /// @brief returns a const reverse iterator to one before the first
+ /// @sa https://json.nlohmann.me/api/basic_json/crend/
+ const_reverse_iterator crend() const noexcept
+ {
+ return const_reverse_iterator(cbegin());
+ }
+
+ public:
+ /// @brief wrapper to access iterator member functions in range-based for
+ /// @sa https://json.nlohmann.me/api/basic_json/items/
+ /// @deprecated This function is deprecated since 3.1.0 and will be removed in
+ /// version 4.0.0 of the library. Please use @ref items() instead;
+ /// that is, replace `json::iterator_wrapper(j)` with `j.items()`.
+ JSON_HEDLEY_DEPRECATED_FOR(3.1.0, items())
+ static iteration_proxy<iterator> iterator_wrapper(reference ref) noexcept
+ {
+ return ref.items();
+ }
+
+ /// @brief wrapper to access iterator member functions in range-based for
+ /// @sa https://json.nlohmann.me/api/basic_json/items/
+ /// @deprecated This function is deprecated since 3.1.0 and will be removed in
+ /// version 4.0.0 of the library. Please use @ref items() instead;
+ /// that is, replace `json::iterator_wrapper(j)` with `j.items()`.
+ JSON_HEDLEY_DEPRECATED_FOR(3.1.0, items())
+ static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) noexcept
+ {
+ return ref.items();
+ }
+
+ /// @brief helper to access iterator member functions in range-based for
+ /// @sa https://json.nlohmann.me/api/basic_json/items/
+ iteration_proxy<iterator> items() noexcept
+ {
+ return iteration_proxy<iterator>(*this);
+ }
+
+ /// @brief helper to access iterator member functions in range-based for
+ /// @sa https://json.nlohmann.me/api/basic_json/items/
+ iteration_proxy<const_iterator> items() const noexcept
+ {
+ return iteration_proxy<const_iterator>(*this);
+ }
+
+ /// @}
+
+ //////////////
+ // capacity //
+ //////////////
+
+ /// @name capacity
+ /// @{
+
+ /// @brief checks whether the container is empty.
+ /// @sa https://json.nlohmann.me/api/basic_json/empty/
+ bool empty() const noexcept
+ {
+ switch (m_data.m_type)
+ {
+ case value_t::null:
+ {
+ // null values are empty
+ return true;
+ }
+
+ case value_t::array:
+ {
+ // delegate call to array_t::empty()
+ return m_data.m_value.array->empty();
+ }
+
+ case value_t::object:
+ {
+ // delegate call to object_t::empty()
+ return m_data.m_value.object->empty();
+ }
+
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ // all other types are nonempty
+ return false;
+ }
+ }
+ }
+
+ /// @brief returns the number of elements
+ /// @sa https://json.nlohmann.me/api/basic_json/size/
+ size_type size() const noexcept
+ {
+ switch (m_data.m_type)
+ {
+ case value_t::null:
+ {
+ // null values are empty
+ return 0;
+ }
+
+ case value_t::array:
+ {
+ // delegate call to array_t::size()
+ return m_data.m_value.array->size();
+ }
+
+ case value_t::object:
+ {
+ // delegate call to object_t::size()
+ return m_data.m_value.object->size();
+ }
+
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ // all other types have size 1
+ return 1;
+ }
+ }
+ }
+
+ /// @brief returns the maximum possible number of elements
+ /// @sa https://json.nlohmann.me/api/basic_json/max_size/
+ size_type max_size() const noexcept
+ {
+ switch (m_data.m_type)
+ {
+ case value_t::array:
+ {
+ // delegate call to array_t::max_size()
+ return m_data.m_value.array->max_size();
+ }
+
+ case value_t::object:
+ {
+ // delegate call to object_t::max_size()
+ return m_data.m_value.object->max_size();
+ }
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ // all other types have max_size() == size()
+ return size();
+ }
+ }
+ }
+
+ /// @}
+
+ ///////////////
+ // modifiers //
+ ///////////////
+
+ /// @name modifiers
+ /// @{
+
+ /// @brief clears the contents
+ /// @sa https://json.nlohmann.me/api/basic_json/clear/
+ void clear() noexcept
+ {
+ switch (m_data.m_type)
+ {
+ case value_t::number_integer:
+ {
+ m_data.m_value.number_integer = 0;
+ break;
+ }
+
+ case value_t::number_unsigned:
+ {
+ m_data.m_value.number_unsigned = 0;
+ break;
+ }
+
+ case value_t::number_float:
+ {
+ m_data.m_value.number_float = 0.0;
+ break;
+ }
+
+ case value_t::boolean:
+ {
+ m_data.m_value.boolean = false;
+ break;
+ }
+
+ case value_t::string:
+ {
+ m_data.m_value.string->clear();
+ break;
+ }
+
+ case value_t::binary:
+ {
+ m_data.m_value.binary->clear();
+ break;
+ }
+
+ case value_t::array:
+ {
+ m_data.m_value.array->clear();
+ break;
+ }
+
+ case value_t::object:
+ {
+ m_data.m_value.object->clear();
+ break;
+ }
+
+ case value_t::null:
+ case value_t::discarded:
+ default:
+ break;
+ }
+ }
+
+ /// @brief add an object to an array
+ /// @sa https://json.nlohmann.me/api/basic_json/push_back/
+ void push_back(basic_json&& val)
+ {
+ // push_back only works for null objects or arrays
+ if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array())))
+ {
+ JSON_THROW(type_error::create(308, detail::concat("cannot use push_back() with ", type_name()), this));
+ }
+
+ // transform null object into an array
+ if (is_null())
+ {
+ m_data.m_type = value_t::array;
+ m_data.m_value = value_t::array;
+ assert_invariant();
+ }
+
+ // add element to array (move semantics)
+ const auto old_capacity = m_data.m_value.array->capacity();
+ m_data.m_value.array->push_back(std::move(val));
+ set_parent(m_data.m_value.array->back(), old_capacity);
+ // if val is moved from, basic_json move constructor marks it null, so we do not call the destructor
+ }
+
+ /// @brief add an object to an array
+ /// @sa https://json.nlohmann.me/api/basic_json/operator+=/
+ reference operator+=(basic_json&& val)
+ {
+ push_back(std::move(val));
+ return *this;
+ }
+
+ /// @brief add an object to an array
+ /// @sa https://json.nlohmann.me/api/basic_json/push_back/
+ void push_back(const basic_json& val)
+ {
+ // push_back only works for null objects or arrays
+ if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array())))
+ {
+ JSON_THROW(type_error::create(308, detail::concat("cannot use push_back() with ", type_name()), this));
+ }
+
+ // transform null object into an array
+ if (is_null())
+ {
+ m_data.m_type = value_t::array;
+ m_data.m_value = value_t::array;
+ assert_invariant();
+ }
+
+ // add element to array
+ const auto old_capacity = m_data.m_value.array->capacity();
+ m_data.m_value.array->push_back(val);
+ set_parent(m_data.m_value.array->back(), old_capacity);
+ }
+
+ /// @brief add an object to an array
+ /// @sa https://json.nlohmann.me/api/basic_json/operator+=/
+ reference operator+=(const basic_json& val)
+ {
+ push_back(val);
+ return *this;
+ }
+
+ /// @brief add an object to an object
+ /// @sa https://json.nlohmann.me/api/basic_json/push_back/
+ void push_back(const typename object_t::value_type& val)
+ {
+ // push_back only works for null objects or objects
+ if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_object())))
+ {
+ JSON_THROW(type_error::create(308, detail::concat("cannot use push_back() with ", type_name()), this));
+ }
+
+ // transform null object into an object
+ if (is_null())
+ {
+ m_data.m_type = value_t::object;
+ m_data.m_value = value_t::object;
+ assert_invariant();
+ }
+
+ // add element to object
+ auto res = m_data.m_value.object->insert(val);
+ set_parent(res.first->second);
+ }
+
+ /// @brief add an object to an object
+ /// @sa https://json.nlohmann.me/api/basic_json/operator+=/
+ reference operator+=(const typename object_t::value_type& val)
+ {
+ push_back(val);
+ return *this;
+ }
+
+ /// @brief add an object to an object
+ /// @sa https://json.nlohmann.me/api/basic_json/push_back/
+ void push_back(initializer_list_t init)
+ {
+ if (is_object() && init.size() == 2 && (*init.begin())->is_string())
+ {
+ basic_json&& key = init.begin()->moved_or_copied();
+ push_back(typename object_t::value_type(
+ std::move(key.get_ref<string_t&>()), (init.begin() + 1)->moved_or_copied()));
+ }
+ else
+ {
+ push_back(basic_json(init));
+ }
+ }
+
+ /// @brief add an object to an object
+ /// @sa https://json.nlohmann.me/api/basic_json/operator+=/
+ reference operator+=(initializer_list_t init)
+ {
+ push_back(init);
+ return *this;
+ }
+
+ /// @brief add an object to an array
+ /// @sa https://json.nlohmann.me/api/basic_json/emplace_back/
+ template<class... Args>
+ reference emplace_back(Args&& ... args)
+ {
+ // emplace_back only works for null objects or arrays
+ if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array())))
+ {
+ JSON_THROW(type_error::create(311, detail::concat("cannot use emplace_back() with ", type_name()), this));
+ }
+
+ // transform null object into an array
+ if (is_null())
+ {
+ m_data.m_type = value_t::array;
+ m_data.m_value = value_t::array;
+ assert_invariant();
+ }
+
+ // add element to array (perfect forwarding)
+ const auto old_capacity = m_data.m_value.array->capacity();
+ m_data.m_value.array->emplace_back(std::forward<Args>(args)...);
+ return set_parent(m_data.m_value.array->back(), old_capacity);
+ }
+
+ /// @brief add an object to an object if key does not exist
+ /// @sa https://json.nlohmann.me/api/basic_json/emplace/
+ template<class... Args>
+ std::pair<iterator, bool> emplace(Args&& ... args)
+ {
+ // emplace only works for null objects or arrays
+ if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_object())))
+ {
+ JSON_THROW(type_error::create(311, detail::concat("cannot use emplace() with ", type_name()), this));
+ }
+
+ // transform null object into an object
+ if (is_null())
+ {
+ m_data.m_type = value_t::object;
+ m_data.m_value = value_t::object;
+ assert_invariant();
+ }
+
+ // add element to array (perfect forwarding)
+ auto res = m_data.m_value.object->emplace(std::forward<Args>(args)...);
+ set_parent(res.first->second);
+
+ // create result iterator and set iterator to the result of emplace
+ auto it = begin();
+ it.m_it.object_iterator = res.first;
+
+ // return pair of iterator and boolean
+ return {it, res.second};
+ }
+
+ /// Helper for insertion of an iterator
+ /// @note: This uses std::distance to support GCC 4.8,
+ /// see https://github.com/nlohmann/json/pull/1257
+ template<typename... Args>
+ iterator insert_iterator(const_iterator pos, Args&& ... args)
+ {
+ iterator result(this);
+ JSON_ASSERT(m_data.m_value.array != nullptr);
+
+ auto insert_pos = std::distance(m_data.m_value.array->begin(), pos.m_it.array_iterator);
+ m_data.m_value.array->insert(pos.m_it.array_iterator, std::forward<Args>(args)...);
+ result.m_it.array_iterator = m_data.m_value.array->begin() + insert_pos;
+
+ // This could have been written as:
+ // result.m_it.array_iterator = m_data.m_value.array->insert(pos.m_it.array_iterator, cnt, val);
+ // but the return value of insert is missing in GCC 4.8, so it is written this way instead.
+
+ set_parents();
+ return result;
+ }
+
+ /// @brief inserts element into array
+ /// @sa https://json.nlohmann.me/api/basic_json/insert/
+ iterator insert(const_iterator pos, const basic_json& val)
+ {
+ // insert only works for arrays
+ if (JSON_HEDLEY_LIKELY(is_array()))
+ {
+ // check if iterator pos fits to this JSON value
+ if (JSON_HEDLEY_UNLIKELY(pos.m_object != this))
+ {
+ JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", this));
+ }
+
+ // insert to array and return iterator
+ return insert_iterator(pos, val);
+ }
+
+ JSON_THROW(type_error::create(309, detail::concat("cannot use insert() with ", type_name()), this));
+ }
+
+ /// @brief inserts element into array
+ /// @sa https://json.nlohmann.me/api/basic_json/insert/
+ iterator insert(const_iterator pos, basic_json&& val)
+ {
+ return insert(pos, val);
+ }
+
+ /// @brief inserts copies of element into array
+ /// @sa https://json.nlohmann.me/api/basic_json/insert/
+ iterator insert(const_iterator pos, size_type cnt, const basic_json& val)
+ {
+ // insert only works for arrays
+ if (JSON_HEDLEY_LIKELY(is_array()))
+ {
+ // check if iterator pos fits to this JSON value
+ if (JSON_HEDLEY_UNLIKELY(pos.m_object != this))
+ {
+ JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", this));
+ }
+
+ // insert to array and return iterator
+ return insert_iterator(pos, cnt, val);
+ }
+
+ JSON_THROW(type_error::create(309, detail::concat("cannot use insert() with ", type_name()), this));
+ }
+
+ /// @brief inserts range of elements into array
+ /// @sa https://json.nlohmann.me/api/basic_json/insert/
+ iterator insert(const_iterator pos, const_iterator first, const_iterator last)
+ {
+ // insert only works for arrays
+ if (JSON_HEDLEY_UNLIKELY(!is_array()))
+ {
+ JSON_THROW(type_error::create(309, detail::concat("cannot use insert() with ", type_name()), this));
+ }
+
+ // check if iterator pos fits to this JSON value
+ if (JSON_HEDLEY_UNLIKELY(pos.m_object != this))
+ {
+ JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", this));
+ }
+
+ // check if range iterators belong to the same JSON object
+ if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object))
+ {
+ JSON_THROW(invalid_iterator::create(210, "iterators do not fit", this));
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(first.m_object == this))
+ {
+ JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container", this));
+ }
+
+ // insert to array and return iterator
+ return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator);
+ }
+
+ /// @brief inserts elements from initializer list into array
+ /// @sa https://json.nlohmann.me/api/basic_json/insert/
+ iterator insert(const_iterator pos, initializer_list_t ilist)
+ {
+ // insert only works for arrays
+ if (JSON_HEDLEY_UNLIKELY(!is_array()))
+ {
+ JSON_THROW(type_error::create(309, detail::concat("cannot use insert() with ", type_name()), this));
+ }
+
+ // check if iterator pos fits to this JSON value
+ if (JSON_HEDLEY_UNLIKELY(pos.m_object != this))
+ {
+ JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", this));
+ }
+
+ // insert to array and return iterator
+ return insert_iterator(pos, ilist.begin(), ilist.end());
+ }
+
+ /// @brief inserts range of elements into object
+ /// @sa https://json.nlohmann.me/api/basic_json/insert/
+ void insert(const_iterator first, const_iterator last)
+ {
+ // insert only works for objects
+ if (JSON_HEDLEY_UNLIKELY(!is_object()))
+ {
+ JSON_THROW(type_error::create(309, detail::concat("cannot use insert() with ", type_name()), this));
+ }
+
+ // check if range iterators belong to the same JSON object
+ if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object))
+ {
+ JSON_THROW(invalid_iterator::create(210, "iterators do not fit", this));
+ }
+
+ // passed iterators must belong to objects
+ if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object()))
+ {
+ JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects", this));
+ }
+
+ m_data.m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator);
+ }
+
+ /// @brief updates a JSON object from another object, overwriting existing keys
+ /// @sa https://json.nlohmann.me/api/basic_json/update/
+ void update(const_reference j, bool merge_objects = false)
+ {
+ update(j.begin(), j.end(), merge_objects);
+ }
+
+ /// @brief updates a JSON object from another object, overwriting existing keys
+ /// @sa https://json.nlohmann.me/api/basic_json/update/
+ void update(const_iterator first, const_iterator last, bool merge_objects = false)
+ {
+ // implicitly convert null value to an empty object
+ if (is_null())
+ {
+ m_data.m_type = value_t::object;
+ m_data.m_value.object = create<object_t>();
+ assert_invariant();
+ }
+
+ if (JSON_HEDLEY_UNLIKELY(!is_object()))
+ {
+ JSON_THROW(type_error::create(312, detail::concat("cannot use update() with ", type_name()), this));
+ }
+
+ // check if range iterators belong to the same JSON object
+ if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object))
+ {
+ JSON_THROW(invalid_iterator::create(210, "iterators do not fit", this));
+ }
+
+ // passed iterators must belong to objects
+ if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object()))
+ {
+ JSON_THROW(type_error::create(312, detail::concat("cannot use update() with ", first.m_object->type_name()), first.m_object));
+ }
+
+ for (auto it = first; it != last; ++it)
+ {
+ if (merge_objects && it.value().is_object())
+ {
+ auto it2 = m_data.m_value.object->find(it.key());
+ if (it2 != m_data.m_value.object->end())
+ {
+ it2->second.update(it.value(), true);
+ continue;
+ }
+ }
+ m_data.m_value.object->operator[](it.key()) = it.value();
+#if JSON_DIAGNOSTICS
+ m_data.m_value.object->operator[](it.key()).m_parent = this;
+#endif
+ }
+ }
+
+ /// @brief exchanges the values
+ /// @sa https://json.nlohmann.me/api/basic_json/swap/
+ void swap(reference other) noexcept (
+ std::is_nothrow_move_constructible<value_t>::value&&
+ std::is_nothrow_move_assignable<value_t>::value&&
+ std::is_nothrow_move_constructible<json_value>::value&& // NOLINT(cppcoreguidelines-noexcept-swap,performance-noexcept-swap)
+ std::is_nothrow_move_assignable<json_value>::value
+ )
+ {
+ std::swap(m_data.m_type, other.m_data.m_type);
+ std::swap(m_data.m_value, other.m_data.m_value);
+
+ set_parents();
+ other.set_parents();
+ assert_invariant();
+ }
+
+ /// @brief exchanges the values
+ /// @sa https://json.nlohmann.me/api/basic_json/swap/
+ friend void swap(reference left, reference right) noexcept (
+ std::is_nothrow_move_constructible<value_t>::value&&
+ std::is_nothrow_move_assignable<value_t>::value&&
+ std::is_nothrow_move_constructible<json_value>::value&& // NOLINT(cppcoreguidelines-noexcept-swap,performance-noexcept-swap)
+ std::is_nothrow_move_assignable<json_value>::value
+ )
+ {
+ left.swap(right);
+ }
+
+ /// @brief exchanges the values
+ /// @sa https://json.nlohmann.me/api/basic_json/swap/
+ void swap(array_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap)
+ {
+ // swap only works for arrays
+ if (JSON_HEDLEY_LIKELY(is_array()))
+ {
+ using std::swap;
+ swap(*(m_data.m_value.array), other);
+ }
+ else
+ {
+ JSON_THROW(type_error::create(310, detail::concat("cannot use swap(array_t&) with ", type_name()), this));
+ }
+ }
+
+ /// @brief exchanges the values
+ /// @sa https://json.nlohmann.me/api/basic_json/swap/
+ void swap(object_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap)
+ {
+ // swap only works for objects
+ if (JSON_HEDLEY_LIKELY(is_object()))
+ {
+ using std::swap;
+ swap(*(m_data.m_value.object), other);
+ }
+ else
+ {
+ JSON_THROW(type_error::create(310, detail::concat("cannot use swap(object_t&) with ", type_name()), this));
+ }
+ }
+
+ /// @brief exchanges the values
+ /// @sa https://json.nlohmann.me/api/basic_json/swap/
+ void swap(string_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap)
+ {
+ // swap only works for strings
+ if (JSON_HEDLEY_LIKELY(is_string()))
+ {
+ using std::swap;
+ swap(*(m_data.m_value.string), other);
+ }
+ else
+ {
+ JSON_THROW(type_error::create(310, detail::concat("cannot use swap(string_t&) with ", type_name()), this));
+ }
+ }
+
+ /// @brief exchanges the values
+ /// @sa https://json.nlohmann.me/api/basic_json/swap/
+ void swap(binary_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap)
+ {
+ // swap only works for strings
+ if (JSON_HEDLEY_LIKELY(is_binary()))
+ {
+ using std::swap;
+ swap(*(m_data.m_value.binary), other);
+ }
+ else
+ {
+ JSON_THROW(type_error::create(310, detail::concat("cannot use swap(binary_t&) with ", type_name()), this));
+ }
+ }
+
+ /// @brief exchanges the values
+ /// @sa https://json.nlohmann.me/api/basic_json/swap/
+ void swap(typename binary_t::container_type& other) // NOLINT(bugprone-exception-escape)
+ {
+ // swap only works for strings
+ if (JSON_HEDLEY_LIKELY(is_binary()))
+ {
+ using std::swap;
+ swap(*(m_data.m_value.binary), other);
+ }
+ else
+ {
+ JSON_THROW(type_error::create(310, detail::concat("cannot use swap(binary_t::container_type&) with ", type_name()), this));
+ }
+ }
+
+ /// @}
+
+ //////////////////////////////////////////
+ // lexicographical comparison operators //
+ //////////////////////////////////////////
+
+ /// @name lexicographical comparison operators
+ /// @{
+
+ // note parentheses around operands are necessary; see
+ // https://github.com/nlohmann/json/issues/1530
+#define JSON_IMPLEMENT_OPERATOR(op, null_result, unordered_result, default_result) \
+ const auto lhs_type = lhs.type(); \
+ const auto rhs_type = rhs.type(); \
+ \
+ if (lhs_type == rhs_type) /* NOLINT(readability/braces) */ \
+ { \
+ switch (lhs_type) \
+ { \
+ case value_t::array: \
+ return (*lhs.m_data.m_value.array) op (*rhs.m_data.m_value.array); \
+ \
+ case value_t::object: \
+ return (*lhs.m_data.m_value.object) op (*rhs.m_data.m_value.object); \
+ \
+ case value_t::null: \
+ return (null_result); \
+ \
+ case value_t::string: \
+ return (*lhs.m_data.m_value.string) op (*rhs.m_data.m_value.string); \
+ \
+ case value_t::boolean: \
+ return (lhs.m_data.m_value.boolean) op (rhs.m_data.m_value.boolean); \
+ \
+ case value_t::number_integer: \
+ return (lhs.m_data.m_value.number_integer) op (rhs.m_data.m_value.number_integer); \
+ \
+ case value_t::number_unsigned: \
+ return (lhs.m_data.m_value.number_unsigned) op (rhs.m_data.m_value.number_unsigned); \
+ \
+ case value_t::number_float: \
+ return (lhs.m_data.m_value.number_float) op (rhs.m_data.m_value.number_float); \
+ \
+ case value_t::binary: \
+ return (*lhs.m_data.m_value.binary) op (*rhs.m_data.m_value.binary); \
+ \
+ case value_t::discarded: \
+ default: \
+ return (unordered_result); \
+ } \
+ } \
+ else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_float) \
+ { \
+ return static_cast<number_float_t>(lhs.m_data.m_value.number_integer) op rhs.m_data.m_value.number_float; \
+ } \
+ else if (lhs_type == value_t::number_float && rhs_type == value_t::number_integer) \
+ { \
+ return lhs.m_data.m_value.number_float op static_cast<number_float_t>(rhs.m_data.m_value.number_integer); \
+ } \
+ else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_float) \
+ { \
+ return static_cast<number_float_t>(lhs.m_data.m_value.number_unsigned) op rhs.m_data.m_value.number_float; \
+ } \
+ else if (lhs_type == value_t::number_float && rhs_type == value_t::number_unsigned) \
+ { \
+ return lhs.m_data.m_value.number_float op static_cast<number_float_t>(rhs.m_data.m_value.number_unsigned); \
+ } \
+ else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_integer) \
+ { \
+ return static_cast<number_integer_t>(lhs.m_data.m_value.number_unsigned) op rhs.m_data.m_value.number_integer; \
+ } \
+ else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_unsigned) \
+ { \
+ return lhs.m_data.m_value.number_integer op static_cast<number_integer_t>(rhs.m_data.m_value.number_unsigned); \
+ } \
+ else if(compares_unordered(lhs, rhs))\
+ {\
+ return (unordered_result);\
+ }\
+ \
+ return (default_result);
+
+ JSON_PRIVATE_UNLESS_TESTED:
+ // returns true if:
+ // - any operand is NaN and the other operand is of number type
+ // - any operand is discarded
+ // in legacy mode, discarded values are considered ordered if
+ // an operation is computed as an odd number of inverses of others
+ static bool compares_unordered(const_reference lhs, const_reference rhs, bool inverse = false) noexcept
+ {
+ if ((lhs.is_number_float() && std::isnan(lhs.m_data.m_value.number_float) && rhs.is_number())
+ || (rhs.is_number_float() && std::isnan(rhs.m_data.m_value.number_float) && lhs.is_number()))
+ {
+ return true;
+ }
+#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
+ return (lhs.is_discarded() || rhs.is_discarded()) && !inverse;
+#else
+ static_cast<void>(inverse);
+ return lhs.is_discarded() || rhs.is_discarded();
+#endif
+ }
+
+ private:
+ bool compares_unordered(const_reference rhs, bool inverse = false) const noexcept
+ {
+ return compares_unordered(*this, rhs, inverse);
+ }
+
+ public:
+#if JSON_HAS_THREE_WAY_COMPARISON
+ /// @brief comparison: equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/
+ bool operator==(const_reference rhs) const noexcept
+ {
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wfloat-equal"
+#endif
+ const_reference lhs = *this;
+ JSON_IMPLEMENT_OPERATOR( ==, true, false, false)
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+ }
+
+ /// @brief comparison: equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/
+ template<typename ScalarType>
+ requires std::is_scalar_v<ScalarType>
+ bool operator==(ScalarType rhs) const noexcept
+ {
+ return *this == basic_json(rhs);
+ }
+
+ /// @brief comparison: not equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/
+ bool operator!=(const_reference rhs) const noexcept
+ {
+ if (compares_unordered(rhs, true))
+ {
+ return false;
+ }
+ return !operator==(rhs);
+ }
+
+ /// @brief comparison: 3-way
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_spaceship/
+ std::partial_ordering operator<=>(const_reference rhs) const noexcept // *NOPAD*
+ {
+ const_reference lhs = *this;
+ // default_result is used if we cannot compare values. In that case,
+ // we compare types.
+ JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD*
+ std::partial_ordering::equivalent,
+ std::partial_ordering::unordered,
+ lhs_type <=> rhs_type) // *NOPAD*
+ }
+
+ /// @brief comparison: 3-way
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_spaceship/
+ template<typename ScalarType>
+ requires std::is_scalar_v<ScalarType>
+ std::partial_ordering operator<=>(ScalarType rhs) const noexcept // *NOPAD*
+ {
+ return *this <=> basic_json(rhs); // *NOPAD*
+ }
+
+#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
+ // all operators that are computed as an odd number of inverses of others
+ // need to be overloaded to emulate the legacy comparison behavior
+
+ /// @brief comparison: less than or equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_le/
+ JSON_HEDLEY_DEPRECATED_FOR(3.11.0, undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON)
+ bool operator<=(const_reference rhs) const noexcept
+ {
+ if (compares_unordered(rhs, true))
+ {
+ return false;
+ }
+ return !(rhs < *this);
+ }
+
+ /// @brief comparison: less than or equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_le/
+ template<typename ScalarType>
+ requires std::is_scalar_v<ScalarType>
+ bool operator<=(ScalarType rhs) const noexcept
+ {
+ return *this <= basic_json(rhs);
+ }
+
+ /// @brief comparison: greater than or equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/
+ JSON_HEDLEY_DEPRECATED_FOR(3.11.0, undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON)
+ bool operator>=(const_reference rhs) const noexcept
+ {
+ if (compares_unordered(rhs, true))
+ {
+ return false;
+ }
+ return !(*this < rhs);
+ }
+
+ /// @brief comparison: greater than or equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/
+ template<typename ScalarType>
+ requires std::is_scalar_v<ScalarType>
+ bool operator>=(ScalarType rhs) const noexcept
+ {
+ return *this >= basic_json(rhs);
+ }
+#endif
+#else
+ /// @brief comparison: equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/
+ friend bool operator==(const_reference lhs, const_reference rhs) noexcept
+ {
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wfloat-equal"
+#endif
+ JSON_IMPLEMENT_OPERATOR( ==, true, false, false)
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+ }
+
+ /// @brief comparison: equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator==(const_reference lhs, ScalarType rhs) noexcept
+ {
+ return lhs == basic_json(rhs);
+ }
+
+ /// @brief comparison: equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator==(ScalarType lhs, const_reference rhs) noexcept
+ {
+ return basic_json(lhs) == rhs;
+ }
+
+ /// @brief comparison: not equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/
+ friend bool operator!=(const_reference lhs, const_reference rhs) noexcept
+ {
+ if (compares_unordered(lhs, rhs, true))
+ {
+ return false;
+ }
+ return !(lhs == rhs);
+ }
+
+ /// @brief comparison: not equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator!=(const_reference lhs, ScalarType rhs) noexcept
+ {
+ return lhs != basic_json(rhs);
+ }
+
+ /// @brief comparison: not equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator!=(ScalarType lhs, const_reference rhs) noexcept
+ {
+ return basic_json(lhs) != rhs;
+ }
+
+ /// @brief comparison: less than
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/
+ friend bool operator<(const_reference lhs, const_reference rhs) noexcept
+ {
+ // default_result is used if we cannot compare values. In that case,
+ // we compare types. Note we have to call the operator explicitly,
+ // because MSVC has problems otherwise.
+ JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type))
+ }
+
+ /// @brief comparison: less than
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator<(const_reference lhs, ScalarType rhs) noexcept
+ {
+ return lhs < basic_json(rhs);
+ }
+
+ /// @brief comparison: less than
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator<(ScalarType lhs, const_reference rhs) noexcept
+ {
+ return basic_json(lhs) < rhs;
+ }
+
+ /// @brief comparison: less than or equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_le/
+ friend bool operator<=(const_reference lhs, const_reference rhs) noexcept
+ {
+ if (compares_unordered(lhs, rhs, true))
+ {
+ return false;
+ }
+ return !(rhs < lhs);
+ }
+
+ /// @brief comparison: less than or equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_le/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator<=(const_reference lhs, ScalarType rhs) noexcept
+ {
+ return lhs <= basic_json(rhs);
+ }
+
+ /// @brief comparison: less than or equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_le/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept
+ {
+ return basic_json(lhs) <= rhs;
+ }
+
+ /// @brief comparison: greater than
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/
+ friend bool operator>(const_reference lhs, const_reference rhs) noexcept
+ {
+ // double inverse
+ if (compares_unordered(lhs, rhs))
+ {
+ return false;
+ }
+ return !(lhs <= rhs);
+ }
+
+ /// @brief comparison: greater than
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator>(const_reference lhs, ScalarType rhs) noexcept
+ {
+ return lhs > basic_json(rhs);
+ }
+
+ /// @brief comparison: greater than
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator>(ScalarType lhs, const_reference rhs) noexcept
+ {
+ return basic_json(lhs) > rhs;
+ }
+
+ /// @brief comparison: greater than or equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/
+ friend bool operator>=(const_reference lhs, const_reference rhs) noexcept
+ {
+ if (compares_unordered(lhs, rhs, true))
+ {
+ return false;
+ }
+ return !(lhs < rhs);
+ }
+
+ /// @brief comparison: greater than or equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator>=(const_reference lhs, ScalarType rhs) noexcept
+ {
+ return lhs >= basic_json(rhs);
+ }
+
+ /// @brief comparison: greater than or equal
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/
+ template<typename ScalarType, typename std::enable_if<
+ std::is_scalar<ScalarType>::value, int>::type = 0>
+ friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept
+ {
+ return basic_json(lhs) >= rhs;
+ }
+#endif
+
+#undef JSON_IMPLEMENT_OPERATOR
+
+ /// @}
+
+ ///////////////////
+ // serialization //
+ ///////////////////
+
+ /// @name serialization
+ /// @{
+#ifndef JSON_NO_IO
+ /// @brief serialize to stream
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/
+ friend std::ostream& operator<<(std::ostream& o, const basic_json& j)
+ {
+ // read width member and use it as indentation parameter if nonzero
+ const bool pretty_print = o.width() > 0;
+ const auto indentation = pretty_print ? o.width() : 0;
+
+ // reset width to 0 for subsequent calls to this stream
+ o.width(0);
+
+ // do the actual serialization
+ serializer s(detail::output_adapter<char>(o), o.fill());
+ s.dump(j, pretty_print, false, static_cast<unsigned int>(indentation));
+ return o;
+ }
+
+ /// @brief serialize to stream
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/
+ /// @deprecated This function is deprecated since 3.0.0 and will be removed in
+ /// version 4.0.0 of the library. Please use
+ /// operator<<(std::ostream&, const basic_json&) instead; that is,
+ /// replace calls like `j >> o;` with `o << j;`.
+ JSON_HEDLEY_DEPRECATED_FOR(3.0.0, operator<<(std::ostream&, const basic_json&))
+ friend std::ostream& operator>>(const basic_json& j, std::ostream& o)
+ {
+ return o << j;
+ }
+#endif // JSON_NO_IO
+ /// @}
+
+ /////////////////////
+ // deserialization //
+ /////////////////////
+
+ /// @name deserialization
+ /// @{
+
+ /// @brief deserialize from a compatible input
+ /// @sa https://json.nlohmann.me/api/basic_json/parse/
+ template<typename InputType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json parse(InputType&& i,
+ const parser_callback_t cb = nullptr,
+ const bool allow_exceptions = true,
+ const bool ignore_comments = false)
+ {
+ basic_json result;
+ parser(detail::input_adapter(std::forward<InputType>(i)), cb, allow_exceptions, ignore_comments).parse(true, result);
+ return result;
+ }
+
+ /// @brief deserialize from a pair of character iterators
+ /// @sa https://json.nlohmann.me/api/basic_json/parse/
+ template<typename IteratorType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json parse(IteratorType first,
+ IteratorType last,
+ const parser_callback_t cb = nullptr,
+ const bool allow_exceptions = true,
+ const bool ignore_comments = false)
+ {
+ basic_json result;
+ parser(detail::input_adapter(std::move(first), std::move(last)), cb, allow_exceptions, ignore_comments).parse(true, result);
+ return result;
+ }
+
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ JSON_HEDLEY_DEPRECATED_FOR(3.8.0, parse(ptr, ptr + len))
+ static basic_json parse(detail::span_input_adapter&& i,
+ const parser_callback_t cb = nullptr,
+ const bool allow_exceptions = true,
+ const bool ignore_comments = false)
+ {
+ basic_json result;
+ parser(i.get(), cb, allow_exceptions, ignore_comments).parse(true, result);
+ return result;
+ }
+
+ /// @brief check if the input is valid JSON
+ /// @sa https://json.nlohmann.me/api/basic_json/accept/
+ template<typename InputType>
+ static bool accept(InputType&& i,
+ const bool ignore_comments = false)
+ {
+ return parser(detail::input_adapter(std::forward<InputType>(i)), nullptr, false, ignore_comments).accept(true);
+ }
+
+ /// @brief check if the input is valid JSON
+ /// @sa https://json.nlohmann.me/api/basic_json/accept/
+ template<typename IteratorType>
+ static bool accept(IteratorType first, IteratorType last,
+ const bool ignore_comments = false)
+ {
+ return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments).accept(true);
+ }
+
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ JSON_HEDLEY_DEPRECATED_FOR(3.8.0, accept(ptr, ptr + len))
+ static bool accept(detail::span_input_adapter&& i,
+ const bool ignore_comments = false)
+ {
+ return parser(i.get(), nullptr, false, ignore_comments).accept(true);
+ }
+
+ /// @brief generate SAX events
+ /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/
+ template <typename InputType, typename SAX>
+ JSON_HEDLEY_NON_NULL(2)
+ static bool sax_parse(InputType&& i, SAX* sax,
+ input_format_t format = input_format_t::json,
+ const bool strict = true,
+ const bool ignore_comments = false)
+ {
+ auto ia = detail::input_adapter(std::forward<InputType>(i));
+ return format == input_format_t::json
+ ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict)
+ : detail::binary_reader<basic_json, decltype(ia), SAX>(std::move(ia), format).sax_parse(format, sax, strict);
+ }
+
+ /// @brief generate SAX events
+ /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/
+ template<class IteratorType, class SAX>
+ JSON_HEDLEY_NON_NULL(3)
+ static bool sax_parse(IteratorType first, IteratorType last, SAX* sax,
+ input_format_t format = input_format_t::json,
+ const bool strict = true,
+ const bool ignore_comments = false)
+ {
+ auto ia = detail::input_adapter(std::move(first), std::move(last));
+ return format == input_format_t::json
+ ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict)
+ : detail::binary_reader<basic_json, decltype(ia), SAX>(std::move(ia), format).sax_parse(format, sax, strict);
+ }
+
+ /// @brief generate SAX events
+ /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/
+ /// @deprecated This function is deprecated since 3.8.0 and will be removed in
+ /// version 4.0.0 of the library. Please use
+ /// sax_parse(ptr, ptr + len) instead.
+ template <typename SAX>
+ JSON_HEDLEY_DEPRECATED_FOR(3.8.0, sax_parse(ptr, ptr + len, ...))
+ JSON_HEDLEY_NON_NULL(2)
+ static bool sax_parse(detail::span_input_adapter&& i, SAX* sax,
+ input_format_t format = input_format_t::json,
+ const bool strict = true,
+ const bool ignore_comments = false)
+ {
+ auto ia = i.get();
+ return format == input_format_t::json
+ // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
+ ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict)
+ // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
+ : detail::binary_reader<basic_json, decltype(ia), SAX>(std::move(ia), format).sax_parse(format, sax, strict);
+ }
+#ifndef JSON_NO_IO
+ /// @brief deserialize from stream
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/
+ /// @deprecated This stream operator is deprecated since 3.0.0 and will be removed in
+ /// version 4.0.0 of the library. Please use
+ /// operator>>(std::istream&, basic_json&) instead; that is,
+ /// replace calls like `j << i;` with `i >> j;`.
+ JSON_HEDLEY_DEPRECATED_FOR(3.0.0, operator>>(std::istream&, basic_json&))
+ friend std::istream& operator<<(basic_json& j, std::istream& i)
+ {
+ return operator>>(i, j);
+ }
+
+ /// @brief deserialize from stream
+ /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/
+ friend std::istream& operator>>(std::istream& i, basic_json& j)
+ {
+ parser(detail::input_adapter(i)).parse(false, j);
+ return i;
+ }
+#endif // JSON_NO_IO
+ /// @}
+
+ ///////////////////////////
+ // convenience functions //
+ ///////////////////////////
+
+ /// @brief return the type as string
+ /// @sa https://json.nlohmann.me/api/basic_json/type_name/
+ JSON_HEDLEY_RETURNS_NON_NULL
+ const char* type_name() const noexcept
+ {
+ switch (m_data.m_type)
+ {
+ case value_t::null:
+ return "null";
+ case value_t::object:
+ return "object";
+ case value_t::array:
+ return "array";
+ case value_t::string:
+ return "string";
+ case value_t::boolean:
+ return "boolean";
+ case value_t::binary:
+ return "binary";
+ case value_t::discarded:
+ return "discarded";
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ default:
+ return "number";
+ }
+ }
+
+ JSON_PRIVATE_UNLESS_TESTED:
+ //////////////////////
+ // member variables //
+ //////////////////////
+
+ struct data
+ {
+ /// the type of the current element
+ value_t m_type = value_t::null;
+
+ /// the value of the current element
+ json_value m_value = {};
+
+ data(const value_t v)
+ : m_type(v), m_value(v)
+ {
+ }
+
+ data(size_type cnt, const basic_json& val)
+ : m_type(value_t::array)
+ {
+ m_value.array = create<array_t>(cnt, val);
+ }
+
+ data() noexcept = default;
+ data(data&&) noexcept = default;
+ data(const data&) noexcept = delete;
+ data& operator=(data&&) noexcept = delete;
+ data& operator=(const data&) noexcept = delete;
+
+ ~data() noexcept
+ {
+ m_value.destroy(m_type);
+ }
+ };
+
+ data m_data = {};
+
+#if JSON_DIAGNOSTICS
+ /// a pointer to a parent value (for debugging purposes)
+ basic_json* m_parent = nullptr;
+#endif
+
+ //////////////////////////////////////////
+ // binary serialization/deserialization //
+ //////////////////////////////////////////
+
+ /// @name binary serialization/deserialization support
+ /// @{
+
+ public:
+ /// @brief create a CBOR serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/
+ static std::vector<std::uint8_t> to_cbor(const basic_json& j)
+ {
+ std::vector<std::uint8_t> result;
+ to_cbor(j, result);
+ return result;
+ }
+
+ /// @brief create a CBOR serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/
+ static void to_cbor(const basic_json& j, detail::output_adapter<std::uint8_t> o)
+ {
+ binary_writer<std::uint8_t>(o).write_cbor(j);
+ }
+
+ /// @brief create a CBOR serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/
+ static void to_cbor(const basic_json& j, detail::output_adapter<char> o)
+ {
+ binary_writer<char>(o).write_cbor(j);
+ }
+
+ /// @brief create a MessagePack serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/
+ static std::vector<std::uint8_t> to_msgpack(const basic_json& j)
+ {
+ std::vector<std::uint8_t> result;
+ to_msgpack(j, result);
+ return result;
+ }
+
+ /// @brief create a MessagePack serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/
+ static void to_msgpack(const basic_json& j, detail::output_adapter<std::uint8_t> o)
+ {
+ binary_writer<std::uint8_t>(o).write_msgpack(j);
+ }
+
+ /// @brief create a MessagePack serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/
+ static void to_msgpack(const basic_json& j, detail::output_adapter<char> o)
+ {
+ binary_writer<char>(o).write_msgpack(j);
+ }
+
+ /// @brief create a UBJSON serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/
+ static std::vector<std::uint8_t> to_ubjson(const basic_json& j,
+ const bool use_size = false,
+ const bool use_type = false)
+ {
+ std::vector<std::uint8_t> result;
+ to_ubjson(j, result, use_size, use_type);
+ return result;
+ }
+
+ /// @brief create a UBJSON serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/
+ static void to_ubjson(const basic_json& j, detail::output_adapter<std::uint8_t> o,
+ const bool use_size = false, const bool use_type = false)
+ {
+ binary_writer<std::uint8_t>(o).write_ubjson(j, use_size, use_type);
+ }
+
+ /// @brief create a UBJSON serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/
+ static void to_ubjson(const basic_json& j, detail::output_adapter<char> o,
+ const bool use_size = false, const bool use_type = false)
+ {
+ binary_writer<char>(o).write_ubjson(j, use_size, use_type);
+ }
+
+ /// @brief create a BJData serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/
+ static std::vector<std::uint8_t> to_bjdata(const basic_json& j,
+ const bool use_size = false,
+ const bool use_type = false)
+ {
+ std::vector<std::uint8_t> result;
+ to_bjdata(j, result, use_size, use_type);
+ return result;
+ }
+
+ /// @brief create a BJData serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/
+ static void to_bjdata(const basic_json& j, detail::output_adapter<std::uint8_t> o,
+ const bool use_size = false, const bool use_type = false)
+ {
+ binary_writer<std::uint8_t>(o).write_ubjson(j, use_size, use_type, true, true);
+ }
+
+ /// @brief create a BJData serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/
+ static void to_bjdata(const basic_json& j, detail::output_adapter<char> o,
+ const bool use_size = false, const bool use_type = false)
+ {
+ binary_writer<char>(o).write_ubjson(j, use_size, use_type, true, true);
+ }
+
+ /// @brief create a BSON serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_bson/
+ static std::vector<std::uint8_t> to_bson(const basic_json& j)
+ {
+ std::vector<std::uint8_t> result;
+ to_bson(j, result);
+ return result;
+ }
+
+ /// @brief create a BSON serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_bson/
+ static void to_bson(const basic_json& j, detail::output_adapter<std::uint8_t> o)
+ {
+ binary_writer<std::uint8_t>(o).write_bson(j);
+ }
+
+ /// @brief create a BSON serialization of a given JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/to_bson/
+ static void to_bson(const basic_json& j, detail::output_adapter<char> o)
+ {
+ binary_writer<char>(o).write_bson(j);
+ }
+
+ /// @brief create a JSON value from an input in CBOR format
+ /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/
+ template<typename InputType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json from_cbor(InputType&& i,
+ const bool strict = true,
+ const bool allow_exceptions = true,
+ const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = detail::input_adapter(std::forward<InputType>(i));
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ /// @brief create a JSON value from an input in CBOR format
+ /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/
+ template<typename IteratorType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json from_cbor(IteratorType first, IteratorType last,
+ const bool strict = true,
+ const bool allow_exceptions = true,
+ const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = detail::input_adapter(std::move(first), std::move(last));
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ template<typename T>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len))
+ static basic_json from_cbor(const T* ptr, std::size_t len,
+ const bool strict = true,
+ const bool allow_exceptions = true,
+ const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)
+ {
+ return from_cbor(ptr, ptr + len, strict, allow_exceptions, tag_handler);
+ }
+
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len))
+ static basic_json from_cbor(detail::span_input_adapter&& i,
+ const bool strict = true,
+ const bool allow_exceptions = true,
+ const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = i.get();
+ // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ /// @brief create a JSON value from an input in MessagePack format
+ /// @sa https://json.nlohmann.me/api/basic_json/from_msgpack/
+ template<typename InputType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json from_msgpack(InputType&& i,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = detail::input_adapter(std::forward<InputType>(i));
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ /// @brief create a JSON value from an input in MessagePack format
+ /// @sa https://json.nlohmann.me/api/basic_json/from_msgpack/
+ template<typename IteratorType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json from_msgpack(IteratorType first, IteratorType last,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = detail::input_adapter(std::move(first), std::move(last));
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ template<typename T>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_msgpack(ptr, ptr + len))
+ static basic_json from_msgpack(const T* ptr, std::size_t len,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ return from_msgpack(ptr, ptr + len, strict, allow_exceptions);
+ }
+
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_msgpack(ptr, ptr + len))
+ static basic_json from_msgpack(detail::span_input_adapter&& i,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = i.get();
+ // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ /// @brief create a JSON value from an input in UBJSON format
+ /// @sa https://json.nlohmann.me/api/basic_json/from_ubjson/
+ template<typename InputType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json from_ubjson(InputType&& i,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = detail::input_adapter(std::forward<InputType>(i));
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ /// @brief create a JSON value from an input in UBJSON format
+ /// @sa https://json.nlohmann.me/api/basic_json/from_ubjson/
+ template<typename IteratorType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json from_ubjson(IteratorType first, IteratorType last,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = detail::input_adapter(std::move(first), std::move(last));
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ template<typename T>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_ubjson(ptr, ptr + len))
+ static basic_json from_ubjson(const T* ptr, std::size_t len,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ return from_ubjson(ptr, ptr + len, strict, allow_exceptions);
+ }
+
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_ubjson(ptr, ptr + len))
+ static basic_json from_ubjson(detail::span_input_adapter&& i,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = i.get();
+ // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ /// @brief create a JSON value from an input in BJData format
+ /// @sa https://json.nlohmann.me/api/basic_json/from_bjdata/
+ template<typename InputType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json from_bjdata(InputType&& i,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = detail::input_adapter(std::forward<InputType>(i));
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ /// @brief create a JSON value from an input in BJData format
+ /// @sa https://json.nlohmann.me/api/basic_json/from_bjdata/
+ template<typename IteratorType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json from_bjdata(IteratorType first, IteratorType last,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = detail::input_adapter(std::move(first), std::move(last));
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ /// @brief create a JSON value from an input in BSON format
+ /// @sa https://json.nlohmann.me/api/basic_json/from_bson/
+ template<typename InputType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json from_bson(InputType&& i,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = detail::input_adapter(std::forward<InputType>(i));
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ /// @brief create a JSON value from an input in BSON format
+ /// @sa https://json.nlohmann.me/api/basic_json/from_bson/
+ template<typename IteratorType>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json from_bson(IteratorType first, IteratorType last,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = detail::input_adapter(std::move(first), std::move(last));
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict);
+ return res ? result : basic_json(value_t::discarded);
+ }
+
+ template<typename T>
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_bson(ptr, ptr + len))
+ static basic_json from_bson(const T* ptr, std::size_t len,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ return from_bson(ptr, ptr + len, strict, allow_exceptions);
+ }
+
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_bson(ptr, ptr + len))
+ static basic_json from_bson(detail::span_input_adapter&& i,
+ const bool strict = true,
+ const bool allow_exceptions = true)
+ {
+ basic_json result;
+ detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+ auto ia = i.get();
+ // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
+ const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict);
+ return res ? result : basic_json(value_t::discarded);
+ }
+ /// @}
+
+ //////////////////////////
+ // JSON Pointer support //
+ //////////////////////////
+
+ /// @name JSON Pointer functions
+ /// @{
+
+ /// @brief access specified element via JSON Pointer
+ /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/
+ reference operator[](const json_pointer& ptr)
+ {
+ return ptr.get_unchecked(this);
+ }
+
+ template<typename BasicJsonType, detail::enable_if_t<detail::is_basic_json<BasicJsonType>::value, int> = 0>
+ JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)
+ reference operator[](const ::nlohmann::json_pointer<BasicJsonType>& ptr)
+ {
+ return ptr.get_unchecked(this);
+ }
+
+ /// @brief access specified element via JSON Pointer
+ /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/
+ const_reference operator[](const json_pointer& ptr) const
+ {
+ return ptr.get_unchecked(this);
+ }
+
+ template<typename BasicJsonType, detail::enable_if_t<detail::is_basic_json<BasicJsonType>::value, int> = 0>
+ JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)
+ const_reference operator[](const ::nlohmann::json_pointer<BasicJsonType>& ptr) const
+ {
+ return ptr.get_unchecked(this);
+ }
+
+ /// @brief access specified element via JSON Pointer
+ /// @sa https://json.nlohmann.me/api/basic_json/at/
+ reference at(const json_pointer& ptr)
+ {
+ return ptr.get_checked(this);
+ }
+
+ template<typename BasicJsonType, detail::enable_if_t<detail::is_basic_json<BasicJsonType>::value, int> = 0>
+ JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)
+ reference at(const ::nlohmann::json_pointer<BasicJsonType>& ptr)
+ {
+ return ptr.get_checked(this);
+ }
+
+ /// @brief access specified element via JSON Pointer
+ /// @sa https://json.nlohmann.me/api/basic_json/at/
+ const_reference at(const json_pointer& ptr) const
+ {
+ return ptr.get_checked(this);
+ }
+
+ template<typename BasicJsonType, detail::enable_if_t<detail::is_basic_json<BasicJsonType>::value, int> = 0>
+ JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)
+ const_reference at(const ::nlohmann::json_pointer<BasicJsonType>& ptr) const
+ {
+ return ptr.get_checked(this);
+ }
+
+ /// @brief return flattened JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/flatten/
+ basic_json flatten() const
+ {
+ basic_json result(value_t::object);
+ json_pointer::flatten("", *this, result);
+ return result;
+ }
+
+ /// @brief unflatten a previously flattened JSON value
+ /// @sa https://json.nlohmann.me/api/basic_json/unflatten/
+ basic_json unflatten() const
+ {
+ return json_pointer::unflatten(*this);
+ }
+
+ /// @}
+
+ //////////////////////////
+ // JSON Patch functions //
+ //////////////////////////
+
+ /// @name JSON Patch functions
+ /// @{
+
+ /// @brief applies a JSON patch in-place without copying the object
+ /// @sa https://json.nlohmann.me/api/basic_json/patch/
+ void patch_inplace(const basic_json& json_patch)
+ {
+ basic_json& result = *this;
+ // the valid JSON Patch operations
+ enum class patch_operations {add, remove, replace, move, copy, test, invalid};
+
+ const auto get_op = [](const std::string & op)
+ {
+ if (op == "add")
+ {
+ return patch_operations::add;
+ }
+ if (op == "remove")
+ {
+ return patch_operations::remove;
+ }
+ if (op == "replace")
+ {
+ return patch_operations::replace;
+ }
+ if (op == "move")
+ {
+ return patch_operations::move;
+ }
+ if (op == "copy")
+ {
+ return patch_operations::copy;
+ }
+ if (op == "test")
+ {
+ return patch_operations::test;
+ }
+
+ return patch_operations::invalid;
+ };
+
+ // wrapper for "add" operation; add value at ptr
+ const auto operation_add = [&result](json_pointer & ptr, basic_json val)
+ {
+ // adding to the root of the target document means replacing it
+ if (ptr.empty())
+ {
+ result = val;
+ return;
+ }
+
+ // make sure the top element of the pointer exists
+ json_pointer const top_pointer = ptr.top();
+ if (top_pointer != ptr)
+ {
+ result.at(top_pointer);
+ }
+
+ // get reference to parent of JSON pointer ptr
+ const auto last_path = ptr.back();
+ ptr.pop_back();
+ // parent must exist when performing patch add per RFC6902 specs
+ basic_json& parent = result.at(ptr);
+
+ switch (parent.m_data.m_type)
+ {
+ case value_t::null:
+ case value_t::object:
+ {
+ // use operator[] to add value
+ parent[last_path] = val;
+ break;
+ }
+
+ case value_t::array:
+ {
+ if (last_path == "-")
+ {
+ // special case: append to back
+ parent.push_back(val);
+ }
+ else
+ {
+ const auto idx = json_pointer::template array_index<basic_json_t>(last_path);
+ if (JSON_HEDLEY_UNLIKELY(idx > parent.size()))
+ {
+ // avoid undefined behavior
+ JSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), &parent));
+ }
+
+ // default case: insert add offset
+ parent.insert(parent.begin() + static_cast<difference_type>(idx), val);
+ }
+ break;
+ }
+
+ // if there exists a parent it cannot be primitive
+ case value_t::string: // LCOV_EXCL_LINE
+ case value_t::boolean: // LCOV_EXCL_LINE
+ case value_t::number_integer: // LCOV_EXCL_LINE
+ case value_t::number_unsigned: // LCOV_EXCL_LINE
+ case value_t::number_float: // LCOV_EXCL_LINE
+ case value_t::binary: // LCOV_EXCL_LINE
+ case value_t::discarded: // LCOV_EXCL_LINE
+ default: // LCOV_EXCL_LINE
+ JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
+ }
+ };
+
+ // wrapper for "remove" operation; remove value at ptr
+ const auto operation_remove = [this, & result](json_pointer & ptr)
+ {
+ // get reference to parent of JSON pointer ptr
+ const auto last_path = ptr.back();
+ ptr.pop_back();
+ basic_json& parent = result.at(ptr);
+
+ // remove child
+ if (parent.is_object())
+ {
+ // perform range check
+ auto it = parent.find(last_path);
+ if (JSON_HEDLEY_LIKELY(it != parent.end()))
+ {
+ parent.erase(it);
+ }
+ else
+ {
+ JSON_THROW(out_of_range::create(403, detail::concat("key '", last_path, "' not found"), this));
+ }
+ }
+ else if (parent.is_array())
+ {
+ // note erase performs range check
+ parent.erase(json_pointer::template array_index<basic_json_t>(last_path));
+ }
+ };
+
+ // type check: top level value must be an array
+ if (JSON_HEDLEY_UNLIKELY(!json_patch.is_array()))
+ {
+ JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects", &json_patch));
+ }
+
+ // iterate and apply the operations
+ for (const auto& val : json_patch)
+ {
+ // wrapper to get a value for an operation
+ const auto get_value = [&val](const std::string & op,
+ const std::string & member,
+ bool string_type) -> basic_json &
+ {
+ // find value
+ auto it = val.m_data.m_value.object->find(member);
+
+ // context-sensitive error message
+ const auto error_msg = (op == "op") ? "operation" : detail::concat("operation '", op, '\''); // NOLINT(bugprone-unused-local-non-trivial-variable)
+
+ // check if desired value is present
+ if (JSON_HEDLEY_UNLIKELY(it == val.m_data.m_value.object->end()))
+ {
+ // NOLINTNEXTLINE(performance-inefficient-string-concatenation)
+ JSON_THROW(parse_error::create(105, 0, detail::concat(error_msg, " must have member '", member, "'"), &val));
+ }
+
+ // check if result is of type string
+ if (JSON_HEDLEY_UNLIKELY(string_type && !it->second.is_string()))
+ {
+ // NOLINTNEXTLINE(performance-inefficient-string-concatenation)
+ JSON_THROW(parse_error::create(105, 0, detail::concat(error_msg, " must have string member '", member, "'"), &val));
+ }
+
+ // no error: return value
+ return it->second;
+ };
+
+ // type check: every element of the array must be an object
+ if (JSON_HEDLEY_UNLIKELY(!val.is_object()))
+ {
+ JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects", &val));
+ }
+
+ // collect mandatory members
+ const auto op = get_value("op", "op", true).template get<std::string>();
+ const auto path = get_value(op, "path", true).template get<std::string>();
+ json_pointer ptr(path);
+
+ switch (get_op(op))
+ {
+ case patch_operations::add:
+ {
+ operation_add(ptr, get_value("add", "value", false));
+ break;
+ }
+
+ case patch_operations::remove:
+ {
+ operation_remove(ptr);
+ break;
+ }
+
+ case patch_operations::replace:
+ {
+ // the "path" location must exist - use at()
+ result.at(ptr) = get_value("replace", "value", false);
+ break;
+ }
+
+ case patch_operations::move:
+ {
+ const auto from_path = get_value("move", "from", true).template get<std::string>();
+ json_pointer from_ptr(from_path);
+
+ // the "from" location must exist - use at()
+ basic_json const v = result.at(from_ptr);
+
+ // The move operation is functionally identical to a
+ // "remove" operation on the "from" location, followed
+ // immediately by an "add" operation at the target
+ // location with the value that was just removed.
+ operation_remove(from_ptr);
+ operation_add(ptr, v);
+ break;
+ }
+
+ case patch_operations::copy:
+ {
+ const auto from_path = get_value("copy", "from", true).template get<std::string>();
+ const json_pointer from_ptr(from_path);
+
+ // the "from" location must exist - use at()
+ basic_json const v = result.at(from_ptr);
+
+ // The copy is functionally identical to an "add"
+ // operation at the target location using the value
+ // specified in the "from" member.
+ operation_add(ptr, v);
+ break;
+ }
+
+ case patch_operations::test:
+ {
+ bool success = false;
+ JSON_TRY
+ {
+ // check if "value" matches the one at "path"
+ // the "path" location must exist - use at()
+ success = (result.at(ptr) == get_value("test", "value", false));
+ }
+ JSON_INTERNAL_CATCH (out_of_range&)
+ {
+ // ignore out of range errors: success remains false
+ }
+
+ // throw an exception if test fails
+ if (JSON_HEDLEY_UNLIKELY(!success))
+ {
+ JSON_THROW(other_error::create(501, detail::concat("unsuccessful: ", val.dump()), &val));
+ }
+
+ break;
+ }
+
+ case patch_operations::invalid:
+ default:
+ {
+ // op must be "add", "remove", "replace", "move", "copy", or
+ // "test"
+ JSON_THROW(parse_error::create(105, 0, detail::concat("operation value '", op, "' is invalid"), &val));
+ }
+ }
+ }
+ }
+
+ /// @brief applies a JSON patch to a copy of the current object
+ /// @sa https://json.nlohmann.me/api/basic_json/patch/
+ basic_json patch(const basic_json& json_patch) const
+ {
+ basic_json result = *this;
+ result.patch_inplace(json_patch);
+ return result;
+ }
+
+ /// @brief creates a diff as a JSON patch
+ /// @sa https://json.nlohmann.me/api/basic_json/diff/
+ JSON_HEDLEY_WARN_UNUSED_RESULT
+ static basic_json diff(const basic_json& source, const basic_json& target,
+ const std::string& path = "")
+ {
+ // the patch
+ basic_json result(value_t::array);
+
+ // if the values are the same, return empty patch
+ if (source == target)
+ {
+ return result;
+ }
+
+ if (source.type() != target.type())
+ {
+ // different types: replace value
+ result.push_back(
+ {
+ {"op", "replace"}, {"path", path}, {"value", target}
+ });
+ return result;
+ }
+
+ switch (source.type())
+ {
+ case value_t::array:
+ {
+ // first pass: traverse common elements
+ std::size_t i = 0;
+ while (i < source.size() && i < target.size())
+ {
+ // recursive call to compare array values at index i
+ auto temp_diff = diff(source[i], target[i], detail::concat(path, '/', std::to_string(i)));
+ result.insert(result.end(), temp_diff.begin(), temp_diff.end());
+ ++i;
+ }
+
+ // We now reached the end of at least one array
+ // in a second pass, traverse the remaining elements
+
+ // remove my remaining elements
+ const auto end_index = static_cast<difference_type>(result.size());
+ while (i < source.size())
+ {
+ // add operations in reverse order to avoid invalid
+ // indices
+ result.insert(result.begin() + end_index, object(
+ {
+ {"op", "remove"},
+ {"path", detail::concat(path, '/', std::to_string(i))}
+ }));
+ ++i;
+ }
+
+ // add other remaining elements
+ while (i < target.size())
+ {
+ result.push_back(
+ {
+ {"op", "add"},
+ {"path", detail::concat(path, "/-")},
+ {"value", target[i]}
+ });
+ ++i;
+ }
+
+ break;
+ }
+
+ case value_t::object:
+ {
+ // first pass: traverse this object's elements
+ for (auto it = source.cbegin(); it != source.cend(); ++it)
+ {
+ // escape the key name to be used in a JSON patch
+ const auto path_key = detail::concat(path, '/', detail::escape(it.key()));
+
+ if (target.find(it.key()) != target.end())
+ {
+ // recursive call to compare object values at key it
+ auto temp_diff = diff(it.value(), target[it.key()], path_key);
+ result.insert(result.end(), temp_diff.begin(), temp_diff.end());
+ }
+ else
+ {
+ // found a key that is not in o -> remove it
+ result.push_back(object(
+ {
+ {"op", "remove"}, {"path", path_key}
+ }));
+ }
+ }
+
+ // second pass: traverse other object's elements
+ for (auto it = target.cbegin(); it != target.cend(); ++it)
+ {
+ if (source.find(it.key()) == source.end())
+ {
+ // found a key that is not in this -> add it
+ const auto path_key = detail::concat(path, '/', detail::escape(it.key()));
+ result.push_back(
+ {
+ {"op", "add"}, {"path", path_key},
+ {"value", it.value()}
+ });
+ }
+ }
+
+ break;
+ }
+
+ case value_t::null:
+ case value_t::string:
+ case value_t::boolean:
+ case value_t::number_integer:
+ case value_t::number_unsigned:
+ case value_t::number_float:
+ case value_t::binary:
+ case value_t::discarded:
+ default:
+ {
+ // both primitive type: replace value
+ result.push_back(
+ {
+ {"op", "replace"}, {"path", path}, {"value", target}
+ });
+ break;
+ }
+ }
+
+ return result;
+ }
+ /// @}
+
+ ////////////////////////////////
+ // JSON Merge Patch functions //
+ ////////////////////////////////
+
+ /// @name JSON Merge Patch functions
+ /// @{
+
+ /// @brief applies a JSON Merge Patch
+ /// @sa https://json.nlohmann.me/api/basic_json/merge_patch/
+ void merge_patch(const basic_json& apply_patch)
+ {
+ if (apply_patch.is_object())
+ {
+ if (!is_object())
+ {
+ *this = object();
+ }
+ for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it)
+ {
+ if (it.value().is_null())
+ {
+ erase(it.key());
+ }
+ else
+ {
+ operator[](it.key()).merge_patch(it.value());
+ }
+ }
+ }
+ else
+ {
+ *this = apply_patch;
+ }
+ }
+
+ /// @}
+};
+
+/// @brief user-defined to_string function for JSON values
+/// @sa https://json.nlohmann.me/api/basic_json/to_string/
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+std::string to_string(const NLOHMANN_BASIC_JSON_TPL& j)
+{
+ return j.dump();
+}
+
+inline namespace literals
+{
+inline namespace json_literals
+{
+
+/// @brief user-defined string literal for JSON values
+/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json/
+JSON_HEDLEY_NON_NULL(1)
+#if !defined(JSON_HEDLEY_GCC_VERSION) || JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0)
+ inline nlohmann::json operator ""_json(const char* s, std::size_t n)
+#else
+ inline nlohmann::json operator "" _json(const char* s, std::size_t n)
+#endif
+{
+ return nlohmann::json::parse(s, s + n);
+}
+
+/// @brief user-defined string literal for JSON pointer
+/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json_pointer/
+JSON_HEDLEY_NON_NULL(1)
+#if !defined(JSON_HEDLEY_GCC_VERSION) || JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0)
+ inline nlohmann::json::json_pointer operator ""_json_pointer(const char* s, std::size_t n)
+#else
+ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n)
+#endif
+{
+ return nlohmann::json::json_pointer(std::string(s, n));
+}
+
+} // namespace json_literals
+} // namespace literals
+NLOHMANN_JSON_NAMESPACE_END
+
+///////////////////////
+// nonmember support //
+///////////////////////
+
+namespace std // NOLINT(cert-dcl58-cpp)
+{
+
+/// @brief hash value for JSON objects
+/// @sa https://json.nlohmann.me/api/basic_json/std_hash/
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+struct hash<nlohmann::NLOHMANN_BASIC_JSON_TPL> // NOLINT(cert-dcl58-cpp)
+{
+ std::size_t operator()(const nlohmann::NLOHMANN_BASIC_JSON_TPL& j) const
+ {
+ return nlohmann::detail::hash(j);
+ }
+};
+
+// specialization for std::less<value_t>
+template<>
+struct less< ::nlohmann::detail::value_t> // do not remove the space after '<', see https://github.com/nlohmann/json/pull/679
+{
+ /*!
+ @brief compare two value_t enum values
+ @since version 3.0.0
+ */
+ bool operator()(::nlohmann::detail::value_t lhs,
+ ::nlohmann::detail::value_t rhs) const noexcept
+ {
+#if JSON_HAS_THREE_WAY_COMPARISON
+ return std::is_lt(lhs <=> rhs); // *NOPAD*
+#else
+ return ::nlohmann::detail::operator<(lhs, rhs);
+#endif
+ }
+};
+
+// C++20 prohibit function specialization in the std namespace.
+#ifndef JSON_HAS_CPP_20
+
+/// @brief exchanges the values of two JSON objects
+/// @sa https://json.nlohmann.me/api/basic_json/std_swap/
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC_JSON_TPL& j2) noexcept( // NOLINT(readability-inconsistent-declaration-parameter-name, cert-dcl58-cpp)
+ is_nothrow_move_constructible<nlohmann::NLOHMANN_BASIC_JSON_TPL>::value&& // NOLINT(misc-redundant-expression,cppcoreguidelines-noexcept-swap,performance-noexcept-swap)
+ is_nothrow_move_assignable<nlohmann::NLOHMANN_BASIC_JSON_TPL>::value)
+{
+ j1.swap(j2);
+}
+
+#endif
+
+} // namespace std
+
+#if JSON_USE_GLOBAL_UDLS
+ #if !defined(JSON_HEDLEY_GCC_VERSION) || JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0)
+ using nlohmann::literals::json_literals::operator ""_json; // NOLINT(misc-unused-using-decls,google-global-names-in-headers)
+ using nlohmann::literals::json_literals::operator ""_json_pointer; //NOLINT(misc-unused-using-decls,google-global-names-in-headers)
+ #else
+ using nlohmann::literals::json_literals::operator "" _json; // NOLINT(misc-unused-using-decls,google-global-names-in-headers)
+ using nlohmann::literals::json_literals::operator "" _json_pointer; //NOLINT(misc-unused-using-decls,google-global-names-in-headers)
+ #endif
+#endif
+
+// #include <nlohmann/detail/macro_unscope.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+// restore clang diagnostic settings
+#if defined(__clang__)
+ #pragma clang diagnostic pop
+#endif
+
+// clean up
+#undef JSON_ASSERT
+#undef JSON_INTERNAL_CATCH
+#undef JSON_THROW
+#undef JSON_PRIVATE_UNLESS_TESTED
+#undef NLOHMANN_BASIC_JSON_TPL_DECLARATION
+#undef NLOHMANN_BASIC_JSON_TPL
+#undef JSON_EXPLICIT
+#undef NLOHMANN_CAN_CALL_STD_FUNC_IMPL
+#undef JSON_INLINE_VARIABLE
+#undef JSON_NO_UNIQUE_ADDRESS
+#undef JSON_DISABLE_ENUM_SERIALIZATION
+#undef JSON_USE_GLOBAL_UDLS
+
+#ifndef JSON_TEST_KEEP_MACROS
+ #undef JSON_CATCH
+ #undef JSON_TRY
+ #undef JSON_HAS_CPP_11
+ #undef JSON_HAS_CPP_14
+ #undef JSON_HAS_CPP_17
+ #undef JSON_HAS_CPP_20
+ #undef JSON_HAS_FILESYSTEM
+ #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM
+ #undef JSON_HAS_THREE_WAY_COMPARISON
+ #undef JSON_HAS_RANGES
+ #undef JSON_HAS_STATIC_RTTI
+ #undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
+#endif
+
+// #include <nlohmann/thirdparty/hedley/hedley_undef.hpp>
+// __ _____ _____ _____
+// __| | __| | | | JSON for Modern C++
+// | | |__ | | | | | | version 3.11.3
+// |_____|_____|_____|_|___| https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+#undef JSON_HEDLEY_ALWAYS_INLINE
+#undef JSON_HEDLEY_ARM_VERSION
+#undef JSON_HEDLEY_ARM_VERSION_CHECK
+#undef JSON_HEDLEY_ARRAY_PARAM
+#undef JSON_HEDLEY_ASSUME
+#undef JSON_HEDLEY_BEGIN_C_DECLS
+#undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE
+#undef JSON_HEDLEY_CLANG_HAS_BUILTIN
+#undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE
+#undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE
+#undef JSON_HEDLEY_CLANG_HAS_EXTENSION
+#undef JSON_HEDLEY_CLANG_HAS_FEATURE
+#undef JSON_HEDLEY_CLANG_HAS_WARNING
+#undef JSON_HEDLEY_COMPCERT_VERSION
+#undef JSON_HEDLEY_COMPCERT_VERSION_CHECK
+#undef JSON_HEDLEY_CONCAT
+#undef JSON_HEDLEY_CONCAT3
+#undef JSON_HEDLEY_CONCAT3_EX
+#undef JSON_HEDLEY_CONCAT_EX
+#undef JSON_HEDLEY_CONST
+#undef JSON_HEDLEY_CONSTEXPR
+#undef JSON_HEDLEY_CONST_CAST
+#undef JSON_HEDLEY_CPP_CAST
+#undef JSON_HEDLEY_CRAY_VERSION
+#undef JSON_HEDLEY_CRAY_VERSION_CHECK
+#undef JSON_HEDLEY_C_DECL
+#undef JSON_HEDLEY_DEPRECATED
+#undef JSON_HEDLEY_DEPRECATED_FOR
+#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL
+#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_
+#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED
+#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES
+#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS
+#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION
+#undef JSON_HEDLEY_DIAGNOSTIC_POP
+#undef JSON_HEDLEY_DIAGNOSTIC_PUSH
+#undef JSON_HEDLEY_DMC_VERSION
+#undef JSON_HEDLEY_DMC_VERSION_CHECK
+#undef JSON_HEDLEY_EMPTY_BASES
+#undef JSON_HEDLEY_EMSCRIPTEN_VERSION
+#undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK
+#undef JSON_HEDLEY_END_C_DECLS
+#undef JSON_HEDLEY_FLAGS
+#undef JSON_HEDLEY_FLAGS_CAST
+#undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE
+#undef JSON_HEDLEY_GCC_HAS_BUILTIN
+#undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE
+#undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE
+#undef JSON_HEDLEY_GCC_HAS_EXTENSION
+#undef JSON_HEDLEY_GCC_HAS_FEATURE
+#undef JSON_HEDLEY_GCC_HAS_WARNING
+#undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK
+#undef JSON_HEDLEY_GCC_VERSION
+#undef JSON_HEDLEY_GCC_VERSION_CHECK
+#undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE
+#undef JSON_HEDLEY_GNUC_HAS_BUILTIN
+#undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE
+#undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE
+#undef JSON_HEDLEY_GNUC_HAS_EXTENSION
+#undef JSON_HEDLEY_GNUC_HAS_FEATURE
+#undef JSON_HEDLEY_GNUC_HAS_WARNING
+#undef JSON_HEDLEY_GNUC_VERSION
+#undef JSON_HEDLEY_GNUC_VERSION_CHECK
+#undef JSON_HEDLEY_HAS_ATTRIBUTE
+#undef JSON_HEDLEY_HAS_BUILTIN
+#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE
+#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS
+#undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE
+#undef JSON_HEDLEY_HAS_EXTENSION
+#undef JSON_HEDLEY_HAS_FEATURE
+#undef JSON_HEDLEY_HAS_WARNING
+#undef JSON_HEDLEY_IAR_VERSION
+#undef JSON_HEDLEY_IAR_VERSION_CHECK
+#undef JSON_HEDLEY_IBM_VERSION
+#undef JSON_HEDLEY_IBM_VERSION_CHECK
+#undef JSON_HEDLEY_IMPORT
+#undef JSON_HEDLEY_INLINE
+#undef JSON_HEDLEY_INTEL_CL_VERSION
+#undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK
+#undef JSON_HEDLEY_INTEL_VERSION
+#undef JSON_HEDLEY_INTEL_VERSION_CHECK
+#undef JSON_HEDLEY_IS_CONSTANT
+#undef JSON_HEDLEY_IS_CONSTEXPR_
+#undef JSON_HEDLEY_LIKELY
+#undef JSON_HEDLEY_MALLOC
+#undef JSON_HEDLEY_MCST_LCC_VERSION
+#undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK
+#undef JSON_HEDLEY_MESSAGE
+#undef JSON_HEDLEY_MSVC_VERSION
+#undef JSON_HEDLEY_MSVC_VERSION_CHECK
+#undef JSON_HEDLEY_NEVER_INLINE
+#undef JSON_HEDLEY_NON_NULL
+#undef JSON_HEDLEY_NO_ESCAPE
+#undef JSON_HEDLEY_NO_RETURN
+#undef JSON_HEDLEY_NO_THROW
+#undef JSON_HEDLEY_NULL
+#undef JSON_HEDLEY_PELLES_VERSION
+#undef JSON_HEDLEY_PELLES_VERSION_CHECK
+#undef JSON_HEDLEY_PGI_VERSION
+#undef JSON_HEDLEY_PGI_VERSION_CHECK
+#undef JSON_HEDLEY_PREDICT
+#undef JSON_HEDLEY_PRINTF_FORMAT
+#undef JSON_HEDLEY_PRIVATE
+#undef JSON_HEDLEY_PUBLIC
+#undef JSON_HEDLEY_PURE
+#undef JSON_HEDLEY_REINTERPRET_CAST
+#undef JSON_HEDLEY_REQUIRE
+#undef JSON_HEDLEY_REQUIRE_CONSTEXPR
+#undef JSON_HEDLEY_REQUIRE_MSG
+#undef JSON_HEDLEY_RESTRICT
+#undef JSON_HEDLEY_RETURNS_NON_NULL
+#undef JSON_HEDLEY_SENTINEL
+#undef JSON_HEDLEY_STATIC_ASSERT
+#undef JSON_HEDLEY_STATIC_CAST
+#undef JSON_HEDLEY_STRINGIFY
+#undef JSON_HEDLEY_STRINGIFY_EX
+#undef JSON_HEDLEY_SUNPRO_VERSION
+#undef JSON_HEDLEY_SUNPRO_VERSION_CHECK
+#undef JSON_HEDLEY_TINYC_VERSION
+#undef JSON_HEDLEY_TINYC_VERSION_CHECK
+#undef JSON_HEDLEY_TI_ARMCL_VERSION
+#undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK
+#undef JSON_HEDLEY_TI_CL2000_VERSION
+#undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK
+#undef JSON_HEDLEY_TI_CL430_VERSION
+#undef JSON_HEDLEY_TI_CL430_VERSION_CHECK
+#undef JSON_HEDLEY_TI_CL6X_VERSION
+#undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK
+#undef JSON_HEDLEY_TI_CL7X_VERSION
+#undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK
+#undef JSON_HEDLEY_TI_CLPRU_VERSION
+#undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK
+#undef JSON_HEDLEY_TI_VERSION
+#undef JSON_HEDLEY_TI_VERSION_CHECK
+#undef JSON_HEDLEY_UNAVAILABLE
+#undef JSON_HEDLEY_UNLIKELY
+#undef JSON_HEDLEY_UNPREDICTABLE
+#undef JSON_HEDLEY_UNREACHABLE
+#undef JSON_HEDLEY_UNREACHABLE_RETURN
+#undef JSON_HEDLEY_VERSION
+#undef JSON_HEDLEY_VERSION_DECODE_MAJOR
+#undef JSON_HEDLEY_VERSION_DECODE_MINOR
+#undef JSON_HEDLEY_VERSION_DECODE_REVISION
+#undef JSON_HEDLEY_VERSION_ENCODE
+#undef JSON_HEDLEY_WARNING
+#undef JSON_HEDLEY_WARN_UNUSED_RESULT
+#undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG
+#undef JSON_HEDLEY_FALL_THROUGH
+
+
+
+#endif // INCLUDE_NLOHMANN_JSON_HPP_
diff --git a/llama/llama-grammar.cpp b/llama/llama-grammar.cpp
new file mode 100644
index 00000000..a106428d
--- /dev/null
+++ b/llama/llama-grammar.cpp
@@ -0,0 +1,565 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "llama-grammar.h"
+
+#include "llama-vocab.h"
+#include "llama-sampling.h"
+
+#include <algorithm>
+
+// Decodes a UTF-8 string which may end in an incomplete sequence. Adds a terminating 0 for use as
+// pointer. If an invalid sequence is encountered, returns `llama_partial_utf8.n_remain == -1`.
+std::pair<std::vector<uint32_t>, llama_partial_utf8> decode_utf8(
+ const std::string & src,
+ llama_partial_utf8 partial_start) {
+ static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 };
+ const char * pos = src.c_str();
+ std::vector<uint32_t> code_points;
+
+ // common english strings have the same number of codepoints and bytes. `+ 1` for the terminating 0.
+ code_points.reserve(src.size() + 1);
+ uint32_t value = partial_start.value;
+ int n_remain = partial_start.n_remain;
+
+ // continue previous decode, if applicable
+ while (*pos != 0 && n_remain > 0) {
+ uint8_t next_byte = static_cast<uint8_t>(*pos);
+ if ((next_byte >> 6) != 2) {
+ // invalid sequence, abort
+ code_points.push_back(0);
+ return std::make_pair(std::move(code_points), llama_partial_utf8{ 0, -1 });
+ }
+ value = (value << 6) + (next_byte & 0x3F);
+ ++pos;
+ --n_remain;
+ }
+
+ if (partial_start.n_remain > 0 && n_remain == 0) {
+ code_points.push_back(value);
+ }
+
+ // decode any subsequent utf-8 sequences, which may end in an incomplete one
+ while (*pos != 0) {
+ uint8_t first_byte = static_cast<uint8_t>(*pos);
+ uint8_t highbits = first_byte >> 4;
+ n_remain = lookup[highbits] - 1;
+
+ if (n_remain < 0) {
+ // invalid sequence, abort
+ code_points.clear();
+ code_points.push_back(0);
+ return std::make_pair(std::move(code_points), llama_partial_utf8{ 0, n_remain });
+ }
+
+ uint8_t mask = (1 << (7 - n_remain)) - 1;
+ value = first_byte & mask;
+
+ ++pos;
+ while (*pos != 0 && n_remain > 0) {
+ value = (value << 6) + (static_cast<uint8_t>(*pos) & 0x3F);
+ ++pos;
+ --n_remain;
+ }
+ if (n_remain == 0) {
+ code_points.push_back(value);
+ }
+ }
+ code_points.push_back(0);
+
+ return std::make_pair(std::move(code_points), llama_partial_utf8{ value, n_remain });
+}
+
+const llama_grammar_rules & llama_grammar_get_rules(const struct llama_grammar * grammar) {
+ return grammar->rules;
+}
+
+llama_grammar_stacks & llama_grammar_get_stacks(struct llama_grammar * grammar) {
+ return grammar->stacks;
+}
+
+// returns true iff pos points to the end of one of the definitions of a rule
+static bool llama_grammar_is_end_of_sequence(const llama_grammar_element * pos) {
+ switch (pos->type) {
+ case LLAMA_GRETYPE_END: return true; // NOLINT
+ case LLAMA_GRETYPE_ALT: return true; // NOLINT
+ default: return false;
+ }
+}
+
+// returns true iff chr satisfies the char range at pos (regular or inverse range)
+// asserts that pos is pointing to a char range element
+static std::pair<bool, const llama_grammar_element *> llama_grammar_match_char(
+ const llama_grammar_element * pos,
+ const uint32_t chr) {
+
+ bool found = false;
+ bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR || pos->type == LLAMA_GRETYPE_CHAR_ANY;
+
+ GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT); // NOLINT
+
+ do {
+ if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {
+ // inclusive range, e.g. [a-z]
+ found = found || (pos->value <= chr && chr <= pos[1].value);
+ pos += 2;
+ } else if (pos->type == LLAMA_GRETYPE_CHAR_ANY) {
+ // Any character matches "."
+ found = true;
+ pos += 1;
+ } else {
+ // exact char match, e.g. [a] or "a"
+ found = found || pos->value == chr;
+ pos += 1;
+ }
+ } while (pos->type == LLAMA_GRETYPE_CHAR_ALT);
+
+ return std::make_pair(found == is_positive_char, pos);
+}
+
+// returns true iff some continuation of the given partial UTF-8 sequence could satisfy the char
+// range at pos (regular or inverse range)
+// asserts that pos is pointing to a char range element
+static bool llama_grammar_match_partial_char(
+ const llama_grammar_element * pos,
+ const llama_partial_utf8 partial_utf8) {
+ bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR || pos->type == LLAMA_GRETYPE_CHAR_ANY;
+ GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT);
+
+ uint32_t partial_value = partial_utf8.value;
+ int n_remain = partial_utf8.n_remain;
+
+ // invalid sequence or 7-bit char split across 2 bytes (overlong)
+ if (n_remain < 0 || (n_remain == 1 && partial_value < 2)) {
+ return false;
+ }
+
+ // range of possible code points this partial UTF-8 sequence could complete to
+ uint32_t low = partial_value << (n_remain * 6);
+ uint32_t high = low | ((1 << (n_remain * 6)) - 1);
+
+ if (low == 0) {
+ if (n_remain == 2) {
+ low = 1 << 11;
+ } else if (n_remain == 3) {
+ low = 1 << 16;
+ }
+ }
+
+ do {
+ if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {
+ // inclusive range, e.g. [a-z]
+ if (pos->value <= high && low <= pos[1].value) {
+ return is_positive_char;
+ }
+ pos += 2;
+ } else if (pos->type == LLAMA_GRETYPE_CHAR_ANY) {
+ // Any character matches "."
+ return true;
+ } else {
+ // exact char match, e.g. [a] or "a"
+ if (low <= pos->value && pos->value <= high) {
+ return is_positive_char;
+ }
+ pos += 1;
+ }
+ } while (pos->type == LLAMA_GRETYPE_CHAR_ALT);
+
+ return !is_positive_char;
+}
+
+// transforms a grammar pushdown stack into N possible stacks, all ending
+// at a character range (terminal element)
+static void llama_grammar_advance_stack(
+ const llama_grammar_rules & rules,
+ const llama_grammar_stack & stack,
+ llama_grammar_stacks & new_stacks) {
+ if (stack.empty()) {
+ if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) {
+ new_stacks.emplace_back(stack);
+ }
+ return;
+ }
+
+ const llama_grammar_element * pos = stack.back();
+
+ switch (pos->type) {
+ case LLAMA_GRETYPE_RULE_REF: {
+ const size_t rule_id = static_cast<size_t>(pos->value);
+ const llama_grammar_element * subpos = rules[rule_id].data();
+ do {
+ // init new stack without the top (pos)
+ llama_grammar_stack new_stack(stack.begin(), stack.end() - 1);
+ if (!llama_grammar_is_end_of_sequence(pos + 1)) {
+ // if this rule ref is followed by another element, add that to stack
+ new_stack.push_back(pos + 1);
+ }
+ if (!llama_grammar_is_end_of_sequence(subpos)) {
+ // if alternate is nonempty, add to stack
+ new_stack.push_back(subpos);
+ }
+ llama_grammar_advance_stack(rules, new_stack, new_stacks);
+ while (!llama_grammar_is_end_of_sequence(subpos)) {
+ // scan to end of alternate def
+ subpos++;
+ }
+ if (subpos->type == LLAMA_GRETYPE_ALT) {
+ // there's another alternate def of this rule to process
+ subpos++;
+ } else {
+ break;
+ }
+ } while (true);
+ break;
+ }
+ case LLAMA_GRETYPE_CHAR:
+ case LLAMA_GRETYPE_CHAR_NOT:
+ case LLAMA_GRETYPE_CHAR_ANY:
+ if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) {
+ // only add the stack if it's not a duplicate of one we already have
+ new_stacks.emplace_back(stack);
+ }
+ break;
+ default:
+ // end of alternate (LLAMA_GRETYPE_END, LLAMA_GRETYPE_ALT) or middle of char range
+ // (LLAMA_GRETYPE_CHAR_ALT, LLAMA_GRETYPE_CHAR_RNG_UPPER); stack should never be left on
+ // those
+ GGML_ABORT("fatal error");
+ }
+}
+
+// takes a set of possible pushdown stacks on a grammar, which are required to
+// be positioned at a character range (see `llama_grammar_advance_stack`), and
+// produces the N possible stacks if the given char is accepted at those
+// positions
+void llama_grammar_accept(
+ const llama_grammar_rules & rules,
+ const llama_grammar_stacks & stacks,
+ const uint32_t chr,
+ llama_grammar_stacks & new_stacks) {
+ new_stacks.clear();
+
+ for (const auto & stack : stacks) {
+ if (stack.empty()) {
+ continue;
+ }
+
+ auto match = llama_grammar_match_char(stack.back(), chr);
+ if (match.first) {
+ const llama_grammar_element * pos = match.second;
+
+ // update top of stack to next element, if any
+ llama_grammar_stack new_stack(stack.begin(), stack.end() - 1);
+ if (!llama_grammar_is_end_of_sequence(pos)) {
+ new_stack.push_back(pos);
+ }
+ llama_grammar_advance_stack(rules, new_stack, new_stacks);
+ }
+ }
+}
+
+static llama_grammar_candidates llama_grammar_reject_candidates(
+ const llama_grammar_rules & rules,
+ const llama_grammar_stacks & stacks,
+ const llama_grammar_candidates & candidates) {
+ GGML_ASSERT(!stacks.empty()); // REVIEW
+
+ if (candidates.empty()) {
+ return {};
+ }
+
+ auto rejects = llama_grammar_reject_candidates_for_stack(rules, stacks.front(), candidates);
+
+ for (size_t i = 1, size = stacks.size(); i < size; ++i) {
+ rejects = llama_grammar_reject_candidates_for_stack(rules, stacks[i], rejects);
+ }
+ return rejects;
+}
+
+llama_grammar_candidates llama_grammar_reject_candidates_for_stack(
+ const llama_grammar_rules & rules,
+ const llama_grammar_stack & stack,
+ const llama_grammar_candidates & candidates) {
+
+ llama_grammar_candidates rejects;
+ rejects.reserve(candidates.size());
+
+ if (stack.empty()) {
+ for (const auto & tok : candidates) {
+ if (*tok.code_points != 0 || tok.partial_utf8.n_remain != 0) {
+ rejects.push_back(tok);
+ }
+ }
+ return rejects;
+ }
+
+ const llama_grammar_element * stack_pos = stack.back();
+
+ llama_grammar_candidates next_candidates;
+ next_candidates.reserve(candidates.size());
+
+ for (const auto & tok : candidates) {
+ if (*tok.code_points == 0) {
+ // reached end of full codepoints in token, reject iff it ended in a partial sequence
+ // that cannot satisfy this position in grammar
+ if (tok.partial_utf8.n_remain != 0 &&
+ !llama_grammar_match_partial_char(stack_pos, tok.partial_utf8)) {
+ rejects.push_back(tok);
+ }
+ } else if (llama_grammar_match_char(stack_pos, *tok.code_points).first) {
+ next_candidates.push_back({ tok.index, tok.code_points + 1, tok.partial_utf8 });
+ } else {
+ rejects.push_back(tok);
+ }
+ }
+
+ const auto * stack_pos_after = llama_grammar_match_char(stack_pos, 0).second;
+
+ // update top of stack to next element, if any
+ llama_grammar_stack stack_after(stack.begin(), stack.end() - 1);
+ if (!llama_grammar_is_end_of_sequence(stack_pos_after)) {
+ stack_after.push_back(stack_pos_after);
+ }
+ llama_grammar_stacks next_stacks;
+ llama_grammar_advance_stack(rules, stack_after, next_stacks);
+
+ auto next_rejects = llama_grammar_reject_candidates(rules, next_stacks, next_candidates);
+ for (const auto & tok : next_rejects) {
+ rejects.push_back({ tok.index, tok.code_points - 1, tok.partial_utf8 });
+ }
+
+ return rejects;
+}
+
+static bool llama_grammar_detect_left_recursion(
+ const llama_grammar_rules & rules,
+ size_t rule_index,
+ std::vector<bool> * rules_visited,
+ std::vector<bool> * rules_in_progress,
+ std::vector<bool> * rules_may_be_empty) {
+ if ((*rules_in_progress)[rule_index]) {
+ return true;
+ }
+
+ (*rules_in_progress)[rule_index] = true;
+
+ const llama_grammar_rule & rule = rules[rule_index];
+
+ // First check if the rule might produce the empty string. This could be done combined with the second
+ // step but it's more readable as two steps.
+ bool at_rule_start = true;
+ for (size_t i = 0; i < rule.size(); i++) {
+ if (llama_grammar_is_end_of_sequence(&rule[i])) {
+ if (at_rule_start) {
+ (*rules_may_be_empty)[rule_index] = true;
+ break;
+ }
+ at_rule_start = true;
+ } else {
+ at_rule_start = false;
+ }
+ }
+
+ // Second, recurse into leftmost nonterminals (or next-leftmost as long as the previous nonterminal may
+ // be empty)
+ bool recurse_into_nonterminal = true;
+ for (size_t i = 0; i < rule.size(); i++) {
+ if (rule[i].type == LLAMA_GRETYPE_RULE_REF && recurse_into_nonterminal) {
+ if (llama_grammar_detect_left_recursion(rules, (size_t)rule[i].value, rules_visited, rules_in_progress, rules_may_be_empty)) {
+ return true;
+ }
+ if (!((*rules_may_be_empty)[(size_t)rule[i].value])) {
+ recurse_into_nonterminal = false;
+ }
+ } else if (llama_grammar_is_end_of_sequence(&rule[i])) {
+ recurse_into_nonterminal = true;
+ } else {
+ recurse_into_nonterminal = false;
+ }
+ }
+
+ (*rules_in_progress)[rule_index] = false;
+ (*rules_visited)[rule_index] = true;
+ return false;
+}
+
+//
+// grammar - external
+//
+
+struct llama_grammar * llama_grammar_init_impl(
+ const llama_grammar_element ** rules,
+ size_t n_rules,
+ size_t start_rule_index) {
+ const llama_grammar_element * pos;
+
+ // copy rule definitions into vectors
+ llama_grammar_rules vec_rules(n_rules);
+ for (size_t i = 0; i < n_rules; i++) {
+ for (pos = rules[i]; pos->type != LLAMA_GRETYPE_END; pos++) {
+ vec_rules[i].push_back(*pos);
+ }
+ vec_rules[i].push_back({LLAMA_GRETYPE_END, 0});
+ }
+
+ // Check for left recursion
+ std::vector<bool> rules_visited(n_rules);
+ std::vector<bool> rules_in_progress(n_rules);
+ std::vector<bool> rules_may_be_empty(n_rules);
+ for (size_t i = 0; i < n_rules; i++) {
+ if (rules_visited[i]) {
+ continue;
+ }
+ if (llama_grammar_detect_left_recursion(vec_rules, i, &rules_visited, &rules_in_progress, &rules_may_be_empty)) {
+ LLAMA_LOG_ERROR("unsupported grammar, left recursion detected for nonterminal at index %zu", i);
+ return nullptr;
+ }
+ }
+
+ // loop over alternates of start rule to build initial stacks
+ llama_grammar_stacks stacks;
+ pos = vec_rules[start_rule_index].data();
+ do {
+ llama_grammar_stack stack;
+ if (!llama_grammar_is_end_of_sequence(pos)) {
+ // if alternate is nonempty, add to stack
+ stack.push_back(pos);
+ }
+ llama_grammar_advance_stack(vec_rules, stack, stacks);
+ while (!llama_grammar_is_end_of_sequence(pos)) {
+ // scan to end of alternate def
+ pos++;
+ }
+ if (pos->type == LLAMA_GRETYPE_ALT) {
+ // there's another alternate def of this rule to process
+ pos++;
+ } else {
+ break;
+ }
+ } while (true);
+
+ // Important: vec_rules has to be moved here, not copied, because stacks contains
+ // pointers to elements of vec_rules. If vec_rules were copied into llama_grammar
+ // then the pointers would be invalidated when the local vec_rules goes out of scope.
+ return new llama_grammar{ std::move(vec_rules), std::move(stacks), {} };
+}
+
+void llama_grammar_free_impl(struct llama_grammar * grammar) {
+ delete grammar;
+}
+
+struct llama_grammar * llama_grammar_copy_impl(const struct llama_grammar * grammar) {
+ llama_grammar * result = new llama_grammar{ grammar->rules, grammar->stacks, grammar->partial_utf8 };
+
+ // redirect elements in stacks to point to new rules
+ for (size_t is = 0; is < result->stacks.size(); is++) {
+ for (size_t ie = 0; ie < result->stacks[is].size(); ie++) {
+ for (size_t ir0 = 0; ir0 < grammar->rules.size(); ir0++) {
+ for (size_t ir1 = 0; ir1 < grammar->rules[ir0].size(); ir1++) {
+ if (grammar->stacks[is][ie] == &grammar->rules[ir0][ir1]) {
+ result->stacks[is][ie] = &result->rules[ir0][ir1];
+ }
+ }
+ }
+ }
+ }
+
+ return result;
+}
+
+void llama_grammar_sample_impl(const struct llama_grammar * grammar, const struct llama_vocab * vocab, const struct llama_sampling * smpl, llama_token_data_array * candidates) {
+ GGML_ASSERT(grammar);
+ GGML_ASSERT(vocab);
+
+ int64_t t_start_sample_us = ggml_time_us();
+
+ bool allow_eog = false;
+ for (const auto & stack : grammar->stacks) {
+ if (stack.empty()) {
+ allow_eog = true;
+ break;
+ }
+ }
+
+ std::vector<std::pair<std::vector<uint32_t>, llama_partial_utf8>> candidates_decoded;
+ candidates_decoded.reserve(candidates->size);
+
+ llama_grammar_candidates candidates_grammar;
+ candidates_grammar.reserve(candidates->size);
+
+ for (size_t i = 0; i < candidates->size; ++i) {
+ const llama_token id = candidates->data[i].id;
+ const std::string & piece = vocab->cache_token_to_piece.at(id);
+
+ if (llama_token_is_eog_impl(*vocab, id)) {
+ if (!allow_eog) {
+ candidates->data[i].logit = -INFINITY;
+ }
+ } else if (piece.empty() || piece[0] == 0) {
+ candidates->data[i].logit = -INFINITY;
+ } else {
+ candidates_decoded.push_back(decode_utf8(piece, grammar->partial_utf8));
+ candidates_grammar.push_back({ i, candidates_decoded.back().first.data(), candidates_decoded.back().second });
+ }
+ }
+
+ const auto rejects = llama_grammar_reject_candidates(grammar->rules, grammar->stacks, candidates_grammar);
+ for (const auto & reject : rejects) {
+ candidates->data[reject.index].logit = -INFINITY;
+ }
+
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+}
+
+void llama_grammar_accept_token_impl(struct llama_grammar * grammar, const struct llama_vocab * vocab, const struct llama_sampling * smpl, llama_token token) {
+ const int64_t t_start_sample_us = ggml_time_us();
+
+ if (llama_token_is_eog_impl(*vocab, token)) {
+ for (const auto & stack : grammar->stacks) {
+ if (stack.empty()) {
+ return;
+ }
+ }
+ GGML_ABORT("fatal error");
+ }
+
+ const std::string & piece = vocab->cache_token_to_piece.at(token);
+
+ // Note terminating 0 in decoded string
+ const auto decoded = decode_utf8(piece, grammar->partial_utf8);
+ const auto & code_points = decoded.first;
+
+ llama_grammar_stacks tmp_new_stacks;
+ for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) {
+ llama_grammar_accept(grammar->rules, grammar->stacks, *it, tmp_new_stacks);
+ grammar->stacks = tmp_new_stacks;
+ }
+
+ grammar->partial_utf8 = decoded.second;
+ GGML_ASSERT(!grammar->stacks.empty());
+
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+}
diff --git a/llama/llama-grammar.h b/llama/llama-grammar.h
new file mode 100644
index 00000000..f16cf336
--- /dev/null
+++ b/llama/llama-grammar.h
@@ -0,0 +1,65 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "llama-impl.h"
+
+struct llama_vocab;
+struct llama_sampling;
+
+struct llama_grammar {
+ const llama_grammar_rules rules;
+ llama_grammar_stacks stacks;
+
+ // buffer for partially generated UTF-8 sequence from accepted tokens
+ llama_partial_utf8 partial_utf8;
+};
+
+//
+// internal API
+//
+
+struct llama_grammar * llama_grammar_init_impl(
+ const llama_grammar_element ** rules,
+ size_t n_rules,
+ size_t start_rule_index);
+
+void llama_grammar_free_impl(struct llama_grammar * grammar);
+
+struct llama_grammar * llama_grammar_copy_impl(const struct llama_grammar * grammar);
+
+void llama_grammar_sample_impl(
+ const struct llama_grammar * grammar,
+ const struct llama_vocab * vocab,
+ const struct llama_sampling * smpl,
+ llama_token_data_array * candidates);
+
+void llama_grammar_accept_token_impl(
+ struct llama_grammar * grammar,
+ const struct llama_vocab * vocab,
+ const struct llama_sampling * smpl,
+ llama_token token);
diff --git a/llama/llama-impl.h b/llama/llama-impl.h
new file mode 100644
index 00000000..3d127b37
--- /dev/null
+++ b/llama/llama-impl.h
@@ -0,0 +1,73 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#define LLAMA_API_INTERNAL
+#include "llama.h"
+
+#ifdef __GNUC__
+#ifdef __MINGW32__
+#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
+#else
+#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
+#endif
+#else
+#define LLAMA_ATTRIBUTE_FORMAT(...)
+#endif
+
+//
+// logging
+//
+
+LLAMA_ATTRIBUTE_FORMAT(2, 3)
+void llama_log_internal (ggml_log_level level, const char * format, ...);
+void llama_log_callback_default(ggml_log_level level, const char * text, void * user_data);
+
+#define LLAMA_LOG_INFO(...) llama_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)
+#define LLAMA_LOG_WARN(...) llama_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__)
+#define LLAMA_LOG_ERROR(...) llama_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
+
+//
+// helpers
+//
+
+static void replace_all(std::string & s, const std::string & search, const std::string & replace) {
+ if (search.empty()) {
+ return;
+ }
+ std::string builder;
+ builder.reserve(s.length());
+ size_t pos = 0;
+ size_t last_pos = 0;
+ while ((pos = s.find(search, last_pos)) != std::string::npos) {
+ builder.append(s, last_pos, pos - last_pos);
+ builder.append(replace);
+ last_pos = pos + search.length();
+ }
+ builder.append(s, last_pos, std::string::npos);
+ s = std::move(builder);
+}
diff --git a/llama/llama-sampling.cpp b/llama/llama-sampling.cpp
new file mode 100644
index 00000000..8cc360d8
--- /dev/null
+++ b/llama/llama-sampling.cpp
@@ -0,0 +1,661 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "llama-sampling.h"
+
+#include <algorithm>
+#include <cstring>
+#include <ctime>
+#include <cfloat>
+#include <numeric>
+#include <unordered_map>
+
+static void llama_log_softmax(float * array, size_t size) {
+ float max_l = *std::max_element(array, array + size);
+ float sum = 0.f;
+ for (size_t i = 0; i < size; ++i) {
+ float p = expf(array[i] - max_l);
+ sum += p;
+ array[i] = p;
+ }
+
+ for (size_t i = 0; i < size; ++i) {
+ array[i] = logf(array[i] / sum);
+ }
+}
+
+void llama_set_rng_seed_impl(struct llama_sampling * smpl, uint32_t seed) {
+ if (seed == LLAMA_DEFAULT_SEED) {
+ seed = time(NULL);
+ }
+
+ smpl->rng.seed(seed);
+}
+
+void llama_sample_softmax_impl(struct llama_sampling * smpl, llama_token_data_array * candidates) {
+ GGML_ASSERT(candidates->size > 0);
+
+ const int64_t t_start_sample_us = ggml_time_us();
+
+ // Sort the logits in descending order
+ if (!candidates->sorted) {
+ std::sort(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) {
+ return a.logit > b.logit;
+ });
+ candidates->sorted = true;
+ }
+
+ float max_l = candidates->data[0].logit;
+ float cum_sum = 0.0f;
+ for (size_t i = 0; i < candidates->size; ++i) {
+ float p = expf(candidates->data[i].logit - max_l);
+ candidates->data[i].p = p;
+ cum_sum += p;
+ }
+ for (size_t i = 0; i < candidates->size; ++i) {
+ candidates->data[i].p /= cum_sum;
+ }
+
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ }
+}
+
+void llama_sample_top_k_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, int32_t k, size_t min_keep) {
+ // TODO: move bucket sort to separate function so that top_p/tail_free/typical/softmax first is equally fast
+ // if (k >= (int32_t)candidates->size) {
+ // return;
+ // }
+
+ const int64_t t_start_sample_us = ggml_time_us();
+
+ if (k <= 0) {
+ k = candidates->size;
+ }
+
+ k = std::max(k, (int) min_keep);
+ k = std::min(k, (int) candidates->size);
+
+ // Sort scores in descending order
+ if (!candidates->sorted) {
+ auto comp = [](const llama_token_data & a, const llama_token_data & b) {
+ return a.logit > b.logit;
+ };
+ if (k <= 128) {
+ std::partial_sort(candidates->data, candidates->data + k, candidates->data + candidates->size, comp);
+ } else {
+ constexpr int nbuckets = 128;
+ constexpr float bucket_low = -10.0f;
+ constexpr float bucket_high = 10.0f;
+ constexpr float bucket_scale = nbuckets/(bucket_high - bucket_low);
+ constexpr float bucket_inter = -bucket_low * bucket_scale;
+
+ std::vector<int> bucket_idx(candidates->size);
+ std::vector<int> histo(nbuckets, 0);
+
+ for (int i = 0; i < (int)candidates->size; ++i) {
+ const float val = candidates->data[i].logit;
+ int ib = int(bucket_scale * val + bucket_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);
+ ib = std::max(0, std::min(nbuckets-1, ib));
+ bucket_idx[i] = ib;
+ ++histo[ib];
+ }
+ int nhave = 0;
+ int ib = nbuckets - 1;
+ for ( ; ib >= 0; --ib) {
+ nhave += histo[ib];
+ if (nhave >= k) break;
+ }
+ std::vector<llama_token_data> tmp_tokens(nhave);
+ auto ptr = tmp_tokens.data();
+ std::vector<llama_token_data*> bucket_ptrs;
+ bucket_ptrs.reserve(nbuckets - ib);
+ for (int j = nbuckets - 1; j >= ib; --j) {
+ bucket_ptrs.push_back(ptr);
+ ptr += histo[j];
+ }
+ for (int i = 0; i < (int)candidates->size; ++i) {
+ int j = bucket_idx[i];
+ if (j >= ib) {
+ *bucket_ptrs[nbuckets-1-j]++ = candidates->data[i];
+ }
+ }
+
+ ptr = tmp_tokens.data();
+ int ndone = 0;
+ for (int j = nbuckets-1; j > ib; --j) {
+ std::sort(ptr, ptr + histo[j], comp);
+ ptr += histo[j];
+ ndone += histo[j];
+ }
+ std::partial_sort(ptr, ptr + k - ndone, ptr + histo[ib], comp);
+
+ std::memcpy(candidates->data, tmp_tokens.data(), k*sizeof(llama_token_data));
+
+ }
+ candidates->sorted = true;
+ }
+ candidates->size = k;
+
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ }
+}
+
+void llama_sample_top_p_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, float p, size_t min_keep) {
+ if (p >= 1.0f) {
+ return;
+ }
+
+ llama_sample_softmax_impl(smpl, candidates);
+
+ const int64_t t_start_sample_us = ggml_time_us();
+
+ // Compute the cumulative probabilities
+ float cum_sum = 0.0f;
+ size_t last_idx = candidates->size;
+
+ for (size_t i = 0; i < candidates->size; ++i) {
+ cum_sum += candidates->data[i].p;
+
+ // Check if the running sum is at least p or if we have kept at least min_keep tokens
+ // we set the last index to i+1 to indicate that the current iterate should be included in the set
+ if (cum_sum >= p && i + 1 >= min_keep) {
+ last_idx = i + 1;
+ break;
+ }
+ }
+
+ // Resize the output vector to keep only the top-p tokens
+ candidates->size = last_idx;
+
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ }
+}
+
+void llama_sample_min_p_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, float p, size_t min_keep) {
+ if (p <= 0.0f || !candidates->size) {
+ return;
+ }
+
+ const int64_t t_start_sample_us = ggml_time_us();
+
+ bool min_p_applied = false;
+
+ // if the candidates aren't sorted, try the unsorted implementation first
+ if (!candidates->sorted) {
+ std::vector<llama_token_data> filtered_tokens;
+
+ float max_logit = -FLT_MAX;
+ for (size_t i = 0; i < candidates->size; ++i) {
+ max_logit = std::max(max_logit, candidates->data[i].logit);
+ }
+ const float min_logit = max_logit + logf(p); // min logit for p_i >= p * p_max
+
+ for (size_t i = 0; i < candidates->size; ++i) {
+ if (candidates->data[i].logit >= min_logit) {
+ filtered_tokens.push_back(candidates->data[i]);
+ }
+ }
+
+ // if we have enough values the operation was a success
+ if (filtered_tokens.size() >= min_keep) {
+ memcpy(candidates->data, filtered_tokens.data(), filtered_tokens.size()*sizeof(llama_token_data));
+ candidates->size = filtered_tokens.size();
+ min_p_applied = true;
+ }
+ }
+
+ // if the candidates are sorted or the unsorted implementation failed, use this implementation
+ if (!min_p_applied) {
+ // Sort the logits in descending order
+ if (!candidates->sorted) {
+ std::sort(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) {
+ return a.logit > b.logit;
+ });
+ candidates->sorted = true;
+ }
+
+ const float min_logit = candidates->data[0].logit + logf(p); // min logit for p_i >= p * p_max
+ size_t i = 1; // first token always matches
+
+ for (; i < candidates->size; ++i) {
+ if (candidates->data[i].logit < min_logit && i >= min_keep) {
+ break; // prob too small
+ }
+ }
+
+ // Resize the output vector to keep only the matching tokens
+ candidates->size = i;
+ }
+
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ }
+}
+
+void llama_sample_tail_free_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, float z, size_t min_keep) {
+ if (z >= 1.0f || candidates->size <= 2) {
+ return;
+ }
+
+ llama_sample_softmax_impl((struct llama_sampling *) nullptr, candidates);
+ const int64_t t_start_sample_us = ggml_time_us();
+
+ // Compute the first and second derivatives
+ std::vector<float> first_derivatives(candidates->size - 1);
+ std::vector<float> second_derivatives(candidates->size - 2);
+
+ for (size_t i = 0; i < first_derivatives.size(); ++i) {
+ first_derivatives[i] = candidates->data[i].p - candidates->data[i + 1].p;
+ }
+ for (size_t i = 0; i < second_derivatives.size(); ++i) {
+ second_derivatives[i] = first_derivatives[i] - first_derivatives[i + 1];
+ }
+
+ // Calculate absolute value of second derivatives
+ for (size_t i = 0; i < second_derivatives.size(); ++i) {
+ second_derivatives[i] = std::abs(second_derivatives[i]);
+ }
+
+ // Normalize the second derivatives
+ {
+ const float second_derivatives_sum = std::accumulate(second_derivatives.begin(), second_derivatives.end(), 0.0f);
+
+ if (second_derivatives_sum > 1e-6f) {
+ for (float & value : second_derivatives) {
+ value /= second_derivatives_sum;
+ }
+ } else {
+ for (float & value : second_derivatives) {
+ value = 1.0f / second_derivatives.size();
+ }
+ }
+ }
+
+ float cum_sum = 0.0f;
+ size_t last_idx = candidates->size;
+ for (size_t i = 0; i < second_derivatives.size(); ++i) {
+ cum_sum += second_derivatives[i];
+
+ // Check if the running sum is greater than z or if we have kept at least min_keep tokens
+ if (cum_sum > z && i >= min_keep) {
+ last_idx = i;
+ break;
+ }
+ }
+
+ // Resize the output vector to keep only the tokens above the tail location
+ candidates->size = last_idx;
+
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ }
+}
+
+void llama_sample_typical_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, float p, size_t min_keep) {
+ // Reference implementation:
+ // https://github.com/huggingface/transformers/compare/main...cimeister:typical-sampling:typical-pr
+ if (p >= 1.0f) {
+ return;
+ }
+
+ // Compute the softmax of logits and calculate entropy
+ llama_sample_softmax_impl((struct llama_sampling *) nullptr, candidates);
+
+ const int64_t t_start_sample_us = ggml_time_us();
+
+ float entropy = 0.0f;
+ for (size_t i = 0; i < candidates->size; ++i) {
+ entropy += -candidates->data[i].p * logf(candidates->data[i].p);
+ }
+
+ // Compute the absolute difference between negative log probability and entropy for each candidate
+ std::vector<float> shifted_scores;
+ for (size_t i = 0; i < candidates->size; ++i) {
+ float shifted_score = fabsf(-logf(candidates->data[i].p) - entropy);
+ shifted_scores.push_back(shifted_score);
+ }
+
+ // Sort tokens based on the shifted_scores and their corresponding indices
+ std::vector<size_t> indices(candidates->size);
+ std::iota(indices.begin(), indices.end(), 0);
+
+ std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) {
+ return shifted_scores[a] < shifted_scores[b];
+ });
+
+ // Compute the cumulative probabilities
+ float cum_sum = 0.0f;
+ size_t last_idx = indices.size();
+
+ for (size_t i = 0; i < indices.size(); ++i) {
+ size_t idx = indices[i];
+ cum_sum += candidates->data[idx].p;
+
+ // Check if the running sum is greater than typical or if we have kept at least min_keep tokens
+ if (cum_sum > p && i >= min_keep - 1) {
+ last_idx = i + 1;
+ break;
+ }
+ }
+
+ // Resize the output vector to keep only the locally typical tokens
+ std::vector<llama_token_data> new_candidates;
+ for (size_t i = 0; i < last_idx; ++i) {
+ size_t idx = indices[i];
+ new_candidates.push_back(candidates->data[idx]);
+ }
+
+ // Replace the data in candidates with the new_candidates data
+ std::copy(new_candidates.begin(), new_candidates.end(), candidates->data);
+ candidates->size = new_candidates.size();
+ candidates->sorted = false;
+
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ }
+}
+
+void llama_sample_entropy_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, float min_temp, float max_temp, float exponent_val) {
+ const int64_t t_start_sample_us = ggml_time_us();
+
+ // no need to do anything if there is only one (or zero) candidates
+ if(candidates->size <= 1) {
+ return;
+ }
+
+ // Calculate maximum possible entropy
+ float max_entropy = -logf(1.0f / candidates->size);
+
+ llama_sample_softmax_impl((struct llama_sampling *) nullptr, candidates);
+
+ // Calculate entropy of the softmax probabilities
+ float entropy = 0.0f;
+ for (size_t i = 0; i < candidates->size; ++i) {
+ float prob = candidates->data[i].p;
+ if (prob > 0.0f) { // Ensure no log(0)
+ entropy -= prob * logf(prob);
+ }
+ }
+
+ // Normalize the entropy (max_entropy cannot be 0 here because we checked candidates->size != 1 above)
+ float normalized_entropy = entropy / max_entropy;
+
+ // Map the normalized entropy to the desired temperature range using the power function
+ float dyn_temp = min_temp + (max_temp - min_temp) * powf(normalized_entropy, exponent_val);
+
+#ifdef DEBUG
+ LLAMA_LOG_INFO("Your text maxtemp value is: %f\n", max_temp);
+ LLAMA_LOG_INFO("Entropy: %f\n", entropy);
+ LLAMA_LOG_INFO("Max Possible Entropy: %f\n", max_entropy);
+ LLAMA_LOG_INFO("Normalized Entropy: %f\n", normalized_entropy);
+ LLAMA_LOG_INFO("Exponent: %f\n", exponent_val);
+ LLAMA_LOG_INFO("Dynamic Temperature (dyn_temp): %f\n", dyn_temp);
+#endif
+
+ // Apply the dynamically calculated temperature scaling
+ for (size_t i = 0; i < candidates->size; ++i) {
+ candidates->data[i].logit /= dyn_temp;
+ }
+
+ // Re-compute softmax probabilities after scaling logits with dynamic temperature
+ double max_l_double = candidates->data[0].logit;
+ double cum_sum_double = 0.0;
+ for (size_t i = 0; i < candidates->size; ++i) {
+ double p = exp(candidates->data[i].logit - max_l_double);
+ candidates->data[i].p = p; // Store the scaled probability
+ cum_sum_double += p;
+ }
+ for (size_t i = 0; i < candidates->size; ++i) {
+ candidates->data[i].p /= cum_sum_double; // Re-normalize the probabilities
+ }
+
+#ifdef DEBUG
+ // Print the updated top 25 probabilities after temperature scaling
+ LLAMA_LOG_INFO("\nUpdated Top 25 Probabilities After Dynamic Temperature Scaling (in percentages):\n");
+ for (size_t i = 0; i < 25 && i < candidates->size; ++i) {
+ LLAMA_LOG_INFO("Token %zu: %f%%\n", i + 1, candidates->data[i].p * 100.0f);
+ }
+#endif
+
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ }
+}
+
+void llama_sample_temp_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, float temp) {
+ const int64_t t_start_sample_us = ggml_time_us();
+
+ for (size_t i = 0; i < candidates->size; ++i) {
+ candidates->data[i].logit /= temp;
+ }
+
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ }
+}
+
+void llama_sample_repetition_penalties_impl(
+ struct llama_sampling * smpl,
+ llama_token_data_array * candidates,
+ const llama_token * last_tokens,
+ size_t penalty_last_n,
+ float penalty_repeat,
+ float penalty_freq,
+ float penalty_present) {
+ if (penalty_last_n == 0 || (penalty_repeat == 1.0f && penalty_freq == 0.0f && penalty_present == 0.0f)) {
+ return;
+ }
+
+ const int64_t t_start_sample_us = ggml_time_us();
+
+ // Create a frequency map to count occurrences of each token in last_tokens
+ std::unordered_map<llama_token, int> token_count;
+ for (size_t i = 0; i < penalty_last_n; ++i) {
+ token_count[last_tokens[i]]++;
+ }
+
+ // Apply frequency and presence penalties to the candidates
+ for (size_t i = 0; i < candidates->size; ++i) {
+ const auto token_iter = token_count.find(candidates->data[i].id);
+ if (token_iter == token_count.end()) {
+ continue;
+ }
+
+ const int count = token_iter->second;
+
+ // The academic publication that described this technique actually just only divided, but that would cause tokens with negative logits to become more likely, which is obviously wrong.
+ // This is common fix for this problem, which is to multiply by the penalty instead of dividing.
+ if (candidates->data[i].logit <= 0) {
+ candidates->data[i].logit *= penalty_repeat;
+ } else {
+ candidates->data[i].logit /= penalty_repeat;
+ }
+
+ candidates->data[i].logit -= float(count) * penalty_freq + float(count > 0) * penalty_present;
+ }
+
+ candidates->sorted = false;
+
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ }
+}
+
+void llama_sample_apply_guidance_impl(
+ struct llama_sampling * smpl,
+ float * logits,
+ float * logits_guidance,
+ float scale) {
+ GGML_ASSERT(smpl);
+
+ const auto t_start_sample_us = ggml_time_us();
+ const auto n_vocab = smpl->n_vocab;
+
+ llama_log_softmax(logits, n_vocab);
+ llama_log_softmax(logits_guidance, n_vocab);
+
+ for (int i = 0; i < n_vocab; ++i) {
+ auto & l = logits[i];
+ const auto & g = logits_guidance[i];
+
+ l = scale * (l - g) + g;
+ }
+
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+}
+
+llama_token llama_sample_token_mirostat_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, float tau, float eta, int32_t m, float * mu) {
+ GGML_ASSERT(smpl);
+
+ const int32_t n_vocab = float(smpl->n_vocab);
+
+ int64_t t_start_sample_us = ggml_time_us();
+
+ llama_sample_softmax_impl((struct llama_sampling *) nullptr, candidates);
+
+ // Estimate s_hat using the most probable m tokens
+ float s_hat = 0.0;
+ float sum_ti_bi = 0.0;
+ float sum_ti_sq = 0.0;
+ for (size_t i = 0; i < size_t(m - 1) && i < candidates->size - 1; ++i) {
+ float t_i = logf(float(i + 2) / float(i + 1));
+ float b_i = logf(candidates->data[i].p / candidates->data[i + 1].p);
+ sum_ti_bi += t_i * b_i;
+ sum_ti_sq += t_i * t_i;
+ }
+ s_hat = sum_ti_bi / sum_ti_sq;
+
+ // Compute k from the estimated s_hat and target surprise value
+ float epsilon_hat = s_hat - 1;
+ float k = powf((epsilon_hat * powf(2, *mu)) / (1 - powf(n_vocab, -epsilon_hat)), 1 / s_hat);
+
+ // Sample the next word X using top-k sampling
+ llama_sample_top_k_impl((struct llama_sampling *) nullptr, candidates, int(k), 1);
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ llama_token X = llama_sample_token_impl(smpl, candidates);
+ t_start_sample_us = ggml_time_us();
+
+ // Compute error as the difference between observed surprise and target surprise value
+ size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) {
+ return candidate.id == X;
+ }));
+ float observed_surprise = -log2f(candidates->data[X_idx].p);
+ float e = observed_surprise - tau;
+
+ // Update mu using the learning rate and error
+ *mu = *mu - eta * e;
+
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ return X;
+}
+
+llama_token llama_sample_token_mirostat_v2_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, float tau, float eta, float * mu) {
+ int64_t t_start_sample_us;
+ t_start_sample_us = ggml_time_us();
+
+ llama_sample_softmax_impl(smpl, candidates);
+
+ // Truncate the words with surprise values greater than mu
+ candidates->size = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) {
+ return -log2f(candidate.p) > *mu;
+ }));
+
+ if (candidates->size == 0) {
+ candidates->size = 1;
+ }
+
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ }
+
+ // Normalize the probabilities of the remaining words
+ llama_sample_softmax_impl(smpl, candidates);
+
+ // Sample the next word X from the remaining words
+ llama_token X = llama_sample_token_impl(smpl, candidates);
+ t_start_sample_us = ggml_time_us();
+
+ // Compute error as the difference between observed surprise and target surprise value
+ size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) {
+ return candidate.id == X;
+ }));
+ float observed_surprise = -log2f(candidates->data[X_idx].p);
+ float e = observed_surprise - tau;
+
+ // Update mu using the learning rate and error
+ *mu = *mu - eta * e;
+
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ }
+ return X;
+}
+
+llama_token llama_sample_token_greedy_impl(struct llama_sampling * smpl, llama_token_data_array * candidates) {
+ const int64_t t_start_sample_us = ggml_time_us();
+
+ // Find max element
+ auto * max_iter = std::max_element(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) {
+ return a.logit < b.logit;
+ });
+
+ llama_token result = max_iter->id;
+ if (smpl) {
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ smpl->n_sample++;
+ }
+ return result;
+}
+
+llama_token llama_sample_token_with_rng_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, std::mt19937 & rng) {
+ GGML_ASSERT(smpl);
+
+ const int64_t t_start_sample_us = ggml_time_us();
+ llama_sample_softmax_impl((struct llama_sampling *) nullptr, candidates);
+
+ std::vector<float> probs;
+ probs.reserve(candidates->size);
+ for (size_t i = 0; i < candidates->size; ++i) {
+ probs.push_back(candidates->data[i].p);
+ }
+
+ std::discrete_distribution<> dist(probs.begin(), probs.end());
+ int idx = dist(rng);
+
+ llama_token result = candidates->data[idx].id;
+
+ smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
+ smpl->n_sample++;
+
+ return result;
+}
+
+llama_token llama_sample_token_impl(struct llama_sampling * smpl, llama_token_data_array * candidates) {
+ return llama_sample_token_with_rng_impl(smpl, candidates, smpl->rng);
+}
diff --git a/llama/llama-sampling.h b/llama/llama-sampling.h
new file mode 100644
index 00000000..614987c7
--- /dev/null
+++ b/llama/llama-sampling.h
@@ -0,0 +1,82 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "llama-impl.h"
+
+struct llama_sampling {
+ llama_sampling(int32_t n_vocab) : n_vocab(n_vocab) {}
+
+ std::mt19937 rng;
+
+ int32_t n_vocab = 0;
+
+ mutable int64_t t_sample_us = 0;
+ mutable int32_t n_sample = 0;
+
+ void reset_timings() const {
+ t_sample_us = 0;
+ n_sample = 0;
+ }
+};
+
+//
+// internal API
+//
+
+void llama_set_rng_seed_impl(struct llama_sampling * smpl, uint32_t seed);
+
+void llama_sample_softmax_impl (struct llama_sampling * smpl, llama_token_data_array * candidates);
+void llama_sample_top_k_impl (struct llama_sampling * smpl, llama_token_data_array * candidates, int32_t k, size_t min_keep);
+void llama_sample_top_p_impl (struct llama_sampling * smpl, llama_token_data_array * candidates, float p, size_t min_keep);
+void llama_sample_min_p_impl (struct llama_sampling * smpl, llama_token_data_array * candidates, float p, size_t min_keep);
+void llama_sample_tail_free_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, float z, size_t min_keep);
+void llama_sample_typical_impl (struct llama_sampling * smpl, llama_token_data_array * candidates, float p, size_t min_keep);
+void llama_sample_entropy_impl (struct llama_sampling * smpl, llama_token_data_array * candidates, float min_temp, float max_temp, float exponent_val);
+void llama_sample_temp_impl (struct llama_sampling * smpl, llama_token_data_array * candidates, float temp);
+
+void llama_sample_repetition_penalties_impl(
+ struct llama_sampling * smpl,
+ llama_token_data_array * candidates,
+ const llama_token * last_tokens,
+ size_t penalty_last_n,
+ float penalty_repeat,
+ float penalty_freq,
+ float penalty_present);
+
+void llama_sample_apply_guidance_impl(
+ struct llama_sampling * smpl,
+ float * logits,
+ float * logits_guidance,
+ float scale);
+
+llama_token llama_sample_token_mirostat_impl (struct llama_sampling * smpl, llama_token_data_array * candidates, float tau, float eta, int32_t m, float * mu);
+llama_token llama_sample_token_mirostat_v2_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, float tau, float eta, float * mu);
+llama_token llama_sample_token_greedy_impl (struct llama_sampling * smpl, llama_token_data_array * candidates);
+llama_token llama_sample_token_with_rng_impl (struct llama_sampling * smpl, llama_token_data_array * candidates, std::mt19937 & rng);
+llama_token llama_sample_token_impl (struct llama_sampling * smpl, llama_token_data_array * candidates);
+
diff --git a/llama/llama-vocab.cpp b/llama/llama-vocab.cpp
new file mode 100644
index 00000000..4bc61196
--- /dev/null
+++ b/llama/llama-vocab.cpp
@@ -0,0 +1,1891 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "llama-vocab.h"
+
+#include "unicode.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cfloat>
+#include <climits>
+#include <cstdarg>
+#include <cstring>
+#include <forward_list>
+#include <queue>
+#include <sstream>
+
+//
+// helpers
+//
+
+LLAMA_ATTRIBUTE_FORMAT(1, 2)
+static std::string format(const char * fmt, ...) {
+ va_list ap;
+ va_list ap2;
+ va_start(ap, fmt);
+ va_copy(ap2, ap);
+ int size = vsnprintf(NULL, 0, fmt, ap);
+ GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT
+ std::vector<char> buf(size + 1);
+ int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
+ GGML_ASSERT(size2 == size);
+ va_end(ap2);
+ va_end(ap);
+ return std::string(buf.data(), size);
+}
+
+struct naive_trie {
+ naive_trie() : has_value(false), value(0) {
+ }
+ void insert(const char * key, size_t len, int32_t value = 0) {
+ if (len == 0) {
+ this->has_value = true;
+ this->value = value;
+ return;
+ }
+ char c = key[0];
+ auto res = children.find(c);
+ if (res != children.end()) {
+ res->second.insert(key + 1, len - 1, value);
+ } else {
+ auto res = children.insert(std::make_pair(c, naive_trie()));
+ res.first->second.insert(key + 1, len - 1, value);
+ }
+ }
+ std::pair<const char *, size_t> get_longest_prefix(const char * key, size_t len, size_t offset = 0) {
+ if (len == 0 || offset == len) {
+ return std::make_pair(key, offset);
+ }
+ char c = key[offset];
+ auto res = children.find(c);
+ if (res != children.end()) {
+ return res->second.get_longest_prefix(key, len, offset + 1);
+ }
+
+ return std::make_pair(key, offset);
+ }
+ const struct naive_trie * traverse(const char c) const {
+ auto res = children.find(c);
+ if (res != children.end()) {
+ return &res->second;
+ }
+
+ return NULL;
+ }
+ std::map<char, struct naive_trie> children;
+ bool has_value;
+ llama_token value;
+};
+
+//
+// impl
+//
+
+int llama_vocab::find_bpe_rank(const std::string & token_left, const std::string & token_right) const {
+ GGML_ASSERT(token_left.find(' ') == std::string::npos);
+ GGML_ASSERT(token_left.find('\n') == std::string::npos);
+ GGML_ASSERT(token_right.find(' ') == std::string::npos);
+ GGML_ASSERT(token_right.find('\n') == std::string::npos);
+
+ auto it = bpe_ranks.find(std::make_pair(token_left, token_right));
+ if (it == bpe_ranks.end()) {
+ return -1;
+ }
+
+ return it->second;
+}
+
+static enum llama_vocab_type llama_vocab_get_type(const llama_vocab & vocab) {
+ return vocab.type;
+}
+
+static bool llama_is_normal_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_NORMAL;
+}
+
+static bool llama_is_unknown_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_UNKNOWN;
+}
+
+static bool llama_is_control_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_CONTROL;
+}
+
+static bool llama_is_byte_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_BYTE;
+}
+
+static bool llama_is_user_defined_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_USER_DEFINED;
+}
+
+static bool llama_is_unused_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_UNUSED;
+}
+
+static uint8_t llama_token_to_byte(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE);
+ GGML_ASSERT(llama_is_byte_token(vocab, id));
+ const auto & token_data = vocab.id_to_token.at(id);
+ switch (llama_vocab_get_type(vocab)) {
+ case LLAMA_VOCAB_TYPE_SPM:
+ case LLAMA_VOCAB_TYPE_UGM: {
+ auto buf = token_data.text.substr(3, 2);
+ return strtol(buf.c_str(), NULL, 16);
+ }
+ case LLAMA_VOCAB_TYPE_BPE: {
+ GGML_ABORT("fatal error");
+ //return unicode_utf8_to_byte(token_data.text); // TODO: why is this here after GGML_ASSERT?
+ }
+ case LLAMA_VOCAB_TYPE_WPM: {
+ GGML_ABORT("fatal error");
+ }
+ default:
+ GGML_ABORT("fatal error");
+ }
+}
+
+static void llama_escape_whitespace(std::string & text) {
+ replace_all(text, " ", "\xe2\x96\x81");
+}
+
+static void llama_unescape_whitespace(std::string & word) {
+ replace_all(word, "\xe2\x96\x81", " ");
+}
+
+struct llm_symbol {
+ using index = int;
+ index prev;
+ index next;
+ const char * text;
+ size_t n;
+};
+
+static_assert(std::is_trivially_copyable<llm_symbol>::value, "llm_symbol is not trivially copyable");
+
+//
+// SPM tokenizer
+// original implementation:
+// https://github.com/ggerganov/llama.cpp/commit/074bea2eb1f1349a0118239c4152914aecaa1be4
+//
+
+struct llm_bigram_spm {
+ struct comparator {
+ bool operator()(llm_bigram_spm & l, llm_bigram_spm & r) {
+ return (l.score < r.score) || (l.score == r.score && l.left > r.left);
+ }
+ };
+ using queue_storage = std::vector<llm_bigram_spm>;
+ using queue = std::priority_queue<llm_bigram_spm, queue_storage, comparator>;
+ llm_symbol::index left;
+ llm_symbol::index right;
+ float score;
+ size_t size;
+};
+
+struct llm_tokenizer_spm {
+ llm_tokenizer_spm(const llama_vocab & vocab) : vocab(vocab) {}
+
+ void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
+ // split string into utf8 chars
+ int index = 0;
+ size_t offs = 0;
+ while (offs < text.size()) {
+ llm_symbol sym;
+ size_t len = unicode_len_utf8(text[offs]);
+ sym.text = text.c_str() + offs;
+ sym.n = std::min(len, text.size() - offs);
+ offs += sym.n;
+ sym.prev = index - 1;
+ sym.next = offs == text.size() ? -1 : index + 1;
+ index++;
+ symbols.emplace_back(sym);
+ }
+
+ // seed the work queue with all possible 2-character tokens.
+ for (size_t i = 1; i < symbols.size(); ++i) {
+ try_add_bigram(i - 1, i);
+ }
+
+ // keep substituting the highest frequency pairs for as long as we can.
+ while (!work_queue.empty()) {
+ auto bigram = work_queue.top();
+ work_queue.pop();
+
+ auto & left_sym = symbols[bigram.left];
+ auto & right_sym = symbols[bigram.right];
+
+ // if one of the symbols already got merged, skip it.
+ if (left_sym.n == 0 || right_sym.n == 0 ||
+ left_sym.n + right_sym.n != bigram.size) {
+ continue;
+ }
+
+ // merge the right sym into the left one
+ left_sym.n += right_sym.n;
+ right_sym.n = 0;
+
+ //LLAMA_LOG_INFO("left = '%*s' size = %zu\n", (int) left_sym.n, left_sym.text, bigram.size);
+
+ // remove the right sym from the chain
+ left_sym.next = right_sym.next;
+ if (right_sym.next >= 0) {
+ symbols[right_sym.next].prev = bigram.left;
+ }
+
+ // find more substitutions
+ try_add_bigram(left_sym.prev, bigram.left);
+ try_add_bigram(bigram.left, left_sym.next);
+ }
+
+ for (int i = 0; i != -1; i = symbols[i].next) {
+ auto & symbol = symbols[i];
+ resegment(symbol, output);
+ }
+ }
+
+private:
+ void resegment(llm_symbol & symbol, std::vector<llama_vocab::id> & output) {
+ auto text = std::string(symbol.text, symbol.n);
+ auto token = vocab.token_to_id.find(text);
+
+ // Do we need to support is_unused?
+ if (token != vocab.token_to_id.end()) {
+ output.push_back((*token).second);
+ return;
+ }
+
+ const auto p = rev_merge.find(text);
+
+ if (p == rev_merge.end()) {
+ // output any symbols that did not form tokens as bytes.
+ output.reserve(output.size() + symbol.n);
+ for (int j = 0; j < (int)symbol.n; ++j) {
+ llama_vocab::id token_id = llama_byte_to_token_impl(vocab, symbol.text[j]);
+ output.push_back(token_id);
+ }
+ return;
+ }
+
+ resegment(symbols[p->second.first], output);
+ resegment(symbols[p->second.second], output);
+ }
+
+ void try_add_bigram(int left, int right) {
+ if (left == -1 || right == -1) {
+ return;
+ }
+
+ const std::string text = std::string(symbols[left].text, symbols[left].n + symbols[right].n);
+ auto token = vocab.token_to_id.find(text);
+
+ if (token == vocab.token_to_id.end()) {
+ return;
+ }
+
+ if (static_cast<size_t>((*token).second) >= vocab.id_to_token.size()) {
+ return;
+ }
+
+ const auto & tok_data = vocab.id_to_token[(*token).second];
+
+ llm_bigram_spm bigram;
+ bigram.left = left;
+ bigram.right = right;
+ bigram.score = tok_data.score;
+ bigram.size = text.size();
+
+ work_queue.push(bigram);
+
+ // Do we need to support is_unused?
+ rev_merge[text] = std::make_pair(left, right);
+ }
+
+ const llama_vocab & vocab;
+
+ std::vector<llm_symbol> symbols;
+ llm_bigram_spm::queue work_queue;
+
+ std::map<std::string, std::pair<int, int>> rev_merge;
+};
+
+//
+// BPE tokenizer
+// adapted from https://github.com/cmp-nct/ggllm.cpp [MIT License]
+// tried to simplify unicode stuff, so most likely does not work 100% correctly!
+//
+
+// TODO: there are a lot of common parts between spm and bpe tokenizers, should be refactored and reused
+
+template<typename T, typename Container = std::vector<T>, typename Compare = std::less<typename Container::value_type>>
+class llama_priority_queue : public std::priority_queue<T, Container, Compare> {
+public:
+ using std::priority_queue<T, Container, Compare>::priority_queue;
+
+ T pop_move() {
+ T item = std::move(this->c.front());
+ std::pop_heap(this->c.begin(), this->c.end(), this->comp);
+ this->c.pop_back();
+ return item;
+ }
+
+ void pop() = delete;
+};
+
+struct llm_bigram_bpe {
+ struct comparator {
+ bool operator()(const llm_bigram_bpe & l, const llm_bigram_bpe & r) const {
+ return l.rank > r.rank || (l.rank == r.rank && l.left > r.left);
+ }
+ };
+
+ using queue_storage = std::vector<llm_bigram_bpe>;
+ using queue = llama_priority_queue<llm_bigram_bpe, queue_storage, comparator>;
+ llm_symbol::index left;
+ llm_symbol::index right;
+ std::string text;
+ int rank;
+ size_t size;
+};
+
+struct llm_tokenizer_bpe {
+ llm_tokenizer_bpe(const llama_vocab & vocab): vocab(vocab) {
+ GGML_ASSERT(vocab.type == LLAMA_VOCAB_TYPE_BPE);
+ switch (vocab.type_pre) {
+ case LLAMA_VOCAB_PRE_TYPE_LLAMA3:
+ regex_exprs = {
+ // original regex from tokenizer.json
+ //"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+
+ // adapted: https://github.com/ggerganov/llama.cpp/pull/6920#issuecomment-2080233989
+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_DBRX:
+ case LLAMA_VOCAB_PRE_TYPE_SMAUG:
+ regex_exprs = {
+ // same as llama3
+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM:
+ regex_exprs = {
+ "[\r\n]",
+ "\\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿDŽ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿff-stﬓ-ﬗＡ-Ｚａ-ｚ𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+",
+ "\\s?[!-/:-~！-／：-～‘-‟ -。]+",
+ "\\s+$",
+ "[一-龥ࠀ-一가-]+",
+ "\\p{N}+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER:
+ regex_exprs = {
+ "[\r\n]",
+ "\\s?\\p{L}+",
+ "\\s?\\p{P}+",
+ "[一-龥ࠀ-一가-]+",
+ "\\p{N}",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_FALCON:
+ regex_exprs = {
+ "[\\p{P}\\$\\+<=>\\^~\\|`]+",
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ "[0-9][0-9][0-9]",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_STARCODER:
+ case LLAMA_VOCAB_PRE_TYPE_REFACT:
+ case LLAMA_VOCAB_PRE_TYPE_COMMAND_R:
+ case LLAMA_VOCAB_PRE_TYPE_SMOLLM:
+ case LLAMA_VOCAB_PRE_TYPE_CODESHELL:
+ case LLAMA_VOCAB_PRE_TYPE_EXAONE:
+ regex_exprs = {
+ "\\p{N}",
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_GPT2:
+ case LLAMA_VOCAB_PRE_TYPE_MPT:
+ case LLAMA_VOCAB_PRE_TYPE_OLMO:
+ case LLAMA_VOCAB_PRE_TYPE_JAIS:
+ regex_exprs = {
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_STABLELM2:
+ case LLAMA_VOCAB_PRE_TYPE_QWEN2:
+ regex_exprs = {
+ // original regex from tokenizer.json
+ // "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_PORO:
+ case LLAMA_VOCAB_PRE_TYPE_BLOOM:
+ case LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH:
+ regex_exprs = {
+ " ?[^(\\s|.,!?…。，、।۔،)]+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_CHATGLM4:
+ regex_exprs = {
+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_VIKING:
+ regex_exprs = {
+ " ?[^(\\s|.,!?…。，、।۔،)]+",
+ "\\p{N}",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_TEKKEN:
+ // original regex from tokenizer.json
+ // "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+ regex_exprs = {
+ "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ };
+ break;
+ default:
+ // default regex for BPE tokenization pre-processing
+ regex_exprs = {
+ "[\\p{P}\\$\\+<=>\\^~\\|]+",
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ "\\p{N}+",
+ "[0-9][0-9][0-9]",
+ };
+ break;
+ }
+ }
+
+ void append(const llama_vocab::id token_id, std::vector<llama_vocab::id> & output) const {
+ output.push_back(token_id);
+ }
+
+ bool append_bos(std::vector<llama_vocab::id> & output) const {
+ if (vocab.tokenizer_add_bos) {
+ GGML_ASSERT(vocab.special_bos_id != -1);
+ output.push_back(vocab.special_bos_id);
+ return true;
+ }
+ return false;
+ }
+
+ bool append_eos(std::vector<llama_vocab::id> & output) const {
+ if (vocab.tokenizer_add_eos) {
+ GGML_ASSERT(vocab.special_eos_id != -1);
+ output.push_back(vocab.special_eos_id);
+ return true;
+ }
+ return false;
+ }
+
+ void check_double_bos_eos(const std::vector<llama_vocab::id> & output) const {
+ if (vocab.tokenizer_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) {
+ LLAMA_LOG_WARN(
+ "%s: Added a BOS token to the prompt as specified by the model but the prompt "
+ "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
+ "Are you sure this is what you want?\n", __FUNCTION__);
+ }
+ if (vocab.tokenizer_add_eos && output.size() >= 2 && *(output.end()-2) == vocab.special_eos_id) {
+ LLAMA_LOG_WARN(
+ "%s: Added a EOS token to the prompt as specified by the model but the prompt "
+ "also ends with a EOS token. So now the final prompt ends with 2 EOS tokens. "
+ "Are you sure this is what you want?\n", __FUNCTION__);
+ }
+ }
+
+ void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
+ int final_prev_index = -1;
+
+ const auto word_collection = unicode_regex_split(text, regex_exprs);
+
+ symbols_final.clear();
+
+ for (auto & word : word_collection) {
+ work_queue = llm_bigram_bpe::queue();
+ symbols.clear();
+
+ int index = 0;
+ size_t offset = 0;
+
+ if (vocab.tokenizer_ignore_merges && vocab.token_to_id.find(word) != vocab.token_to_id.end()) {
+ symbols.emplace_back(llm_symbol{-1, -1, word.c_str(), word.size()});
+ offset = word.size();
+ }
+
+ while (offset < word.size()) {
+ llm_symbol sym;
+ size_t char_len = std::min(word.size() - offset, (size_t) unicode_len_utf8(word[offset]));
+ sym.text = word.c_str() + offset;
+ sym.n = char_len;
+ offset += sym.n;
+ sym.prev = index - 1;
+ sym.next = offset == word.size() ? -1 : index + 1;
+ index++;
+ symbols.emplace_back(sym);
+ }
+ for (size_t i = 1; i < symbols.size(); ++i) {
+ add_new_bigram(i - 1, i);
+ }
+
+ // build token(s)
+ while (!work_queue.empty()) {
+ auto bigram = work_queue.pop_move();
+
+ auto & left_symbol = symbols[bigram.left];
+ auto & right_symbol = symbols[bigram.right];
+
+ if (left_symbol.n == 0 || right_symbol.n == 0) {
+ continue;
+ }
+ std::string left_token = std::string(left_symbol.text, left_symbol.n);
+ std::string right_token = std::string(right_symbol.text, right_symbol.n);
+ if (left_token + right_token != bigram.text) {
+ continue; // Skip this bigram if it's outdated
+ }
+
+ // merge the right sym into the left one
+ left_symbol.n += right_symbol.n;
+ right_symbol.n = 0;
+
+ // remove the right sym from the chain
+ left_symbol.next = right_symbol.next;
+ if (right_symbol.next >= 0) {
+ symbols[right_symbol.next].prev = bigram.left;
+ }
+
+ add_new_bigram(left_symbol.prev, bigram.left); // left side of current symbol
+ add_new_bigram(bigram.left, left_symbol.next); // right side of current symbol
+ }
+
+ // add the finished tokens to the final list keeping correct order for next and prev
+ for (auto & sym : symbols) {
+ if (sym.n > 0) {
+ sym.prev = final_prev_index;
+ sym.next = -1;
+ if (final_prev_index != -1) {
+ symbols_final[final_prev_index].next = symbols_final.size();
+ }
+ symbols_final.emplace_back(sym);
+ final_prev_index = symbols_final.size() - 1;
+ }
+ }
+ }
+
+ symbols = symbols_final;
+
+ if (!symbols.empty()) {
+ for (int i = 0; i != -1; i = symbols[i].next) {
+ auto & symbol = symbols[i];
+ if (symbol.n == 0) {
+ continue;
+ }
+
+ const std::string str = std::string(symbol.text, symbol.n);
+ const auto token = vocab.token_to_id.find(str);
+
+ if (token == vocab.token_to_id.end()) {
+ for (auto j = str.begin(); j != str.end(); ++j) {
+ std::string byte_str(1, *j);
+ auto token_multibyte = vocab.token_to_id.find(byte_str);
+ if (token_multibyte != vocab.token_to_id.end()) {
+ output.push_back(token_multibyte->second);
+ }
+ }
+ } else {
+ output.push_back((*token).second);
+ }
+ }
+ }
+ }
+
+private:
+ void add_new_bigram(int left, int right) {
+ if (left == -1 || right == -1) {
+ return;
+ }
+
+ std::string left_token = std::string(symbols[left].text, symbols[left].n);
+ std::string right_token = std::string(symbols[right].text, symbols[right].n);
+
+ int rank_found = -1;
+
+ rank_found = vocab.find_bpe_rank(left_token, right_token);
+
+ if (rank_found < 0) {
+ return;
+ }
+
+ llm_bigram_bpe bigram;
+
+ bigram.left = left;
+ bigram.right = right;
+ bigram.text = left_token + right_token;
+ bigram.size = left_token.size() + right_token.size();
+ bigram.rank = rank_found;
+
+ work_queue.push(bigram);
+ }
+
+ const llama_vocab & vocab;
+
+ std::vector<std::string> regex_exprs;
+
+ std::vector<llm_symbol> symbols;
+ std::vector<llm_symbol> symbols_final;
+
+ llm_bigram_bpe::queue work_queue;
+};
+
+//
+// WPM tokenizer
+//
+
+struct llm_tokenizer_wpm {
+ llm_tokenizer_wpm(const llama_vocab & vocab): vocab(vocab) {}
+
+ void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) const {
+ const auto & token_map = vocab.token_to_id;
+
+ // normalize and split by whitespace
+ std::vector<std::string> words = preprocess(text);
+
+ // bos token prepended already
+
+ // find the longest tokens that form the words
+ for (const std::string & word : words) {
+ // skip empty words
+ if (word.size() == 0) {
+ continue;
+ }
+
+ // prepend phantom space
+ const std::string word1 = "\xe2\x96\x81" + word;
+ const int n = word1.size();
+
+ const size_t current_tokens = output.size();
+
+ // we're at the start of a new word
+ // move through character position in word
+ for (int i = 0; i < n; ++i) {
+ // loop through possible match length
+ bool match = false;
+ for (int j = std::min(n, i + vocab.max_token_len + 1); j > i; j--) {
+ auto it = token_map.find(word1.substr(i, j - i));
+ if (it != token_map.end()) {
+ output.push_back(it->second);
+ match = true;
+ i = j - 1;
+ break;
+ }
+ }
+
+ if (!match) { // discard all
+ output.resize(current_tokens);
+ break; // and discard next tokens
+ }
+ }
+
+ // we didn't find any matches for this word
+ if (current_tokens == output.size()) {
+ output.push_back(vocab.special_unk_id);
+ }
+ }
+ }
+
+ // TODO: reduce string copies by using cpts_offs array
+ std::vector<std::string> preprocess(const std::string & text) const {
+ const std::vector<uint32_t> cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text));
+ std::vector<std::string> words(1, "");
+
+ for (const uint32_t cpt : cpts_nfd) {
+ const auto flags = unicode_cpt_flags(cpt);
+
+ if (flags.is_whitespace) {
+ if (words.back().size()) { // finish previous word if any
+ words.emplace_back();
+ }
+ continue;
+ }
+
+ assert (!flags.is_separator);
+ if (cpt == 0 || cpt == 0xFFFD || flags.is_control) {
+ continue;
+ }
+
+ const std::string s = unicode_cpt_to_utf8(unicode_tolower(cpt));
+ if (flags.is_punctuation || ( cpt < 0x7F && flags.is_symbol ) || is_chinese_char(cpt)) {
+ if (words.back().size()) { // finish previous word if any
+ words.emplace_back();
+ }
+ words.back() = s; // single char word
+ words.emplace_back(); // start a new word
+ } else {
+ words.back() += s; // append char to word
+ }
+ }
+
+ if (!words.back().size()) {
+ words.pop_back();
+ }
+
+ return words;
+ }
+
+ static bool is_chinese_char(uint32_t cpt) {
+ return
+ (cpt >= 0x04E00 && cpt <= 0x09FFF) ||
+ (cpt >= 0x03400 && cpt <= 0x04DBF) ||
+ (cpt >= 0x20000 && cpt <= 0x2A6DF) ||
+ (cpt >= 0x2A700 && cpt <= 0x2B73F) ||
+ (cpt >= 0x2B740 && cpt <= 0x2B81F) ||
+ (cpt >= 0x2B920 && cpt <= 0x2CEAF) || // this should be 0x2B820 but in hf rust code it is 0x2B920
+ (cpt >= 0x0F900 && cpt <= 0x0FAFF) ||
+ (cpt >= 0x2F800 && cpt <= 0x2FA1F);
+ //(cpt >= 0x3000 && cpt <= 0x303F) ||
+ //(cpt >= 0xFF00 && cpt <= 0xFFEF);
+ }
+
+ const llama_vocab & vocab;
+};
+
+//
+// UGM tokenizer
+//
+
+struct llm_tokenizer_ugm {
+ llm_tokenizer_ugm(const llama_vocab & vocab) : vocab(vocab) {
+ if (vocab.precompiled_charsmap.size() > 0) {
+ size_t charsmap_offset = 0;
+
+ // First four bytes of precompiled_charsmap contains length of binary
+ // blob containing XOR-compressed compact double array (XCDA) entries
+ uint32_t xcda_blob_size = *(const uint32_t *) &vocab.precompiled_charsmap[0];
+ charsmap_offset += sizeof(xcda_blob_size);
+ if (xcda_blob_size + charsmap_offset >= vocab.precompiled_charsmap.size()) {
+ throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
+ }
+
+ // Next xcda_blob_size bytes contain entries of XOR-compressed compact
+ // double array (XCDA). Each entry is bit-packed into a 32-bit integer.
+ xcda_array = (const uint32_t *) &vocab.precompiled_charsmap[charsmap_offset];
+ xcda_array_size = xcda_blob_size / sizeof(uint32_t);
+ charsmap_offset += xcda_blob_size;
+
+ // Remaining bytes of precompiled charsmap contain null-terminated
+ // replacement strings for prefixes matched by the XCDA.
+ prefix_replacements = &vocab.precompiled_charsmap[charsmap_offset];
+ prefix_replacements_size = vocab.precompiled_charsmap.size() - charsmap_offset;
+ }
+
+ for (unsigned int id = 0; id < vocab.id_to_token.size(); ++id) {
+ const auto &token_data = vocab.id_to_token[id];
+
+ if (llama_is_normal_token(vocab, id)) {
+ min_score = std::min<float>(min_score, token_data.score);
+ max_score = std::max<float>(max_score, token_data.score);
+ }
+
+ if (llama_is_normal_token(vocab, id) ||
+ llama_is_user_defined_token(vocab, id) ||
+ llama_is_unused_token(vocab, id)) {
+ token_matcher.insert(token_data.text.data(), token_data.text.size(), id);
+ }
+
+ if (llama_is_user_defined_token(vocab, id)) {
+ user_defined_token_matcher.insert(token_data.text.data(), token_data.text.size());
+ }
+ }
+
+ unknown_token_score = min_score - unknown_token_score_penalty;
+ }
+
+ /* This implementation is based on SentencePiece optimized Viterbi algorithm for
+ * unigram language models. The general idea is to:
+ * - move along the input sequence in steps of one UTF code point,
+ * - at each step find all possible tokenizations of the prefix by
+ * traversing the tokens trie,
+ * - for each tokenization store the best one so far (by higher score)
+ * - use the position in sequence after given token as an index to store
+ * results
+ * - if there was no valid tokenization of the current UTF code point
+ * then use unknown token with additional score penalty
+ * After processing the whole sequence we backtrack from the end to get
+ * the best tokenization.
+ */
+ void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
+ // get current size of output (for reversal later)
+ size_t output_size = output.size();
+
+ // normalize the input first
+ std::string normalized;
+ normalize(text, &normalized);
+ size_t input_len = normalized.size();
+ if (input_len == 0) {
+ return;
+ }
+
+ // initialize score_sum to -FLT_MAX so it will be always lower than sums of token scores
+ std::vector<struct best_tokenization> tokenization_results(input_len + 1, {vocab.special_unk_id, 0, -FLT_MAX});
+ // at the beginning tokenization score is zero
+ tokenization_results[0] = { vocab.special_unk_id, 0, 0 };
+
+ for (size_t input_offset = 0; input_offset < input_len;) {
+ size_t prefix_offset = input_offset;
+ // calculate how many code units are in the currently processed UTF code point
+ size_t n_utf8_code_units = std::min<size_t>(unicode_len_utf8(normalized[input_offset]), input_len - input_offset);
+
+ // traverse the token matcher trie to find a matching token
+ bool single_codepoint_token_found = false;
+ const struct best_tokenization & current_best = tokenization_results[input_offset];
+ const struct naive_trie * node = token_matcher.traverse(normalized[prefix_offset++]);
+
+ while (prefix_offset <= input_len && node != NULL) {
+ // check if we found valid token in prefix
+ if (node->has_value) {
+ // check if it corresponds to the whole UTF code point
+ if (prefix_offset - input_offset == n_utf8_code_units) {
+ single_codepoint_token_found = true;
+ }
+ llama_token token_id = node->value;
+ const auto & token_data = vocab.id_to_token[token_id];
+
+ // we set the user-defined token scores to 0 to make them more likely to be selected
+ // (normal token scores are log probabilities, so they are negative)
+ // score type is double here to make tokenization results exactly
+ // the same as in the HF tokenizer using SentencePiece
+ const double token_score = llama_is_user_defined_token(vocab, token_id) ? 0.0 : token_data.score;
+ const double challenger_score = current_best.score_sum + token_score;
+ struct best_tokenization & current_champ = tokenization_results[prefix_offset];
+ if (challenger_score > current_champ.score_sum) {
+ struct best_tokenization challenger = { token_id, input_offset, (float) challenger_score };
+ current_champ = challenger;
+ }
+ }
+ node = node->traverse(normalized[prefix_offset++]);
+ }
+
+ // if we didn't find a valid token corresponding to the whole UTF code point
+ // then use unknown token as the tokenization of this UTF code point
+ if (!single_codepoint_token_found) {
+ const double challenger_score = current_best.score_sum + unknown_token_score;
+ prefix_offset = input_offset + n_utf8_code_units;
+ struct best_tokenization & current_champ = tokenization_results[prefix_offset];
+ if (challenger_score > current_champ.score_sum) {
+ struct best_tokenization challenger = { vocab.special_unk_id, input_offset, (float) challenger_score };
+ current_champ = challenger;
+ }
+ }
+
+ // move to the next UTF code point
+ input_offset += n_utf8_code_units;
+ }
+
+ // now backtrack from the end to gather token ids of the best tokenization
+ // merge sequences of consecutive unknown tokens into single unknown tokens
+ bool is_prev_unknown = false;
+ for (struct best_tokenization & tokenization = tokenization_results[input_len]; ; tokenization = tokenization_results[tokenization.input_offset]) {
+ bool is_unknown = tokenization.token_id == vocab.special_unk_id;
+ if (!(is_prev_unknown && is_unknown)) {
+ output.push_back(tokenization.token_id);
+ }
+ if (tokenization.input_offset == 0) {
+ break;
+ }
+ is_prev_unknown = is_unknown;
+ }
+
+ // reverse the output since we added tokens starting from the end of the input
+ std::reverse(output.begin() + output_size, output.end());
+ }
+
+private:
+ const llama_vocab & vocab;
+
+ // helper structure for returning normalization results
+ struct normalization_result {
+ const char * normalized;
+ size_t normalized_len;
+ size_t consumed_input;
+ };
+
+ void normalize(const std::string& input, std::string * normalized) {
+ normalized->clear();
+ normalized->reserve(input.size() * 3);
+
+ const std::string space = vocab.tokenizer_escape_whitespaces ? escaped_space : " ";
+
+ bool shall_prepend_space = !vocab.tokenizer_treat_whitespace_as_suffix && vocab.tokenizer_add_space_prefix;
+ bool shall_append_space = vocab.tokenizer_treat_whitespace_as_suffix && vocab.tokenizer_add_space_prefix;
+ bool shall_merge_spaces = vocab.tokenizer_remove_extra_whitespaces;
+
+ bool is_space_prepended = false;
+ bool processing_non_ws = false;
+
+ size_t input_len = input.size();
+
+ for (size_t input_offset = 0; input_offset < input_len; ) {
+ auto norm_res = normalize_prefix(input, input_offset);
+ for (size_t i = 0; i < norm_res.normalized_len; i++) {
+ char c = norm_res.normalized[i];
+ if (c != ' ') {
+ if (!processing_non_ws) {
+ processing_non_ws = true;
+ if ((shall_prepend_space && !is_space_prepended) || shall_merge_spaces) {
+ normalized->append(space);
+ is_space_prepended = true;
+ }
+ }
+ normalized->push_back(c);
+ } else {
+ if (processing_non_ws) {
+ processing_non_ws = false;
+ }
+ if (!shall_merge_spaces) {
+ normalized->append(space);
+ }
+ }
+ }
+
+ input_offset += norm_res.consumed_input;
+ }
+
+ if (shall_append_space) {
+ normalized->append(space);
+ }
+ }
+
+ /*
+ * This structure is a view wrapper for XOR-compressed double array (XCDA)
+ * See Shunsuke Kanda (2018). Space- and Time-Efficient String Dictionaries.
+ * Each bit-packed entry contains:
+ * - BASE array value in bits 10-30
+ * - LCHECK array value in bits 0-7
+ * - LEAF array value in bit 9
+ * Entries containing indexes of replacement sequences have set bit 31
+ */
+ struct xcda_array_view {
+ public:
+ xcda_array_view(const uint32_t * xcda_array, size_t xcda_array_size) : xcda_array(xcda_array), xcda_array_size(xcda_array_size) {
+ }
+ uint32_t get_base(size_t index) {
+ uint32_t packed_node = get_node(index);
+ return (packed_node >> 10) << ((packed_node & (1U << 9)) >> 6);
+ }
+ uint32_t get_lcheck(size_t index) {
+ uint32_t packed_node = get_node(index);
+ return packed_node & ((1U << 31) | 0xff);
+ }
+ bool get_leaf(size_t index) {
+ uint32_t packed_node = get_node(index);
+ return (packed_node >> 8) & 1;
+ }
+ uint32_t get_value(size_t index) {
+ uint32_t packed_node = get_node(index);
+ return packed_node & ((1U << 31) - 1);
+ }
+ private:
+ uint32_t get_node(size_t index) {
+ if (index > xcda_array_size) {
+ throw std::runtime_error("Index out of array bounds in XCDA array!");
+ }
+ return xcda_array[index];
+ }
+ const uint32_t * xcda_array;
+ size_t xcda_array_size;
+ };
+
+ struct normalization_result normalize_prefix(const std::string & input, size_t input_offset) {
+ if (input_offset == input.size()) {
+ return { &input[input_offset], 0, 0 };
+ }
+
+ // if input prefix matches some user-defined token return this token as normalization result
+ auto user_defined_token_match = user_defined_token_matcher.get_longest_prefix(&input[input_offset], input.size() - input_offset);
+ if (user_defined_token_match.second > 0) {
+ return { &input[input_offset], user_defined_token_match.second, user_defined_token_match.second };
+ }
+
+ size_t longest_prefix_length = 0;
+ size_t longest_prefix_offset = 0;
+
+ if (xcda_array_size > 0) {
+ struct xcda_array_view xcda_view(xcda_array, xcda_array_size);
+
+ // Find the longest normalized sequence matching the input prefix by walking
+ // the XOR-compressed compact double array (XCDA) starting from the root node
+ // We find the index of the next node by calculating BASE[s] ^ c where s is
+ // the index of the previous node and c is a numerical character value
+ uint32_t node_index = 0;
+ // get BASE of the root node
+ node_index = xcda_view.get_base(node_index);
+ for (size_t prefix_offset = input_offset; prefix_offset < input.size(); prefix_offset++) {
+ unsigned char c = input[prefix_offset];
+ if (c == 0) {
+ break;
+ }
+ node_index ^= c;
+ // if value of LCHECK is not c it means that this is not a child of
+ // the previous node, so we stop matching
+ if (xcda_view.get_lcheck(node_index) != c) {
+ break;
+ }
+ bool is_leaf = xcda_view.get_leaf(node_index);
+ // get BASE of the current node
+ node_index ^= xcda_view.get_base(node_index);
+ // if LEAF of the current node is true, it means that its BASE points to the node
+ // containing index of replacement sequence for currently matched input prefix
+ if (is_leaf)
+ {
+ longest_prefix_length = prefix_offset - input_offset + 1;
+ // get index of replacement sequence for currently matched input prefix
+ longest_prefix_offset = xcda_view.get_value(node_index);
+ }
+ }
+ }
+
+ if (longest_prefix_length > 0) {
+ // we have a match, so return the replacement sequence
+ if (longest_prefix_offset >= prefix_replacements_size) {
+ throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
+ }
+ const char * prefix_replacement = &prefix_replacements[longest_prefix_offset];
+ return { prefix_replacement, strlen(prefix_replacement), longest_prefix_length };
+ } else {
+ // check if the input prefix contains a valid sequence of UTF-8 code units
+ try {
+ // if yes, return this sequence unmodified
+ size_t prefix_offset = input_offset;
+ unicode_cpt_from_utf8(input, prefix_offset);
+ return { &input[input_offset], prefix_offset - input_offset, prefix_offset - input_offset };
+ } catch (std::invalid_argument & /*ex*/) {
+ // if no, consume 1 byte and return U+FFFD - REPLACEMENT CHARACTER
+ return { "\xEF\xBF\xBD", 3, 1 };
+ }
+ }
+ }
+
+ // escaped space symbol - U+2581 (Lower One Eighth Block)
+ const std::string escaped_space = "\xE2\x96\x81";
+
+ const char * prefix_replacements = NULL;
+ size_t prefix_replacements_size = 0;
+
+ const uint32_t * xcda_array = NULL;
+ size_t xcda_array_size = 0;
+
+ struct naive_trie user_defined_token_matcher;
+
+ // this structure stores the best tokenization so far at input_offset
+ struct best_tokenization {
+ llama_token token_id;
+ size_t input_offset;
+ float score_sum;
+ };
+
+ float min_score = FLT_MAX;
+ float max_score = -FLT_MAX;
+
+ float unknown_token_score_penalty = 10.0;
+ float unknown_token_score;
+
+ struct naive_trie token_matcher;
+};
+
+//
+// RWKV tokenizer
+//
+
+static std::vector<uint8_t> llama_unescape_rwkv_token(const std::string & escaped) {
+ std::vector<uint8_t> output;
+ output.reserve(escaped.size());
+
+ // Parser state
+ bool escaping = false;
+ uint8_t hex_remaining = 0;
+ uint8_t hex_acc = 0;
+
+ // Step through characters, performing parsing
+ for (const char & c : escaped) {
+ // If we're parsing a hex code, interpret the next character
+ if (hex_remaining != 0) {
+ uint8_t value = (c >= 'a') ? (c - 'a' + 10) : (c - '0');
+ hex_acc = (hex_acc << 4) + value;
+
+ hex_remaining -= 1;
+ if (hex_remaining == 0) {
+ output.push_back(hex_acc);
+ hex_acc = 0;
+ }
+
+ continue;
+ }
+
+ // If we got an escape character, interpret it
+ if (escaping) {
+ if (c == 't') {
+ output.push_back('\t');
+ } else if (c == 'n') {
+ output.push_back('\n');
+ } else if (c == 'r') {
+ output.push_back('\r');
+ } else if (c == 'x') {
+ hex_remaining = 2;
+ } else {
+ output.push_back(c);
+ }
+
+ escaping = false;
+ continue;
+ }
+
+ if (c == '\\') {
+ escaping = true;
+ continue;
+ }
+
+ output.push_back(c);
+ }
+
+ return output;
+}
+
+struct llm_tokenizer_rwkv {
+ llm_tokenizer_rwkv(const llama_vocab & vocab): vocab(vocab) {
+ // RWKV supports arbitrary byte tokens, but the vocab struct only supports string tokens.
+ // For now, we decode the vocab here into the lookup we'll use for tokenization.
+
+ // build trie
+ for (unsigned int id = 0; id < vocab.id_to_token.size(); ++id) {
+ const auto & token = vocab.id_to_token[id];
+ const auto data = llama_unescape_rwkv_token(token.text);
+ token_matcher.insert((const char *) data.data(), data.size(), id);
+ }
+ }
+
+ void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
+ uint32_t position = 0;
+
+ while (position < text.size()) {
+ const struct naive_trie * node = token_matcher.traverse(text[position]);
+ if (node == NULL) {
+ // no matching token found, add unknown token
+ output.push_back(vocab.special_unk_id);
+ position += 1;
+ continue;
+ }
+
+ // traverse the trie to find the longest matching token
+ uint32_t token_id = 0;
+ uint32_t token_length = 0;
+ while (node != NULL) {
+ if (node->has_value) {
+ token_id = node->value;
+ token_length = position + 1;
+ }
+ node = node->traverse(text[++position]);
+ }
+
+ // add the longest matching token
+ output.push_back(token_id);
+ position = token_length;
+ }
+ }
+
+ const llama_vocab & vocab;
+
+ struct naive_trie token_matcher;
+};
+
+//
+// (de-) tokenize
+//
+
+typedef enum FRAGMENT_BUFFER_VARIANT_TYPE {
+ FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN,
+ FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT
+} FRAGMENT_BUFFER_VARIANT_TYPE;
+
+struct fragment_buffer_variant {
+ fragment_buffer_variant(llama_vocab::id _token)
+ :
+ type(FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN),
+ token(_token),
+ raw_text(_dummy),
+ offset(0),
+ length(0) {}
+
+ fragment_buffer_variant(const std::string & _raw_text, int64_t _offset, int64_t _length)
+ :
+ type(FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT),
+ token((llama_vocab::id) - 1),
+ raw_text(_raw_text),
+ offset(_offset),
+ length(_length){
+ GGML_ASSERT(_offset >= 0);
+ GGML_ASSERT(_length >= 1);
+ GGML_ASSERT(offset + length <= raw_text.length());
+ }
+
+ const FRAGMENT_BUFFER_VARIANT_TYPE type;
+ const llama_vocab::id token;
+ const std::string _dummy;
+ const std::string & raw_text;
+ const uint64_t offset;
+ const uint64_t length;
+};
+
+// #define PRETOKENIZERDEBUG
+
+static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<fragment_buffer_variant> & buffer, bool parse_special) {
+ // for each special token
+ for (const llama_vocab::id special_id : vocab.cache_special_tokens) {
+ const auto & data = vocab.id_to_token[special_id];
+ const auto & special_token = data.text;
+
+ if (!parse_special && (data.attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_UNKNOWN))) {
+ // Ignore control and unknown tokens when parse_special == false
+ continue;
+ // User-defined tokens are still pre-tokenized before everything else
+ // ref: https://github.com/huggingface/tokenizers/blob/fdd26ba9a3f0c133427aab0423888cbde91362d7/tokenizers/src/tokenizer/mod.rs#L726
+ // This is mostly relevant for neox-style tokenizers (mpt, olmo, stablelm, etc.)
+ }
+
+ // for each text fragment
+ std::forward_list<fragment_buffer_variant>::iterator it = buffer.begin();
+ while (it != buffer.end()) {
+ auto & fragment = (*it);
+
+ // if a fragment is text ( not yet processed )
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ auto & raw_text = fragment.raw_text;
+
+ auto raw_text_base_offset = fragment.offset;
+ auto raw_text_base_length = fragment.length;
+
+ // loop over the text
+ while (true) {
+ // find the first occurrence of a given special token in this fragment
+ // passing offset argument only limit the "search area" but match coordinates
+ // are still relative to the source full raw_text
+ auto match = raw_text.find(special_token, raw_text_base_offset);
+
+ // no occurrences found, stop processing this fragment for a given special token
+ if (match == std::string::npos) break;
+
+ // check if match is within bounds of offset <-> length
+ if (match + special_token.length() > raw_text_base_offset + raw_text_base_length) break;
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());
+#endif
+ auto source = std::distance(buffer.begin(), it);
+
+ // if match is further than base offset
+ // then we have some text to the left of it
+ if (match > raw_text_base_offset) {
+ // left
+ const int64_t left_reminder_offset = raw_text_base_offset + 0;
+ int64_t left_reminder_length = match - raw_text_base_offset;
+
+ if (data.attr & LLAMA_TOKEN_ATTR_LSTRIP) {
+ while (left_reminder_length > 0 && isspace(raw_text[left_reminder_offset + left_reminder_length - 1])) {
+ left_reminder_length--;
+ }
+ }
+
+ if (left_reminder_length > 0) {
+ buffer.emplace_after(it, raw_text, left_reminder_offset, left_reminder_length);
+ it++;
+ }
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("FL: (%ld %ld) '%s'\n", left_reminder_offset, left_reminder_length, raw_text->substr(left_reminder_offset, left_reminder_length).c_str());
+#endif
+ }
+
+ // special token
+ buffer.emplace_after(it, special_id);
+ it++;
+
+ // right
+ if (match + special_token.length() < raw_text_base_offset + raw_text_base_length) {
+ int64_t right_reminder_offset = match + special_token.length();
+ int64_t right_reminder_length = raw_text_base_length - ((match - raw_text_base_offset) + special_token.length());
+
+ if (data.attr & LLAMA_TOKEN_ATTR_RSTRIP) {
+ while (right_reminder_length > 0 && isspace(raw_text[right_reminder_offset])) {
+ right_reminder_offset++;
+ right_reminder_length--;
+ }
+ }
+
+ if (right_reminder_length > 0) {
+ buffer.emplace_after(it, raw_text, right_reminder_offset, right_reminder_length);
+ it++;
+ }
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("FR: (%ld %ld) '%s'\n", right_reminder_offset, right_reminder_length, raw_text->substr(right_reminder_offset, right_reminder_length).c_str());
+#endif
+
+ if (source == 0) {
+ buffer.erase_after(buffer.before_begin());
+ } else {
+ buffer.erase_after(std::next(buffer.begin(), (source-1)));
+ }
+
+ // repeat for the right side
+ raw_text_base_offset = right_reminder_offset;
+ raw_text_base_length = right_reminder_length;
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("RR: (%ld %ld) '%s'\n", raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());
+#endif
+ } else {
+ if (source == 0) {
+ buffer.erase_after(buffer.before_begin());
+ } else {
+ buffer.erase_after(std::next(buffer.begin(), (source-1)));
+ }
+ break;
+ }
+ }
+ }
+ it++;
+ }
+ }
+}
+
+std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab & vocab, std::string raw_text, bool add_special, bool parse_special) {
+ std::vector<llama_vocab::id> output;
+ std::forward_list<fragment_buffer_variant> fragment_buffer;
+
+ if (!raw_text.empty()) {
+ fragment_buffer.emplace_front(raw_text, 0, raw_text.length());
+ tokenizer_st_partition(vocab, fragment_buffer, parse_special);
+ }
+
+ switch (vocab.type) {
+ case LLAMA_VOCAB_TYPE_SPM:
+ {
+ // OG tokenizer behavior:
+ //
+ // tokenizer.encode('', add_special_tokens=True) returns [1]
+ // tokenizer.encode('', add_special_tokens=False) returns []
+
+ bool is_prev_special = true; // prefix with space if first token
+
+ if (add_special && vocab.tokenizer_add_bos) {
+ GGML_ASSERT(vocab.special_bos_id != -1);
+ output.push_back(vocab.special_bos_id);
+ is_prev_special = true;
+ }
+
+ for (const auto & fragment : fragment_buffer) {
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
+
+ // prefix with space if previous is special
+ if (vocab.tokenizer_add_space_prefix && is_prev_special) {
+ raw_text = " " + raw_text;
+ }
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
+#endif
+ llm_tokenizer_spm tokenizer(vocab);
+ llama_escape_whitespace(raw_text);
+ tokenizer.tokenize(raw_text, output);
+ is_prev_special = false;
+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
+ output.push_back(fragment.token);
+ is_prev_special = true;
+ }
+ }
+
+ if (add_special && vocab.tokenizer_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) {
+ LLAMA_LOG_WARN(
+ "%s: Added a BOS token to the prompt as specified by the model but the prompt "
+ "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
+ "Are you sure this is what you want?\n", __FUNCTION__);
+ }
+
+ if (add_special && vocab.tokenizer_add_eos) {
+ GGML_ASSERT(vocab.special_eos_id != -1);
+ output.push_back(vocab.special_eos_id);
+ }
+ } break;
+ case LLAMA_VOCAB_TYPE_BPE:
+ {
+ llm_tokenizer_bpe tokenizer(vocab);
+
+ if (add_special) {
+ tokenizer.append_bos(output);
+ }
+ for (const auto & fragment : fragment_buffer) {
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
+#endif
+ tokenizer.tokenize(raw_text, output);
+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
+ tokenizer.append(fragment.token, output);
+ }
+ }
+
+ if (add_special) {
+ tokenizer.append_eos(output);
+ tokenizer.check_double_bos_eos(output);
+ }
+ } break;
+ case LLAMA_VOCAB_TYPE_WPM:
+ {
+ if (add_special) {
+ GGML_ASSERT(vocab.special_cls_id != -1);
+ output.push_back(vocab.special_cls_id);
+ }
+
+ llm_tokenizer_wpm tokenizer(vocab);
+
+ for (const auto & fragment : fragment_buffer) {
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
+#endif
+ tokenizer.tokenize(raw_text, output);
+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
+ output.push_back(fragment.token);
+ }
+ }
+
+ if (add_special) {
+ GGML_ASSERT(vocab.special_sep_id != -1);
+ output.push_back(vocab.special_sep_id);
+ }
+ } break;
+ case LLAMA_VOCAB_TYPE_UGM:
+ {
+ llm_tokenizer_ugm tokenizer(vocab);
+
+ if (add_special && vocab.tokenizer_add_bos != 0) {
+ GGML_ASSERT(vocab.special_bos_id != -1);
+ output.push_back(vocab.special_bos_id);
+ }
+
+ for (const auto & fragment : fragment_buffer) {
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
+#endif
+ tokenizer.tokenize(raw_text, output);
+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
+ output.push_back(fragment.token);
+ }
+ }
+
+ if (add_special && vocab.tokenizer_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) {
+ LLAMA_LOG_WARN(
+ "%s: Added a BOS token to the prompt as specified by the model but the prompt "
+ "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
+ "Are you sure this is what you want?\n", __FUNCTION__);
+ }
+
+ if (add_special && vocab.tokenizer_add_eos == 1) {
+ GGML_ASSERT(vocab.special_eos_id != -1);
+ output.push_back(vocab.special_eos_id);
+ }
+ } break;
+ case LLAMA_VOCAB_TYPE_RWKV:
+ {
+ for (const auto & fragment : fragment_buffer) {
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
+#endif
+
+ llm_tokenizer_rwkv tokenizer(vocab);
+ tokenizer.tokenize(raw_text, output);
+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
+ output.push_back(fragment.token);
+ }
+ }
+ } break;
+ case LLAMA_VOCAB_TYPE_NONE:
+ GGML_ABORT("fatal error");
+ }
+
+ return output;
+}
+
+llama_token llama_byte_to_token_impl(const llama_vocab & vocab, uint8_t ch) {
+ GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE);
+ static const char * hex = "0123456789ABCDEF";
+ switch (llama_vocab_get_type(vocab)) {
+ case LLAMA_VOCAB_TYPE_SPM:
+ case LLAMA_VOCAB_TYPE_UGM: {
+ const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 };
+ auto token = vocab.token_to_id.find(buf);
+ if (token != vocab.token_to_id.end()) {
+ return (*token).second;
+ }
+ // Try to fall back to just the byte as a string
+ const char buf2[2] = { (char)ch, 0 };
+ return vocab.token_to_id.at(buf2);
+ }
+ case LLAMA_VOCAB_TYPE_WPM:
+ case LLAMA_VOCAB_TYPE_BPE: {
+ return vocab.token_to_id.at(unicode_byte_to_utf8(ch));
+ }
+ default:
+ GGML_ABORT("fatal error");
+ }
+}
+
+const char * llama_token_get_text_impl(const struct llama_vocab & vocab, llama_token token) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[token].text.c_str();
+}
+
+float llama_token_get_score_impl(const struct llama_vocab & vocab, llama_token token) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[token].score;
+}
+
+llama_token_attr llama_token_get_attr_impl(const struct llama_vocab & vocab, llama_token token) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[token].attr;
+}
+
+bool llama_token_is_eog_impl(const struct llama_vocab & vocab, llama_token token) {
+ return token != -1 && (
+ token == llama_token_eos_impl(vocab) ||
+ token == llama_token_eot_impl(vocab) ||
+ token == llama_token_eom_impl(vocab)
+ );
+}
+
+bool llama_token_is_control_impl(const struct llama_vocab & vocab, llama_token token) {
+ return llama_is_control_token(vocab, token);
+}
+
+llama_token llama_token_bos_impl(const struct llama_vocab & vocab) {
+ return vocab.special_bos_id;
+}
+
+llama_token llama_token_eos_impl(const struct llama_vocab & vocab) {
+ return vocab.special_eos_id;
+}
+
+llama_token llama_token_cls_impl(const struct llama_vocab & vocab) {
+ return vocab.special_cls_id;
+}
+
+llama_token llama_token_sep_impl(const struct llama_vocab & vocab) {
+ return vocab.special_sep_id;
+}
+
+llama_token llama_token_nl_impl(const struct llama_vocab & vocab) {
+ return vocab.linefeed_id;
+}
+
+llama_token llama_token_pad_impl(const struct llama_vocab & vocab) {
+ return vocab.special_pad_id;
+}
+
+bool llama_add_bos_token_impl(const struct llama_vocab & vocab) {
+ return vocab.tokenizer_add_bos;
+}
+
+bool llama_add_eos_token_impl(const struct llama_vocab & vocab) {
+ return vocab.tokenizer_add_eos;
+}
+
+llama_token llama_token_prefix_impl(const struct llama_vocab & vocab) {
+ return vocab.special_prefix_id;
+}
+
+llama_token llama_token_middle_impl(const struct llama_vocab & vocab) {
+ return vocab.special_middle_id;
+}
+
+llama_token llama_token_suffix_impl(const struct llama_vocab & vocab) {
+ return vocab.special_suffix_id;
+}
+
+llama_token llama_token_eot_impl(const struct llama_vocab & vocab) {
+ return vocab.special_eot_id;
+}
+
+llama_token llama_token_eom_impl(const struct llama_vocab & vocab) {
+ return vocab.special_eom_id;
+}
+
+int32_t llama_tokenize_impl(
+ const struct llama_vocab & vocab,
+ const char * text,
+ int32_t text_len,
+ llama_token * tokens,
+ int32_t n_tokens_max,
+ bool add_special,
+ bool parse_special) {
+ auto res = llama_tokenize_internal(vocab, std::string(text, text_len), add_special, parse_special);
+ if (n_tokens_max < (int) res.size()) {
+ // LLAMA_LOG_ERROR("%s: too many tokens\n", __func__);
+ return -((int) res.size());
+ }
+
+ for (size_t i = 0; i < res.size(); i++) {
+ tokens[i] = res[i];
+ }
+
+ return res.size();
+}
+
+static std::string llama_decode_text(const std::string & text) {
+ std::string decoded_text;
+
+ const auto cpts = unicode_cpts_from_utf8(text);
+ for (const auto cpt : cpts) {
+ const auto utf8 = unicode_cpt_to_utf8(cpt);
+ try {
+ decoded_text += unicode_utf8_to_byte(utf8);
+ } catch (const std::out_of_range & /*e*/) {
+ decoded_text += "[UNK_BYTE_0x";
+ for (const auto c : utf8) {
+ decoded_text += format("%02x", (uint8_t) c);
+ }
+ decoded_text += text + "]";
+ }
+ }
+
+ return decoded_text;
+}
+
+// does not write null-terminator to buf
+int32_t llama_token_to_piece_impl(const struct llama_vocab & vocab, llama_token token, char * buf, int32_t length, int32_t lstrip, bool special) {
+ // ref: https://github.com/ggerganov/llama.cpp/pull/7587#discussion_r1620983843
+ static const int attr_special = LLAMA_TOKEN_ATTR_UNKNOWN | LLAMA_TOKEN_ATTR_CONTROL;
+ const llama_token_attr attr = llama_token_get_attr_impl(vocab, token);
+ if (!special && (attr & attr_special)) {
+ return 0;
+ }
+
+ // copy piece chars to output text buffer
+ // skip up to 'lstrip' leading spaces before copying
+ auto _try_copy = [=] (const char * token, size_t size) -> int32_t {
+ for (int32_t i = 0; i < lstrip && size && *token == ' '; ++i) {
+ token++;
+ size--;
+ }
+ if (length < (int32_t)size) {
+ return -(int32_t) size;
+ }
+ memcpy(buf, token, size);
+ return (int32_t) size;
+ };
+
+ // if we have a cache - use it
+ {
+ const auto & cache = vocab.cache_token_to_piece;
+
+ if (!cache.empty()) {
+ const auto & result = cache.at(token);
+ return _try_copy(result.data(), result.size());
+ }
+ }
+
+ if (0 <= token && token < (int32_t) vocab.id_to_token.size()) {
+ const std::string & token_text = vocab.id_to_token[token].text;
+ switch (llama_vocab_get_type(vocab)) {
+ case LLAMA_VOCAB_TYPE_WPM:
+ case LLAMA_VOCAB_TYPE_SPM:
+ case LLAMA_VOCAB_TYPE_UGM: {
+ // NOTE: we accept all unsupported token types,
+ // suppressing them like CONTROL tokens.
+ if (attr & (attr_special | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
+ return _try_copy(token_text.data(), token_text.size());
+ } else if (attr & LLAMA_TOKEN_ATTR_NORMAL) {
+ std::string result = token_text;
+ llama_unescape_whitespace(result);
+ return _try_copy(result.data(), result.size());
+ } else if (attr & LLAMA_TOKEN_ATTR_BYTE) {
+ char byte = (char) llama_token_to_byte(vocab, token);
+ return _try_copy((char*) &byte, 1);
+ }
+ break;
+ }
+ case LLAMA_VOCAB_TYPE_BPE: {
+ // NOTE: we accept all unsupported token types,
+ // suppressing them like CONTROL tokens.
+ if (attr & (attr_special | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
+ return _try_copy(token_text.data(), token_text.size());
+ } else if (attr & LLAMA_TOKEN_ATTR_NORMAL) {
+ std::string result = llama_decode_text(token_text);
+ return _try_copy(result.data(), result.size());
+ }
+ break;
+ }
+ case LLAMA_VOCAB_TYPE_RWKV: {
+ std::vector<uint8_t> result = llama_unescape_rwkv_token(token_text);
+
+ // If we don't have enough space, return an error
+ if (result.size() > (size_t)length) {
+ return -(int)result.size();
+ }
+
+ memcpy(buf, result.data(), result.size());
+ return (int)result.size();
+ }
+ default:
+ GGML_ABORT("fatal error");
+ }
+ }
+
+ return 0;
+}
+
+int32_t llama_detokenize_impl(
+ const struct llama_vocab & vocab,
+ const llama_token * tokens,
+ int32_t n_tokens,
+ char * text,
+ int32_t text_len_max,
+ bool remove_special,
+ bool unparse_special) {
+ int32_t avail = text_len_max;
+ int32_t total = 0;
+
+ // remove the leading space
+ bool remove_space = vocab.tokenizer_add_space_prefix;
+
+ if (remove_special && vocab.tokenizer_add_bos) {
+ if (n_tokens > 0 && tokens[0] == vocab.special_bos_id) {
+ remove_space = false;
+ n_tokens--;
+ tokens++;
+ }
+ }
+
+ if (remove_special && vocab.tokenizer_add_eos) {
+ if (n_tokens > 0 && tokens[n_tokens-1] == vocab.special_eos_id) {
+ n_tokens--;
+ }
+ }
+
+ for (int32_t i = 0; i < n_tokens; ++i) {
+ GGML_ASSERT(avail >= 0);
+ int32_t n_chars = llama_token_to_piece_impl(vocab, tokens[i], text, avail, remove_space, unparse_special);
+ remove_space = false;
+ if (n_chars < 0) {
+ avail = 0;
+ total -= n_chars;
+ } else if (n_chars > 0) {
+ avail -= n_chars;
+ text += n_chars;
+ total += n_chars;
+ }
+ }
+
+ if (total > text_len_max) {
+ return -total;
+ }
+
+ if (vocab.tokenizer_clean_spaces) {
+ text -= total; // restart text
+
+ // first pass: characters ?!., //TODO: where do these characters come from?
+ const int32_t total1 = total;
+ total = total ? 1 : 0;
+ for (int32_t i = 1; i < total1; ++i) {
+ const char x = text[i];
+ if (text[i - 1] == ' ') {
+ if (x == '?' || x == '!' || x == '.' || x == ',') { // " ?", " !", " .", " ,"
+ total--; // remove space
+ }
+ }
+ text[total++] = x;
+ }
+
+ // second pass: strip single apostrophe between spaces
+ const int32_t total2 = total;
+ total = total ? 1 : 0;
+ for (int32_t i = 1; i < total2; ++i) {
+ const char x = text[i];
+ if (x == '\'' && i + 1 < total2 && text[i - 1] == ' ' && text[i + 1] == ' ') { // " ' "
+ total--; // remove prev space
+ text[++i] = '\0'; // remove next space
+ }
+ text[total++] = x;
+ }
+
+ // third pass: apostrophe contractions //NOTE: this makes sense?
+ const int32_t total3 = total;
+ total = total ? 1 : 0;
+ for (int32_t i = 1; i < total3; ++i) {
+ const char x = text[i];
+ if (text[i - 1] == ' ') {
+ if (x == '\'' && i + 1 < total3) {
+ const char x1 = text[i + 1];
+ if (x1 == 't' || x1 == 'd') { // " 't", " 'd"
+ //total--; // remove space
+ } else if (x1 == 's' || x1 == 'm') { // " 's", " 'm"
+ total--; // remove space
+ } else if (i + 2 < total3) {
+ const char x2 = text[i + 2];
+ if ((x1 == 'l' && x2 == 'l')) { // " 'll"
+ //total--; // remove space
+ } else if ((x1 == 'r' && x2 == 'e') || (x1 == 'v' && x2 == 'e')) { // " 're", " 've"
+ total--; // remove space
+ } else {
+ //total--; // remove space
+ }
+ } else {
+ //total--; // remove space
+ }
+ }
+ }
+ text[total++] = x;
+ }
+ }
+
+ return total <= text_len_max ? total : -total;
+}
diff --git a/llama/llama-vocab.h b/llama/llama-vocab.h
new file mode 100644
index 00000000..3b935909
--- /dev/null
+++ b/llama/llama-vocab.h
@@ -0,0 +1,158 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "llama-impl.h"
+
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <map>
+
+struct llama_vocab {
+ using id = llama_token;
+ using token = std::string;
+ using tattr = llama_token_attr;
+
+ struct token_data {
+ token text;
+ float score;
+ tattr attr;
+ };
+
+ enum llama_vocab_type type = LLAMA_VOCAB_TYPE_SPM;
+ enum llama_vocab_pre_type type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+
+ int max_token_len = 0; // used for optimizing longest token search
+
+ std::unordered_map<token, id> token_to_id;
+ std::vector<token_data> id_to_token;
+
+ std::vector<id> cache_special_tokens;
+ std::vector<token> cache_token_to_piece; // llama_token_to_piece(special = true);
+
+ std::map<std::pair<std::string, std::string>, int> bpe_ranks;
+
+ // default LLaMA special tokens
+ id special_bos_id = 1;
+ id special_eos_id = 2;
+ id special_unk_id = 0;
+ id special_sep_id = -1;
+ id special_pad_id = -1;
+ id special_cls_id = -1;
+ id special_mask_id = -1;
+
+ id linefeed_id = 13;
+ id special_prefix_id = -1;
+ id special_suffix_id = -1;
+ id special_middle_id = -1;
+ id special_eot_id = -1; // TODO: move above after "eos_id", and here add "file separator" token
+ id special_eom_id = -1;
+
+ // tokenizer flags
+ bool tokenizer_add_space_prefix = false;
+ bool tokenizer_add_bos = false;
+ bool tokenizer_add_eos = false;
+ bool tokenizer_ignore_merges = false;
+ bool tokenizer_clean_spaces = false; // clean_up_tokenization_spaces
+ bool tokenizer_remove_extra_whitespaces = false;
+ bool tokenizer_escape_whitespaces = true;
+ bool tokenizer_treat_whitespace_as_suffix = false;
+
+ std::vector<char> precompiled_charsmap;
+
+ int find_bpe_rank(const std::string & token_left, const std::string & token_right) const;
+};
+
+const struct llama_vocab * llama_get_vocab(const struct llama_context * ctx);
+
+//
+// internal API
+//
+
+// TODO: rename to llama_tokenize_impl
+// TODO: This should probably be in llama.h
+std::vector<llama_vocab::id> llama_tokenize_internal(
+ const llama_vocab & vocab,
+ std::string raw_text,
+ bool add_special,
+ bool parse_special = false);
+
+llama_token llama_byte_to_token_impl(const llama_vocab & vocab, uint8_t ch);
+
+const char * llama_token_get_text_impl(const struct llama_vocab & vocab, llama_token token);
+
+float llama_token_get_score_impl(const struct llama_vocab & vocab, llama_token token);
+
+llama_token_attr llama_token_get_attr_impl(const struct llama_vocab & vocab, llama_token token);
+
+bool llama_token_is_eog_impl(const struct llama_vocab & vocab, llama_token token);
+
+bool llama_token_is_control_impl(const struct llama_vocab & vocab, llama_token token);
+
+llama_token llama_token_bos_impl(const struct llama_vocab & vocab);
+llama_token llama_token_eos_impl(const struct llama_vocab & vocab);
+llama_token llama_token_cls_impl(const struct llama_vocab & vocab);
+llama_token llama_token_sep_impl(const struct llama_vocab & vocab);
+llama_token llama_token_nl_impl (const struct llama_vocab & vocab);
+llama_token llama_token_pad_impl(const struct llama_vocab & vocab);
+
+bool llama_add_bos_token_impl(const struct llama_vocab & vocab);
+bool llama_add_eos_token_impl(const struct llama_vocab & vocab);
+
+llama_token llama_token_prefix_impl(const struct llama_vocab & vocab);
+llama_token llama_token_middle_impl(const struct llama_vocab & vocab);
+llama_token llama_token_suffix_impl(const struct llama_vocab & vocab);
+llama_token llama_token_eot_impl (const struct llama_vocab & vocab);
+llama_token llama_token_eom_impl (const struct llama_vocab & vocab);
+
+int32_t llama_tokenize_impl(
+ const struct llama_vocab & vocab,
+ const char * text,
+ int32_t text_len,
+ llama_token * tokens,
+ int32_t n_tokens_max,
+ bool add_special,
+ bool parse_special);
+
+// does not write null-terminator to buf
+int32_t llama_token_to_piece_impl(
+ const struct llama_vocab & vocab,
+ llama_token token,
+ char * buf,
+ int32_t length,
+ int32_t lstrip,
+ bool special);
+
+int32_t llama_detokenize_impl(
+ const struct llama_vocab & vocab,
+ const llama_token * tokens,
+ int32_t n_tokens,
+ char * text,
+ int32_t text_len_max,
+ bool remove_special,
+ bool unparse_special);
diff --git a/llama/llama.cpp b/llama/llama.cpp
new file mode 100644
index 00000000..d284c081
--- /dev/null
+++ b/llama/llama.cpp
@@ -0,0 +1,21115 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "llama-impl.h"
+#include "llama-vocab.h"
+#include "llama-grammar.h"
+#include "llama-sampling.h"
+
+#include "unicode.h"
+
+#include "ggml.h"
+#include "ggml-alloc.h"
+#include "ggml-backend.h"
+
+#ifdef GGML_USE_RPC
+# include "ggml-rpc.h"
+#endif
+
+#ifdef GGML_USE_CUDA
+# include "ggml-cuda.h"
+#elif defined(GGML_USE_VULKAN)
+# include "ggml-vulkan.h"
+#elif defined(GGML_USE_SYCL)
+# include "ggml-sycl.h"
+#elif defined(GGML_USE_KOMPUTE)
+# include "ggml-kompute.h"
+#elif defined(GGML_USE_CANN)
+# include "ggml-cann.h"
+#endif
+
+#ifdef GGML_USE_BLAS
+# include "ggml-blas.h"
+#endif
+
+#ifdef GGML_USE_METAL
+# include "ggml-metal.h"
+#endif
+
+// TODO: replace with ggml API call
+#define QK_K 256
+
+#ifdef __has_include
+ #if __has_include(<unistd.h>)
+ #include <unistd.h>
+ #if defined(_POSIX_MAPPED_FILES)
+ #include <sys/mman.h>
+ #include <fcntl.h>
+ #endif
+ #if defined(_POSIX_MEMLOCK_RANGE)
+ #include <sys/resource.h>
+ #endif
+ #endif
+#endif
+
+#if defined(_WIN32)
+ #define WIN32_LEAN_AND_MEAN
+ #ifndef NOMINMAX
+ #define NOMINMAX
+ #endif
+ #include <windows.h>
+ #ifndef PATH_MAX
+ #define PATH_MAX MAX_PATH
+ #endif
+ #include <io.h>
+#endif
+
+#if __cplusplus >= 202000L
+ #define LU8(x) (const char*)(u8##x)
+#else
+ #define LU8(x) u8##x
+#endif
+
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <cctype>
+#include <cfloat>
+#include <cinttypes>
+#include <climits>
+#include <cmath>
+#include <cstdarg>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <ctime>
+#include <fstream>
+#include <functional>
+#include <future>
+#include <initializer_list>
+#include <locale>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <numeric>
+#include <set>
+#include <sstream>
+#include <thread>
+#include <type_traits>
+#include <unordered_map>
+
+#if defined(_MSC_VER)
+#pragma warning(disable: 4244 4267) // possible loss of data
+#endif
+
+// bump if necessary
+#define LLAMA_MAX_LAYERS 512
+#define LLAMA_MAX_EXPERTS 160 // DeepSeekV2
+
+//
+// helpers
+//
+
+// trim whitespace from the beginning and end of a string
+static std::string trim(const std::string & str) {
+ size_t start = 0;
+ size_t end = str.size();
+ while (start < end && isspace(str[start])) {
+ start += 1;
+ }
+ while (end > start && isspace(str[end - 1])) {
+ end -= 1;
+ }
+ return str.substr(start, end - start);
+}
+
+static bool is_float_close(float a, float b, float abs_tol) {
+ // Check for non-negative tolerance
+ if (abs_tol < 0.0) {
+ throw std::invalid_argument("Tolerance must be non-negative");
+ }
+
+ // Exact equality check
+ if (a == b) {
+ return true;
+ }
+
+ // Check for infinities
+ if (std::isinf(a) || std::isinf(b)) {
+ return false;
+ }
+
+ // Regular comparison using the provided absolute tolerance
+ return std::fabs(b - a) <= abs_tol;
+}
+
+static void zeros(std::ofstream & file, size_t n) {
+ char zero = 0;
+ for (size_t i = 0; i < n; ++i) {
+ file.write(&zero, 1);
+ }
+}
+
+LLAMA_ATTRIBUTE_FORMAT(1, 2)
+static std::string format(const char * fmt, ...) {
+ va_list ap;
+ va_list ap2;
+ va_start(ap, fmt);
+ va_copy(ap2, ap);
+ int size = vsnprintf(NULL, 0, fmt, ap);
+ GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT
+ std::vector<char> buf(size + 1);
+ int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
+ GGML_ASSERT(size2 == size);
+ va_end(ap2);
+ va_end(ap);
+ return std::string(buf.data(), size);
+}
+
+//
+// gguf constants (sync with gguf.py)
+//
+
+enum llm_arch {
+ LLM_ARCH_LLAMA,
+ LLM_ARCH_FALCON,
+ LLM_ARCH_BAICHUAN,
+ LLM_ARCH_GROK,
+ LLM_ARCH_GPT2,
+ LLM_ARCH_GPTJ,
+ LLM_ARCH_GPTNEOX,
+ LLM_ARCH_MPT,
+ LLM_ARCH_STARCODER,
+ LLM_ARCH_REFACT,
+ LLM_ARCH_BERT,
+ LLM_ARCH_NOMIC_BERT,
+ LLM_ARCH_JINA_BERT_V2,
+ LLM_ARCH_BLOOM,
+ LLM_ARCH_STABLELM,
+ LLM_ARCH_QWEN,
+ LLM_ARCH_QWEN2,
+ LLM_ARCH_QWEN2MOE,
+ LLM_ARCH_PHI2,
+ LLM_ARCH_PHI3,
+ LLM_ARCH_PLAMO,
+ LLM_ARCH_CODESHELL,
+ LLM_ARCH_ORION,
+ LLM_ARCH_INTERNLM2,
+ LLM_ARCH_MINICPM,
+ LLM_ARCH_GEMMA,
+ LLM_ARCH_GEMMA2,
+ LLM_ARCH_STARCODER2,
+ LLM_ARCH_MAMBA,
+ LLM_ARCH_XVERSE,
+ LLM_ARCH_COMMAND_R,
+ LLM_ARCH_DBRX,
+ LLM_ARCH_OLMO,
+ LLM_ARCH_OPENELM,
+ LLM_ARCH_ARCTIC,
+ LLM_ARCH_DEEPSEEK2,
+ LLM_ARCH_CHATGLM,
+ LLM_ARCH_BITNET,
+ LLM_ARCH_T5,
+ LLM_ARCH_T5ENCODER,
+ LLM_ARCH_JAIS,
+ LLM_ARCH_NEMOTRON,
+ LLM_ARCH_EXAONE,
+ LLM_ARCH_RWKV6,
+ LLM_ARCH_SOLAR,
+ LLM_ARCH_UNKNOWN,
+};
+
+static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
+ { LLM_ARCH_LLAMA, "llama" },
+ { LLM_ARCH_FALCON, "falcon" },
+ { LLM_ARCH_GROK, "grok" },
+ { LLM_ARCH_GPT2, "gpt2" },
+ { LLM_ARCH_GPTJ, "gptj" },
+ { LLM_ARCH_GPTNEOX, "gptneox" },
+ { LLM_ARCH_MPT, "mpt" },
+ { LLM_ARCH_BAICHUAN, "baichuan" },
+ { LLM_ARCH_STARCODER, "starcoder" },
+ { LLM_ARCH_REFACT, "refact" },
+ { LLM_ARCH_BERT, "bert" },
+ { LLM_ARCH_NOMIC_BERT, "nomic-bert" },
+ { LLM_ARCH_JINA_BERT_V2, "jina-bert-v2" },
+ { LLM_ARCH_BLOOM, "bloom" },
+ { LLM_ARCH_STABLELM, "stablelm" },
+ { LLM_ARCH_QWEN, "qwen" },
+ { LLM_ARCH_QWEN2, "qwen2" },
+ { LLM_ARCH_QWEN2MOE, "qwen2moe" },
+ { LLM_ARCH_PHI2, "phi2" },
+ { LLM_ARCH_PHI3, "phi3" },
+ { LLM_ARCH_PLAMO, "plamo" },
+ { LLM_ARCH_CODESHELL, "codeshell" },
+ { LLM_ARCH_ORION, "orion" },
+ { LLM_ARCH_INTERNLM2, "internlm2" },
+ { LLM_ARCH_MINICPM, "minicpm" },
+ { LLM_ARCH_GEMMA, "gemma" },
+ { LLM_ARCH_GEMMA2, "gemma2" },
+ { LLM_ARCH_STARCODER2, "starcoder2" },
+ { LLM_ARCH_MAMBA, "mamba" },
+ { LLM_ARCH_XVERSE, "xverse" },
+ { LLM_ARCH_COMMAND_R, "command-r" },
+ { LLM_ARCH_DBRX, "dbrx" },
+ { LLM_ARCH_OLMO, "olmo" },
+ { LLM_ARCH_OPENELM, "openelm" },
+ { LLM_ARCH_ARCTIC, "arctic" },
+ { LLM_ARCH_DEEPSEEK2, "deepseek2" },
+ { LLM_ARCH_CHATGLM, "chatglm" },
+ { LLM_ARCH_BITNET, "bitnet" },
+ { LLM_ARCH_T5, "t5" },
+ { LLM_ARCH_T5ENCODER, "t5encoder" },
+ { LLM_ARCH_JAIS, "jais" },
+ { LLM_ARCH_NEMOTRON, "nemotron" },
+ { LLM_ARCH_EXAONE, "exaone" },
+ { LLM_ARCH_RWKV6, "rwkv6" },
+ { LLM_ARCH_SOLAR, "solar" },
+ { LLM_ARCH_UNKNOWN, "(unknown)" },
+};
+
+enum llm_kv {
+ LLM_KV_GENERAL_TYPE,
+ LLM_KV_GENERAL_ARCHITECTURE,
+ LLM_KV_GENERAL_QUANTIZATION_VERSION,
+ LLM_KV_GENERAL_ALIGNMENT,
+ LLM_KV_GENERAL_NAME,
+ LLM_KV_GENERAL_AUTHOR,
+ LLM_KV_GENERAL_VERSION,
+ LLM_KV_GENERAL_URL,
+ LLM_KV_GENERAL_DESCRIPTION,
+ LLM_KV_GENERAL_LICENSE,
+ LLM_KV_GENERAL_SOURCE_URL,
+ LLM_KV_GENERAL_SOURCE_HF_REPO,
+
+ LLM_KV_VOCAB_SIZE,
+ LLM_KV_CONTEXT_LENGTH,
+ LLM_KV_EMBEDDING_LENGTH,
+ LLM_KV_BLOCK_COUNT,
+ LLM_KV_LEADING_DENSE_BLOCK_COUNT,
+ LLM_KV_FEED_FORWARD_LENGTH,
+ LLM_KV_EXPERT_FEED_FORWARD_LENGTH,
+ LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH,
+ LLM_KV_USE_PARALLEL_RESIDUAL,
+ LLM_KV_TENSOR_DATA_LAYOUT,
+ LLM_KV_EXPERT_COUNT,
+ LLM_KV_EXPERT_USED_COUNT,
+ LLM_KV_EXPERT_SHARED_COUNT,
+ LLM_KV_EXPERT_WEIGHTS_SCALE,
+ LLM_KV_POOLING_TYPE,
+ LLM_KV_LOGIT_SCALE,
+ LLM_KV_DECODER_START_TOKEN_ID,
+ LLM_KV_ATTN_LOGIT_SOFTCAPPING,
+ LLM_KV_FINAL_LOGIT_SOFTCAPPING,
+ LLM_KV_RESCALE_EVERY_N_LAYERS,
+ LLM_KV_TIME_MIX_EXTRA_DIM,
+ LLM_KV_TIME_DECAY_EXTRA_DIM,
+
+ LLM_KV_ATTENTION_HEAD_COUNT,
+ LLM_KV_ATTENTION_HEAD_COUNT_KV,
+ LLM_KV_ATTENTION_MAX_ALIBI_BIAS,
+ LLM_KV_ATTENTION_CLAMP_KQV,
+ LLM_KV_ATTENTION_KEY_LENGTH,
+ LLM_KV_ATTENTION_VALUE_LENGTH,
+ LLM_KV_ATTENTION_LAYERNORM_EPS,
+ LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,
+ LLM_KV_ATTENTION_CAUSAL,
+ LLM_KV_ATTENTION_Q_LORA_RANK,
+ LLM_KV_ATTENTION_KV_LORA_RANK,
+ LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT,
+ LLM_KV_ATTENTION_SLIDING_WINDOW,
+ LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION,
+
+ LLM_KV_ROPE_DIMENSION_COUNT,
+ LLM_KV_ROPE_FREQ_BASE,
+ LLM_KV_ROPE_SCALE_LINEAR,
+ LLM_KV_ROPE_SCALING_TYPE,
+ LLM_KV_ROPE_SCALING_FACTOR,
+ LLM_KV_ROPE_SCALING_ATTN_FACTOR,
+ LLM_KV_ROPE_SCALING_ORIG_CTX_LEN,
+ LLM_KV_ROPE_SCALING_FINETUNED,
+ LLM_KV_ROPE_SCALING_YARN_LOG_MUL,
+
+ LLM_KV_SPLIT_NO,
+ LLM_KV_SPLIT_COUNT,
+ LLM_KV_SPLIT_TENSORS_COUNT,
+
+ LLM_KV_SSM_INNER_SIZE,
+ LLM_KV_SSM_CONV_KERNEL,
+ LLM_KV_SSM_STATE_SIZE,
+ LLM_KV_SSM_TIME_STEP_RANK,
+ LLM_KV_SSM_DT_B_C_RMS,
+
+ LLM_KV_WKV_HEAD_SIZE,
+
+ LLM_KV_TOKENIZER_MODEL,
+ LLM_KV_TOKENIZER_PRE,
+ LLM_KV_TOKENIZER_LIST,
+ LLM_KV_TOKENIZER_TOKEN_TYPE,
+ LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT,
+ LLM_KV_TOKENIZER_SCORES,
+ LLM_KV_TOKENIZER_MERGES,
+ LLM_KV_TOKENIZER_BOS_ID,
+ LLM_KV_TOKENIZER_EOS_ID,
+ LLM_KV_TOKENIZER_UNK_ID,
+ LLM_KV_TOKENIZER_SEP_ID,
+ LLM_KV_TOKENIZER_PAD_ID,
+ LLM_KV_TOKENIZER_CLS_ID,
+ LLM_KV_TOKENIZER_MASK_ID,
+ LLM_KV_TOKENIZER_ADD_BOS,
+ LLM_KV_TOKENIZER_ADD_EOS,
+ LLM_KV_TOKENIZER_ADD_PREFIX,
+ LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,
+ LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,
+ LLM_KV_TOKENIZER_HF_JSON,
+ LLM_KV_TOKENIZER_RWKV,
+ LLM_KV_TOKENIZER_PREFIX_ID,
+ LLM_KV_TOKENIZER_SUFFIX_ID,
+ LLM_KV_TOKENIZER_MIDDLE_ID,
+ LLM_KV_TOKENIZER_EOT_ID,
+ LLM_KV_TOKENIZER_EOM_ID,
+
+ LLM_KV_ADAPTER_TYPE,
+ LLM_KV_ADAPTER_LORA_ALPHA,
+};
+
+static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
+ { LLM_KV_GENERAL_TYPE, "general.type" },
+ { LLM_KV_GENERAL_ARCHITECTURE, "general.architecture" },
+ { LLM_KV_GENERAL_QUANTIZATION_VERSION, "general.quantization_version" },
+ { LLM_KV_GENERAL_ALIGNMENT, "general.alignment" },
+ { LLM_KV_GENERAL_NAME, "general.name" },
+ { LLM_KV_GENERAL_AUTHOR, "general.author" },
+ { LLM_KV_GENERAL_VERSION, "general.version" },
+ { LLM_KV_GENERAL_URL, "general.url" },
+ { LLM_KV_GENERAL_DESCRIPTION, "general.description" },
+ { LLM_KV_GENERAL_LICENSE, "general.license" },
+ { LLM_KV_GENERAL_SOURCE_URL, "general.source.url" },
+ { LLM_KV_GENERAL_SOURCE_HF_REPO, "general.source.huggingface.repository" },
+
+ { LLM_KV_VOCAB_SIZE, "%s.vocab_size" },
+ { LLM_KV_CONTEXT_LENGTH, "%s.context_length" },
+ { LLM_KV_EMBEDDING_LENGTH, "%s.embedding_length" },
+ { LLM_KV_BLOCK_COUNT, "%s.block_count" },
+ { LLM_KV_LEADING_DENSE_BLOCK_COUNT, "%s.leading_dense_block_count" },
+ { LLM_KV_FEED_FORWARD_LENGTH, "%s.feed_forward_length" },
+ { LLM_KV_EXPERT_FEED_FORWARD_LENGTH, "%s.expert_feed_forward_length" },
+ { LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, "%s.expert_shared_feed_forward_length" },
+ { LLM_KV_USE_PARALLEL_RESIDUAL, "%s.use_parallel_residual" },
+ { LLM_KV_TENSOR_DATA_LAYOUT, "%s.tensor_data_layout" },
+ { LLM_KV_EXPERT_COUNT, "%s.expert_count" },
+ { LLM_KV_EXPERT_USED_COUNT, "%s.expert_used_count" },
+ { LLM_KV_EXPERT_SHARED_COUNT, "%s.expert_shared_count" },
+ { LLM_KV_EXPERT_WEIGHTS_SCALE, "%s.expert_weights_scale" },
+ { LLM_KV_POOLING_TYPE, "%s.pooling_type" },
+ { LLM_KV_LOGIT_SCALE, "%s.logit_scale" },
+ { LLM_KV_DECODER_START_TOKEN_ID, "%s.decoder_start_token_id" },
+ { LLM_KV_ATTN_LOGIT_SOFTCAPPING, "%s.attn_logit_softcapping" },
+ { LLM_KV_FINAL_LOGIT_SOFTCAPPING, "%s.final_logit_softcapping" },
+ { LLM_KV_RESCALE_EVERY_N_LAYERS, "%s.rescale_every_n_layers" },
+ { LLM_KV_TIME_MIX_EXTRA_DIM, "%s.time_mix_extra_dim" },
+ { LLM_KV_TIME_DECAY_EXTRA_DIM, "%s.time_decay_extra_dim" },
+
+ { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },
+ { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },
+ { LLM_KV_ATTENTION_MAX_ALIBI_BIAS, "%s.attention.max_alibi_bias" },
+ { LLM_KV_ATTENTION_CLAMP_KQV, "%s.attention.clamp_kqv" },
+ { LLM_KV_ATTENTION_KEY_LENGTH, "%s.attention.key_length" },
+ { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" },
+ { LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" },
+ { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" },
+ { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" },
+ { LLM_KV_ATTENTION_Q_LORA_RANK, "%s.attention.q_lora_rank" },
+ { LLM_KV_ATTENTION_KV_LORA_RANK, "%s.attention.kv_lora_rank" },
+ { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" },
+ { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" },
+ { LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION, "%s.attention.block_skip_connection.%d" },
+
+ { LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" },
+ { LLM_KV_ROPE_FREQ_BASE, "%s.rope.freq_base" },
+ { LLM_KV_ROPE_SCALE_LINEAR, "%s.rope.scale_linear" },
+ { LLM_KV_ROPE_SCALING_TYPE, "%s.rope.scaling.type" },
+ { LLM_KV_ROPE_SCALING_FACTOR, "%s.rope.scaling.factor" },
+ { LLM_KV_ROPE_SCALING_ATTN_FACTOR, "%s.rope.scaling.attn_factor" },
+ { LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, "%s.rope.scaling.original_context_length" },
+ { LLM_KV_ROPE_SCALING_FINETUNED, "%s.rope.scaling.finetuned" },
+ { LLM_KV_ROPE_SCALING_YARN_LOG_MUL, "%s.rope.scaling.yarn_log_multiplier" },
+
+ { LLM_KV_SPLIT_NO, "split.no" },
+ { LLM_KV_SPLIT_COUNT, "split.count" },
+ { LLM_KV_SPLIT_TENSORS_COUNT, "split.tensors.count" },
+
+ { LLM_KV_SSM_CONV_KERNEL, "%s.ssm.conv_kernel" },
+ { LLM_KV_SSM_INNER_SIZE, "%s.ssm.inner_size" },
+ { LLM_KV_SSM_STATE_SIZE, "%s.ssm.state_size" },
+ { LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" },
+ { LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" },
+
+ { LLM_KV_WKV_HEAD_SIZE, "%s.wkv.head_size" },
+
+ { LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },
+ { LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" },
+ { LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },
+ { LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },
+ { LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" },
+ { LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" },
+ { LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" },
+ { LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" },
+ { LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" },
+ { LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" },
+ { LLM_KV_TOKENIZER_SEP_ID, "tokenizer.ggml.seperator_token_id" },
+ { LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" },
+ { LLM_KV_TOKENIZER_CLS_ID, "tokenizer.ggml.cls_token_id" },
+ { LLM_KV_TOKENIZER_MASK_ID, "tokenizer.ggml.mask_token_id" },
+ { LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },
+ { LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },
+ { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },
+ { LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },
+ { LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },
+ { LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },
+ { LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },
+ { LLM_KV_TOKENIZER_PREFIX_ID, "tokenizer.ggml.prefix_token_id" },
+ { LLM_KV_TOKENIZER_SUFFIX_ID, "tokenizer.ggml.suffix_token_id" },
+ { LLM_KV_TOKENIZER_MIDDLE_ID, "tokenizer.ggml.middle_token_id" },
+ { LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" },
+ { LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" },
+
+ { LLM_KV_ADAPTER_TYPE, "adapter.type" },
+ { LLM_KV_ADAPTER_LORA_ALPHA, "adapter.lora.alpha" },
+};
+
+struct LLM_KV {
+ LLM_KV(llm_arch arch) : arch(arch) {}
+
+ llm_arch arch;
+
+ std::string operator()(llm_kv kv) const {
+ return ::format(LLM_KV_NAMES.at(kv), LLM_ARCH_NAMES.at(arch));
+ }
+};
+
+enum llm_tensor {
+ LLM_TENSOR_TOKEN_EMBD,
+ LLM_TENSOR_TOKEN_EMBD_NORM,
+ LLM_TENSOR_TOKEN_TYPES,
+ LLM_TENSOR_POS_EMBD,
+ LLM_TENSOR_OUTPUT,
+ LLM_TENSOR_OUTPUT_NORM,
+ LLM_TENSOR_ROPE_FREQS,
+ LLM_TENSOR_ROPE_FACTORS_LONG,
+ LLM_TENSOR_ROPE_FACTORS_SHORT,
+ LLM_TENSOR_ATTN_Q,
+ LLM_TENSOR_ATTN_K,
+ LLM_TENSOR_ATTN_V,
+ LLM_TENSOR_ATTN_QKV,
+ LLM_TENSOR_ATTN_OUT,
+ LLM_TENSOR_ATTN_NORM,
+ LLM_TENSOR_ATTN_NORM_2,
+ LLM_TENSOR_ATTN_OUT_NORM,
+ LLM_TENSOR_ATTN_POST_NORM,
+ LLM_TENSOR_ATTN_ROT_EMBD,
+ LLM_TENSOR_FFN_GATE_INP,
+ LLM_TENSOR_FFN_GATE_INP_SHEXP,
+ LLM_TENSOR_FFN_NORM,
+ LLM_TENSOR_FFN_POST_NORM,
+ LLM_TENSOR_FFN_GATE,
+ LLM_TENSOR_FFN_DOWN,
+ LLM_TENSOR_FFN_UP,
+ LLM_TENSOR_FFN_ACT,
+ LLM_TENSOR_FFN_DOWN_EXP, // split experts for backward compatibility
+ LLM_TENSOR_FFN_GATE_EXP,
+ LLM_TENSOR_FFN_UP_EXP,
+ LLM_TENSOR_FFN_NORM_EXPS,
+ LLM_TENSOR_FFN_DOWN_EXPS, // merged experts
+ LLM_TENSOR_FFN_GATE_EXPS,
+ LLM_TENSOR_FFN_UP_EXPS,
+ LLM_TENSOR_FFN_DOWN_SHEXP,
+ LLM_TENSOR_FFN_GATE_SHEXP,
+ LLM_TENSOR_FFN_UP_SHEXP,
+ LLM_TENSOR_ATTN_Q_NORM,
+ LLM_TENSOR_ATTN_K_NORM,
+ LLM_TENSOR_LAYER_OUT_NORM,
+ LLM_TENSOR_SSM_IN,
+ LLM_TENSOR_SSM_CONV1D,
+ LLM_TENSOR_SSM_X,
+ LLM_TENSOR_SSM_DT,
+ LLM_TENSOR_SSM_A,
+ LLM_TENSOR_SSM_D,
+ LLM_TENSOR_SSM_OUT,
+ LLM_TENSOR_TIME_MIX_W1,
+ LLM_TENSOR_TIME_MIX_W2,
+ LLM_TENSOR_TIME_MIX_LERP_X,
+ LLM_TENSOR_TIME_MIX_LERP_W,
+ LLM_TENSOR_TIME_MIX_LERP_K,
+ LLM_TENSOR_TIME_MIX_LERP_V,
+ LLM_TENSOR_TIME_MIX_LERP_R,
+ LLM_TENSOR_TIME_MIX_LERP_G,
+ LLM_TENSOR_TIME_MIX_FIRST,
+ LLM_TENSOR_TIME_MIX_DECAY,
+ LLM_TENSOR_TIME_MIX_DECAY_W1,
+ LLM_TENSOR_TIME_MIX_DECAY_W2,
+ LLM_TENSOR_TIME_MIX_KEY,
+ LLM_TENSOR_TIME_MIX_VALUE,
+ LLM_TENSOR_TIME_MIX_RECEPTANCE,
+ LLM_TENSOR_TIME_MIX_GATE,
+ LLM_TENSOR_TIME_MIX_LN,
+ LLM_TENSOR_TIME_MIX_OUTPUT,
+ LLM_TENSOR_CHANNEL_MIX_LERP_K,
+ LLM_TENSOR_CHANNEL_MIX_LERP_R,
+ LLM_TENSOR_CHANNEL_MIX_KEY,
+ LLM_TENSOR_CHANNEL_MIX_RECEPTANCE,
+ LLM_TENSOR_CHANNEL_MIX_VALUE,
+ LLM_TENSOR_ATTN_Q_A,
+ LLM_TENSOR_ATTN_Q_B,
+ LLM_TENSOR_ATTN_KV_A_MQA,
+ LLM_TENSOR_ATTN_KV_B,
+ LLM_TENSOR_ATTN_Q_A_NORM,
+ LLM_TENSOR_ATTN_KV_A_NORM,
+ LLM_TENSOR_ATTN_SUB_NORM,
+ LLM_TENSOR_FFN_SUB_NORM,
+ LLM_TENSOR_DEC_ATTN_NORM,
+ LLM_TENSOR_DEC_ATTN_Q,
+ LLM_TENSOR_DEC_ATTN_K,
+ LLM_TENSOR_DEC_ATTN_V,
+ LLM_TENSOR_DEC_ATTN_OUT,
+ LLM_TENSOR_DEC_ATTN_REL_B,
+ LLM_TENSOR_DEC_CROSS_ATTN_NORM,
+ LLM_TENSOR_DEC_CROSS_ATTN_Q,
+ LLM_TENSOR_DEC_CROSS_ATTN_K,
+ LLM_TENSOR_DEC_CROSS_ATTN_V,
+ LLM_TENSOR_DEC_CROSS_ATTN_OUT,
+ LLM_TENSOR_DEC_CROSS_ATTN_REL_B,
+ LLM_TENSOR_DEC_FFN_NORM,
+ LLM_TENSOR_DEC_FFN_GATE,
+ LLM_TENSOR_DEC_FFN_DOWN,
+ LLM_TENSOR_DEC_FFN_UP,
+ LLM_TENSOR_DEC_OUTPUT_NORM,
+ LLM_TENSOR_ENC_ATTN_NORM,
+ LLM_TENSOR_ENC_ATTN_Q,
+ LLM_TENSOR_ENC_ATTN_K,
+ LLM_TENSOR_ENC_ATTN_V,
+ LLM_TENSOR_ENC_ATTN_OUT,
+ LLM_TENSOR_ENC_ATTN_REL_B,
+ LLM_TENSOR_ENC_FFN_NORM,
+ LLM_TENSOR_ENC_FFN_GATE,
+ LLM_TENSOR_ENC_FFN_DOWN,
+ LLM_TENSOR_ENC_FFN_UP,
+ LLM_TENSOR_ENC_OUTPUT_NORM,
+ LLM_TENSOR_BSKCN_TV,
+};
+
+static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES = {
+ {
+ LLM_ARCH_LLAMA,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_GATE_EXP, "blk.%d.ffn_gate.%d" },
+ { LLM_TENSOR_FFN_DOWN_EXP, "blk.%d.ffn_down.%d" },
+ { LLM_TENSOR_FFN_UP_EXP, "blk.%d.ffn_up.%d" },
+ { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" },
+ { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },
+ { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },
+ },
+ },
+ {
+ LLM_ARCH_BAICHUAN,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_FALCON,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_GROK,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE_EXP, "blk.%d.ffn_gate.%d" },
+ { LLM_TENSOR_FFN_DOWN_EXP, "blk.%d.ffn_down.%d" },
+ { LLM_TENSOR_FFN_UP_EXP, "blk.%d.ffn_up.%d" },
+ { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" },
+ { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },
+ { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },
+ { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" },
+ { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" },
+ },
+ },
+ {
+ LLM_ARCH_GPT2,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_POS_EMBD, "position_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ },
+ },
+ {
+ LLM_ARCH_GPTJ,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ },
+ },
+ {
+ LLM_ARCH_GPTNEOX,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_MPT,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output"},
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_ACT, "blk.%d.ffn.act" },
+ { LLM_TENSOR_POS_EMBD, "position_embd" },
+ { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm"},
+ { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm"},
+ },
+ },
+ {
+ LLM_ARCH_STARCODER,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_POS_EMBD, "position_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ },
+ },
+ {
+ LLM_ARCH_REFACT,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_BERT,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },
+ { LLM_TENSOR_TOKEN_TYPES, "token_types" },
+ { LLM_TENSOR_POS_EMBD, "position_embd" },
+ { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_NOMIC_BERT,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },
+ { LLM_TENSOR_TOKEN_TYPES, "token_types" },
+ { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_JINA_BERT_V2,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },
+ { LLM_TENSOR_TOKEN_TYPES, "token_types" },
+ { LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" },
+ { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_BLOOM,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ },
+ },
+ {
+ LLM_ARCH_STABLELM,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" },
+ { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" },
+ },
+ },
+ {
+ LLM_ARCH_QWEN,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_QWEN2,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_QWEN2MOE,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },
+ { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" },
+ { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },
+ { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },
+ { LLM_TENSOR_FFN_GATE_INP_SHEXP, "blk.%d.ffn_gate_inp_shexp" },
+ { LLM_TENSOR_FFN_GATE_SHEXP, "blk.%d.ffn_gate_shexp" },
+ { LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" },
+ { LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" },
+ },
+ },
+ {
+ LLM_ARCH_PHI2,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_PHI3,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FACTORS_LONG, "rope_factors_long" },
+ { LLM_TENSOR_ROPE_FACTORS_SHORT, "rope_factors_short" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_PLAMO,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_CODESHELL,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_ORION,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_INTERNLM2,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_MINICPM,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_GATE_EXP, "blk.%d.ffn_gate.%d" },
+ { LLM_TENSOR_FFN_DOWN_EXP, "blk.%d.ffn_down.%d" },
+ { LLM_TENSOR_FFN_UP_EXP, "blk.%d.ffn_up.%d" },
+ },
+ },
+ {
+ LLM_ARCH_GEMMA,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_GEMMA2,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_POST_NORM, "blk.%d.post_attention_norm" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_POST_NORM, "blk.%d.post_ffw_norm" },
+ },
+ },
+ {
+ LLM_ARCH_STARCODER2,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_MAMBA,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_SSM_IN, "blk.%d.ssm_in" },
+ { LLM_TENSOR_SSM_CONV1D, "blk.%d.ssm_conv1d" },
+ { LLM_TENSOR_SSM_X, "blk.%d.ssm_x" },
+ { LLM_TENSOR_SSM_DT, "blk.%d.ssm_dt" },
+ { LLM_TENSOR_SSM_A, "blk.%d.ssm_a" },
+ { LLM_TENSOR_SSM_D, "blk.%d.ssm_d" },
+ { LLM_TENSOR_SSM_OUT, "blk.%d.ssm_out" },
+ },
+ },
+ {
+ LLM_ARCH_XVERSE,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_COMMAND_R,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" },
+ { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" },
+ },
+ },
+ {
+ LLM_ARCH_DBRX,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" },
+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },
+ { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" },
+ { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },
+ { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },
+ },
+ },
+ {
+ LLM_ARCH_OLMO,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_OPENELM,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" },
+ { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_ARCTIC,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_NORM_EXPS, "blk.%d.ffn_norm_exps" },
+ { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" },
+ { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },
+ { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },
+ },
+ },
+ {
+ LLM_ARCH_DEEPSEEK2,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q_A_NORM, "blk.%d.attn_q_a_norm" },
+ { LLM_TENSOR_ATTN_KV_A_NORM, "blk.%d.attn_kv_a_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_Q_A, "blk.%d.attn_q_a" },
+ { LLM_TENSOR_ATTN_Q_B, "blk.%d.attn_q_b" },
+ { LLM_TENSOR_ATTN_KV_A_MQA, "blk.%d.attn_kv_a_mqa" },
+ { LLM_TENSOR_ATTN_KV_B, "blk.%d.attn_kv_b" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },
+ { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" },
+ { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },
+ { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },
+ { LLM_TENSOR_FFN_GATE_INP_SHEXP, "blk.%d.ffn_gate_inp_shexp" },
+ { LLM_TENSOR_FFN_GATE_SHEXP, "blk.%d.ffn_gate_shexp" },
+ { LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" },
+ { LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" },
+ },
+ },
+ {
+ LLM_ARCH_CHATGLM,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ },
+ },
+ {
+ LLM_ARCH_BITNET,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_SUB_NORM, "blk.%d.attn_sub_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_SUB_NORM, "blk.%d.ffn_sub_norm" },
+ },
+ },
+ {
+ LLM_ARCH_T5,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_DEC_OUTPUT_NORM, "dec.output_norm" },
+ { LLM_TENSOR_DEC_ATTN_NORM, "dec.blk.%d.attn_norm" },
+ { LLM_TENSOR_DEC_ATTN_Q, "dec.blk.%d.attn_q" },
+ { LLM_TENSOR_DEC_ATTN_K, "dec.blk.%d.attn_k" },
+ { LLM_TENSOR_DEC_ATTN_V, "dec.blk.%d.attn_v" },
+ { LLM_TENSOR_DEC_ATTN_OUT, "dec.blk.%d.attn_o" },
+ { LLM_TENSOR_DEC_ATTN_REL_B, "dec.blk.%d.attn_rel_b" },
+ { LLM_TENSOR_DEC_CROSS_ATTN_NORM, "dec.blk.%d.cross_attn_norm" },
+ { LLM_TENSOR_DEC_CROSS_ATTN_Q, "dec.blk.%d.cross_attn_q" },
+ { LLM_TENSOR_DEC_CROSS_ATTN_K, "dec.blk.%d.cross_attn_k" },
+ { LLM_TENSOR_DEC_CROSS_ATTN_V, "dec.blk.%d.cross_attn_v" },
+ { LLM_TENSOR_DEC_CROSS_ATTN_OUT, "dec.blk.%d.cross_attn_o" },
+ { LLM_TENSOR_DEC_CROSS_ATTN_REL_B, "dec.blk.%d.cross_attn_rel_b" },
+ { LLM_TENSOR_DEC_FFN_NORM, "dec.blk.%d.ffn_norm" },
+ { LLM_TENSOR_DEC_FFN_GATE, "dec.blk.%d.ffn_gate" },
+ { LLM_TENSOR_DEC_FFN_DOWN, "dec.blk.%d.ffn_down" },
+ { LLM_TENSOR_DEC_FFN_UP, "dec.blk.%d.ffn_up" },
+ { LLM_TENSOR_ENC_OUTPUT_NORM, "enc.output_norm" },
+ { LLM_TENSOR_ENC_ATTN_NORM, "enc.blk.%d.attn_norm" },
+ { LLM_TENSOR_ENC_ATTN_Q, "enc.blk.%d.attn_q" },
+ { LLM_TENSOR_ENC_ATTN_K, "enc.blk.%d.attn_k" },
+ { LLM_TENSOR_ENC_ATTN_V, "enc.blk.%d.attn_v" },
+ { LLM_TENSOR_ENC_ATTN_OUT, "enc.blk.%d.attn_o" },
+ { LLM_TENSOR_ENC_ATTN_REL_B, "enc.blk.%d.attn_rel_b" },
+ { LLM_TENSOR_ENC_FFN_NORM, "enc.blk.%d.ffn_norm" },
+ { LLM_TENSOR_ENC_FFN_GATE, "enc.blk.%d.ffn_gate" },
+ { LLM_TENSOR_ENC_FFN_DOWN, "enc.blk.%d.ffn_down" },
+ { LLM_TENSOR_ENC_FFN_UP, "enc.blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_T5ENCODER,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ENC_OUTPUT_NORM, "enc.output_norm" },
+ { LLM_TENSOR_ENC_ATTN_NORM, "enc.blk.%d.attn_norm" },
+ { LLM_TENSOR_ENC_ATTN_Q, "enc.blk.%d.attn_q" },
+ { LLM_TENSOR_ENC_ATTN_K, "enc.blk.%d.attn_k" },
+ { LLM_TENSOR_ENC_ATTN_V, "enc.blk.%d.attn_v" },
+ { LLM_TENSOR_ENC_ATTN_OUT, "enc.blk.%d.attn_o" },
+ { LLM_TENSOR_ENC_ATTN_REL_B, "enc.blk.%d.attn_rel_b" },
+ { LLM_TENSOR_ENC_FFN_NORM, "enc.blk.%d.ffn_norm" },
+ { LLM_TENSOR_ENC_FFN_GATE, "enc.blk.%d.ffn_gate" },
+ { LLM_TENSOR_ENC_FFN_DOWN, "enc.blk.%d.ffn_down" },
+ { LLM_TENSOR_ENC_FFN_UP, "enc.blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_JAIS,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ },
+ },
+ {
+ LLM_ARCH_NEMOTRON,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_EXAONE,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_RWKV6,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" },
+ { LLM_TENSOR_TIME_MIX_W1, "blk.%d.time_mix_w1" },
+ { LLM_TENSOR_TIME_MIX_W2, "blk.%d.time_mix_w2" },
+ { LLM_TENSOR_TIME_MIX_LERP_X, "blk.%d.time_mix_lerp_x" },
+ { LLM_TENSOR_TIME_MIX_LERP_W, "blk.%d.time_mix_lerp_w" },
+ { LLM_TENSOR_TIME_MIX_LERP_K, "blk.%d.time_mix_lerp_k" },
+ { LLM_TENSOR_TIME_MIX_LERP_V, "blk.%d.time_mix_lerp_v" },
+ { LLM_TENSOR_TIME_MIX_LERP_R, "blk.%d.time_mix_lerp_r" },
+ { LLM_TENSOR_TIME_MIX_LERP_G, "blk.%d.time_mix_lerp_g" },
+ { LLM_TENSOR_TIME_MIX_FIRST, "blk.%d.time_mix_first" },
+ { LLM_TENSOR_TIME_MIX_DECAY, "blk.%d.time_mix_decay" },
+ { LLM_TENSOR_TIME_MIX_DECAY_W1, "blk.%d.time_mix_decay_w1" },
+ { LLM_TENSOR_TIME_MIX_DECAY_W2, "blk.%d.time_mix_decay_w2" },
+ { LLM_TENSOR_TIME_MIX_KEY, "blk.%d.time_mix_key" },
+ { LLM_TENSOR_TIME_MIX_VALUE, "blk.%d.time_mix_value" },
+ { LLM_TENSOR_TIME_MIX_RECEPTANCE, "blk.%d.time_mix_receptance" },
+ { LLM_TENSOR_TIME_MIX_GATE, "blk.%d.time_mix_gate" },
+ { LLM_TENSOR_TIME_MIX_LN, "blk.%d.time_mix_ln" },
+ { LLM_TENSOR_TIME_MIX_OUTPUT, "blk.%d.time_mix_output" },
+ { LLM_TENSOR_CHANNEL_MIX_LERP_K, "blk.%d.channel_mix_lerp_k" },
+ { LLM_TENSOR_CHANNEL_MIX_LERP_R, "blk.%d.channel_mix_lerp_r" },
+ { LLM_TENSOR_CHANNEL_MIX_KEY, "blk.%d.channel_mix_key" },
+ { LLM_TENSOR_CHANNEL_MIX_VALUE, "blk.%d.channel_mix_value" },
+ { LLM_TENSOR_CHANNEL_MIX_RECEPTANCE, "blk.%d.channel_mix_receptance" },
+ },
+ },
+ {
+ LLM_ARCH_SOLAR,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ { LLM_TENSOR_BSKCN_TV, "bskcn_tv" },
+ },
+ },
+ {
+ LLM_ARCH_UNKNOWN,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ },
+ },
+};
+
+static llm_arch llm_arch_from_string(const std::string & name) {
+ for (const auto & kv : LLM_ARCH_NAMES) { // NOLINT
+ if (kv.second == name) {
+ return kv.first;
+ }
+ }
+
+ return LLM_ARCH_UNKNOWN;
+}
+
+// helper to handle gguf constants
+// usage:
+//
+// const auto tn = LLM_TN(LLM_ARCH_LLAMA);
+//
+// std::string name = tn(LLM_TENSOR_OUTPUT); -> "output"
+// std::string name = tn(LLM_TENSOR_TOKEN_EMBD, "bias"); -> "token_embd.bias"
+// std::string name = tn(LLM_TENSOR_ATTN_NORM, "weight", 3); -> "blk.3.attn_norm.weight"
+//
+struct LLM_TN {
+ LLM_TN(llm_arch arch) : arch(arch) {}
+
+ llm_arch arch;
+
+ std::string operator()(llm_tensor tensor) const {
+ if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) {
+ return "__missing__";
+ }
+ return LLM_TENSOR_NAMES.at(arch).at(tensor);
+ }
+
+ std::string operator()(llm_tensor tensor, const std::string & suffix) const {
+ if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) {
+ return "__missing__";
+ }
+ return LLM_TENSOR_NAMES.at(arch).at(tensor) + "." + suffix;
+ }
+
+ std::string operator()(llm_tensor tensor, int bid) const {
+ if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) {
+ return "__missing__";
+ }
+ return ::format(LLM_TENSOR_NAMES.at(arch).at(tensor).c_str(), bid);
+ }
+
+ std::string operator()(llm_tensor tensor, const std::string & suffix, int bid) const {
+ if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) {
+ return "__missing__";
+ }
+ return ::format(LLM_TENSOR_NAMES.at(arch).at(tensor).c_str(), bid) + "." + suffix;
+ }
+
+ std::string operator()(llm_tensor tensor, const std::string & suffix, int bid, int xid) const {
+ if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) {
+ return "__missing__";
+ }
+ return ::format(LLM_TENSOR_NAMES.at(arch).at(tensor).c_str(), bid, xid) + "." + suffix;
+ }
+};
+
+//
+// gguf helpers
+//
+
+static const std::map<llama_rope_scaling_type, const char *> LLAMA_ROPE_SCALING_TYPES = {
+ { LLAMA_ROPE_SCALING_TYPE_NONE, "none" },
+ { LLAMA_ROPE_SCALING_TYPE_LINEAR, "linear" },
+ { LLAMA_ROPE_SCALING_TYPE_YARN, "yarn" },
+};
+
+static llama_rope_scaling_type llama_rope_scaling_type_from_string(const std::string & name) {
+ for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) {
+ if (kv.second == name) {
+ return (llama_rope_scaling_type) kv.first;
+ }
+ }
+
+ return LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;
+}
+
+static std::string gguf_data_to_str(enum gguf_type type, const void * data, int i) {
+ switch (type) {
+ case GGUF_TYPE_UINT8: return std::to_string(((const uint8_t *)data)[i]);
+ case GGUF_TYPE_INT8: return std::to_string(((const int8_t *)data)[i]);
+ case GGUF_TYPE_UINT16: return std::to_string(((const uint16_t *)data)[i]);
+ case GGUF_TYPE_INT16: return std::to_string(((const int16_t *)data)[i]);
+ case GGUF_TYPE_UINT32: return std::to_string(((const uint32_t *)data)[i]);
+ case GGUF_TYPE_INT32: return std::to_string(((const int32_t *)data)[i]);
+ case GGUF_TYPE_UINT64: return std::to_string(((const uint64_t *)data)[i]);
+ case GGUF_TYPE_INT64: return std::to_string(((const int64_t *)data)[i]);
+ case GGUF_TYPE_FLOAT32: return std::to_string(((const float *)data)[i]);
+ case GGUF_TYPE_FLOAT64: return std::to_string(((const double *)data)[i]);
+ case GGUF_TYPE_BOOL: return ((const bool *)data)[i] ? "true" : "false";
+ default: return format("unknown type %d", type);
+ }
+}
+
+static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {
+ const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);
+
+ switch (type) {
+ case GGUF_TYPE_STRING:
+ return gguf_get_val_str(ctx_gguf, i);
+ case GGUF_TYPE_ARRAY:
+ {
+ const enum gguf_type arr_type = gguf_get_arr_type(ctx_gguf, i);
+ int arr_n = gguf_get_arr_n(ctx_gguf, i);
+ const void * data = gguf_get_arr_data(ctx_gguf, i);
+ std::stringstream ss;
+ ss << "[";
+ for (int j = 0; j < arr_n; j++) {
+ if (arr_type == GGUF_TYPE_STRING) {
+ std::string val = gguf_get_arr_str(ctx_gguf, i, j);
+ // escape quotes
+ replace_all(val, "\\", "\\\\");
+ replace_all(val, "\"", "\\\"");
+ ss << '"' << val << '"';
+ } else if (arr_type == GGUF_TYPE_ARRAY) {
+ ss << "???";
+ } else {
+ ss << gguf_data_to_str(arr_type, data, j);
+ }
+ if (j < arr_n - 1) {
+ ss << ", ";
+ }
+ }
+ ss << "]";
+ return ss.str();
+ }
+ default:
+ return gguf_data_to_str(type, gguf_get_val_data(ctx_gguf, i), 0);
+ }
+}
+
+//
+// llama helpers
+//
+
+#if defined(_WIN32)
+static std::string llama_format_win_err(DWORD err) {
+ LPSTR buf;
+ size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+ NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);
+ if (!size) {
+ return "FormatMessageA failed";
+ }
+ std::string ret(buf, size);
+ LocalFree(buf);
+ return ret;
+}
+#endif
+
+template <typename T>
+struct no_init {
+ T value;
+ no_init() { /* do nothing */ }
+};
+
+struct llama_file {
+
+#if defined(_WIN32)
+ // use FILE * so we don't have to re-open the file to mmap
+ FILE * fp;
+ HANDLE fp_win32;
+ size_t size;
+
+private:
+ std::string GetErrorMessageWin32(DWORD error_code) const {
+ std::string ret;
+ LPSTR lpMsgBuf = NULL;
+ DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+ NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL);
+ if (!bufLen) {
+ ret = format("Win32 error code: %s", error_code);
+ } else {
+ ret = lpMsgBuf;
+ LocalFree(lpMsgBuf);
+ }
+
+ return ret;
+ }
+
+public:
+
+ llama_file(const char * fname, const char * mode) {
+ fp = ggml_fopen(fname, mode);
+ if (fp == NULL) {
+ throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
+ }
+ fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));
+ seek(0, SEEK_END);
+ size = tell();
+ seek(0, SEEK_SET);
+ }
+
+ size_t tell() const {
+ // SetFilePointerEx returns the current position when seeking relative 0 bytes
+ LARGE_INTEGER li;
+ li.QuadPart = 0;
+ BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);
+ if (!ret) {
+ throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
+ }
+
+ return li.QuadPart;
+ }
+
+ void seek(size_t offset, int whence) const {
+ // no need to convert SEEK_* to FILE_*. The enums are the same.
+ // Still, keep static asserts to avoid failures in the future.
+ static_assert(SEEK_SET == FILE_BEGIN, "SEEK_SET != FILE_BEGIN");
+ static_assert(SEEK_CUR == FILE_CURRENT, "SEEK_CUR != FILE_CURRENT");
+ static_assert(SEEK_END == FILE_END, "SEEK_END != FILE_END");
+
+ LARGE_INTEGER li;
+ li.QuadPart = offset;
+ BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);
+ if (!ret) {
+ throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
+ }
+ }
+
+ void read_raw(void * ptr, size_t len) const {
+ // On Win32 ReadFile is significant faster than fread which is again significant faster than std::fstream. Thus
+ // use the Win32 API to do file io instead of the C/C++ library functions.
+
+ // There are conditions under which ReadFile cannot read chunks >64MB.
+ // Thus split the operation into smaller chunks if len exceeds this limit.
+ size_t bytes_read = 0;
+ while (bytes_read < len) {
+ size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);
+ DWORD chunk_read = 0;
+ BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);
+ if (!result) {
+ throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
+ }
+ if (chunk_read < chunk_size || chunk_read == 0) {
+ throw std::runtime_error("unexpectedly reached end of file");
+ }
+
+ bytes_read += chunk_read;
+ } ;
+ }
+
+ uint32_t read_u32() const {
+ uint32_t val;
+ read_raw(&val, sizeof(val));
+ return val;
+ }
+
+ void write_raw(const void * ptr, size_t len) const {
+ // There are conditions under which WriteFile cannot write chunks >64MB.
+ // Thus split the operation into smaller chunks if len exceeds this limit.
+ size_t bytes_written = 0;
+ while (bytes_written < len) {
+ size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);
+ DWORD chunk_written = 0;
+ BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);
+ if (!result) {
+ throw std::runtime_error(format("write error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
+ }
+ if (chunk_written < chunk_size || chunk_written == 0) {
+ throw std::runtime_error("unexpectedly failed to write bytes");
+ }
+
+ bytes_written += chunk_written;
+ }
+ }
+
+ void write_u32(std::uint32_t val) const {
+ write_raw(&val, sizeof(val));
+ }
+
+ ~llama_file() {
+ if (fp) {
+ std::fclose(fp);
+ }
+ }
+#else
+ // use FILE * so we don't have to re-open the file to mmap
+ FILE * fp;
+ size_t size;
+
+ llama_file(const char * fname, const char * mode) {
+ fp = ggml_fopen(fname, mode);
+ if (fp == NULL) {
+ throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
+ }
+ seek(0, SEEK_END);
+ size = tell();
+ seek(0, SEEK_SET);
+ }
+
+ size_t tell() const {
+#ifdef _WIN32
+ __int64 ret = _ftelli64(fp);
+#else
+ long ret = std::ftell(fp);
+#endif
+ if (ret == -1) {
+ throw std::runtime_error(format("ftell error: %s", strerror(errno)));
+ }
+
+ return (size_t) ret;
+ }
+
+ void seek(size_t offset, int whence) const {
+#ifdef _WIN32
+ int ret = _fseeki64(fp, (__int64) offset, whence);
+#else
+ int ret = std::fseek(fp, (long) offset, whence);
+#endif
+ if (ret != 0) {
+ throw std::runtime_error(format("seek error: %s", strerror(errno)));
+ }
+ }
+
+ void read_raw(void * ptr, size_t len) const {
+ if (len == 0) {
+ return;
+ }
+ errno = 0;
+ std::size_t ret = std::fread(ptr, len, 1, fp);
+ if (ferror(fp)) {
+ throw std::runtime_error(format("read error: %s", strerror(errno)));
+ }
+ if (ret != 1) {
+ throw std::runtime_error("unexpectedly reached end of file");
+ }
+ }
+
+ uint32_t read_u32() const {
+ uint32_t ret;
+ read_raw(&ret, sizeof(ret));
+ return ret;
+ }
+
+ void write_raw(const void * ptr, size_t len) const {
+ if (len == 0) {
+ return;
+ }
+ errno = 0;
+ size_t ret = std::fwrite(ptr, len, 1, fp);
+ if (ret != 1) {
+ throw std::runtime_error(format("write error: %s", strerror(errno)));
+ }
+ }
+
+ void write_u32(std::uint32_t val) const {
+ write_raw(&val, sizeof(val));
+ }
+
+ ~llama_file() {
+ if (fp) {
+ std::fclose(fp);
+ }
+ }
+#endif
+};
+using llama_files = std::vector<std::unique_ptr<llama_file>>;
+
+struct llama_mmap {
+ void * addr;
+ size_t size;
+
+ llama_mmap(const llama_mmap &) = delete;
+
+#ifdef _POSIX_MAPPED_FILES
+ static constexpr bool SUPPORTED = true;
+
+ // list of mapped fragments (first_offset, last_offset)
+ std::vector<std::pair<size_t, size_t>> mapped_fragments;
+
+ llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) {
+ size = file->size;
+ int fd = fileno(file->fp);
+ int flags = MAP_SHARED;
+ // prefetch/readahead impairs performance on NUMA systems
+ if (numa) { prefetch = 0; }
+#ifdef __linux__
+ // advise the kernel to read the file sequentially (increases readahead)
+ if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {
+ LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",
+ strerror(errno));
+ }
+ if (prefetch) { flags |= MAP_POPULATE; }
+#endif
+ addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0);
+ if (addr == MAP_FAILED) { // NOLINT
+ throw std::runtime_error(format("mmap failed: %s", strerror(errno)));
+ }
+
+ if (prefetch > 0) {
+ // advise the kernel to preload the mapped memory
+ if (posix_madvise(addr, std::min(file->size, prefetch), POSIX_MADV_WILLNEED)) {
+ LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",
+ strerror(errno));
+ }
+ }
+ if (numa) {
+ // advise the kernel not to use readahead
+ // (because the next page might not belong on the same node)
+ if (posix_madvise(addr, file->size, POSIX_MADV_RANDOM)) {
+ LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",
+ strerror(errno));
+ }
+ }
+
+ // initialize list of mapped_fragments
+ mapped_fragments.emplace_back(0, file->size);
+ }
+
+ static void align_range(size_t * first, size_t * last, size_t page_size) {
+ // align first to the next page
+ size_t offset_in_page = *first & (page_size - 1);
+ size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;
+ *first += offset_to_page;
+
+ // align last to the previous page
+ *last = *last & ~(page_size - 1);
+
+ if (*last <= *first) {
+ *last = *first;
+ }
+ }
+
+ // partially unmap the file in the range [first, last)
+ void unmap_fragment(size_t first, size_t last) {
+ // note: this function must not be called multiple times with overlapping ranges
+ // otherwise, there is a risk of invalidating addresses that have been repurposed for other mappings
+ int page_size = sysconf(_SC_PAGESIZE);
+ align_range(&first, &last, page_size);
+ size_t len = last - first;
+
+ if (len == 0) {
+ return;
+ }
+
+ GGML_ASSERT(first % page_size == 0);
+ GGML_ASSERT(last % page_size == 0);
+ GGML_ASSERT(last > first);
+
+ void * next_page_start = (uint8_t *) addr + first;
+
+ // unmap the range
+ if (munmap(next_page_start, len)) {
+ LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));
+ }
+
+ // update the list of mapped fragments to avoid unmapping the same range again in the destructor
+ std::vector<std::pair<size_t, size_t>> new_mapped_fragments;
+ for (const auto & frag : mapped_fragments) {
+ if (frag.first < first && frag.second > last) {
+ // the range is in the middle of the fragment, split it
+ new_mapped_fragments.emplace_back(frag.first, first);
+ new_mapped_fragments.emplace_back(last, frag.second);
+ } else if (frag.first < first && frag.second > first) {
+ // the range starts in the middle of the fragment
+ new_mapped_fragments.emplace_back(frag.first, first);
+ } else if (frag.first < last && frag.second > last) {
+ // the range ends in the middle of the fragment
+ new_mapped_fragments.emplace_back(last, frag.second);
+ } else if (frag.first >= first && frag.second <= last) {
+ // the range covers the entire fragment
+ } else {
+ // the range is outside the fragment
+ new_mapped_fragments.push_back(frag);
+ }
+ }
+ mapped_fragments = std::move(new_mapped_fragments);
+ }
+
+ ~llama_mmap() {
+ for (const auto & frag : mapped_fragments) {
+ if (munmap((char *) addr + frag.first, frag.second - frag.first)) {
+ LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));
+ }
+ }
+ }
+#elif defined(_WIN32)
+ static constexpr bool SUPPORTED = true;
+
+ llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false) {
+ GGML_UNUSED(numa);
+
+ size = file->size;
+
+ HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp));
+
+ HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
+
+ if (hMapping == NULL) {
+ DWORD error = GetLastError();
+ throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));
+ }
+
+ addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
+ DWORD error = GetLastError();
+ CloseHandle(hMapping);
+
+ if (addr == NULL) {
+ throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));
+ }
+
+ if (prefetch > 0) {
+#if _WIN32_WINNT >= 0x602
+ // PrefetchVirtualMemory is only present on Windows 8 and above, so we dynamically load it
+ BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
+ HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");
+
+ // may fail on pre-Windows 8 systems
+ pPrefetchVirtualMemory = reinterpret_cast<decltype(pPrefetchVirtualMemory)> (GetProcAddress(hKernel32, "PrefetchVirtualMemory"));
+
+ if (pPrefetchVirtualMemory) {
+ // advise the kernel to preload the mapped memory
+ WIN32_MEMORY_RANGE_ENTRY range;
+ range.VirtualAddress = addr;
+ range.NumberOfBytes = (SIZE_T) std::min(size, prefetch);
+ if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {
+ LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",
+ llama_format_win_err(GetLastError()).c_str());
+ }
+ }
+#else
+ throw std::runtime_error("PrefetchVirtualMemory unavailable");
+#endif
+ }
+ }
+
+ void unmap_fragment(size_t first, size_t last) {
+ // not supported
+ GGML_UNUSED(first);
+ GGML_UNUSED(last);
+ }
+
+ ~llama_mmap() {
+ if (!UnmapViewOfFile(addr)) {
+ LLAMA_LOG_WARN("warning: UnmapViewOfFile failed: %s\n",
+ llama_format_win_err(GetLastError()).c_str());
+ }
+ }
+#else
+ static constexpr bool SUPPORTED = false;
+
+ llama_mmap(struct llama_file * file, size_t prefetch = -1, bool numa = false) {
+ GGML_UNUSED(file);
+ GGML_UNUSED(prefetch);
+ GGML_UNUSED(numa);
+
+ throw std::runtime_error("mmap not supported");
+ }
+
+ void unmap_fragment(size_t first, size_t last) {
+ GGML_UNUSED(first);
+ GGML_UNUSED(last);
+
+ throw std::runtime_error("mmap not supported");
+ }
+#endif
+};
+using llama_mmaps = std::vector<std::unique_ptr<llama_mmap>>;
+
+// Represents some region of memory being locked using mlock or VirtualLock;
+// will automatically unlock on destruction.
+struct llama_mlock {
+ void * addr = NULL;
+ size_t size = 0;
+
+ bool failed_already = false;
+
+ llama_mlock() {}
+ llama_mlock(const llama_mlock &) = delete;
+
+ ~llama_mlock() {
+ if (size) {
+ raw_unlock(addr, size);
+ }
+ }
+
+ void init(void * ptr) {
+ GGML_ASSERT(addr == NULL && size == 0); // NOLINT
+ addr = ptr;
+ }
+
+ void grow_to(size_t target_size) {
+ GGML_ASSERT(addr);
+ if (failed_already) {
+ return;
+ }
+ size_t granularity = lock_granularity();
+ target_size = (target_size + granularity - 1) & ~(granularity - 1);
+ if (target_size > size) {
+ if (raw_lock((uint8_t *) addr + size, target_size - size)) {
+ size = target_size;
+ } else {
+ failed_already = true;
+ }
+ }
+ }
+
+#ifdef _POSIX_MEMLOCK_RANGE
+ static constexpr bool SUPPORTED = true;
+
+ static size_t lock_granularity() {
+ return (size_t) sysconf(_SC_PAGESIZE);
+ }
+
+ #ifdef __APPLE__
+ #define MLOCK_SUGGESTION \
+ "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \
+ "decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MEMLOCK (ulimit -l).\n"
+ #else
+ #define MLOCK_SUGGESTION \
+ "Try increasing RLIMIT_MEMLOCK ('ulimit -l' as root).\n"
+ #endif
+
+ bool raw_lock(const void * addr, size_t size) const {
+ if (!mlock(addr, size)) {
+ return true;
+ }
+
+ char* errmsg = std::strerror(errno);
+ bool suggest = (errno == ENOMEM);
+
+ // Check if the resource limit is fine after all
+ struct rlimit lock_limit;
+ if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {
+ suggest = false;
+ }
+ if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size)) {
+ suggest = false;
+ }
+
+ LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",
+ size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");
+ return false;
+ }
+
+ #undef MLOCK_SUGGESTION
+
+ static void raw_unlock(void * addr, size_t size) {
+ if (munlock(addr, size)) {
+ LLAMA_LOG_WARN("warning: failed to munlock buffer: %s\n", std::strerror(errno));
+ }
+ }
+#elif defined(_WIN32)
+ static constexpr bool SUPPORTED = true;
+
+ static size_t lock_granularity() {
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return (size_t) si.dwPageSize;
+ }
+
+ bool raw_lock(void * ptr, size_t len) const {
+ for (int tries = 1; ; tries++) {
+ if (VirtualLock(ptr, len)) {
+ return true;
+ }
+ if (tries == 2) {
+ LLAMA_LOG_WARN("warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",
+ len, size, llama_format_win_err(GetLastError()).c_str());
+ return false;
+ }
+
+ // It failed but this was only the first try; increase the working
+ // set size and try again.
+ SIZE_T min_ws_size, max_ws_size;
+ if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {
+ LLAMA_LOG_WARN("warning: GetProcessWorkingSetSize failed: %s\n",
+ llama_format_win_err(GetLastError()).c_str());
+ return false;
+ }
+ // Per MSDN: "The maximum number of pages that a process can lock
+ // is equal to the number of pages in its minimum working set minus
+ // a small overhead."
+ // Hopefully a megabyte is enough overhead:
+ size_t increment = len + 1048576;
+ // The minimum must be <= the maximum, so we need to increase both:
+ min_ws_size += increment;
+ max_ws_size += increment;
+ if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {
+ LLAMA_LOG_WARN("warning: SetProcessWorkingSetSize failed: %s\n",
+ llama_format_win_err(GetLastError()).c_str());
+ return false;
+ }
+ }
+ }
+
+ static void raw_unlock(void * ptr, size_t len) {
+ if (!VirtualUnlock(ptr, len)) {
+ LLAMA_LOG_WARN("warning: failed to VirtualUnlock buffer: %s\n",
+ llama_format_win_err(GetLastError()).c_str());
+ }
+ }
+#else
+ static constexpr bool SUPPORTED = false;
+
+ static size_t lock_granularity() {
+ return (size_t) 65536;
+ }
+
+ bool raw_lock(const void * addr, size_t len) const {
+ LLAMA_LOG_WARN("warning: mlock not supported on this system\n");
+ return false;
+ }
+
+ static void raw_unlock(const void * addr, size_t len) {}
+#endif
+};
+using llama_mlocks = std::vector<std::unique_ptr<llama_mlock>>;
+
+// NOTE: avoid ever using this except for building the token_to_piece caches
+static std::string llama_token_to_piece(const struct llama_model * model, llama_token token, bool special) {
+ std::string piece;
+ piece.resize(piece.capacity()); // using string internal cache
+ const int n_chars = llama_token_to_piece(model, token, &piece[0], piece.size(), 0, special);
+ if (n_chars < 0) {
+ piece.resize(-n_chars);
+ int check = llama_token_to_piece(model, token, &piece[0], piece.size(), 0, special);
+ GGML_ASSERT(check == -n_chars);
+ }
+ else {
+ piece.resize(n_chars);
+ }
+
+ return piece;
+}
+
+static ggml_backend_buffer_type_t llama_default_buffer_type_cpu(bool host_buffer) {
+ ggml_backend_buffer_type_t buft = nullptr;
+
+#if defined(GGML_USE_CUDA)
+ // host buffers should only be used when data is expected to be copied to/from the GPU
+ if (host_buffer) {
+ buft = ggml_backend_cuda_host_buffer_type();
+ }
+#elif defined(GGML_USE_SYCL)
+ if (host_buffer) {
+ buft = ggml_backend_sycl_host_buffer_type();
+ }
+#elif defined(GGML_USE_CPU_HBM)
+ buft = ggml_backend_cpu_hbm_buffer_type();
+#elif defined(GGML_USE_VULKAN)
+ if (host_buffer) {
+ buft = ggml_backend_vk_host_buffer_type();
+ }
+#endif
+
+ if (buft == nullptr) {
+ buft = ggml_backend_cpu_buffer_type();
+ }
+ return buft;
+
+ GGML_UNUSED(host_buffer);
+}
+
+//
+// globals
+//
+
+struct llama_state {
+ llama_state() {
+#ifdef GGML_USE_METAL
+ ggml_backend_metal_log_set_callback(log_callback, log_callback_user_data);
+#elif defined(GGML_USE_CUDA)
+ ggml_backend_cuda_log_set_callback(log_callback, log_callback_user_data);
+#elif defined(GGML_USE_CANN)
+ ggml_backend_cann_log_set_callback(log_callback, log_callback_user_data);
+#endif
+ }
+
+ // We save the log callback globally
+ ggml_log_callback log_callback = llama_log_callback_default;
+ void * log_callback_user_data = nullptr;
+};
+
+static llama_state g_state;
+
+// available llama models
+enum e_model {
+ MODEL_UNKNOWN,
+ MODEL_14M,
+ MODEL_17M,
+ MODEL_22M,
+ MODEL_33M,
+ MODEL_60M,
+ MODEL_70M,
+ MODEL_80M,
+ MODEL_109M,
+ MODEL_137M,
+ MODEL_160M,
+ MODEL_220M,
+ MODEL_250M,
+ MODEL_270M,
+ MODEL_335M,
+ MODEL_410M,
+ MODEL_450M,
+ MODEL_770M,
+ MODEL_780M,
+ MODEL_0_5B,
+ MODEL_1B,
+ MODEL_1_3B,
+ MODEL_1_4B,
+ MODEL_1_6B,
+ MODEL_2B,
+ MODEL_2_8B,
+ MODEL_3B,
+ MODEL_4B,
+ MODEL_6B,
+ MODEL_6_9B,
+ MODEL_7B,
+ MODEL_8B,
+ MODEL_9B,
+ MODEL_11B,
+ MODEL_12B,
+ MODEL_13B,
+ MODEL_14B,
+ MODEL_15B,
+ MODEL_16B,
+ MODEL_20B,
+ MODEL_22B,
+ MODEL_30B,
+ MODEL_34B,
+ MODEL_35B,
+ MODEL_40B,
+ MODEL_65B,
+ MODEL_70B,
+ MODEL_236B,
+ MODEL_314B,
+ MODEL_SMALL,
+ MODEL_MEDIUM,
+ MODEL_LARGE,
+ MODEL_XL,
+ MODEL_A2_7B,
+ MODEL_8x7B,
+ MODEL_8x22B,
+ MODEL_16x12B,
+ MODEL_10B_128x3_66B,
+ MODEL_57B_A14B,
+ MODEL_27B,
+};
+
+static const size_t kiB = 1024;
+static const size_t MiB = 1024*kiB;
+static const size_t GiB = 1024*MiB;
+
+struct llama_hparams {
+ bool vocab_only;
+ bool rope_finetuned;
+ bool use_par_res;
+
+ uint32_t n_vocab;
+ uint32_t n_ctx_train; // context size the model was trained on
+ uint32_t n_embd;
+ uint32_t n_layer;
+ uint32_t n_rot;
+ uint32_t n_swa = 0; // sliding window attention (SWA)
+ uint32_t n_embd_head_k; // dimension of keys (d_k). d_q is assumed to be the same, but there are n_head q heads, and only n_head_kv k-v heads
+ uint32_t n_embd_head_v; // dimension of values (d_v) aka n_embd_head
+ uint32_t n_expert = 0;
+ uint32_t n_expert_used = 0;
+ uint32_t n_vocab_type = 0; // for BERT-style token types
+ uint32_t n_rel_attn_bkts = 0;
+
+ std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_arr;
+ std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_kv_arr;
+ std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_arr;
+
+ std::array<std::array<uint32_t, LLAMA_MAX_LAYERS>, 4> n_bskcn_arr;
+
+ uint32_t n_layer_dense_lead = 0;
+ uint32_t n_lora_q = 0;
+ uint32_t n_lora_kv = 0;
+ uint32_t n_ff_exp = 0;
+ uint32_t n_ff_shexp = 0;
+ uint32_t n_expert_shared = 0;
+ float expert_weights_scale = 0.0;
+
+ float f_norm_eps;
+ float f_norm_rms_eps;
+
+ float f_attn_logit_softcapping = 50.0f;
+ float f_final_logit_softcapping = 30.0f;
+
+ // for RWKV
+ uint32_t rescale_every_n_layers = 0;
+ uint32_t time_mix_extra_dim = 0;
+ uint32_t time_decay_extra_dim = 0;
+ uint32_t wkv_head_size = 0;
+
+ float rope_attn_factor = 1.0f;
+ float rope_freq_base_train;
+ float rope_freq_scale_train;
+ uint32_t n_ctx_orig_yarn;
+ float rope_yarn_log_mul;
+
+ // for State Space Models
+ uint32_t ssm_d_conv = 0;
+ uint32_t ssm_d_inner = 0;
+ uint32_t ssm_d_state = 0;
+ uint32_t ssm_dt_rank = 0;
+ bool ssm_dt_b_c_rms = false;
+
+ float f_clamp_kqv = 0.0f;
+ float f_max_alibi_bias = 0.0f;
+ float f_logit_scale = 0.0f;
+
+ bool causal_attn = true;
+ bool use_alibi = false;
+ bool attn_soft_cap = false;
+
+ // needed by encoder-decoder models (e.g. T5, FLAN-T5)
+ // ref: https://github.com/ggerganov/llama.cpp/pull/8141
+ llama_token dec_start_token_id = -1;
+
+ enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE;
+ enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE;
+ enum llama_rope_scaling_type rope_scaling_type_train = LLAMA_ROPE_SCALING_TYPE_NONE;
+
+ bool operator!=(const llama_hparams & other) const {
+ if (this->vocab_only != other.vocab_only) return true;
+ if (this->n_vocab != other.n_vocab) return true;
+ if (this->n_ctx_train != other.n_ctx_train) return true;
+ if (this->n_embd != other.n_embd) return true;
+ if (this->n_layer != other.n_layer) return true;
+ if (this->n_rot != other.n_rot) return true;
+ if (this->n_swa != other.n_swa) return true;
+ if (this->n_embd_head_k != other.n_embd_head_k) return true;
+ if (this->n_embd_head_v != other.n_embd_head_v) return true;
+ if (this->n_expert != other.n_expert) return true;
+ if (this->n_expert_used != other.n_expert_used) return true;
+
+ if (this->n_head_arr != other.n_head_arr) return true;
+ if (this->n_head_kv_arr != other.n_head_kv_arr) return true;
+ if (this->n_ff_arr != other.n_ff_arr) return true;
+ if (this->n_bskcn_arr != other.n_bskcn_arr) return true;
+
+ if (this->n_rel_attn_bkts != other.n_rel_attn_bkts) return true;
+ if (this->n_layer_dense_lead != other.n_layer_dense_lead) return true;
+ if (this->n_lora_q != other.n_lora_q) return true;
+ if (this->n_lora_kv != other.n_lora_kv) return true;
+ if (this->n_ff_exp != other.n_ff_exp) return true;
+ if (this->n_ff_shexp != other.n_ff_shexp) return true;
+ if (this->n_expert_shared != other.n_expert_shared) return true;
+
+ if (this->rope_finetuned != other.rope_finetuned) return true;
+ if (this->n_ctx_orig_yarn != other.n_ctx_orig_yarn) return true;
+
+ if (this->ssm_d_conv != other.ssm_d_conv) return true;
+ if (this->ssm_d_inner != other.ssm_d_inner) return true;
+ if (this->ssm_d_state != other.ssm_d_state) return true;
+ if (this->ssm_dt_rank != other.ssm_dt_rank) return true;
+ if (this->ssm_dt_b_c_rms != other.ssm_dt_b_c_rms) return true;
+
+ if (this->rescale_every_n_layers != other.rescale_every_n_layers) return true;
+ if (this->time_mix_extra_dim != other.time_mix_extra_dim) return true;
+ if (this->time_decay_extra_dim != other.time_decay_extra_dim) return true;
+ if (this->wkv_head_size != other.wkv_head_size) return true;
+
+ if (this->dec_start_token_id != other.dec_start_token_id) return true;
+
+ const float EPSILON = 1e-9f;
+
+ if (!is_float_close(this->f_norm_eps, other.f_norm_eps, EPSILON)) return true;
+ if (!is_float_close(this->f_norm_rms_eps, other.f_norm_rms_eps, EPSILON)) return true;
+ if (!is_float_close(this->rope_attn_factor, other.rope_attn_factor, EPSILON)) return true;
+ if (!is_float_close(this->rope_freq_base_train, other.rope_freq_base_train, EPSILON)) return true;
+ if (!is_float_close(this->rope_freq_scale_train, other.rope_freq_scale_train, EPSILON)) return true;
+ if (!is_float_close(this->expert_weights_scale, other.expert_weights_scale, EPSILON)) return true;
+ if (!is_float_close(this->rope_yarn_log_mul, other.rope_yarn_log_mul, EPSILON)) return true;
+
+ return false;
+ }
+
+ uint32_t n_head(uint32_t il = 0) const {
+ if (il < n_layer) {
+ return n_head_arr[il];
+ }
+
+ GGML_ABORT("fatal error");
+ }
+
+ uint32_t n_head_kv(uint32_t il = 0) const {
+ if (il < n_layer) {
+ return n_head_kv_arr[il];
+ }
+
+ GGML_ABORT("fatal error");
+ }
+
+ uint32_t n_ff(uint32_t il = 0) const {
+ if (il < n_layer) {
+ return n_ff_arr[il];
+ }
+
+ GGML_ABORT("fatal error");
+ }
+
+ uint32_t n_gqa(uint32_t il = 0) const {
+ const uint32_t n_head = this->n_head(il);
+ const uint32_t n_head_kv = this->n_head_kv(il);
+
+ if (n_head_kv == 0) {
+ return 0;
+ }
+
+ return n_head/n_head_kv;
+ }
+
+ uint32_t n_embd_k_gqa(uint32_t il = 0) const { // dimension of key embeddings across all k-v heads
+ const uint32_t n_head_kv = this->n_head_kv(il);
+
+ return n_embd_head_k * n_head_kv;
+ }
+
+ uint32_t n_embd_v_gqa(uint32_t il = 0) const { // dimension of value embeddings across all k-v heads
+ const uint32_t n_head_kv = this->n_head_kv(il);
+
+ return n_embd_head_v * n_head_kv;
+ }
+
+ uint32_t n_embd_k_s() const { // dimension of the rolling state embeddings
+ // corresponds to Mamba's conv_states size or RWKV's token_shift states size
+ if (wkv_head_size != 0) {
+ // for RWKV models
+ return 2 * n_embd;
+ } else {
+ // TODO: maybe support other convolution strides than 1
+ // NOTE: since the first column of the conv_state is shifted out each time, it's not actually needed
+ return (ssm_d_conv > 0 ? ssm_d_conv - 1 : 0) * ssm_d_inner;
+ }
+ }
+
+ uint32_t n_embd_v_s() const { // dimension of the recurrent state embeddings
+ if (wkv_head_size != 0) {
+ // corresponds to RWKV's wkv_states size
+ return n_embd * wkv_head_size;
+ } else {
+ // corresponds to Mamba's ssm_states size
+ return ssm_d_state * ssm_d_inner;
+ }
+ }
+
+ bool n_bskcn(uint32_t n, uint32_t il = 0) const {
+ if (il < n_layer) {
+ return n_bskcn_arr[n][il] > 0;
+ }
+
+ GGML_ABORT("fatal error");
+ }
+};
+
+static_assert(std::is_trivially_copyable<llama_hparams>::value, "llama_hparams must be trivially copyable");
+
+struct llama_cparams {
+ uint32_t n_ctx; // context size used during inference
+ uint32_t n_batch;
+ uint32_t n_ubatch;
+ uint32_t n_seq_max;
+ int n_threads; // number of threads to use for generation
+ int n_threads_batch; // number of threads to use for batch processing
+
+ float rope_freq_base;
+ float rope_freq_scale;
+
+ uint32_t n_ctx_orig_yarn;
+ // These hyperparameters are not exposed in GGUF, because all
+ // existing YaRN models use the same values for them.
+ float yarn_ext_factor;
+ float yarn_attn_factor;
+ float yarn_beta_fast;
+ float yarn_beta_slow;
+ float defrag_thold;
+
+ bool embeddings;
+ bool causal_attn;
+ bool offload_kqv;
+ bool flash_attn;
+
+ enum llama_pooling_type pooling_type;
+
+ ggml_backend_sched_eval_callback cb_eval;
+ void * cb_eval_user_data;
+};
+
+// TODO: separate into "llama_layer_enc" and "llama_layer_dec"
+struct llama_layer {
+ // normalization
+ struct ggml_tensor * attn_norm;
+ struct ggml_tensor * attn_norm_b;
+ struct ggml_tensor * attn_norm_2;
+ struct ggml_tensor * attn_norm_2_b;
+ struct ggml_tensor * attn_q_norm;
+ struct ggml_tensor * attn_q_norm_b;
+ struct ggml_tensor * attn_k_norm;
+ struct ggml_tensor * attn_k_norm_b;
+ struct ggml_tensor * attn_out_norm;
+ struct ggml_tensor * attn_out_norm_b;
+ struct ggml_tensor * attn_q_a_norm;
+ struct ggml_tensor * attn_kv_a_norm;
+ struct ggml_tensor * attn_sub_norm;
+ struct ggml_tensor * attn_post_norm;
+ struct ggml_tensor * ffn_sub_norm;
+ struct ggml_tensor * attn_norm_cross;
+ struct ggml_tensor * attn_norm_enc;
+
+ // attention
+ struct ggml_tensor * wq;
+ struct ggml_tensor * wk;
+ struct ggml_tensor * wv;
+ struct ggml_tensor * wo;
+ struct ggml_tensor * wqkv;
+ struct ggml_tensor * wq_a;
+ struct ggml_tensor * wq_b;
+ struct ggml_tensor * wkv_a_mqa;
+ struct ggml_tensor * wkv_b;
+ struct ggml_tensor * wq_cross;
+ struct ggml_tensor * wk_cross;
+ struct ggml_tensor * wv_cross;
+ struct ggml_tensor * wo_cross;
+ struct ggml_tensor * wq_enc;
+ struct ggml_tensor * wk_enc;
+ struct ggml_tensor * wv_enc;
+ struct ggml_tensor * wo_enc;
+
+ // attention bias
+ struct ggml_tensor * bq;
+ struct ggml_tensor * bk;
+ struct ggml_tensor * bv;
+ struct ggml_tensor * bo;
+ struct ggml_tensor * bqkv;
+
+ // relative position bias
+ struct ggml_tensor * attn_rel_b;
+ struct ggml_tensor * attn_rel_b_enc;
+ struct ggml_tensor * attn_rel_b_cross;
+
+ // normalization
+ struct ggml_tensor * ffn_norm;
+ struct ggml_tensor * ffn_norm_b;
+ struct ggml_tensor * ffn_post_norm;
+ struct ggml_tensor * layer_out_norm;
+ struct ggml_tensor * layer_out_norm_b;
+ struct ggml_tensor * ffn_norm_exps;
+ struct ggml_tensor * ffn_norm_enc;
+
+ // ff
+ struct ggml_tensor * ffn_gate; // w1
+ struct ggml_tensor * ffn_down; // w2
+ struct ggml_tensor * ffn_up; // w3
+ struct ggml_tensor * ffn_gate_enc;
+ struct ggml_tensor * ffn_down_enc;
+ struct ggml_tensor * ffn_up_enc;
+
+ // ff MoE
+ struct ggml_tensor * ffn_gate_inp;
+ struct ggml_tensor * ffn_gate_exps;
+ struct ggml_tensor * ffn_down_exps;
+ struct ggml_tensor * ffn_up_exps ;
+
+ // ff shared expert (shexp)
+ struct ggml_tensor * ffn_gate_inp_shexp;
+ struct ggml_tensor * ffn_gate_shexp;
+ struct ggml_tensor * ffn_down_shexp;
+ struct ggml_tensor * ffn_up_shexp;
+
+ // ff bias
+ struct ggml_tensor * ffn_gate_b = nullptr;
+ struct ggml_tensor * ffn_down_b = nullptr; // b2
+ struct ggml_tensor * ffn_up_b = nullptr; // b3
+ struct ggml_tensor * ffn_act;
+
+ // mamba proj
+ struct ggml_tensor * ssm_in;
+ struct ggml_tensor * ssm_x;
+ struct ggml_tensor * ssm_dt;
+ struct ggml_tensor * ssm_out;
+
+ // mamba
+ struct ggml_tensor * ssm_conv1d;
+ struct ggml_tensor * ssm_a;
+ struct ggml_tensor * ssm_d;
+
+ // mamba bias
+ struct ggml_tensor * ssm_conv1d_b;
+ struct ggml_tensor * ssm_dt_b;
+
+ // rwkv
+ struct ggml_tensor * time_mix_w1;
+ struct ggml_tensor * time_mix_w2;
+ struct ggml_tensor * time_mix_lerp_x;
+ struct ggml_tensor * time_mix_lerp_w;
+ struct ggml_tensor * time_mix_lerp_k;
+ struct ggml_tensor * time_mix_lerp_v;
+ struct ggml_tensor * time_mix_lerp_r;
+ struct ggml_tensor * time_mix_lerp_g;
+
+ struct ggml_tensor * time_mix_first;
+ struct ggml_tensor * time_mix_decay;
+ struct ggml_tensor * time_mix_decay_w1;
+ struct ggml_tensor * time_mix_decay_w2;
+ struct ggml_tensor * time_mix_key;
+ struct ggml_tensor * time_mix_value;
+ struct ggml_tensor * time_mix_receptance;
+ struct ggml_tensor * time_mix_gate;
+
+ struct ggml_tensor * time_mix_ln;
+ struct ggml_tensor * time_mix_ln_b;
+ struct ggml_tensor * time_mix_output;
+
+ struct ggml_tensor * channel_mix_lerp_k;
+ struct ggml_tensor * channel_mix_lerp_r;
+
+ struct ggml_tensor * channel_mix_key;
+ struct ggml_tensor * channel_mix_receptance;
+ struct ggml_tensor * channel_mix_value;
+
+ // long rope factors
+ struct ggml_tensor * rope_long = nullptr;
+ struct ggml_tensor * rope_short = nullptr;
+ struct ggml_tensor * rope_freqs = nullptr;
+
+ // bitnet scale
+ struct ggml_tensor * wq_scale;
+ struct ggml_tensor * wk_scale;
+ struct ggml_tensor * wv_scale;
+ struct ggml_tensor * wo_scale;
+ struct ggml_tensor * ffn_gate_scale;
+ struct ggml_tensor * ffn_up_scale;
+ struct ggml_tensor * ffn_down_scale;
+
+ struct ggml_tensor * bskcn_tv;
+};
+
+// very similar to llama_batch,
+// but has more metadata about sequences
+struct llama_ubatch {
+ bool equal_seqs;
+ // TODO: whole_seqs for embeddings?
+
+ uint32_t n_tokens; // total tokens (n_seq_tokens * n_seqs)
+ uint32_t n_seq_tokens; // tokens per sequence
+ uint32_t n_seqs;
+
+ llama_token * token; // [n_tokens]
+ float * embd; // [n_embd, n_tokens]
+ llama_pos * pos; // [n_tokens]
+ int32_t * n_seq_id; // [n_seqs]
+ llama_seq_id ** seq_id; // [n_seqs]
+ int8_t * output; // [n_tokens]
+};
+
+struct llama_kv_cell {
+ llama_pos pos = -1;
+ llama_pos delta = 0;
+ int32_t src = -1; // used by recurrent state models to copy states
+ int32_t tail = -1;
+
+ std::set<llama_seq_id> seq_id;
+
+ bool has_seq_id(const llama_seq_id & id) const {
+ return seq_id.find(id) != seq_id.end();
+ }
+
+ bool is_empty() const {
+ return seq_id.empty();
+ }
+
+ bool is_same_seq(const llama_kv_cell & other) const {
+ return seq_id == other.seq_id;
+ }
+};
+
+// ring-buffer of cached KV data
+struct llama_kv_cache {
+ bool has_shift = false;
+ bool do_defrag = false;
+ bool recurrent = false; // with recurrent state models, a cell can hold the state for more than one past token
+ bool v_trans = true; // the value tensor is transposed
+
+ // Note: The value of head isn't only used to optimize searching
+ // for a free KV slot. llama_decode_internal also uses it, so it
+ // cannot be freely changed after a slot has been allocated.
+ uint32_t head = 0;
+ uint32_t size = 0;
+ uint32_t used = 0; // used cells (i.e. at least one seq_id)
+
+ // computed before each graph build
+ uint32_t n = 0;
+
+ ggml_type type_k = GGML_TYPE_F16;
+ ggml_type type_v = GGML_TYPE_F16;
+
+ std::vector<llama_kv_cell> cells;
+
+ std::vector<struct ggml_tensor *> k_l; // per layer
+ std::vector<struct ggml_tensor *> v_l;
+
+ std::vector<struct ggml_context *> ctxs;
+ std::vector<ggml_backend_buffer_t> bufs;
+
+ size_t total_size() const {
+ size_t size = 0;
+ for (ggml_backend_buffer_t buf : bufs) {
+ size += ggml_backend_buffer_get_size(buf);
+ }
+ return size;
+ }
+
+ ~llama_kv_cache() {
+ for (struct ggml_context * ctx : ctxs) {
+ ggml_free(ctx);
+ }
+ for (ggml_backend_buffer_t buf : bufs) {
+ ggml_backend_buffer_free(buf);
+ }
+ }
+};
+
+struct llama_control_vector {
+ std::vector<struct ggml_tensor *> tensors; // per layer
+ std::vector<struct ggml_context *> ctxs;
+ std::vector<ggml_backend_buffer_t> bufs;
+
+ int32_t layer_start = -1;
+ int32_t layer_end = -1;
+
+ struct ggml_tensor * tensor_for(int il) const {
+ if (il < 0 || il < layer_start || il > layer_end || (size_t) il >= tensors.size()) {
+ return nullptr;
+ }
+ return tensors[il];
+ }
+
+ struct ggml_tensor * apply_to(struct ggml_context * ctx, struct ggml_tensor * cur, int il) const {
+ ggml_tensor * layer_dir = tensor_for(il);
+ if (layer_dir != nullptr) {
+ cur = ggml_add(ctx, cur, layer_dir);
+ }
+ return cur;
+ }
+
+ ~llama_control_vector() {
+ for (struct ggml_context * ctx : ctxs) {
+ ggml_free(ctx);
+ }
+ for (ggml_backend_buffer_t buf : bufs) {
+ ggml_backend_buffer_free(buf);
+ }
+ }
+};
+
+struct llama_model {
+ e_model type = MODEL_UNKNOWN;
+ llm_arch arch = LLM_ARCH_UNKNOWN;
+ llama_ftype ftype = LLAMA_FTYPE_ALL_F32;
+
+ std::string name = "n/a";
+
+ llama_hparams hparams = {};
+ llama_vocab vocab;
+
+ struct ggml_tensor * tok_embd;
+ struct ggml_tensor * type_embd;
+ struct ggml_tensor * pos_embd;
+ struct ggml_tensor * tok_norm;
+ struct ggml_tensor * tok_norm_b;
+
+ struct ggml_tensor * output_norm;
+ struct ggml_tensor * output_norm_b;
+ struct ggml_tensor * output;
+ struct ggml_tensor * output_b;
+ struct ggml_tensor * output_norm_enc;
+
+ std::vector<llama_layer> layers;
+
+ llama_split_mode split_mode;
+ int main_gpu;
+ int n_gpu_layers;
+
+ std::vector<std::string> rpc_servers;
+
+ // gguf metadata
+ std::unordered_map<std::string, std::string> gguf_kv;
+
+ // layer -> buffer type mapping
+ struct layer_buft {
+ layer_buft() : buft_matrix(nullptr), buft(nullptr) {}
+ layer_buft(ggml_backend_buffer_type_t matrix) : buft_matrix(matrix), buft(matrix) {}
+ layer_buft(ggml_backend_buffer_type_t matrix, ggml_backend_buffer_type_t other) : buft_matrix(matrix), buft(other) {}
+
+ ggml_backend_buffer_type_t buft_matrix; // matrices only - used by split buffers and backends that support only matrix multiplication
+ ggml_backend_buffer_type_t buft; // everything else
+ };
+
+ layer_buft buft_input;
+ layer_buft buft_output;
+ std::vector<layer_buft> buft_layer;
+
+ // contexts where the model tensors metadata is stored
+ std::vector<struct ggml_context *> ctxs;
+
+ // the model memory buffers for the tensor data
+ std::vector<ggml_backend_buffer_t> bufs;
+
+ // model memory mapped files
+ llama_mmaps mappings;
+
+ // objects representing data potentially being locked in memory
+ llama_mlocks mlock_bufs;
+ llama_mlocks mlock_mmaps;
+
+ // for quantize-stats only
+ std::vector<std::pair<std::string, struct ggml_tensor *>> tensors_by_name;
+
+ int64_t t_load_us = 0;
+ int64_t t_start_us = 0;
+
+ // keep track of loaded lora adapters
+ std::set<struct llama_lora_adapter *> lora_adapters;
+
+ ~llama_model() {
+ for (struct ggml_context * ctx : ctxs) {
+ ggml_free(ctx);
+ }
+ for (ggml_backend_buffer_t buf : bufs) {
+#ifdef GGML_USE_CUDA
+ if (ggml_backend_buffer_get_type(buf) == ggml_backend_cpu_buffer_type()) {
+ ggml_backend_cuda_unregister_host_buffer(ggml_backend_buffer_get_base(buf));
+ }
+#endif
+ ggml_backend_buffer_free(buf);
+ }
+ while (!lora_adapters.empty()) {
+ llama_lora_adapter_free(*lora_adapters.begin());
+ }
+ }
+};
+
+struct llama_sbatch_seq {
+ int32_t n_seq_id;
+ llama_seq_id * seq_id;
+ size_t offset;
+ size_t length;
+
+ // helper for smoother batch API transition -- can be deprecated in the future
+ llama_seq_id all_seq_id; // used if seq_id == NULL
+};
+
+// sequence-length-aware batch splitting
+struct llama_sbatch {
+ // tokens left in this batch
+ size_t n_tokens;
+
+ size_t n_embd;
+
+ bool logits_all; // TODO: remove once lctx.logits_all is removed too
+
+ // sorted indices into the batch
+ std::vector<size_t> ids;
+ // batch indices of the output
+ std::vector<size_t> out_ids;
+ std::vector<llama_sbatch_seq> seq;
+ const llama_batch * batch = nullptr;
+
+ // buffers for the ubatch
+ std::vector<llama_token> ubatch_token;
+ std::vector<float> ubatch_embd;
+ std::vector<llama_pos> ubatch_pos;
+ std::vector<int32_t> ubatch_n_seq_id;
+ std::vector<llama_seq_id *> ubatch_seq_id;
+ std::vector<int8_t> ubatch_output;
+
+ llama_ubatch reserve_ubatch(size_t n_ubatch, bool has_embd = false) {
+ // clear empty sequences
+ // the previous ubatch is assumed to be gone,
+ // so nothing should refer to values in these sequences anymore.
+ for (size_t i = seq.size(); i-- > 0;) {
+ if (seq[i].length == 0) {
+ seq.pop_back();
+ } else {
+ break;
+ }
+ }
+ ubatch_token.resize(!has_embd ? n_ubatch : 0);
+ ubatch_embd.resize(has_embd ? n_embd * n_ubatch : 0);
+ ubatch_pos.resize(n_ubatch);
+ ubatch_n_seq_id.resize(n_ubatch);
+ ubatch_seq_id.resize(n_ubatch);
+ ubatch_output.resize(n_ubatch);
+ llama_ubatch ubatch = {
+ /*equal_seqs =*/ true,
+ /*n_tokens =*/ 0,
+ /*n_seq_tokens =*/ 0,
+ /*n_seqs =*/ 0,
+ /*token =*/ !has_embd ? ubatch_token.data() : nullptr,
+ /*embd =*/ has_embd ? ubatch_embd.data() : nullptr,
+ /*pos =*/ ubatch_pos.data(),
+ /*n_seq_id =*/ ubatch_n_seq_id.data(),
+ /*seq_id =*/ ubatch_seq_id.data(),
+ /*output =*/ ubatch_output.data(),
+ };
+ return ubatch;
+ }
+
+ void add_seq_to_ubatch(llama_ubatch & ubatch, llama_sbatch_seq & seq, size_t length) {
+ GGML_ASSERT(batch != nullptr);
+ GGML_ASSERT(length <= seq.length);
+ // Can only add sequences of equal lengths to a batch,
+ // otherwise it isn't clear to which sequence a token belongs
+ GGML_ASSERT(seq.n_seq_id == 0 || ubatch.n_seqs == 0 || length == (size_t) ubatch.n_tokens / ubatch.n_seqs);
+ GGML_ASSERT((seq.n_seq_id != 0) == ubatch.equal_seqs);
+ // NOTE: loops are separated for cache-friendliness
+ if (batch->token) {
+ if (ubatch.equal_seqs) {
+ for (size_t i = 0; i < length; ++i) {
+ ubatch.token[ubatch.n_tokens + i] = batch->token[ids[seq.offset + i]];
+ }
+ } else {
+ // simple split
+ ubatch.token = batch->token + seq.offset;
+ }
+ } else {
+ ubatch.token = nullptr;
+ }
+ if (batch->embd) {
+ if (ubatch.equal_seqs) {
+ for (size_t i = 0; i < length; ++i) {
+ memcpy(
+ ubatch.embd + n_embd * (ubatch.n_tokens + i),
+ batch->embd + n_embd * ids[seq.offset + i],
+ n_embd * sizeof(float)
+ );
+ }
+ } else {
+ // simple split
+ ubatch.embd = batch->embd + (n_embd * seq.offset);
+ }
+ } else {
+ ubatch.embd = nullptr;
+ }
+ // from here on, the else branches are deprecated;
+ // they are helpers for smoother batch API transition
+ if (batch->pos) {
+ if (ubatch.equal_seqs) {
+ for (size_t i = 0; i < length; ++i) {
+ ubatch.pos[ubatch.n_tokens + i] = batch->pos[ids[seq.offset + i]];
+ }
+ } else {
+ // simple split
+ ubatch.pos = batch->pos + seq.offset;
+ }
+ } else {
+ for (size_t i = 0; i < length; ++i) {
+ llama_pos bi = ids[seq.offset + i];
+ ubatch.pos[ubatch.n_tokens + i] = batch->all_pos_0 + (bi * batch->all_pos_1);
+ }
+ }
+ if (ubatch.equal_seqs) {
+ ubatch.n_seq_id[ubatch.n_seqs] = seq.n_seq_id;
+ if (seq.seq_id) {
+ ubatch.seq_id[ubatch.n_seqs] = seq.seq_id;
+ } else {
+ GGML_ASSERT(seq.n_seq_id == 1);
+ ubatch.seq_id[ubatch.n_seqs] = &seq.all_seq_id;
+ }
+ } else {
+ // simple split
+ if (batch->n_seq_id) {
+ for (size_t i = 0; i < length; ++i) {
+ ubatch.n_seq_id = batch->n_seq_id + seq.offset;
+ }
+ } else {
+ for (size_t i = 0; i < length; ++i) {
+ ubatch.n_seq_id[ubatch.n_seqs + i] = 1;
+ }
+ }
+ if (batch->seq_id) {
+ for (size_t i = 0; i < length; ++i) {
+ ubatch.seq_id = batch->seq_id + seq.offset;
+ }
+ } else {
+ for (size_t i = 0; i < length; ++i) {
+ ubatch.seq_id[ubatch.n_seqs + i] = &seq.all_seq_id;
+ }
+ }
+ }
+ if (logits_all) {
+ for (size_t i = 0; i < length; ++i) {
+ ubatch.output[ubatch.n_tokens + i] = 1;
+ out_ids.push_back(ids[seq.offset + i]);
+ }
+ } else if (batch->logits) {
+ if (ubatch.equal_seqs) {
+ for (size_t i = 0; i < length; ++i) {
+ size_t id = ids[seq.offset + i];
+ int8_t is_output = batch->logits[id];
+ ubatch.output[ubatch.n_tokens + i] = is_output;
+ if (is_output) { out_ids.push_back(id); }
+ }
+ } else {
+ // simple split
+ ubatch.output = batch->logits + seq.offset;
+ for (size_t i = 0; i < length; ++i) {
+ if (ubatch.output[i] != 0) { out_ids.push_back(seq.offset + i); }
+ }
+ }
+ } else {
+ // only get last output
+ for (size_t i = 0; i < length; ++i) {
+ size_t id = ids[seq.offset + i];
+ int8_t is_last = id == ids.size() - 1;
+ ubatch.output[ubatch.n_tokens + i] = is_last;
+ if (is_last) { out_ids.push_back(id); }
+ }
+ }
+ if (ubatch.n_tokens == 0 && ubatch.n_seqs == 0) {
+ ubatch.n_seq_tokens = ubatch.equal_seqs ? length : 1;
+ }
+ ubatch.n_tokens += length;
+ ubatch.n_seqs += ubatch.equal_seqs ? 1 : length; // virtual sequences for simple splits
+ seq.offset += length;
+ seq.length -= length;
+ n_tokens -= length;
+ GGML_ASSERT(ubatch.n_tokens == ubatch.n_seq_tokens * ubatch.n_seqs);
+ }
+
+ // simple split, unknown number of sequences of unequal lengths
+ llama_ubatch split_simple(size_t n_ubatch) {
+ n_ubatch = n_tokens < n_ubatch ? n_tokens : n_ubatch;
+ llama_ubatch ubatch = reserve_ubatch(n_ubatch, /* has_embd */ batch->embd != nullptr);
+ ubatch.equal_seqs = false;
+ if (!seq.empty()) {
+ llama_sbatch_seq & s = seq[0];
+ size_t length = s.length < n_ubatch ? s.length : n_ubatch;
+ GGML_ASSERT(seq.size() == 1 && s.n_seq_id == 0); // don't mix with other splits
+ add_seq_to_ubatch(ubatch, s, length);
+ }
+ return ubatch;
+ }
+
+ // make batches of equal-length sequences
+ llama_ubatch split_equal(size_t n_ubatch) {
+ n_ubatch = n_tokens < n_ubatch ? n_tokens : n_ubatch;
+ llama_ubatch ubatch = reserve_ubatch(n_ubatch, /* has_embd */ batch->embd != nullptr);
+ if (!seq.empty()) {
+ size_t length = 0;
+ size_t n_tokens_in_ubatch = 0;
+ GGML_ASSERT(seq[0].n_seq_id > 0); // should not be mixed with simple splits
+ // smallest first, because it's easier to split this way;
+ // starting from the end to pop in constant time.
+ for (size_t i = seq.size(); i-- > 0;) {
+ llama_sbatch_seq & s = seq[i];
+ GGML_ASSERT(s.length > 0);
+ if (length == 0) {
+ length = s.length < n_ubatch ? s.length : n_ubatch;
+ }
+ add_seq_to_ubatch(ubatch, s, length);
+ n_tokens_in_ubatch += length;
+ // shared prompts can't be mixed with any of their sequences,
+ // so it's safer to compute them in their own ubatch
+ if (s.n_seq_id > 1) { break; }
+ // stop when there isn't enough space for another sequence
+ if (length + n_tokens_in_ubatch > n_ubatch) { break; }
+ }
+ }
+ return ubatch;
+ }
+
+ // sequence-wise split
+ llama_ubatch split_seq(size_t n_ubatch) {
+ n_ubatch = n_tokens < n_ubatch ? n_tokens : n_ubatch;
+ llama_ubatch ubatch = reserve_ubatch(n_ubatch, /* has_embd */ batch->embd != nullptr);
+ if (!seq.empty()) {
+ llama_sbatch_seq & s = seq[seq.size() - 1];
+ size_t length = s.length < n_ubatch ? s.length : n_ubatch;
+ GGML_ASSERT(s.n_seq_id > 0); // should not be mixed with simple splits
+ add_seq_to_ubatch(ubatch, s, length);
+ }
+ return ubatch;
+ }
+
+ void from_batch(const llama_batch & batch, const size_t n_embd, const bool simple_split = false, const bool logits_all = false) {
+ GGML_ASSERT(batch.n_tokens >= 0);
+ this->batch = &batch;
+ this->n_embd = n_embd;
+ this->logits_all = logits_all;
+
+ n_tokens = batch.n_tokens;
+ ids.resize(n_tokens);
+ out_ids.clear();
+ // TODO: reserve out_ids and seq
+
+ for (size_t i = 0; i < n_tokens; ++i) {
+ ids[i] = i;
+ }
+ if (simple_split) {
+ seq.resize(1);
+ llama_sbatch_seq & s = seq[0];
+ s.n_seq_id = 0;
+ s.seq_id = nullptr;
+ s.offset = 0;
+ s.length = n_tokens;
+ s.all_seq_id = batch.all_seq_id;
+ return;
+ }
+ std::sort(ids.begin(), ids.end(),
+ [&batch](size_t a, size_t b) {
+ int32_t n_seq_a = batch.n_seq_id ? batch.n_seq_id[a] : 1;
+ int32_t n_seq_b = batch.n_seq_id ? batch.n_seq_id[b] : 1;
+ // sort by seq_id, then by pos
+ if (n_seq_a == n_seq_b) {
+ if (batch.seq_id) {
+ for (int32_t i = 0; i < n_seq_a; ++i) {
+ llama_seq_id seq_id_a = batch.seq_id[a][i];
+ llama_seq_id seq_id_b = batch.seq_id[b][i];
+ // smaller seq_ids go first
+ if (seq_id_a != seq_id_b) {
+ return seq_id_a < seq_id_b;
+ }
+ }
+ }
+ // when all else is equal, sort by pos
+ if (batch.pos) {
+ return batch.pos[a] < batch.pos[b];
+ }
+ // no pos, sort by id (assuming batch.all_pos_1 is positive)
+ return a < b;
+ }
+ // shared prompts go first
+ return n_seq_a > n_seq_b;
+ }
+ );
+ // init seq
+ llama_sbatch_seq * last_seq = nullptr;
+
+ if (batch.n_seq_id != nullptr && batch.seq_id != nullptr) {
+ for (size_t i = 0; i < n_tokens; ++i) {
+ const size_t bi = ids[i];
+ const int32_t n_seqs = batch.n_seq_id[bi];
+ llama_seq_id * seq_ids = batch.seq_id[bi];
+ if (last_seq != nullptr) {
+ bool same = n_seqs == last_seq->n_seq_id;
+ for (int32_t j = 0; same && j < n_seqs; ++j) {
+ if (seq_ids[j] != last_seq->seq_id[j]) {
+ same = false;
+ }
+ }
+ if (same) {
+ last_seq->length += 1;
+ continue;
+ }
+ }
+ llama_sbatch_seq new_seq = {n_seqs, seq_ids, i, 1, batch.all_seq_id};
+ seq.push_back(new_seq);
+ last_seq = &seq.back();
+ }
+ } else {
+ llama_sbatch_seq new_seq = {1, nullptr, 0, n_tokens, batch.all_seq_id};
+ seq.push_back(new_seq);
+ }
+ // keep shared prompts first at the end, then sort by length descending.
+ std::sort(seq.begin(), seq.end(),
+ [](llama_sbatch_seq & a, llama_sbatch_seq & b) {
+ if (a.n_seq_id == b.n_seq_id) {
+ return a.length > b.length;
+ }
+ return a.n_seq_id < b.n_seq_id;
+ }
+ );
+ }
+};
+
+struct llama_context {
+ llama_context(const llama_model & model)
+ : model(model)
+ , sampling(llama_n_vocab(&model))
+ , t_start_us(model.t_start_us)
+ , t_load_us(model.t_load_us) {}
+
+ ~llama_context() {
+ ggml_backend_sched_free(sched);
+
+ for (ggml_backend_t backend : backends) {
+ ggml_backend_free(backend);
+ }
+
+ ggml_backend_buffer_free(buf_output);
+ }
+
+ const struct llama_model & model;
+
+ struct llama_cparams cparams;
+ struct llama_sampling sampling;
+ struct llama_sbatch sbatch;
+ struct llama_kv_cache kv_self;
+ struct llama_control_vector cvec;
+
+ std::unordered_map<struct llama_lora_adapter *, float> lora_adapters;
+
+ std::vector<ggml_backend_t> backends;
+#ifdef GGML_USE_METAL
+ ggml_backend_t backend_metal = nullptr;
+#endif
+#ifdef GGML_USE_BLAS
+ ggml_backend_t backend_blas = nullptr;
+#endif
+ ggml_backend_t backend_cpu = nullptr;
+
+ ggml_threadpool_t threadpool = nullptr;
+ ggml_threadpool_t threadpool_batch = nullptr;
+
+ bool has_evaluated_once = false;
+
+ int64_t t_start_us;
+ int64_t t_load_us;
+ int64_t t_p_eval_us = 0;
+ int64_t t_eval_us = 0;
+
+ int64_t t_compute_start_us = 0;
+ int64_t n_queued_tokens = 0;
+
+ int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1)
+ int32_t n_eval = 0; // number of eval calls
+
+ // host buffer for the model output (logits and embeddings)
+ ggml_backend_buffer_t buf_output = nullptr;
+
+ // decode output (2-dimensional array: [n_outputs][n_vocab])
+ size_t logits_size = 0; // capacity (of floats) for logits
+ float * logits = nullptr;
+
+ std::vector<int32_t> output_ids; // map batch token positions to ids of the logits and embd buffers
+ size_t output_size = 0; // capacity (of tokens positions) for the output buffers
+ int32_t n_outputs = 0; // number of actually-used outputs in the current ubatch or last logical batch
+
+ bool logits_all = false;
+
+ // embeddings output (2-dimensional array: [n_outputs][n_embd])
+ // populated only when pooling_type == LLAMA_POOLING_TYPE_NONE
+ size_t embd_size = 0; // capacity (of floats) for embeddings
+ float * embd = nullptr;
+
+ // sequence embeddings output (map of [n_embd] vectors)
+ // populated only when pooling_type != LLAMA_POOLING_TYPE_NONE
+ std::map<llama_seq_id, std::vector<float>> embd_seq;
+
+ // whether we are computing encoder output or decoder output
+ bool is_encoding = false;
+
+ // output of the encoder part of the encoder-decoder models
+ std::vector<float> embd_enc;
+ std::vector<std::set<llama_seq_id>> seq_ids_enc;
+
+ // memory buffers used to evaluate the model
+ std::vector<uint8_t> buf_compute_meta;
+ ggml_backend_sched_t sched = nullptr;
+
+ ggml_abort_callback abort_callback = nullptr;
+ void * abort_callback_data = nullptr;
+
+ // input tensors
+ struct ggml_tensor * inp_tokens; // I32 [n_batch]
+ struct ggml_tensor * inp_embd; // F32 [n_embd, n_batch]
+ struct ggml_tensor * inp_pos; // I32 [n_batch]
+ struct ggml_tensor * inp_out_ids; // I32 [n_outputs]
+ struct ggml_tensor * inp_KQ_mask; // F32 [kv_size, n_batch]
+ struct ggml_tensor * inp_KQ_mask_swa; // F32 [kv_size, n_batch]
+ struct ggml_tensor * inp_K_shift; // I32 [kv_size]
+ struct ggml_tensor * inp_mean; // F32 [n_batch, n_batch]
+ struct ggml_tensor * inp_cls; // I32 [n_batch]
+ struct ggml_tensor * inp_s_copy; // I32 [kv_size]
+ struct ggml_tensor * inp_s_mask; // F32 [1, n_kv]
+ struct ggml_tensor * inp_s_seq; // I32 [n_kv, n_batch]
+ struct ggml_tensor * inp_pos_bucket; // I32 [n_batch|n_kv, n_batch]
+ struct ggml_tensor * inp_embd_enc; // F32 [n_embd, n_outputs_enc]
+ struct ggml_tensor * inp_KQ_mask_cross; // F32 [n_outputs_enc, n_batch]
+};
+
+struct llama_lora_weight {
+ struct ggml_tensor * a = nullptr;
+ struct ggml_tensor * b = nullptr;
+ llama_lora_weight() = default;
+ llama_lora_weight(struct ggml_tensor * a, struct ggml_tensor * b): a(a), b(b) {}
+};
+
+struct llama_lora_adapter {
+ struct llama_model * base_model;
+ // map tensor name to lora_a_b
+ std::unordered_map<std::string, struct llama_lora_weight> ab_map;
+ std::vector<struct ggml_context *> ctxs;
+ std::vector<ggml_backend_buffer_t> bufs;
+
+ float alpha;
+
+ llama_lora_adapter(struct llama_model * base_model): base_model(base_model) {
+ base_model->lora_adapters.insert(this);
+ }
+
+ llama_lora_weight * get_weight(struct ggml_tensor * w) {
+ std::string name(w->name);
+ auto pos = ab_map.find(name);
+ if (ab_map.find(name) != ab_map.end()) {
+ return &pos->second;
+ }
+ return nullptr;
+ }
+
+ ~llama_lora_adapter() {
+ for (struct ggml_context * ctx : ctxs) {
+ ggml_free(ctx);
+ }
+ for (ggml_backend_buffer_t buf : bufs) {
+ ggml_backend_buffer_free(buf);
+ }
+ auto pos = base_model->lora_adapters.find(this);
+ if (pos != base_model->lora_adapters.end()) {
+ base_model->lora_adapters.erase(pos);
+ }
+ }
+};
+
+static size_t llama_get_device_count(const llama_model & model) {
+ size_t count = 1;
+#if defined(GGML_USE_CUDA)
+ count = ggml_backend_cuda_get_device_count();
+#elif defined(GGML_USE_SYCL)
+ count = ggml_backend_sycl_get_device_count();
+#elif defined(GGML_USE_VULKAN)
+ count = ggml_backend_vk_get_device_count();
+#elif defined(GGML_USE_CANN)
+ return ggml_backend_cann_get_device_count();
+#endif
+#if defined(GGML_USE_RPC)
+ count += model.rpc_servers.size();
+#endif
+ return count;
+ GGML_UNUSED(model);
+}
+
+static ggml_backend_buffer_type_t llama_default_buffer_type_offload(const llama_model & model, int gpu) {
+ ggml_backend_buffer_type_t buft = nullptr;
+
+#if defined(GGML_USE_RPC)
+ int dev_count = (int)llama_get_device_count(model);
+ int rpc_count = (int)model.rpc_servers.size();
+ if (gpu >= dev_count - rpc_count) {
+ const char * endpoint = model.rpc_servers[gpu - dev_count + rpc_count].c_str();
+ return ggml_backend_rpc_buffer_type(endpoint);
+ }
+#endif
+#if defined(GGML_USE_METAL)
+ buft = ggml_backend_metal_buffer_type();
+#elif defined(GGML_USE_CUDA)
+ buft = ggml_backend_cuda_buffer_type(gpu);
+#elif defined(GGML_USE_VULKAN)
+ buft = ggml_backend_vk_buffer_type(gpu);
+#elif defined(GGML_USE_SYCL)
+ buft = ggml_backend_sycl_buffer_type(gpu);
+#elif defined(GGML_USE_KOMPUTE)
+ buft = ggml_backend_kompute_buffer_type(gpu);
+ if (buft == nullptr) {
+ LLAMA_LOG_WARN("%s: cannot use GPU %d, check `vulkaninfo --summary`\n", __func__, gpu);
+ }
+#elif defined(GGML_USE_CANN)
+ buft = ggml_backend_cann_buffer_type(gpu);
+#endif
+
+ if (buft == nullptr) {
+ buft = llama_default_buffer_type_cpu(true);
+ }
+ return buft;
+ GGML_UNUSED(model);
+ GGML_UNUSED(gpu);
+}
+
+static ggml_backend_buffer_type_t llama_default_buffer_type_split(const llama_model & model, int fallback_gpu, const float * tensor_split) {
+ ggml_backend_buffer_type_t buft = nullptr;
+
+#ifdef GGML_USE_CUDA
+ if (ggml_backend_cuda_get_device_count() > 1) {
+ buft = ggml_backend_cuda_split_buffer_type(tensor_split);
+ }
+#endif
+
+#ifdef GGML_USE_SYCL
+ if (ggml_backend_sycl_get_device_count() > 1) {
+ buft = ggml_backend_sycl_split_buffer_type(tensor_split);
+ }
+#endif
+
+ if (buft == nullptr) {
+ buft = llama_default_buffer_type_offload(model, fallback_gpu);
+ }
+ return buft;
+
+ GGML_UNUSED(tensor_split);
+}
+
+static size_t llama_get_device_memory(const llama_model & model, int device) {
+#if defined(GGML_USE_RPC)
+ int dev_count = (int)llama_get_device_count(model);
+ int rpc_count = (int)model.rpc_servers.size();
+ if (device >= dev_count - rpc_count) {
+ size_t total;
+ size_t free;
+ const char * endpoint = model.rpc_servers[device - dev_count + rpc_count].c_str();
+ ggml_backend_rpc_get_device_memory(endpoint, &free, &total);
+ return free;
+ }
+#endif
+#if defined(GGML_USE_CUDA)
+ size_t total;
+ size_t free;
+ ggml_backend_cuda_get_device_memory(device, &free, &total);
+ return free;
+#elif defined(GGML_USE_SYCL)
+ size_t total;
+ size_t free;
+ ggml_backend_sycl_get_device_memory(device, &free, &total);
+ return free;
+#elif defined(GGML_USE_VULKAN)
+ size_t total;
+ size_t free;
+ ggml_backend_vk_get_device_memory(device, &free, &total);
+ return free;
+#elif defined(GGML_USE_CANN)
+ size_t total;
+ size_t free;
+ ggml_backend_cann_get_device_memory(device, &free, &total);
+ return free;
+#else
+ return 1;
+#endif
+ GGML_UNUSED(model);
+ GGML_UNUSED(device);
+}
+
+//
+// kv cache helpers
+//
+
+static bool llama_kv_cache_init(
+ struct llama_kv_cache & cache,
+ const llama_context * ctx,
+ ggml_type type_k,
+ ggml_type type_v,
+ uint32_t kv_size,
+ bool offload) {
+ const llama_model & model = ctx->model;
+ const llama_cparams & cparams = ctx->cparams;
+
+ const struct llama_hparams & hparams = model.hparams;
+
+ const int64_t n_layer = hparams.n_layer;
+
+ cache.has_shift = false;
+
+ cache.recurrent = llama_model_is_recurrent(&model);
+ cache.v_trans = !cache.recurrent && !cparams.flash_attn;
+
+ cache.head = 0;
+ cache.size = kv_size;
+ cache.used = 0;
+
+ cache.type_k = type_k;
+ cache.type_v = type_v;
+
+ cache.cells.clear();
+ cache.cells.resize(kv_size);
+
+ // count used buffer types
+ std::map<ggml_backend_buffer_type_t, int> buft_layer_count;
+ if (offload) {
+ for (int64_t i = 0; i < n_layer; ++i) {
+ buft_layer_count[model.buft_layer[i].buft]++;
+ }
+ } else {
+ buft_layer_count[llama_default_buffer_type_cpu(true)] = n_layer;
+ }
+
+ // create a context for each buffer type
+ std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;
+ for (auto & it : buft_layer_count) {
+ int n_layers = it.second;
+ struct ggml_init_params params = {
+ /*.mem_size =*/ 2u*n_layers*ggml_tensor_overhead(),
+ /*.mem_buffer =*/ NULL,
+ /*.no_alloc =*/ true,
+ };
+ ggml_context * ctx = ggml_init(params);
+ if (!ctx) {
+ LLAMA_LOG_ERROR("%s: failed to allocate context for kv cache\n", __func__);
+ return false;
+ }
+ ctx_map[it.first] = ctx;
+ cache.ctxs.push_back(ctx);
+ }
+
+ cache.k_l.reserve(n_layer);
+ cache.v_l.reserve(n_layer);
+
+ for (int i = 0; i < (int) n_layer; i++) {
+ const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(i) + hparams.n_embd_k_s();
+ const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(i) + hparams.n_embd_v_s();
+
+ struct ggml_context * ctx = offload ? ctx_map.at(model.buft_layer[i].buft) : cache.ctxs.front();
+ ggml_tensor * k = ggml_new_tensor_1d(ctx, type_k, n_embd_k_gqa*kv_size);
+ ggml_tensor * v = ggml_new_tensor_1d(ctx, type_v, n_embd_v_gqa*kv_size);
+ ggml_format_name(k, "cache_k_l%d", i);
+ ggml_format_name(v, "cache_v_l%d", i);
+ cache.k_l.push_back(k);
+ cache.v_l.push_back(v);
+ }
+
+ // allocate tensors and initialize the buffers to avoid NaNs in the padding
+ for (auto it : ctx_map) {
+ ggml_backend_buffer_type_t buft = it.first;
+ ggml_context * ctx = it.second;
+ ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);
+ if (!buf) {
+ LLAMA_LOG_ERROR("%s: failed to allocate buffer for kv cache\n", __func__);
+ return false;
+ }
+ ggml_backend_buffer_clear(buf, 0);
+ LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
+ cache.bufs.push_back(buf);
+ }
+
+ return true;
+}
+
+// find an empty slot of size "n_tokens" in the cache
+// updates the cache head
+// Note: On success, it's important that cache.head points
+// to the first cell of the slot.
+static bool llama_kv_cache_find_slot(
+ struct llama_kv_cache & cache,
+ const struct llama_ubatch & batch) {
+ const uint32_t n_tokens = batch.n_tokens;
+ const uint32_t n_seqs = batch.n_seqs;
+ const uint32_t n_seq_tokens = batch.n_seq_tokens;
+
+ if (cache.recurrent) {
+ // For recurrent state architectures (like Mamba or RWKV),
+ // each cache cell can store the state for a whole sequence.
+ // A slot should be always be contiguous.
+
+ // can only process batches with an equal number of new tokens in each sequence
+ GGML_ASSERT(batch.equal_seqs);
+
+ int32_t min = cache.size - 1;
+ int32_t max = 0;
+
+ // everything should fit if all seq_ids are smaller than the max
+ for (uint32_t s = 0; s < n_seqs; ++s) {
+ const uint32_t n_seq_id = batch.n_seq_id[s];
+ for (uint32_t j = 0; j < n_seq_id; ++j) {
+ const llama_seq_id seq_id = batch.seq_id[s][j];
+
+ if (seq_id < 0 || (uint32_t) seq_id >= cache.size) {
+ // too big seq_id
+ // TODO: would it be possible to resize the cache instead?
+ LLAMA_LOG_ERROR("%s: seq_id=%d >= n_seq_max=%d Try using a bigger --parallel value\n", __func__, seq_id, cache.size);
+ return false;
+ }
+ if (j > 0) {
+ llama_kv_cell & seq = cache.cells[seq_id];
+ if (seq.tail >= 0) {
+ llama_kv_cell & cell = cache.cells[seq.tail];
+ // clear cells from seq_ids that become shared
+ // (should not normally happen, but let's handle it anyway)
+ cell.seq_id.erase(seq_id);
+ seq.tail = -1;
+ if (cell.seq_id.empty()) {
+ cell.pos = -1;
+ cell.src = -1;
+ cache.used -= 1;
+ }
+ }
+ }
+ }
+ }
+
+#ifndef NDEBUG
+ {
+ std::vector<int32_t> tails_verif;
+ tails_verif.assign(cache.size, -1);
+ for (uint32_t i = 0; i < cache.size; ++i) {
+ llama_kv_cell & cell = cache.cells[i];
+ for (llama_seq_id seq_id : cell.seq_id) {
+ if (tails_verif[seq_id] != -1) {
+ LLAMA_LOG_ERROR("%s: duplicate tail for seq_id %d in cell %d and %d\n", __func__, seq_id, i, tails_verif[seq_id]);
+ }
+ tails_verif[seq_id] = i;
+ }
+ }
+ for (uint32_t i = 0; i < cache.size; ++i) {
+ if (tails_verif[i] != cache.cells[i].tail) {
+ LLAMA_LOG_ERROR("%s: wrong tail for seq_id %d, (%d instead of %d)\n", __func__, i, cache.cells[i].tail, tails_verif[i]);
+ }
+ }
+ }
+#endif
+
+ // find next empty cell
+ uint32_t next_empty_cell = cache.head;
+
+ for (uint32_t i = 0; i < cache.size; ++i) {
+ if (next_empty_cell >= cache.size) { next_empty_cell -= cache.size; }
+ llama_kv_cell & cell = cache.cells[next_empty_cell];
+ if (cell.is_empty()) { break; }
+ next_empty_cell += 1;
+ }
+
+ // find usable cell range
+ for (uint32_t s = 0; s < n_seqs; ++s) {
+ const llama_seq_id seq_id = batch.seq_id[s][0];
+ llama_kv_cell & seq_meta = cache.cells[seq_id];
+ bool has_cell = false;
+ if (seq_meta.tail >= 0) {
+ llama_kv_cell & cell = cache.cells[seq_meta.tail];
+ GGML_ASSERT(cell.has_seq_id(seq_id));
+ // does this seq_id "own" the cell?
+ if (cell.seq_id.size() == 1) { has_cell = true; }
+ }
+ if (!has_cell) {
+ llama_kv_cell & empty_cell = cache.cells[next_empty_cell];
+ GGML_ASSERT(empty_cell.is_empty());
+ // copy old tail into the empty cell
+ if (seq_meta.tail >= 0) {
+ llama_kv_cell & orig_cell = cache.cells[seq_meta.tail];
+ empty_cell.pos = orig_cell.pos;
+ empty_cell.src = orig_cell.src;
+ orig_cell.seq_id.erase(seq_id);
+ empty_cell.seq_id.insert(seq_id); // will be overwritten
+ }
+ seq_meta.tail = next_empty_cell;
+ // find next empty cell
+ if (s + 1 < n_seqs) {
+ next_empty_cell += 1;
+ for (uint32_t i = 0; i < cache.size; ++i) {
+ if (next_empty_cell >= cache.size) { next_empty_cell -= cache.size; }
+ llama_kv_cell & cell = cache.cells[next_empty_cell];
+ if (cell.is_empty()) { break; }
+ next_empty_cell += 1;
+ }
+ }
+ }
+ if (min > seq_meta.tail) { min = seq_meta.tail; }
+ if (max < seq_meta.tail) { max = seq_meta.tail; }
+ }
+
+ // gather and re-order
+ for (uint32_t s = 0; s < n_seqs; ++s) {
+ int32_t dst_id = s + min;
+ int32_t src_id = cache.cells[batch.seq_id[s][0]].tail;
+ if (dst_id != src_id) {
+ llama_kv_cell & dst_cell = cache.cells[dst_id];
+ llama_kv_cell & src_cell = cache.cells[src_id];
+
+ std::swap(dst_cell.pos, src_cell.pos);
+ std::swap(dst_cell.src, src_cell.src);
+ std::swap(dst_cell.seq_id, src_cell.seq_id);
+
+ // swap tails (assuming they NEVER overlap)
+ for (const llama_seq_id seq_id : src_cell.seq_id) {
+ cache.cells[seq_id].tail = src_id;
+ }
+ for (const llama_seq_id seq_id : dst_cell.seq_id) {
+ cache.cells[seq_id].tail = dst_id;
+ }
+ }
+ }
+
+ // update the pos of the used seqs
+ for (uint32_t s = 0; s < n_seqs; ++s) {
+ const llama_pos last_pos = batch.pos[n_seq_tokens * s + n_seq_tokens - 1];
+ int32_t cell_id = s + min;
+ llama_kv_cell & cell = cache.cells[cell_id];
+
+ if (cell.pos >= 0 && last_pos != cell.pos + (llama_pos) n_seq_tokens) {
+ // What should happen when the pos backtracks or skips a value?
+ // Clearing the state mid-batch would require special-casing which isn't done.
+ LLAMA_LOG_WARN("%s: non-consecutive token position %d after %d for sequence %d with %u new tokens\n",
+ __func__, last_pos, cell.pos, batch.seq_id[s][0], n_seq_tokens);
+ }
+ cell.pos = last_pos;
+ cell.seq_id.clear();
+ for (int32_t j = 0; j < batch.n_seq_id[s]; ++j) {
+ const llama_seq_id seq_id = batch.seq_id[s][j];
+ cell.seq_id.insert(seq_id);
+ cache.cells[seq_id].tail = cell_id;
+ }
+ }
+
+ // allow getting the range of used cells, from head to head + n
+ cache.head = min;
+ cache.n = max - min + 1;
+
+ // sanity check
+ return cache.n >= n_seqs;
+ }
+ // otherwise, one cell per token.
+
+ if (n_tokens > cache.size) {
+ LLAMA_LOG_ERROR("%s: n_tokens=%d > cache.size=%d\n", __func__, n_tokens, cache.size);
+ return false;
+ }
+
+ uint32_t n_tested = 0;
+
+ while (true) {
+ if (cache.head + n_tokens > cache.size) {
+ n_tested += cache.size - cache.head;
+ cache.head = 0;
+ continue;
+ }
+
+ bool found = true;
+ for (uint32_t i = 0; i < n_tokens; i++) {
+ if (cache.cells[cache.head + i].pos >= 0) {
+ found = false;
+ cache.head += i + 1;
+ n_tested += i + 1;
+ break;
+ }
+ }
+
+ if (found) {
+ break;
+ }
+
+ if (n_tested >= cache.size) {
+ //LLAMA_LOG_ERROR("%s: failed to find a slot for %d tokens\n", __func__, n_tokens);
+ return false;
+ }
+ }
+
+ for (uint32_t s = 0; s < n_seqs; s++) {
+ for (uint32_t i = 0; i < n_seq_tokens; ++i) {
+ uint32_t k = s*n_seq_tokens + i;
+ cache.cells[cache.head + k].pos = batch.pos[k];
+
+ for (int32_t j = 0; j < batch.n_seq_id[s]; j++) {
+ cache.cells[cache.head + k].seq_id.insert(batch.seq_id[s][j]);
+ }
+ }
+ }
+
+ cache.used += n_tokens;
+
+ return true;
+}
+
+// find how many cells are currently in use
+static uint32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache) {
+ for (uint32_t i = cache.size; i > 0; --i) {
+ const llama_kv_cell & cell = cache.cells[i - 1];
+
+ if (cell.pos >= 0 && !cell.is_empty()) {
+ return i;
+ }
+ }
+
+ return 0;
+}
+
+static void llama_kv_cache_clear(struct llama_kv_cache & cache) {
+ for (int32_t i = 0; i < (int32_t) cache.size; ++i) {
+ cache.cells[i].pos = -1;
+ cache.cells[i].seq_id.clear();
+ cache.cells[i].src = -1;
+ cache.cells[i].tail = -1;
+ }
+ cache.head = 0;
+ cache.used = 0;
+
+ for (auto & buf : cache.bufs) {
+ ggml_backend_buffer_clear(buf, 0);
+ }
+}
+
+static bool llama_kv_cache_seq_rm(
+ struct llama_kv_cache & cache,
+ llama_seq_id seq_id,
+ llama_pos p0,
+ llama_pos p1) {
+ uint32_t new_head = cache.size;
+
+ if (p0 < 0) p0 = 0;
+ if (p1 < 0) p1 = std::numeric_limits<llama_pos>::max();
+
+ // models like Mamba or RWKV can't have a state partially erased
+ if (cache.recurrent) {
+ if (seq_id >= (int64_t) cache.size) {
+ // could be fatal
+ return false;
+ }
+ if (0 <= seq_id) {
+ int32_t & tail_id = cache.cells[seq_id].tail;
+ if (tail_id >= 0) {
+ const llama_kv_cell & cell = cache.cells[tail_id];
+ // partial intersection is invalid
+ if ((0 < p0 && p0 <= cell.pos) || (0 < p1 && p1 <= cell.pos)) {
+ return false;
+ }
+ // invalidate tails which will be cleared
+ if (p0 <= cell.pos && cell.pos < p1) {
+ tail_id = -1;
+ }
+ }
+ } else {
+ // seq_id is negative, then the range should include everything or nothing
+ if (p0 != p1 && (p0 != 0 || p1 != std::numeric_limits<llama_pos>::max())) {
+ return false;
+ }
+ }
+ }
+
+ for (uint32_t i = 0; i < cache.size; ++i) {
+ if (cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) {
+ if (seq_id < 0) {
+ cache.cells[i].seq_id.clear();
+ } else if (cache.cells[i].has_seq_id(seq_id)) {
+ cache.cells[i].seq_id.erase(seq_id);
+ } else {
+ continue;
+ }
+ if (cache.cells[i].is_empty()) {
+ // keep count of the number of used cells
+ if (cache.cells[i].pos >= 0) cache.used--;
+
+ cache.cells[i].pos = -1;
+ cache.cells[i].src = -1;
+ if (new_head == cache.size) new_head = i;
+ }
+ }
+ }
+
+ // If we freed up a slot, set head to it so searching can start there.
+ if (new_head != cache.size && new_head < cache.head) cache.head = new_head;
+
+ return true;
+}
+
+static void llama_kv_cache_seq_cp(
+ struct llama_kv_cache & cache,
+ llama_seq_id seq_id_src,
+ llama_seq_id seq_id_dst,
+ llama_pos p0,
+ llama_pos p1) {
+ if (p0 < 0) p0 = 0;
+ if (p1 < 0) p1 = std::numeric_limits<llama_pos>::max();
+
+ if (cache.recurrent) {
+ if ((uint32_t) seq_id_dst < cache.size && (uint32_t) seq_id_src < cache.size) {
+ llama_kv_cell & tail_src = cache.cells[seq_id_src];
+ llama_kv_cell & tail_dst = cache.cells[seq_id_dst];
+ if (tail_dst.tail >= 0) {
+ // clear destination seq_id if it wasn't empty
+ llama_kv_cell & cell_dst = cache.cells[tail_dst.tail];
+
+ cell_dst.seq_id.erase(seq_id_dst);
+ tail_dst.tail = -1;
+ if (cell_dst.seq_id.empty()) {
+ cell_dst.pos = -1;
+ cell_dst.delta = -1;
+ cell_dst.src = -1;
+ cache.used -= 1;
+ }
+ }
+ if (tail_src.tail >= 0) {
+ llama_kv_cell & cell_src = cache.cells[tail_src.tail];
+
+ cell_src.seq_id.insert(seq_id_dst);
+ tail_dst.tail = tail_src.tail;
+ }
+ }
+
+ return;
+ }
+ // otherwise, this is the KV cache of a Transformer-like model
+
+ cache.head = 0;
+
+ for (uint32_t i = 0; i < cache.size; ++i) {
+ if (cache.cells[i].has_seq_id(seq_id_src) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) {
+ cache.cells[i].seq_id.insert(seq_id_dst);
+ }
+ }
+}
+
+static void llama_kv_cache_seq_keep(struct llama_kv_cache & cache, llama_seq_id seq_id) {
+ uint32_t new_head = cache.size;
+
+ for (uint32_t i = 0; i < cache.size; ++i) {
+ if (cache.recurrent && (llama_seq_id) i != seq_id) {
+ cache.cells[i].tail = -1;
+ }
+ if (!cache.cells[i].has_seq_id(seq_id)) {
+ if (cache.cells[i].pos >= 0) cache.used--;
+ cache.cells[i].pos = -1;
+ cache.cells[i].src = -1;
+ cache.cells[i].seq_id.clear();
+ if (new_head == cache.size) new_head = i;
+ } else {
+ cache.cells[i].seq_id.clear();
+ cache.cells[i].seq_id.insert(seq_id);
+ }
+ }
+
+ // If we freed up a slot, set head to it so searching can start there.
+ if (new_head != cache.size && new_head < cache.head) cache.head = new_head;
+}
+
+static void llama_kv_cache_seq_add(
+ struct llama_kv_cache & cache,
+ llama_seq_id seq_id,
+ llama_pos p0,
+ llama_pos p1,
+ llama_pos delta) {
+ uint32_t new_head = cache.size;
+
+ if (p0 < 0) p0 = 0;
+ if (p1 < 0) p1 = std::numeric_limits<llama_pos>::max();
+ // If there is no range then return early to avoid looping over the cache.
+ if (p0 == p1) return;
+
+ if (cache.recurrent) {
+ // for Mamba-like or RWKV models, only the pos needs to be shifted
+ if (0 <= seq_id && seq_id < (int64_t) cache.size) {
+ const int32_t tail_id = cache.cells[seq_id].tail;
+ if (tail_id >= 0) {
+ llama_kv_cell & cell = cache.cells[tail_id];
+ if (cell.has_seq_id(seq_id) && p0 <= cell.pos && cell.pos < p1) {
+ cell.pos += delta;
+ }
+ }
+ }
+ return;
+ }
+
+ for (uint32_t i = 0; i < cache.size; ++i) {
+ if (cache.cells[i].has_seq_id(seq_id) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) {
+ cache.has_shift = true;
+ cache.cells[i].pos += delta;
+ cache.cells[i].delta += delta;
+
+ if (cache.cells[i].pos < 0) {
+ if (!cache.cells[i].is_empty()) {
+ cache.used--;
+ }
+ cache.cells[i].pos = -1;
+ cache.cells[i].seq_id.clear();
+ if (new_head == cache.size) {
+ new_head = i;
+ }
+ }
+ }
+ }
+
+ // If we freed up a slot, set head to it so searching can start there.
+ // Otherwise we just start the next search from the beginning.
+ cache.head = new_head != cache.size ? new_head : 0;
+}
+
+static void llama_kv_cache_seq_div(
+ struct llama_kv_cache & cache,
+ llama_seq_id seq_id,
+ llama_pos p0,
+ llama_pos p1,
+ int d) {
+ if (p0 < 0) p0 = 0;
+ if (p1 < 0) p1 = std::numeric_limits<llama_pos>::max();
+ // If there is no range then return early to avoid looping over the cache.
+ if (p0 == p1) return;
+
+ if (cache.recurrent) {
+ // for Mamba-like or RWKV models, only the pos needs to be changed
+ if (0 <= seq_id && seq_id < (int64_t) cache.size) {
+ const int32_t tail_id = cache.cells[seq_id].tail;
+ if (tail_id >= 0) {
+ llama_kv_cell & cell = cache.cells[tail_id];
+ if (cell.has_seq_id(seq_id) && p0 <= cell.pos && cell.pos < p1) {
+ cell.pos /= d;
+ }
+ }
+ }
+ return;
+ }
+
+ for (uint32_t i = 0; i < cache.size; ++i) {
+ if (cache.cells[i].has_seq_id(seq_id) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) {
+ cache.has_shift = true;
+
+ {
+ llama_pos p_old = cache.cells[i].pos;
+ cache.cells[i].pos /= d;
+ cache.cells[i].delta += cache.cells[i].pos - p_old;
+ }
+ }
+ }
+}
+
+static llama_pos llama_kv_cache_seq_pos_max(struct llama_kv_cache & cache, llama_seq_id seq_id) {
+ llama_pos result = 0;
+
+ for (uint32_t i = 0; i < cache.size; ++i) {
+ if (cache.cells[i].has_seq_id(seq_id)) {
+ result = std::max(result, cache.cells[i].pos);
+ }
+ }
+
+ return result;
+}
+
+static void llama_kv_cache_defrag(struct llama_kv_cache & cache) {
+ if (!cache.recurrent) {
+ cache.do_defrag = true;
+ }
+}
+
+static uint32_t llama_kv_cache_get_padding(const struct llama_cparams & cparams) {
+ // the FA kernels require padding to avoid extra runtime boundary checks
+ return cparams.flash_attn ? 256u : 32u;
+}
+
+//
+// model loading and saving
+//
+
+enum llama_fver {
+ GGUF_FILE_VERSION_V1 = 1,
+ GGUF_FILE_VERSION_V2 = 2,
+ GGUF_FILE_VERSION_V3 = 3,
+};
+
+static const char * llama_file_version_name(llama_fver version) {
+ switch (version) {
+ case GGUF_FILE_VERSION_V1: return "GGUF V1 (support until nov 2023)";
+ case GGUF_FILE_VERSION_V2: return "GGUF V2";
+ case GGUF_FILE_VERSION_V3: return "GGUF V3 (latest)";
+ }
+
+ return "unknown";
+}
+
+static std::string llama_format_tensor_shape(const std::vector<int64_t> & ne) {
+ char buf[256];
+ snprintf(buf, sizeof(buf), "%5" PRId64, ne.at(0));
+ for (size_t i = 1; i < ne.size(); i++) {
+ snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), ", %5" PRId64, ne.at(i));
+ }
+ return buf;
+}
+
+static std::string llama_format_tensor_shape(const struct ggml_tensor * t) {
+ char buf[256];
+ snprintf(buf, sizeof(buf), "%5" PRId64, t->ne[0]);
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
+ snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), ", %5" PRId64, t->ne[i]);
+ }
+ return buf;
+}
+
+namespace GGUFMeta {
+ template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int)>
+ struct GKV_Base_Type {
+ static constexpr gguf_type gt = gt_;
+
+ static T getter(const gguf_context * ctx, const int kid) {
+ return gfun(ctx, kid);
+ }
+ };
+
+ template<typename T> struct GKV_Base;
+
+ template<> struct GKV_Base<bool >: GKV_Base_Type<bool, GGUF_TYPE_BOOL, gguf_get_val_bool> {};
+ template<> struct GKV_Base<uint8_t >: GKV_Base_Type<uint8_t, GGUF_TYPE_UINT8, gguf_get_val_u8 > {};
+ template<> struct GKV_Base<uint16_t >: GKV_Base_Type<uint16_t, GGUF_TYPE_UINT16, gguf_get_val_u16 > {};
+ template<> struct GKV_Base<uint32_t >: GKV_Base_Type<uint32_t, GGUF_TYPE_UINT32, gguf_get_val_u32 > {};
+ template<> struct GKV_Base<uint64_t >: GKV_Base_Type<uint64_t, GGUF_TYPE_UINT64, gguf_get_val_u64 > {};
+ template<> struct GKV_Base<int8_t >: GKV_Base_Type<int8_t, GGUF_TYPE_INT8, gguf_get_val_i8 > {};
+ template<> struct GKV_Base<int16_t >: GKV_Base_Type<int16_t, GGUF_TYPE_INT16, gguf_get_val_i16 > {};
+ template<> struct GKV_Base<int32_t >: GKV_Base_Type<int32_t, GGUF_TYPE_INT32, gguf_get_val_i32 > {};
+ template<> struct GKV_Base<int64_t >: GKV_Base_Type<int64_t, GGUF_TYPE_INT64, gguf_get_val_i64 > {};
+ template<> struct GKV_Base<float >: GKV_Base_Type<float, GGUF_TYPE_FLOAT32, gguf_get_val_f32 > {};
+ template<> struct GKV_Base<double >: GKV_Base_Type<double, GGUF_TYPE_FLOAT64, gguf_get_val_f64 > {};
+ template<> struct GKV_Base<const char *>: GKV_Base_Type<const char *, GGUF_TYPE_STRING, gguf_get_val_str > {};
+
+ template<> struct GKV_Base<std::string> {
+ static constexpr gguf_type gt = GGUF_TYPE_STRING;
+
+ static std::string getter(const gguf_context * ctx, const int kid) {
+ return gguf_get_val_str(ctx, kid);
+ }
+ };
+
+ struct ArrayInfo {
+ const gguf_type gt;
+ const size_t length;
+ const void * data;
+ };
+
+ template<> struct GKV_Base<ArrayInfo> {
+ public:
+ static constexpr gguf_type gt = GGUF_TYPE_ARRAY;
+ static ArrayInfo getter(const gguf_context *ctx, const int k) {
+ return ArrayInfo {
+ gguf_get_arr_type(ctx, k),
+ size_t(gguf_get_arr_n(ctx, k)),
+ gguf_get_arr_data(ctx, k),
+ };
+ }
+ };
+
+ template<typename T>
+ class GKV : public GKV_Base<T> {
+ GKV() = delete;
+
+ public:
+ static T get_kv(const gguf_context * ctx, const int k) {
+ const enum gguf_type kt = gguf_get_kv_type(ctx, k);
+
+ if (kt != GKV::gt) {
+ throw std::runtime_error(format("key %s has wrong type %s but expected type %s",
+ gguf_get_key(ctx, k), gguf_type_name(kt), gguf_type_name(GKV::gt)));
+ }
+ return GKV::getter(ctx, k);
+ }
+
+ static const char * override_type_to_str(const llama_model_kv_override_type ty) {
+ switch (ty) {
+ case LLAMA_KV_OVERRIDE_TYPE_BOOL: return "bool";
+ case LLAMA_KV_OVERRIDE_TYPE_INT: return "int";
+ case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return "float";
+ case LLAMA_KV_OVERRIDE_TYPE_STR: return "str";
+ }
+ return "unknown";
+ }
+
+ static bool validate_override(const llama_model_kv_override_type expected_type, const struct llama_model_kv_override * ovrd) {
+ if (!ovrd) { return false; }
+ if (ovrd->tag == expected_type) {
+ LLAMA_LOG_INFO("%s: Using metadata override (%5s) '%s' = ",
+ __func__, override_type_to_str(ovrd->tag), ovrd->key);
+ switch (ovrd->tag) {
+ case LLAMA_KV_OVERRIDE_TYPE_BOOL: {
+ LLAMA_LOG_INFO("%s\n", ovrd->val_bool ? "true" : "false");
+ } break;
+ case LLAMA_KV_OVERRIDE_TYPE_INT: {
+ LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->val_i64);
+ } break;
+ case LLAMA_KV_OVERRIDE_TYPE_FLOAT: {
+ LLAMA_LOG_INFO("%.6f\n", ovrd->val_f64);
+ } break;
+ case LLAMA_KV_OVERRIDE_TYPE_STR: {
+ LLAMA_LOG_INFO("%s\n", ovrd->val_str);
+ } break;
+ default:
+ // Shouldn't be possible to end up here, but just in case...
+ throw std::runtime_error(
+ format("Unsupported attempt to override %s type for metadata key %s\n",
+ override_type_to_str(ovrd->tag), ovrd->key));
+ }
+ return true;
+ }
+ LLAMA_LOG_WARN("%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\n",
+ __func__, ovrd->key, override_type_to_str(expected_type), override_type_to_str(ovrd->tag));
+ return false;
+ }
+
+ template<typename OT>
+ static typename std::enable_if<std::is_same<OT, bool>::value, bool>::type
+ try_override(OT & target, const struct llama_model_kv_override * ovrd) {
+ if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, ovrd)) {
+ target = ovrd->val_bool;
+ return true;
+ }
+ return false;
+ }
+
+ template<typename OT>
+ static typename std::enable_if<!std::is_same<OT, bool>::value && std::is_integral<OT>::value, bool>::type
+ try_override(OT & target, const struct llama_model_kv_override * ovrd) {
+ if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, ovrd)) {
+ target = ovrd->val_i64;
+ return true;
+ }
+ return false;
+ }
+
+ template<typename OT>
+ static typename std::enable_if<std::is_floating_point<OT>::value, bool>::type
+ try_override(T & target, const struct llama_model_kv_override * ovrd) {
+ if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, ovrd)) {
+ target = ovrd->val_f64;
+ return true;
+ }
+ return false;
+ }
+
+ template<typename OT>
+ static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type
+ try_override(T & target, const struct llama_model_kv_override * ovrd) {
+ if (validate_override(LLAMA_KV_OVERRIDE_TYPE_STR, ovrd)) {
+ target = ovrd->val_str;
+ return true;
+ }
+ return false;
+ }
+
+ static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override * ovrd = nullptr) {
+ if (try_override<T>(target, ovrd)) {
+ return true;
+ }
+ if (k < 0) { return false; }
+ target = get_kv(ctx, k);
+ return true;
+ }
+
+ static bool set(const gguf_context * ctx, const char * key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {
+ return set(ctx, gguf_find_key(ctx, key), target, ovrd);
+ }
+
+ static bool set(const gguf_context * ctx, const std::string & key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {
+ return set(ctx, key.c_str(), target, ovrd);
+ }
+ };
+}
+
+using llama_buf_map = std::unordered_map<uint32_t, ggml_backend_buffer_t>;
+
+static size_t llama_model_max_nodes(const llama_model & model) {
+ return std::max<size_t>(8192, model.tensors_by_name.size()*5);
+}
+
+struct llama_model_loader {
+ int n_kv = 0;
+ int n_tensors = 0;
+ int n_created = 0;
+
+ int64_t n_elements = 0;
+ size_t n_bytes = 0;
+
+ bool use_mmap = false;
+ bool check_tensors;
+
+ llama_files files;
+ llama_ftype ftype;
+ llama_fver fver;
+
+ llama_mmaps mappings;
+
+ // Holds information on a model weight
+ struct llama_tensor_weight {
+ uint16_t idx; // source file index
+ size_t offs; // tensor data offset in the original file
+
+ ggml_tensor * tensor;
+
+ llama_tensor_weight(const llama_file * file, uint16_t idx, const char * name, const struct gguf_context * gguf_ctx, ggml_tensor * tensor) : idx(idx), tensor(tensor) {
+ const int tensor_idx = gguf_find_tensor(gguf_ctx, name);
+ offs = gguf_get_data_offset(gguf_ctx) + gguf_get_tensor_offset(gguf_ctx, tensor_idx);
+
+ if (offs + ggml_nbytes(tensor) < offs || offs + ggml_nbytes(tensor) > file->size) {
+ throw std::runtime_error(format("tensor '%s' data is not within the file bounds, model is corrupted or incomplete", name));
+ }
+ }
+ };
+ std::vector<llama_tensor_weight> weights;
+
+ std::unordered_map<std::string, struct llama_model_kv_override> kv_overrides;
+
+ struct gguf_context * meta = NULL;
+ std::vector<ggml_context *> contexts;
+
+ std::string arch_name;
+ LLM_KV llm_kv = LLM_KV(LLM_ARCH_UNKNOWN);
+
+ llama_model_loader(const std::string & fname, bool use_mmap, bool check_tensors, const struct llama_model_kv_override * param_overrides_p) {
+ int trace = 0;
+ if (getenv("LLAMA_TRACE")) {
+ trace = atoi(getenv("LLAMA_TRACE"));
+ }
+
+ if (param_overrides_p != nullptr) {
+ for (const struct llama_model_kv_override * p = param_overrides_p; p->key[0] != 0; p++) {
+ kv_overrides.insert({std::string(p->key), *p});
+ }
+ }
+
+ struct ggml_context * ctx = NULL;
+ struct gguf_init_params params = {
+ /*.no_alloc = */ true,
+ /*.ctx = */ &ctx,
+ };
+
+ meta = gguf_init_from_file(fname.c_str(), params);
+ if (!meta) {
+ throw std::runtime_error(format("%s: failed to load model from %s\n", __func__, fname.c_str()));
+ }
+
+ get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);
+ llm_kv = LLM_KV(llm_arch_from_string(arch_name));
+
+ files.emplace_back(new llama_file(fname.c_str(), "rb"));
+ contexts.emplace_back(ctx);
+
+ // Save tensors data offset of the main file.
+ // For subsidiary files, `meta` tensor data offset must not be used,
+ // so we build a unified tensors index for weights.
+ for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
+ weights.emplace_back(files.back().get(), 0, cur->name, meta, cur);
+ }
+ uint16_t n_split = 0;
+ get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false);
+
+ // Load additional GGML contexts
+ if (n_split > 1) {
+ uint16_t idx = 0;
+ get_key(llm_kv(LLM_KV_SPLIT_NO), idx);
+ if (idx != 0) {
+ throw std::runtime_error(format("illegal split file: %d, model must be loaded with the first split", idx));
+ }
+
+ char split_prefix[PATH_MAX] = {0};
+ if (!llama_split_prefix(split_prefix, sizeof(split_prefix), fname.c_str(), idx, n_split)) {
+ throw std::runtime_error(format("invalid split file: %s", fname.c_str()));
+ }
+
+ if (trace > 0) {
+ LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split);
+ }
+
+ char split_path[PATH_MAX] = {0};
+ for (idx = 1; idx < n_split; idx++) {
+ llama_split_path(split_path, sizeof(split_path), split_prefix, idx, n_split);
+
+ struct gguf_init_params split_params = {
+ /*.no_alloc = */ true,
+ /*.ctx = */ &ctx,
+ };
+ struct gguf_context * ctx_gguf = gguf_init_from_file(split_path, split_params);
+ if (!ctx_gguf) {
+ throw std::runtime_error(format("%s: failed to load GGUF split from %s\n", __func__, split_path));
+ }
+
+ files.emplace_back(new llama_file(split_path, "rb"));
+ contexts.emplace_back(ctx);
+
+ // Save tensors data offset info of the shard.
+ for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
+ weights.emplace_back(files.back().get(), idx, cur->name, ctx_gguf, cur);
+ }
+
+ gguf_free(ctx_gguf);
+ }
+
+ get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors);
+
+ // sanity check
+ {
+ const int n_tensors_loaded = (int) weights.size();
+ if (n_tensors != n_tensors_loaded) {
+ throw std::runtime_error(format("corrupted model: %d tensors expected but %d found", n_tensors, n_tensors_loaded));
+ }
+ }
+
+ LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1);
+ }
+
+ n_kv = gguf_get_n_kv(meta);
+ n_tensors = weights.size();
+
+ fver = (enum llama_fver) gguf_get_version(meta);
+
+ std::set<std::string> tensor_names;
+ for (auto & w : weights) {
+ n_elements += ggml_nelements(w.tensor);
+ n_bytes += ggml_nbytes(w.tensor);
+ // make sure there is no duplicated tensor names
+ const std::string name(w.tensor->name);
+ auto found = tensor_names.find(name);
+ if (found != tensor_names.end()) {
+ throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", w.tensor->name));
+ }
+ tensor_names.insert(name);
+ }
+
+ LLAMA_LOG_INFO("%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n",
+ __func__, n_kv, n_tensors, fname.c_str(), llama_file_version_name(fver));
+
+ // determine file type based on the number of tensors for each quantization and print meta data
+ // TODO: make optional
+ {
+ std::map<enum ggml_type, uint32_t> n_type;
+
+ uint32_t n_type_max = 0;
+ enum ggml_type type_max = GGML_TYPE_F32;
+
+ for (int i = 0; i < n_tensors; i++) {
+ const ggml_tensor * tensor = weights.at(i).tensor;
+ enum ggml_type type = tensor->type;
+
+ n_type[type]++;
+
+ if (n_type_max < n_type[type]) {
+ n_type_max = n_type[type];
+ type_max = type;
+ }
+
+ if (trace > 0) {
+ const uint16_t sid = weights.at(i).idx;
+ LLAMA_LOG_INFO("%s: - tensor %4d, split %2d: %32s %-8s [ %s ]\n", __func__, i, sid, ggml_get_name(tensor), ggml_type_name(type), llama_format_tensor_shape(tensor).c_str());
+ }
+ }
+
+ switch (type_max) {
+ case GGML_TYPE_F32: ftype = LLAMA_FTYPE_ALL_F32; break;
+ case GGML_TYPE_F16: ftype = LLAMA_FTYPE_MOSTLY_F16; break;
+ case GGML_TYPE_BF16: ftype = LLAMA_FTYPE_MOSTLY_BF16; break;
+ case GGML_TYPE_Q4_0: ftype = LLAMA_FTYPE_MOSTLY_Q4_0; break;
+ case GGML_TYPE_Q4_1: ftype = LLAMA_FTYPE_MOSTLY_Q4_1; break;
+ case GGML_TYPE_Q5_0: ftype = LLAMA_FTYPE_MOSTLY_Q5_0; break;
+ case GGML_TYPE_Q5_1: ftype = LLAMA_FTYPE_MOSTLY_Q5_1; break;
+ case GGML_TYPE_Q8_0: ftype = LLAMA_FTYPE_MOSTLY_Q8_0; break;
+ case GGML_TYPE_Q2_K: ftype = LLAMA_FTYPE_MOSTLY_Q2_K; break;
+ case GGML_TYPE_Q3_K: ftype = LLAMA_FTYPE_MOSTLY_Q3_K_M; break;
+ case GGML_TYPE_Q4_K: ftype = LLAMA_FTYPE_MOSTLY_Q4_K_M; break;
+ case GGML_TYPE_Q5_K: ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M; break;
+ case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break;
+ case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break;
+ case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break;
+ case GGML_TYPE_IQ2_S: ftype = LLAMA_FTYPE_MOSTLY_IQ2_S; break;
+ case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break;
+ case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break;
+ case GGML_TYPE_IQ1_M: ftype = LLAMA_FTYPE_MOSTLY_IQ1_M; break;
+ case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break;
+ case GGML_TYPE_IQ4_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS; break;
+ case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break;
+ case GGML_TYPE_Q4_0_4_4: ftype = LLAMA_FTYPE_MOSTLY_Q4_0_4_4; break;
+ case GGML_TYPE_Q4_0_4_8: ftype = LLAMA_FTYPE_MOSTLY_Q4_0_4_8; break;
+ case GGML_TYPE_Q4_0_8_8: ftype = LLAMA_FTYPE_MOSTLY_Q4_0_8_8; break;
+ default:
+ {
+ LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));
+ ftype = LLAMA_FTYPE_ALL_F32;
+ } break;
+ }
+
+ // this is a way to mark that we have "guessed" the file type
+ ftype = (llama_ftype) (ftype | LLAMA_FTYPE_GUESSED);
+
+ {
+ const int kid = gguf_find_key(meta, "general.file_type"); // TODO: use LLM_KV
+ if (kid >= 0) {
+ ftype = (llama_ftype) gguf_get_val_u32(meta, kid);
+ }
+ }
+
+ LLAMA_LOG_INFO("%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", __func__);
+
+ for (int i = 0; i < n_kv; i++) {
+ const char * name = gguf_get_key(meta, i);
+ const enum gguf_type type = gguf_get_kv_type(meta, i);
+ const std::string type_name =
+ type == GGUF_TYPE_ARRAY
+ ? format("%s[%s,%d]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(meta, i)), gguf_get_arr_n(meta, i))
+ : gguf_type_name(type);
+
+ std::string value = gguf_kv_to_str(meta, i);
+ const size_t MAX_VALUE_LEN = 40;
+ if (value.size() > MAX_VALUE_LEN) {
+ value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str());
+ }
+ replace_all(value, "\n", "\\n");
+
+ LLAMA_LOG_INFO("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), value.c_str());
+ }
+
+ // print type counts
+ for (auto & kv : n_type) {
+ if (kv.second == 0) {
+ continue;
+ }
+
+ LLAMA_LOG_INFO("%s: - type %4s: %4d tensors\n", __func__, ggml_type_name(kv.first), kv.second);
+ }
+ }
+
+ if (!llama_mmap::SUPPORTED) {
+ LLAMA_LOG_WARN("%s: mmap is not supported on this platform\n", __func__);
+ use_mmap = false;
+ }
+
+ this->use_mmap = use_mmap;
+ this->check_tensors = check_tensors;
+ }
+
+ ~llama_model_loader() {
+ if (meta) {
+ gguf_free(meta);
+ }
+ for (auto * ctx : contexts) {
+ ggml_free(ctx);
+ }
+ }
+
+ template<typename T>
+ typename std::enable_if<std::is_integral<T>::value, bool>::type
+ get_arr_n(const std::string & key, T & result, const bool required = true) {
+ const int kid = gguf_find_key(meta, key.c_str());
+
+ if (kid < 0) {
+ if (required) {
+ throw std::runtime_error(format("key not found in model: %s", key.c_str()));
+ }
+ return false;
+ }
+
+ struct GGUFMeta::ArrayInfo arr_info =
+ GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta, kid);
+
+
+ result = arr_info.length;
+ return true;
+ }
+
+ template<typename T>
+ typename std::enable_if<std::is_integral<T>::value, bool>::type
+ get_arr_n(const enum llm_kv kid, T & result, const bool required = true) {
+ return get_arr_n(llm_kv(kid), result, required);
+ }
+
+ template<typename T>
+ bool get_arr(const std::string & key, std::vector<T> & result, const bool required = true) {
+ const int kid = gguf_find_key(meta, key.c_str());
+
+ if (kid < 0 || gguf_get_kv_type(meta, kid) != GGUF_TYPE_ARRAY) {
+ if (required) {
+ throw std::runtime_error(format("array key not found in model: %s", key.c_str()));
+ }
+ return false;
+ }
+
+ struct GGUFMeta::ArrayInfo arr_info =
+ GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta, kid);
+
+ switch (arr_info.gt) {
+ case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;
+ case GGUF_TYPE_INT32: GGML_ASSERT(
+ (std::is_same<T, int32_t>::value) ||
+ (std::is_same<T, uint32_t>::value)); break;
+ default:
+ throw std::runtime_error(format("%s is not a float32, int32 array", key.c_str()));
+ }
+
+ result.resize(arr_info.length);
+ result.assign((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length);
+
+ return true;
+ }
+
+ template<typename T, size_t N_MAX>
+ bool get_arr(const std::string & key, std::array<T, N_MAX> & result, const bool required = true) {
+ const int kid = gguf_find_key(meta, key.c_str());
+
+ if (kid < 0 || gguf_get_kv_type(meta, kid) != GGUF_TYPE_ARRAY) {
+ if (required) {
+ throw std::runtime_error(format("array key not found in model: %s", key.c_str()));
+ }
+ return false;
+ }
+
+ struct GGUFMeta::ArrayInfo arr_info =
+ GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta, kid);
+
+ switch (arr_info.gt) {
+ case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;
+ case GGUF_TYPE_INT32: GGML_ASSERT(
+ (std::is_same<T, int32_t>::value) ||
+ (std::is_same<T, uint32_t>::value)); break;
+ default:
+ throw std::runtime_error(format("%s is not a float32, int32 array", key.c_str()));
+ }
+
+ if (arr_info.length > N_MAX) {
+ throw std::runtime_error(format("array length %u for key %s exceeds max %u", (uint32_t) arr_info.length, key.c_str(), (uint32_t) N_MAX));
+ }
+
+ std::copy((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length, result.begin());
+
+ return true;
+ }
+
+ template<typename T>
+ bool get_arr(const enum llm_kv kid, T & result, const bool required = true) {
+ return get_arr(llm_kv(kid), result, required);
+ }
+
+ template<typename T>
+ bool get_key(const std::string & key, T & result, const bool required = true) {
+ auto it = kv_overrides.find(key);
+
+ const struct llama_model_kv_override * override =
+ it != kv_overrides.end() ? &it->second : nullptr;
+
+ const bool found = GGUFMeta::GKV<T>::set(meta, key, result, override);
+
+ if (required && !found) {
+ throw std::runtime_error(format("key not found in model: %s", key.c_str()));
+ }
+
+ return found;
+ }
+
+ template<typename T>
+ bool get_key(const enum llm_kv kid, T & result, const bool required = true) {
+ return get_key(llm_kv(kid), result, required);
+ }
+
+ // get array of n <= N_MAX elements, or a single element repeated n times
+ template<typename T, size_t N_MAX>
+ bool get_key_or_arr(const std::string & key, std::array<T, N_MAX> & result, uint32_t n, const bool required = true) {
+ const int kid = gguf_find_key(meta, key.c_str());
+
+ if (kid < 0) {
+ if (required) {
+ throw std::runtime_error(format("key not found in model: %s", key.c_str()));
+ }
+ return false;
+ }
+
+ if (n > N_MAX) {
+ throw std::runtime_error(format("n > N_MAX: %u > %u for key %s", (uint32_t) n, (uint32_t) N_MAX, key.c_str()));
+ }
+
+ if (gguf_get_kv_type(meta, kid) == GGUF_TYPE_ARRAY) {
+ struct GGUFMeta::ArrayInfo arr_info =
+ GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta, kid);
+
+ if (n != arr_info.length) {
+ throw std::runtime_error(format("key %s has wrong array length; expected %u, got %u", key.c_str(), n, (uint32_t) arr_info.length));
+ }
+
+ return get_arr(key, result, required);
+ } else {
+ T value;
+
+ bool ok = get_key(key, value, required);
+ if (!ok) {
+ return false;
+ }
+
+ for (uint32_t i = 0; i < n; i++) {
+ result[i] = value;
+ }
+
+ return true;
+ }
+ }
+
+ template<typename T>
+ bool get_key_or_arr(const enum llm_kv kid, T & result, uint32_t n, const bool required = true) {
+ return get_key_or_arr(llm_kv(kid), result, n, required);
+ }
+
+ std::string get_arch_name() const {
+ return arch_name;
+ }
+
+ enum llm_arch get_arch() const {
+ return llm_kv.arch;
+ }
+
+ const char * get_tensor_name(int i) const {
+ return weights.at(i).tensor->name;
+ }
+
+ const llama_tensor_weight * get_weight(const char * name) const {
+ for (const auto & weight : weights) {
+ if (strcmp(name, weight.tensor->name) == 0) {
+ return &weight;
+ }
+ }
+ return nullptr;
+ }
+
+ const llama_tensor_weight * get_weight(int i) const {
+ return get_weight(get_tensor_name(i));
+ }
+
+ const llama_tensor_weight & require_weight(const char * name) const {
+ const llama_tensor_weight * weight = get_weight(name);
+ if (!weight) {
+ throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name));
+ }
+ return *weight;
+ }
+
+ struct ggml_tensor * get_tensor_meta(const char * name) const {
+ const auto * weight = get_weight(name);
+ if (!weight) {
+ return nullptr;
+ }
+ return weight->tensor;
+ }
+
+ struct ggml_tensor * require_tensor_meta(const char * name) const {
+ struct ggml_tensor * tensor = get_tensor_meta(name);
+ if (!tensor) {
+ throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name));
+ }
+ return tensor;
+ }
+
+ struct ggml_tensor * get_tensor_meta(int i) const {
+ return get_tensor_meta(get_tensor_name(i));
+ }
+
+ struct ggml_tensor * create_tensor_for(struct ggml_context * ctx, const struct ggml_tensor * cur, bool duplicated) {
+ struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur);
+ ggml_set_name(tensor, ggml_get_name(cur));
+
+ if (duplicated) {
+ size_data += ggml_nbytes(cur);
+ } else {
+ n_created++;
+ }
+
+ return tensor;
+ }
+
+ const struct ggml_tensor * check_tensor_dims(const std::string & name, const std::vector<int64_t> & ne, bool required) const {
+ const struct ggml_tensor * cur = get_tensor_meta(name.c_str());
+
+ if (cur == NULL) {
+ if (!required) {
+ return NULL;
+ }
+ throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));
+ }
+
+ {
+ bool is_ok = true;
+ for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {
+ if ((i < ne.size() && ne[i] != cur->ne[i]) || (i >= ne.size() && cur->ne[i] != 1)) {
+ is_ok = false;
+ break;
+ }
+ }
+ if (!is_ok) {
+ throw std::runtime_error(
+ format("%s: tensor '%s' has wrong shape; expected %s, got %s",
+ __func__, name.c_str(),
+ llama_format_tensor_shape(ne).c_str(),
+ llama_format_tensor_shape(cur).c_str()));
+ }
+ }
+
+ return cur;
+ }
+
+ static const int TENSOR_NOT_REQUIRED = 1;
+ static const int TENSOR_DUPLICATED = 2;
+
+ struct ggml_tensor * create_tensor(struct ggml_context * ctx, const std::string & name, const std::vector<int64_t> & ne, int flags = 0) {
+ const struct ggml_tensor * cur = check_tensor_dims(name, ne, !(flags & TENSOR_NOT_REQUIRED));
+
+ if (cur == NULL) {
+ return NULL;
+ }
+
+ return create_tensor_for(ctx, cur, flags & TENSOR_DUPLICATED);
+ }
+
+ struct ggml_tensor * create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::vector<int64_t> & ne, size_t offset, bool required = true) {
+ const struct ggml_tensor * cur = check_tensor_dims(name, ne, required);
+
+ if (cur == NULL) {
+ return NULL;
+ }
+
+ if (cur->type != base->type) {
+ throw std::runtime_error(format("%s: tensor '%s' has wrong type; expected %s, got %s", __func__, name.c_str(), ggml_type_name(base->type), ggml_type_name(cur->type)));
+ }
+
+ std::array<int64_t, GGML_MAX_DIMS> dims;
+ for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {
+ dims[i] = i < ne.size() ? ne[i] : 1;
+ }
+
+ struct ggml_tensor * tensor = ggml_view_4d(ctx, base,
+ dims[0], dims[1], dims[2], dims[3],
+ cur->nb[1], cur->nb[2], cur->nb[3],
+ offset);
+
+ ggml_set_name(tensor, name.c_str());
+
+ n_created++;
+
+ return tensor;
+ }
+
+ void done_getting_tensors() const {
+ if (n_created != n_tensors) {
+ throw std::runtime_error(format("%s: wrong number of tensors; expected %d, got %d", __func__, n_tensors, n_created));
+ }
+ }
+
+ void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr) {
+ if (use_mmap) {
+ mappings.reserve(files.size());
+ mmaps_used.reserve(files.size());
+ for (const auto & file : files) {
+ std::unique_ptr<llama_mmap> mapping(new llama_mmap(file.get(), prefetch ? -1 : 0, ggml_is_numa()));
+ mmaps_used.emplace_back(mapping->size, 0);
+ if (mlock_mmaps) {
+ std::unique_ptr<llama_mlock> mlock_mmap(new llama_mlock());
+ mlock_mmap->init(mapping->addr);
+ mlock_mmaps->emplace_back(std::move(mlock_mmap));
+ }
+ mappings.emplace_back(std::move(mapping));
+ }
+ }
+
+ // compute the total size of all tensors for progress reporting
+ for (auto & w : weights) {
+ size_data += ggml_nbytes(w.tensor);
+ }
+ }
+
+ void get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const {
+ GGML_ASSERT(!mappings.empty());
+ const auto & mapping = mappings.at(idx);
+
+ *first = mapping->size;
+ *last = 0;
+ *addr = mapping->addr;
+ for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) {
+ try {
+ const auto * weight = get_weight(ggml_get_name(tensor));
+ if (!weight) {
+ continue;
+ }
+ if (weight->idx != idx) {
+ continue;
+ }
+ *first = std::min(*first, weight->offs);
+ *last = std::max(*last, weight->offs + ggml_nbytes(tensor));
+ } catch(...) {
+ // the tensor is not in the model
+ }
+ }
+ }
+
+ // for backwards compatibility, does not support ggml-backend
+ void load_data_for(struct ggml_tensor * cur) const {
+ const auto & w = require_weight(ggml_get_name(cur));
+
+ if (use_mmap) {
+ const auto & mapping = mappings.at(w.idx);
+ if (cur->data == nullptr) {
+ cur->data = (uint8_t *)mapping->addr + w.offs;
+ } else {
+ memcpy(cur->data, (uint8_t *)mapping->addr + w.offs, ggml_nbytes(cur));
+ }
+ } else {
+ GGML_ASSERT(cur->data != nullptr);
+ GGML_ASSERT(w.idx < files.size());
+ const auto & file = files.at(w.idx);
+ file->seek(w.offs, SEEK_SET);
+ file->read_raw(cur->data, ggml_nbytes(cur));
+ }
+
+ if (check_tensors && !ggml_validate_row_data(cur->type, cur->data, ggml_nbytes(cur))) {
+ throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
+ }
+ }
+
+ size_t size_done = 0;
+ size_t size_data = 0;
+ std::vector<std::pair<size_t, size_t>> mmaps_used;
+
+ // Returns false if cancelled by progress_callback
+ bool load_all_data(
+ struct ggml_context * ctx,
+ llama_buf_map & bufs_mmap,
+ llama_mlocks * lmlocks,
+ llama_progress_callback progress_callback,
+ void * progress_callback_user_data) {
+ GGML_ASSERT(size_data != 0 && "call init_mappings() first");
+
+ std::vector<no_init<uint8_t>> read_buf;
+ std::vector<std::future<std::pair<ggml_tensor *, bool>>> validation_result;
+
+#if defined(GGML_USE_CUDA)
+ // 4 staging buffers for async uploads, each sized 1MB seems to be a good default for single NVMe drives.
+ // NVMe raid configurations might require more / larger buffers.
+ constexpr size_t n_buffers = 4;
+ constexpr size_t buffer_size = 1 * 1024 * 1024; // 1MB
+
+ std::vector<ggml_backend_buffer_t> host_buffers;
+ std::vector<void*> host_ptrs;
+ std::vector<ggml_backend_event_t> events;
+ size_t buffer_idx = 0; // buffer to use for async loads
+
+ ggml_backend_t cuda_backend = nullptr;
+ if (!use_mmap && !check_tensors) {
+ // When not using mmaped io use async uploads from pinned memory to GPU memory.
+ // First determine if the CUDA backend is active, and if so, determine the device ID.
+ ggml_backend_buffer_t buf = bufs_mmap.count(0) ? bufs_mmap.at(0) : nullptr;
+ if (buf) {
+ ggml_backend_buffer_type_t buffer_type = ggml_backend_buffer_get_type(buf);
+ for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) {
+ auto * cuda_buffer_type = ggml_backend_cuda_buffer_type(i);
+ if (buffer_type == cuda_buffer_type) {
+ cuda_backend = ggml_backend_cuda_init(i);
+ break;
+ }
+ }
+ }
+
+ // If the cuda backend is active create pinned memory buffers and events for synchronisation.
+ if (cuda_backend) {
+ for (size_t idx = 0; idx < n_buffers; ++idx) {
+ host_buffers.emplace_back(ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), buffer_size));
+ host_ptrs.emplace_back(ggml_backend_buffer_get_base(host_buffers[idx]));
+ events.emplace_back(ggml_backend_event_new(cuda_backend));
+ }
+ }
+ }
+#endif
+
+ for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) {
+ const auto * weight = get_weight(ggml_get_name(cur));
+ if (weight == nullptr) {
+ // this can happen with split experts models
+ continue;
+ }
+
+ if (progress_callback) {
+ if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) {
+ return false;
+ }
+ }
+
+ size_t n_size = ggml_nbytes(cur);
+
+ if (use_mmap) {
+ const auto & mapping = mappings.at(weight->idx);
+ ggml_backend_buffer_t buf_mmap = nullptr;
+ if (bufs_mmap.count(weight->idx)) {
+ buf_mmap = bufs_mmap.at(weight->idx);
+ }
+ uint8_t * data = (uint8_t *) mapping->addr + weight->offs;
+
+ if (check_tensors) {
+ validation_result.emplace_back(std::async(std::launch::async, [cur, data, n_size] {
+ return std::make_pair(cur, ggml_validate_row_data(cur->type, data, n_size));
+ }));
+ }
+
+ GGML_ASSERT(buf_mmap || cur->data); // either we have a buffer to allocate the tensor in, or it is already allocated
+ if (buf_mmap && cur->data == nullptr) {
+ ggml_backend_tensor_alloc(buf_mmap, cur, data);
+ if (lmlocks) {
+ const auto & lmlock = lmlocks->at(weight->idx);
+ lmlock->grow_to(weight->offs + n_size);
+ }
+
+ auto & mmap_used = mmaps_used[weight->idx];
+ mmap_used.first = std::min(mmap_used.first, weight->offs);
+ mmap_used.second = std::max(mmap_used.second, weight->offs + n_size);
+ } else {
+ ggml_backend_tensor_set(cur, data, 0, n_size);
+ }
+ } else {
+ GGML_ASSERT(weight->idx < files.size());
+ const auto & file = files.at(weight->idx);
+ if (ggml_backend_buffer_is_host(cur->buffer)) {
+ file->seek(weight->offs, SEEK_SET);
+ file->read_raw(cur->data, n_size);
+ if (check_tensors) {
+ validation_result.emplace_back(std::async(std::launch::async, [cur, n_size] {
+ return std::make_pair(cur, ggml_validate_row_data(cur->type, cur->data, n_size));
+ }));
+ }
+ } else {
+#if defined(GGML_USE_CUDA)
+ // If cuda_backend is valid load the tensor in chunks to pinned memory and upload the buffers asynchronously to the GPU.
+ if (cuda_backend) {
+ file->seek(weight->offs, SEEK_SET);
+
+ size_t bytes_read = 0;
+
+ while (bytes_read < n_size) {
+ size_t read_iteration = std::min<size_t>(buffer_size, n_size - bytes_read);
+
+ ggml_backend_event_synchronize(events[buffer_idx]);
+ file->read_raw(host_ptrs[buffer_idx], read_iteration);
+ ggml_backend_tensor_set_async(cuda_backend, cur, host_ptrs[buffer_idx], bytes_read, read_iteration);
+ ggml_backend_event_record(events[buffer_idx]);
+
+ bytes_read += read_iteration;
+ ++buffer_idx;
+ buffer_idx %= n_buffers;
+ }
+ }
+ else
+#endif
+ {
+ read_buf.resize(n_size);
+ file->seek(weight->offs, SEEK_SET);
+ file->read_raw(read_buf.data(), n_size);
+ ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size);
+ if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) {
+ throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
+ }
+ }
+ }
+ }
+
+ size_done += n_size;
+ }
+
+#if defined(GGML_USE_CUDA)
+ // free temporary resources used for async cuda uploads
+ if (cuda_backend) {
+ for (size_t idx = 0; idx < n_buffers;++idx) {
+ ggml_backend_event_synchronize(events[idx]);
+ ggml_backend_event_free(events[idx]);
+ ggml_backend_buffer_free(host_buffers[idx]);
+ }
+ ggml_backend_free(cuda_backend);
+ }
+#endif
+
+ // check validation results
+ bool validation_failed = false;
+ for (auto & future : validation_result) {
+ auto result = future.get();
+ if (!result.second) {
+ LLAMA_LOG_ERROR("%s: tensor '%s' has invalid data\n", __func__, ggml_get_name(result.first));
+ validation_failed = true;
+ }
+ }
+ if (validation_failed) {
+ throw std::runtime_error("found tensors with invalid data");
+ }
+
+ // check if this is the last call and do final cleanup
+ if (size_done >= size_data) {
+ // unmap offloaded tensors and metadata
+ if (use_mmap) {
+ for (uint32_t idx = 0; idx < mappings.size(); idx++) {
+ const auto & mmap_used = mmaps_used.at(idx);
+ auto & mapping = mappings.at(idx);
+ mapping->unmap_fragment(0, mmap_used.first);
+ if (mmap_used.second != 0) {
+ mapping->unmap_fragment(mmap_used.second, mapping->size);
+ }
+ }
+ }
+ if (progress_callback) {
+ // Even though the model is done loading, we still honor
+ // cancellation since we need to free allocations.
+ return progress_callback(1.0f, progress_callback_user_data);
+ }
+ }
+
+ return true;
+ }
+};
+
+template<>
+bool llama_model_loader::get_key(const enum llm_kv kid, enum llama_pooling_type & result, const bool required) {
+ uint32_t tmp;
+ const bool found = get_key(kid, tmp, required);
+ if (found) {
+ result = (enum llama_pooling_type) tmp;
+ } else {
+ result = LLAMA_POOLING_TYPE_UNSPECIFIED;
+ }
+ return found;
+}
+
+
+//
+// load LLaMA models
+//
+
+static const char * llama_model_arch_name(llm_arch arch) {
+ auto it = LLM_ARCH_NAMES.find(arch);
+ if (it == LLM_ARCH_NAMES.end()) {
+ return "unknown";
+ }
+ return it->second;
+}
+
+static std::string llama_model_ftype_name(llama_ftype ftype) {
+ if (ftype & LLAMA_FTYPE_GUESSED) {
+ return llama_model_ftype_name((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) + " (guessed)";
+ }
+
+ switch (ftype) {
+ case LLAMA_FTYPE_ALL_F32: return "all F32";
+ case LLAMA_FTYPE_MOSTLY_F16: return "F16";
+ case LLAMA_FTYPE_MOSTLY_BF16: return "BF16";
+ case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";
+ case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";
+ case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0";
+ case LLAMA_FTYPE_MOSTLY_Q5_1: return "Q5_1";
+ case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0";
+ case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium";
+ case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small";
+ case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small";
+ case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium";
+ case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large";
+ case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "Q4_K - Small";
+ case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "Q4_K - Medium";
+ case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small";
+ case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium";
+ case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K";
+ case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return "IQ2_XXS - 2.0625 bpw";
+ case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw";
+ case LLAMA_FTYPE_MOSTLY_IQ2_S: return "IQ2_S - 2.5 bpw";
+ case LLAMA_FTYPE_MOSTLY_IQ2_M: return "IQ2_M - 2.7 bpw";
+ case LLAMA_FTYPE_MOSTLY_IQ3_XS: return "IQ3_XS - 3.3 bpw";
+ case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return "IQ3_XXS - 3.0625 bpw";
+ case LLAMA_FTYPE_MOSTLY_IQ1_S: return "IQ1_S - 1.5625 bpw";
+ case LLAMA_FTYPE_MOSTLY_IQ1_M: return "IQ1_M - 1.75 bpw";
+ case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw";
+ case LLAMA_FTYPE_MOSTLY_IQ4_XS: return "IQ4_XS - 4.25 bpw";
+ case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw";
+ case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw";
+ case LLAMA_FTYPE_MOSTLY_Q4_0_4_4: return "Q4_0_4_4";
+ case LLAMA_FTYPE_MOSTLY_Q4_0_4_8: return "Q4_0_4_8";
+ case LLAMA_FTYPE_MOSTLY_Q4_0_8_8: return "Q4_0_8_8";
+
+ default: return "unknown, may not work";
+ }
+}
+
+static const char * llama_model_type_name(e_model type) {
+ switch (type) {
+ case MODEL_14M: return "14M";
+ case MODEL_17M: return "17M";
+ case MODEL_22M: return "22M";
+ case MODEL_33M: return "33M";
+ case MODEL_60M: return "60M";
+ case MODEL_70M: return "70M";
+ case MODEL_80M: return "80M";
+ case MODEL_109M: return "109M";
+ case MODEL_137M: return "137M";
+ case MODEL_160M: return "160M";
+ case MODEL_220M: return "220M";
+ case MODEL_250M: return "250M";
+ case MODEL_270M: return "270M";
+ case MODEL_335M: return "335M";
+ case MODEL_410M: return "410M";
+ case MODEL_450M: return "450M";
+ case MODEL_770M: return "770M";
+ case MODEL_780M: return "780M";
+ case MODEL_0_5B: return "0.5B";
+ case MODEL_1B: return "1B";
+ case MODEL_1_3B: return "1.3B";
+ case MODEL_1_4B: return "1.4B";
+ case MODEL_1_6B: return "1.6B";
+ case MODEL_2B: return "2B";
+ case MODEL_2_8B: return "2.8B";
+ case MODEL_3B: return "3B";
+ case MODEL_4B: return "4B";
+ case MODEL_6B: return "6B";
+ case MODEL_6_9B: return "6.9B";
+ case MODEL_7B: return "7B";
+ case MODEL_8B: return "8B";
+ case MODEL_9B: return "9B";
+ case MODEL_11B: return "11B";
+ case MODEL_12B: return "12B";
+ case MODEL_13B: return "13B";
+ case MODEL_14B: return "14B";
+ case MODEL_15B: return "15B";
+ case MODEL_16B: return "16B";
+ case MODEL_20B: return "20B";
+ case MODEL_30B: return "30B";
+ case MODEL_34B: return "34B";
+ case MODEL_35B: return "35B";
+ case MODEL_40B: return "40B";
+ case MODEL_65B: return "65B";
+ case MODEL_70B: return "70B";
+ case MODEL_236B: return "236B";
+ case MODEL_314B: return "314B";
+ case MODEL_SMALL: return "0.1B";
+ case MODEL_MEDIUM: return "0.4B";
+ case MODEL_LARGE: return "0.8B";
+ case MODEL_XL: return "1.5B";
+ case MODEL_A2_7B: return "A2.7B";
+ case MODEL_8x7B: return "8x7B";
+ case MODEL_8x22B: return "8x22B";
+ case MODEL_16x12B: return "16x12B";
+ case MODEL_10B_128x3_66B: return "10B+128x3.66B";
+ case MODEL_57B_A14B: return "57B.A14B";
+ case MODEL_27B: return "27B";
+ default: return "?B";
+ }
+}
+
+static const char * llama_model_vocab_type_name(enum llama_vocab_type type){
+ switch (type) {
+ case LLAMA_VOCAB_TYPE_NONE: return "no vocab";
+ case LLAMA_VOCAB_TYPE_SPM: return "SPM";
+ case LLAMA_VOCAB_TYPE_BPE: return "BPE";
+ case LLAMA_VOCAB_TYPE_WPM: return "WPM";
+ case LLAMA_VOCAB_TYPE_UGM: return "UGM";
+ case LLAMA_VOCAB_TYPE_RWKV: return "RWKV";
+ default: return "unknown";
+ }
+}
+
+static void llm_load_arch(llama_model_loader & ml, llama_model & model) {
+ model.arch = ml.get_arch();
+ if (model.arch == LLM_ARCH_UNKNOWN) {
+ throw std::runtime_error("unknown model architecture: '" + ml.get_arch_name() + "'");
+ }
+}
+
+static void llm_load_hparams(
+ llama_model_loader & ml,
+ llama_model & model) {
+ auto & hparams = model.hparams;
+ const gguf_context * ctx = ml.meta;
+
+ // get metadata as string
+ for (int i = 0; i < gguf_get_n_kv(ctx); i++) {
+ enum gguf_type type = gguf_get_kv_type(ctx, i);
+ if (type == GGUF_TYPE_ARRAY) {
+ continue;
+ }
+ const char * name = gguf_get_key(ctx, i);
+ const std::string value = gguf_kv_to_str(ctx, i);
+ model.gguf_kv.emplace(name, value);
+ }
+
+ // get general kv
+ ml.get_key(LLM_KV_GENERAL_NAME, model.name, false);
+
+ // get hparams kv
+ ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab);
+
+ // everything past this point is not vocab-related
+ if (hparams.vocab_only) {
+ return;
+ }
+
+ ml.get_key(LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);
+ ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);
+ ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer);
+ ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);
+ ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);
+
+ GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);
+ GGML_ASSERT(hparams.n_expert_used <= hparams.n_expert);
+ if (hparams.n_expert > 0) {
+ GGML_ASSERT(hparams.n_expert_used > 0);
+ } else {
+ GGML_ASSERT(hparams.n_expert_used == 0);
+ }
+
+ // zero-out the per-layer hparams
+ std::fill(hparams.n_head_arr.begin(), hparams.n_head_arr.end(), 0);
+ std::fill(hparams.n_head_kv_arr.begin(), hparams.n_head_kv_arr.end(), 0);
+ std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0);
+
+ ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer);
+ ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer);
+
+ // n_head_kv is optional, default to n_head
+ hparams.n_head_kv_arr = hparams.n_head_arr;
+
+ ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, hparams.n_layer, false);
+
+ bool rope_finetuned = false;
+ ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);
+ hparams.rope_finetuned = rope_finetuned;
+
+ hparams.n_ctx_orig_yarn = hparams.n_ctx_train;
+ ml.get_key(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.n_ctx_orig_yarn, false);
+
+ // rope_freq_base (optional)
+ hparams.rope_freq_base_train = 10000.0f;
+ ml.get_key(LLM_KV_ROPE_FREQ_BASE, hparams.rope_freq_base_train, false);
+
+ std::string rope_scaling("linear");
+ ml.get_key(LLM_KV_ROPE_SCALING_TYPE, rope_scaling, false);
+ hparams.rope_scaling_type_train = llama_rope_scaling_type_from_string(rope_scaling);
+ GGML_ASSERT(hparams.rope_scaling_type_train != LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED);
+
+ // rope_freq_scale (inverse of the kv) is optional
+ float ropescale = 0.0f;
+ if (!ml.get_key(LLM_KV_ROPE_SCALING_FACTOR, ropescale, false)) {
+ // try the old key name
+ ml.get_key(LLM_KV_ROPE_SCALE_LINEAR, ropescale, false);
+ }
+ hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale;
+
+ ml.get_key(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor, false);
+
+ // non-transformer models do not have attention heads
+ if (hparams.n_head() > 0) {
+ // gpt-neox n_rot = rotary_pct * (n_embd / n_head)
+ // gpt-j n_rot = rotary_dim
+
+ hparams.n_embd_head_k = hparams.n_embd / hparams.n_head();
+ ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k, false);
+
+ hparams.n_embd_head_v = hparams.n_embd / hparams.n_head();
+ ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v, false);
+
+ // sanity check for n_rot (optional)
+ hparams.n_rot = hparams.n_embd_head_k;
+
+ ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot, false);
+
+ if (model.arch == LLM_ARCH_LLAMA || model.arch == LLM_ARCH_FALCON) {
+ if (hparams.n_rot != hparams.n_embd_head_k) {
+ throw std::runtime_error(format("invalid n_rot: %u, expected %u", hparams.n_rot, hparams.n_embd_head_k));
+ }
+ }
+ } else {
+ hparams.n_rot = 0;
+ hparams.n_embd_head_k = 0;
+ hparams.n_embd_head_v = 0;
+ }
+
+ // arch-specific KVs
+ switch (model.arch) {
+ case LLM_ARCH_LLAMA:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ if (hparams.n_expert == 8) {
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_8x7B; break;
+ case 56: model.type = e_model::MODEL_8x22B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } else {
+ switch (hparams.n_layer) {
+ case 22: model.type = e_model::MODEL_1B; break;
+ case 26: model.type = e_model::MODEL_3B; break;
+ // granite uses a vocab with len 49152
+ case 32: model.type = hparams.n_vocab == 49152 ? e_model::MODEL_3B : (hparams.n_vocab < 40000 ? e_model::MODEL_7B : e_model::MODEL_8B); break;
+ case 36: model.type = e_model::MODEL_8B; break; // granite
+ case 40: model.type = e_model::MODEL_13B; break;
+ case 48: model.type = e_model::MODEL_34B; break;
+ case 60: model.type = e_model::MODEL_30B; break;
+ case 80: model.type = hparams.n_head() == hparams.n_head_kv() ? e_model::MODEL_65B : e_model::MODEL_70B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ }
+ } break;
+ case LLM_ARCH_MINICPM:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ switch (hparams.n_layer) {
+ case 40: model.type = e_model::MODEL_2B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_GROK:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ switch (hparams.n_layer) {
+ case 64: model.type = e_model::MODEL_314B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_FALCON:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_7B; break;
+ case 60: model.type = e_model::MODEL_40B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_BAICHUAN:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_7B; break;
+ case 40: model.type = e_model::MODEL_13B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+
+ if (model.type == e_model::MODEL_13B) {
+ // TODO: become GGUF KV parameter
+ hparams.f_max_alibi_bias = 8.0f;
+ }
+ } break;
+ case LLM_ARCH_STARCODER:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ switch (hparams.n_layer) {
+ case 24: model.type = e_model::MODEL_1B; break;
+ case 36: model.type = e_model::MODEL_3B; break;
+ case 42: model.type = e_model::MODEL_7B; break;
+ case 40: model.type = e_model::MODEL_15B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_REFACT:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_1B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+
+ // TODO: become GGUF KV parameter
+ hparams.f_max_alibi_bias = 8.0f;
+ } break;
+ case LLM_ARCH_BERT:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);
+ ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type);
+ ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
+
+ switch (hparams.n_layer) {
+ case 3:
+ model.type = e_model::MODEL_17M; break; // bge-micro
+ case 6:
+ model.type = e_model::MODEL_22M; break; // MiniLM-L6
+ case 12:
+ switch (hparams.n_embd) {
+ case 384: model.type = e_model::MODEL_33M; break; // MiniLM-L12, bge-small
+ case 768: model.type = e_model::MODEL_109M; break; // bge-base
+ } break;
+ case 24:
+ model.type = e_model::MODEL_335M; break; // bge-large
+ }
+ } break;
+ case LLM_ARCH_JINA_BERT_V2:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);
+ ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type);
+ ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type);
+ hparams.f_max_alibi_bias = 8.0f;
+
+ switch (hparams.n_layer) {
+ case 4: model.type = e_model::MODEL_33M; break; // jina-embeddings-small
+ case 12: model.type = e_model::MODEL_137M; break; // jina-embeddings-base
+ }
+ } break;
+ case LLM_ARCH_NOMIC_BERT:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);
+ ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type);
+ ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type);
+
+ if (hparams.n_layer == 12 && hparams.n_embd == 768) {
+ model.type = e_model::MODEL_137M;
+ }
+ } break;
+ case LLM_ARCH_BLOOM:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+
+ switch (hparams.n_layer) {
+ case 24: model.type = e_model::MODEL_1B; break;
+ case 30:
+ switch (hparams.n_embd) {
+ case 2560: model.type = e_model::MODEL_3B; break;
+ case 4096: model.type = e_model::MODEL_7B; break;
+ } break;
+ }
+
+ // TODO: become GGUF KV parameter
+ hparams.f_max_alibi_bias = 8.0f;
+ } break;
+ case LLM_ARCH_MPT:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv, false);
+ ml.get_key(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, hparams.f_max_alibi_bias);
+
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_7B; break;
+ case 48: model.type = e_model::MODEL_30B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_STABLELM:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+
+ switch (hparams.n_layer) {
+ case 24: model.type = e_model::MODEL_1B; break;
+ case 32: model.type = e_model::MODEL_3B; break;
+ case 40: model.type = e_model::MODEL_12B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_QWEN:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_7B; break;
+ case 40: model.type = e_model::MODEL_13B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_QWEN2:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ switch (hparams.n_layer) {
+ case 24: model.type = hparams.n_embd == 1024 ? e_model::MODEL_0_5B : e_model::MODEL_1B; break;
+ case 32: model.type = e_model::MODEL_7B; break;
+ case 40: model.type = hparams.n_head() == 20 ? e_model::MODEL_4B : e_model::MODEL_13B; break;
+ case 80: model.type = e_model::MODEL_70B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_QWEN2MOE:
+ {
+ ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);
+ ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);
+
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ switch (hparams.n_layer) {
+ case 24: model.type = e_model::MODEL_A2_7B; break;
+ case 28: model.type = e_model::MODEL_57B_A14B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_PHI2:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+
+ switch (hparams.n_layer) {
+ case 24: model.type = e_model::MODEL_1B; break;
+ case 32: model.type = e_model::MODEL_3B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_PHI3:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ switch (hparams.n_layer) {
+ case 24: model.type = e_model::MODEL_1B; break;
+ case 32: model.type = e_model::MODEL_3B; break;
+ case 40: model.type = e_model::MODEL_14B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+
+ // for backward compatibility ; see: https://github.com/ggerganov/llama.cpp/pull/8931
+ if ((hparams.n_layer == 32 || hparams.n_layer == 40) && hparams.n_ctx_train == 4096) {
+ // default value for Phi-3-mini-4k-instruct and Phi-3-medium-4k-instruct
+ hparams.n_swa = 2047;
+ } else if (hparams.n_layer == 32 && hparams.n_head_kv(0) == 32 && hparams.n_ctx_train == 131072) {
+ // default value for Phi-3-mini-128k-instruct
+ hparams.n_swa = 262144;
+ } else if (hparams.n_layer == 40 && hparams.n_ctx_train == 131072) {
+ // default value for Phi-3-medium-128k-instruct
+ hparams.n_swa = 131072;
+ }
+ bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);
+ if (!found_swa && hparams.n_swa == 0) {
+ throw std::runtime_error("invalid value for sliding_window");
+ }
+ } break;
+ case LLM_ARCH_PLAMO:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ switch (hparams.n_layer) {
+ case 40: model.type = e_model::MODEL_13B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_GPT2:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ switch (hparams.n_layer) {
+ case 12: model.type = e_model::MODEL_SMALL; break;
+ case 24: model.type = e_model::MODEL_MEDIUM; break;
+ case 36: model.type = e_model::MODEL_LARGE; break;
+ case 48: model.type = e_model::MODEL_XL; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_CODESHELL:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ switch (hparams.n_layer) {
+ case 42: model.type = e_model::MODEL_7B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_ORION:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+
+ switch (hparams.n_layer) {
+ case 40: model.type = e_model::MODEL_14B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_INTERNLM2:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_7B; break;
+ case 48: model.type = e_model::MODEL_20B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_GEMMA:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ switch (hparams.n_layer) {
+ case 18: model.type = e_model::MODEL_2B; break;
+ case 28: model.type = e_model::MODEL_7B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_GEMMA2:
+ {
+ hparams.n_swa = 4096; // default value of gemma 2
+ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_ATTN_LOGIT_SOFTCAPPING, hparams.f_attn_logit_softcapping, false);
+ ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);
+ hparams.attn_soft_cap = true;
+
+ switch (hparams.n_layer) {
+ case 26: model.type = e_model::MODEL_2B; break;
+ case 42: model.type = e_model::MODEL_9B; break;
+ case 46: model.type = e_model::MODEL_27B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_STARCODER2:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ switch (hparams.n_layer) {
+ case 30: model.type = e_model::MODEL_3B; break;
+ case 32: model.type = e_model::MODEL_7B; break;
+ case 40: model.type = e_model::MODEL_15B; break;
+ case 52: model.type = e_model::MODEL_20B; break; // granite
+ case 88: model.type = e_model::MODEL_34B; break; // granite
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_MAMBA:
+ {
+ ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
+ ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);
+ ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
+ ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
+ ml.get_key(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms, false);
+
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ switch (hparams.n_layer) {
+ case 24:
+ switch (hparams.n_embd) {
+ case 768: model.type = e_model::MODEL_SMALL; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ case 48:
+ switch (hparams.n_embd) {
+ case 1024: model.type = e_model::MODEL_MEDIUM; break;
+ case 1536: model.type = e_model::MODEL_LARGE; break;
+ case 2048: model.type = e_model::MODEL_XL; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ case 64:
+ switch (hparams.n_embd) {
+ case 2560: model.type = e_model::MODEL_3B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_XVERSE:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_7B; break;
+ case 40: model.type = e_model::MODEL_13B; break;
+ case 80: model.type = e_model::MODEL_65B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_COMMAND_R:
+ {
+ ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ switch (hparams.n_layer) {
+ case 40: model.type = e_model::MODEL_35B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_DBRX:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv);
+
+ switch (hparams.n_layer) {
+ case 40: model.type = e_model::MODEL_16x12B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_OLMO:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv, false);
+
+ switch (hparams.n_layer) {
+ case 22: model.type = e_model::MODEL_1B; break;
+ case 32: model.type = e_model::MODEL_7B; break;
+ case 80: model.type = e_model::MODEL_70B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_OPENELM:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ switch (hparams.n_layer) {
+ case 16: model.type = e_model::MODEL_270M; break;
+ case 20: model.type = e_model::MODEL_450M; break;
+ case 28: model.type = e_model::MODEL_1B; break;
+ case 36: model.type = e_model::MODEL_3B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_GPTNEOX:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ ml.get_key(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res);
+ switch (hparams.n_layer) {
+ case 6:
+ switch (hparams.n_ff()) {
+ case 512: model.type = e_model::MODEL_14M; break;
+ case 2048: model.type = e_model::MODEL_70M; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ case 12:
+ switch (hparams.n_ff()) {
+ case 3072: model.type = e_model::MODEL_160M; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ case 16:
+ switch (hparams.n_ff()) {
+ case 8192: model.type = e_model::MODEL_1B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ case 24:
+ switch (hparams.n_ff()) {
+ case 4096: model.type = e_model::MODEL_410M; break;
+ case 8192: model.type = e_model::MODEL_1_4B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ case 32:
+ switch (hparams.n_ff()) {
+ case 10240: model.type = e_model::MODEL_2_8B; break;
+ case 16384: model.type = e_model::MODEL_6_9B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ case 36:
+ switch (hparams.n_ff()) {
+ case 20480: model.type = e_model::MODEL_12B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ case 44:
+ switch (hparams.n_ff()) {
+ case 24576: model.type = e_model::MODEL_20B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_ARCTIC:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ if (hparams.n_expert == 128) {
+ switch (hparams.n_layer) {
+ case 35: model.type = e_model::MODEL_10B_128x3_66B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } else {
+ model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_DEEPSEEK2:
+ {
+ bool is_lite = (hparams.n_layer == 27);
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);
+ if (!is_lite) {
+ ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);
+ }
+ ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);
+ ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
+ ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);
+ ml.get_key(LLM_KV_ROPE_SCALING_YARN_LOG_MUL, hparams.rope_yarn_log_mul);
+
+ switch (hparams.n_layer) {
+ case 27: model.type = e_model::MODEL_16B; break;
+ case 60: model.type = e_model::MODEL_236B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_CHATGLM:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ switch (hparams.n_layer) {
+ case 28: model.type = e_model::MODEL_6B; break;
+ case 40: model.type = e_model::MODEL_9B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_BITNET:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ switch (hparams.n_layer) {
+ case 26: model.type = e_model::MODEL_3B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_T5:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);
+
+ uint32_t dec_start_token_id;
+ if (ml.get_key(LLM_KV_DECODER_START_TOKEN_ID, dec_start_token_id, false)) {
+ hparams.dec_start_token_id = dec_start_token_id;
+ }
+
+ switch (hparams.n_layer) {
+ case 6: model.type = e_model::MODEL_60M; break; // t5-small
+ case 8: model.type = e_model::MODEL_80M; break; // flan-t5-small
+ case 12:
+ switch (hparams.n_ff()) {
+ case 3072: model.type = e_model::MODEL_220M; break; // t5-base
+ case 2048: model.type = e_model::MODEL_250M; break; // flan-t5-base
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ case 24:
+ switch (hparams.n_ff()) {
+ case 4096: model.type = e_model::MODEL_770M; break; // t5-large
+ case 2816: model.type = e_model::MODEL_780M; break; // flan-t5-large
+ case 16384: model.type = e_model::MODEL_3B; break; // t5-3b
+ case 5120: model.type = e_model::MODEL_3B; break; // flan-t5-xl
+ case 65536: model.type = e_model::MODEL_11B; break; // t5-11b
+ case 10240: model.type = e_model::MODEL_11B; break; // flan-t5-xxl
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_T5ENCODER:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);
+ model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ case LLM_ARCH_JAIS:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ ml.get_key(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, hparams.f_max_alibi_bias);
+
+ switch (hparams.n_layer) {
+ case 24: model.type = e_model::MODEL_1_3B; break;
+ case 40: model.type = e_model::MODEL_13B; break;
+ /* TODO: add variants */
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_NEMOTRON:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_4B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_EXAONE:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_8B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_RWKV6:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ ml.get_key(LLM_KV_WKV_HEAD_SIZE, hparams.wkv_head_size);
+ ml.get_key(LLM_KV_TIME_MIX_EXTRA_DIM, hparams.time_mix_extra_dim);
+ ml.get_key(LLM_KV_TIME_DECAY_EXTRA_DIM, hparams.time_decay_extra_dim);
+ ml.get_key(LLM_KV_RESCALE_EVERY_N_LAYERS, hparams.rescale_every_n_layers, false);
+
+ switch (hparams.n_layer) {
+ case 24: model.type = e_model::MODEL_1_6B; break;
+ case 32:
+ switch (hparams.n_embd) {
+ case 2560: model.type = e_model::MODEL_3B; break;
+ case 4096: model.type = e_model::MODEL_7B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ } break;
+ case 61: model.type = e_model::MODEL_14B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_SOLAR:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ for (int i = 0; i < hparams.n_bskcn_arr.max_size(); ++i) {
+ auto & bskcn = hparams.n_bskcn_arr.at(i);
+ bskcn.fill(0);
+ ml.get_key_or_arr(::format(LLM_KV_NAMES.at(LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION), LLM_ARCH_NAMES.at(ml.llm_kv.arch), i), bskcn, hparams.n_layer, false);
+ }
+
+ switch (hparams.n_layer) {
+ case 64: model.type = e_model::MODEL_22B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ }
+ default: (void)0;
+ }
+
+ model.ftype = ml.ftype;
+
+ if (hparams.f_max_alibi_bias > 0.0f) {
+ hparams.use_alibi = true;
+ }
+
+ hparams.rope_type = llama_rope_type(&model);
+}
+
+static void llm_load_vocab(
+ llama_model_loader & ml,
+ llama_model & model) {
+ auto & vocab = model.vocab;
+
+ struct gguf_context * ctx = ml.meta;
+
+ const auto kv = LLM_KV(model.arch);
+
+ // determine vocab type
+ {
+ std::string tokenizer_model;
+ std::string tokenizer_pre;
+
+ ml.get_key(LLM_KV_TOKENIZER_MODEL, tokenizer_model);
+ ml.get_key(LLM_KV_TOKENIZER_PRE, tokenizer_pre, false);
+
+ if (tokenizer_model == "no_vocab") {
+ vocab.type = LLAMA_VOCAB_TYPE_NONE;
+
+ // default special tokens
+ vocab.special_bos_id = -1;
+ vocab.special_eos_id = -1;
+ vocab.special_unk_id = -1;
+ vocab.special_sep_id = -1;
+ vocab.special_pad_id = -1;
+ vocab.special_cls_id = -1;
+ vocab.special_mask_id = -1;
+ vocab.linefeed_id = -1;
+
+ return;
+ } else if (tokenizer_model == "llama") {
+ vocab.type = LLAMA_VOCAB_TYPE_SPM;
+
+ // default special tokens
+ vocab.special_bos_id = 1;
+ vocab.special_eos_id = 2;
+ vocab.special_unk_id = 0;
+ vocab.special_sep_id = -1;
+ vocab.special_pad_id = -1;
+ vocab.special_cls_id = -1;
+ vocab.special_mask_id = -1;
+ } else if (tokenizer_model == "bert") {
+ vocab.type = LLAMA_VOCAB_TYPE_WPM;
+
+ // default special tokens
+ vocab.special_bos_id = -1;
+ vocab.special_eos_id = -1;
+ vocab.special_unk_id = 100;
+ vocab.special_sep_id = 102;
+ vocab.special_pad_id = 0;
+ vocab.special_cls_id = 101;
+ vocab.special_mask_id = 103;
+ } else if (tokenizer_model == "gpt2") {
+ vocab.type = LLAMA_VOCAB_TYPE_BPE;
+
+ // read bpe merges and populate bpe ranks
+ const int merges_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_MERGES).c_str());
+ if (merges_keyidx == -1) {
+ throw std::runtime_error("cannot find tokenizer merges in model file\n");
+ }
+
+ const int n_merges = gguf_get_arr_n(ctx, merges_keyidx);
+ for (int i = 0; i < n_merges; i++) {
+ const std::string word = gguf_get_arr_str(ctx, merges_keyidx, i);
+ GGML_ASSERT(unicode_cpts_from_utf8(word).size() > 0);
+
+ std::string first;
+ std::string second;
+
+ const size_t pos = word.find(' ', 1);
+
+ if (pos != std::string::npos) {
+ first = word.substr(0, pos);
+ second = word.substr(pos + 1);
+ }
+
+ vocab.bpe_ranks.emplace(std::make_pair(first, second), i);
+ }
+
+ // default special tokens
+ vocab.special_bos_id = 11;
+ vocab.special_eos_id = 11;
+ vocab.special_unk_id = -1;
+ vocab.special_sep_id = -1;
+ vocab.special_pad_id = -1;
+ vocab.special_cls_id = -1;
+ vocab.special_mask_id = -1;
+ } else if (tokenizer_model == "t5") {
+ vocab.type = LLAMA_VOCAB_TYPE_UGM;
+
+ // default special tokens
+ vocab.special_bos_id = -1;
+ vocab.special_eos_id = 1;
+ vocab.special_unk_id = 2;
+ vocab.special_sep_id = -1;
+ vocab.special_pad_id = 0;
+ vocab.special_cls_id = -1;
+ vocab.special_mask_id = -1;
+
+ const int precompiled_charsmap_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP).c_str());
+ if (precompiled_charsmap_keyidx != -1) {
+ size_t n_precompiled_charsmap = gguf_get_arr_n(ctx, precompiled_charsmap_keyidx);
+ const char * precompiled_charsmap = (const char *) gguf_get_arr_data(ctx, precompiled_charsmap_keyidx);
+ vocab.precompiled_charsmap.assign(precompiled_charsmap, precompiled_charsmap + n_precompiled_charsmap);
+#ifdef IS_BIG_ENDIAN
+ // correct endiannes of data in precompiled_charsmap binary blob
+ uint32_t * xcda_blob_size = (uint32_t *) &vocab.precompiled_charsmap[0];
+ *xcda_blob_size = __builtin_bswap32(*xcda_blob_size);
+ assert(*xcda_blob_size + sizeof(uint32_t) < n_precompiled_charsmap);
+ size_t xcda_array_size = *xcda_blob_size / sizeof(uint32_t);
+ uint32_t * xcda_array = (uint32_t *) &vocab.precompiled_charsmap[sizeof(uint32_t)];
+ for (size_t i = 0; i < xcda_array_size; ++i) {
+ xcda_array[i] = __builtin_bswap32(xcda_array[i]);
+ }
+#endif
+ }
+ } else if (tokenizer_model == "rwkv") {
+ vocab.type = LLAMA_VOCAB_TYPE_RWKV;
+
+ // default special tokens
+ vocab.special_bos_id = -1;
+ vocab.special_eos_id = -1;
+ vocab.special_unk_id = -1;
+ vocab.special_sep_id = -1;
+ vocab.special_pad_id = -1;
+ } else {
+ throw std::runtime_error(format("unknown tokenizer: '%s'", tokenizer_model.c_str()));
+ }
+
+ // for now, only BPE models have pre-tokenizers
+ if (vocab.type == LLAMA_VOCAB_TYPE_BPE) {
+ vocab.tokenizer_add_space_prefix = false;
+ vocab.tokenizer_clean_spaces = true;
+ if (tokenizer_pre == "default") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ } else if (
+ tokenizer_pre == "llama3" ||
+ tokenizer_pre == "llama-v3" ||
+ tokenizer_pre == "llama-bpe") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
+ vocab.tokenizer_ignore_merges = true;
+ vocab.tokenizer_add_bos = true;
+ } else if (
+ tokenizer_pre == "deepseek-llm") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM;
+ vocab.tokenizer_clean_spaces = false;
+ } else if (
+ tokenizer_pre == "deepseek-coder") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER;
+ vocab.tokenizer_clean_spaces = false;
+ } else if (
+ tokenizer_pre == "falcon") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_FALCON;
+ } else if (
+ tokenizer_pre == "mpt") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_MPT;
+ } else if (
+ tokenizer_pre == "starcoder") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_STARCODER;
+ } else if (
+ tokenizer_pre == "gpt-2" ||
+ tokenizer_pre == "phi-2" ||
+ tokenizer_pre == "jina-es" ||
+ tokenizer_pre == "jina-de" ||
+ tokenizer_pre == "jina-v2-es" ||
+ tokenizer_pre == "jina-v2-de" ||
+ tokenizer_pre == "jina-v2-code") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;
+ } else if (
+ tokenizer_pre == "refact") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_REFACT;
+ } else if (
+ tokenizer_pre == "command-r") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_COMMAND_R;
+ vocab.tokenizer_clean_spaces = false;
+ } else if (
+ tokenizer_pre == "qwen2") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_QWEN2;
+ vocab.tokenizer_clean_spaces = false;
+ } else if (
+ tokenizer_pre == "stablelm2") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_STABLELM2;
+ } else if (
+ tokenizer_pre == "olmo") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_OLMO;
+ } else if (
+ tokenizer_pre == "dbrx") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DBRX;
+ } else if (
+ tokenizer_pre == "smaug-bpe") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMAUG;
+ } else if (
+ tokenizer_pre == "poro-chat") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_PORO;
+ vocab.tokenizer_clean_spaces = false;
+ } else if (
+ tokenizer_pre == "chatglm-bpe") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_CHATGLM4;
+ vocab.special_bos_id = -1;
+ } else if (
+ tokenizer_pre == "viking") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_VIKING;
+ vocab.tokenizer_clean_spaces = false;
+ } else if (
+ tokenizer_pre == "jais") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_JAIS;
+ } else if (
+ tokenizer_pre == "tekken") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_TEKKEN;
+ vocab.tokenizer_clean_spaces = false;
+ vocab.tokenizer_ignore_merges = true;
+ vocab.tokenizer_add_bos = true;
+ } else if (
+ tokenizer_pre == "smollm") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMOLLM;
+ vocab.tokenizer_clean_spaces = false;
+ } else if (
+ tokenizer_pre == "codeshell") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_CODESHELL;
+ } else if (
+ tokenizer_pre == "bloom") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_BLOOM;
+ } else if (
+ tokenizer_pre == "gpt3-finnish") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH;
+ } else if (
+ tokenizer_pre == "exaone") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_EXAONE;
+ } else {
+ LLAMA_LOG_WARN("%s: missing or unrecognized pre-tokenizer type, using: 'default'\n", __func__);
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ }
+ } else if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ vocab.tokenizer_add_space_prefix = true;
+ vocab.tokenizer_clean_spaces = false;
+ vocab.tokenizer_add_bos = true;
+ vocab.tokenizer_add_eos = false;
+ } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ vocab.tokenizer_add_space_prefix = false;
+ vocab.tokenizer_clean_spaces = true;
+ vocab.tokenizer_add_bos = true;
+ vocab.tokenizer_add_eos = false;
+ } else if (vocab.type == LLAMA_VOCAB_TYPE_UGM) {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ vocab.tokenizer_add_bos = false;
+ vocab.tokenizer_add_eos = true;
+ } else if (vocab.type == LLAMA_VOCAB_TYPE_RWKV) {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ vocab.tokenizer_add_space_prefix = false;
+ vocab.tokenizer_clean_spaces = false;
+ vocab.tokenizer_add_bos = false;
+ vocab.tokenizer_add_eos = false;
+ } else {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ }
+
+ ml.get_key(LLM_KV_TOKENIZER_ADD_PREFIX, vocab.tokenizer_add_space_prefix, false);
+ ml.get_key(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, vocab.tokenizer_remove_extra_whitespaces, false);
+ }
+
+ const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str());
+ if (token_idx == -1) {
+ throw std::runtime_error("cannot find tokenizer vocab in model file\n");
+ }
+
+ const float * scores = nullptr;
+ const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str());
+ if (score_idx != -1) {
+ scores = (const float * ) gguf_get_arr_data(ctx, score_idx);
+ }
+
+ const int * toktypes = nullptr;
+ const int toktype_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str());
+ if (toktype_idx != -1) {
+ toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx);
+ }
+
+ const uint32_t n_vocab = gguf_get_arr_n(ctx, token_idx);
+
+ vocab.id_to_token.resize(n_vocab);
+
+ for (uint32_t i = 0; i < n_vocab; i++) {
+ std::string word = gguf_get_arr_str(ctx, token_idx, i);
+ GGML_ASSERT(unicode_cpts_from_utf8(word).size() > 0);
+
+ vocab.token_to_id[word] = i;
+ vocab.max_token_len = std::max(vocab.max_token_len, (int) word.size());
+
+ auto & token_data = vocab.id_to_token[i];
+ token_data.text = std::move(word);
+ token_data.score = scores ? scores[i] : 0.0f;
+ token_data.attr = LLAMA_TOKEN_ATTR_NORMAL;
+
+ if (toktypes) { //TODO: remove, required until per token attributes are available from GGUF file
+ switch(toktypes[i]) {
+ case LLAMA_TOKEN_TYPE_UNKNOWN: token_data.attr = LLAMA_TOKEN_ATTR_UNKNOWN; break;
+ case LLAMA_TOKEN_TYPE_UNUSED: token_data.attr = LLAMA_TOKEN_ATTR_UNUSED; break;
+ case LLAMA_TOKEN_TYPE_NORMAL: token_data.attr = LLAMA_TOKEN_ATTR_NORMAL; break;
+ case LLAMA_TOKEN_TYPE_CONTROL: token_data.attr = LLAMA_TOKEN_ATTR_CONTROL; break;
+ case LLAMA_TOKEN_TYPE_USER_DEFINED: token_data.attr = LLAMA_TOKEN_ATTR_USER_DEFINED; break;
+ case LLAMA_TOKEN_TYPE_BYTE: token_data.attr = LLAMA_TOKEN_ATTR_BYTE; break;
+ case LLAMA_TOKEN_TYPE_UNDEFINED: token_data.attr = LLAMA_TOKEN_ATTR_UNDEFINED; break;
+ default: token_data.attr = LLAMA_TOKEN_ATTR_UNDEFINED; break;
+ }
+ }
+ }
+ GGML_ASSERT(vocab.id_to_token.size() == vocab.token_to_id.size());
+
+ // determine the newline token: LLaMA "<0x0A>" == 10 == '\n', Falcon 193 == '\n'
+ if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
+ // For Fill-In-the-Middle (FIM)/infill models which where converted
+ // prior to support of FIM special tokens in GGUF, the following
+ // will allow those models to continue to work. The general names
+ // of the known models are currently CodeLlama (LLM_ARCH_LLAMA) and
+ // CodeGemma (LLM_ARCH_GEMMA). This can potentially be removed once
+ // new versions of these models have been published.
+ std::string gen_name;
+ ml.get_key(LLM_KV_GENERAL_NAME, gen_name, false);
+
+ std::transform(gen_name.begin(), gen_name.end(), gen_name.begin(),
+ [](unsigned char c){ return std::tolower(c); });
+
+ if (gen_name.find("code") != std::string::npos) {
+ if (model.arch == LLM_ARCH_LLAMA
+ && 32010 < vocab.id_to_token.size()
+ && vocab.id_to_token[32007].text.find("<PRE>") != std::string::npos
+ && vocab.id_to_token[32008].text.find("<SUF>") != std::string::npos
+ && vocab.id_to_token[32009].text.find("<MID>") != std::string::npos
+ && vocab.id_to_token[32010].text.find("<EOT>") != std::string::npos) {
+ vocab.special_prefix_id = 32007;
+ vocab.special_suffix_id = 32008;
+ vocab.special_middle_id = 32009;
+ vocab.special_eot_id = 32010;
+ } else if (model.arch == LLM_ARCH_GEMMA
+ && 107 < vocab.id_to_token.size()
+ && vocab.id_to_token[67].text == "<|fim_prefix|>"
+ && vocab.id_to_token[69].text == "<|fim_suffix|>"
+ && vocab.id_to_token[68].text == "<|fim_middle|>"
+ && vocab.id_to_token[107].text == "<end_of_turn>") {
+ vocab.special_prefix_id = 67;
+ vocab.special_suffix_id = 69;
+ vocab.special_middle_id = 68;
+ // TODO: this is not EOT, it is "file separator" token, needs fix
+ // https://huggingface.co/google/codegemma-7b-it/blob/9b1d9231388358c04d90bd003458f5070d97db44/tokenizer_config.json#L565-L572
+ //vocab.special_eot_id = 70;
+ vocab.special_eot_id = 107;
+ }
+ }
+ try {
+ vocab.linefeed_id = llama_byte_to_token_impl(vocab, '\n');
+ } catch (const std::exception & e) {
+ LLAMA_LOG_WARN("%s: SPM vocabulary, but newline token not found: %s! Using special_pad_id instead.", __func__, e.what());
+ vocab.linefeed_id = vocab.special_pad_id;
+ }
+ } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) {
+ vocab.linefeed_id = vocab.special_pad_id;
+ } else if (vocab.type == LLAMA_VOCAB_TYPE_RWKV) {
+ const std::vector<int> ids = llama_tokenize_internal(vocab, "\n", false);
+ GGML_ASSERT(!ids.empty() && "model vocab missing newline token");
+ vocab.linefeed_id = ids[0];
+ } else {
+ const std::vector<int> ids = llama_tokenize_internal(vocab, "\xC4\x8A", false); // U+010A
+ GGML_ASSERT(!ids.empty() && "model vocab missing newline token");
+ vocab.linefeed_id = ids[0];
+ }
+
+ // special tokens
+ {
+ const std::vector<std::pair<enum llm_kv, int32_t &>> special_token_types = {
+ { LLM_KV_TOKENIZER_BOS_ID, vocab.special_bos_id },
+ { LLM_KV_TOKENIZER_EOS_ID, vocab.special_eos_id },
+ { LLM_KV_TOKENIZER_UNK_ID, vocab.special_unk_id },
+ { LLM_KV_TOKENIZER_SEP_ID, vocab.special_sep_id },
+ { LLM_KV_TOKENIZER_PAD_ID, vocab.special_pad_id },
+ { LLM_KV_TOKENIZER_CLS_ID, vocab.special_cls_id },
+ { LLM_KV_TOKENIZER_MASK_ID, vocab.special_mask_id },
+ { LLM_KV_TOKENIZER_PREFIX_ID, vocab.special_prefix_id },
+ { LLM_KV_TOKENIZER_SUFFIX_ID, vocab.special_suffix_id },
+ { LLM_KV_TOKENIZER_MIDDLE_ID, vocab.special_middle_id },
+ { LLM_KV_TOKENIZER_EOT_ID, vocab.special_eot_id },
+ { LLM_KV_TOKENIZER_EOM_ID, vocab.special_eom_id },
+ };
+
+ for (const auto & it : special_token_types) {
+ const std::string & key = kv(std::get<0>(it));
+ int32_t & id = std::get<1>(it);
+
+ uint32_t new_id;
+ if (!ml.get_key(std::get<0>(it), new_id, false)) {
+ continue;
+ }
+ if (new_id >= vocab.id_to_token.size()) {
+ LLAMA_LOG_WARN("%s: bad special token: '%s' = %ud, using default id %d\n",
+ __func__, key.c_str(), new_id, id);
+ } else {
+ id = new_id;
+ }
+ }
+
+ // Handle add_bos_token and add_eos_token
+ {
+ bool temp = true;
+
+ if (ml.get_key(LLM_KV_TOKENIZER_ADD_BOS, temp, false)) {
+ vocab.tokenizer_add_bos = temp;
+ }
+ if (ml.get_key(LLM_KV_TOKENIZER_ADD_EOS, temp, false)) {
+ vocab.tokenizer_add_eos = temp;
+ }
+ }
+
+ // find EOT token: "<|eot_id|>", "<|im_end|>", "<end_of_turn>", etc.
+ //
+ // TODO: convert scripts should provide this token through the KV metadata LLAMA_KV_TOKENIZER_EOT_ID
+ // for now, we apply this workaround to find the EOT token based on its text
+ if (vocab.special_eot_id == -1) {
+ for (const auto & t : vocab.token_to_id) {
+ if (
+ // TODO: gemma "<end_of_turn>" is exported as a normal token, so the following check does not work
+ // need to fix convert script
+ //vocab.id_to_token[t.second].type == LLAMA_TOKEN_TYPE_CONTROL &&
+ (t.first == "<|eot_id|>" ||
+ t.first == "<|im_end|>" ||
+ t.first == "<|end|>" ||
+ t.first == "<end_of_turn>" ||
+ t.first == "<|endoftext|>"
+ )
+ ) {
+ vocab.special_eot_id = t.second;
+ break;
+ }
+ }
+ }
+
+ // find EOM token: "<|eom_id|>"
+ //
+ // TODO: convert scripts should provide this token through the KV metadata LLAMA_KV_TOKENIZER_EOM_ID
+ // for now, we apply this workaround to find the EOM token based on its text
+ if (vocab.special_eom_id == -1) {
+ const auto & t = vocab.token_to_id.find("<|eom_id|>");
+ if (t != vocab.token_to_id.end()) {
+ vocab.special_eom_id = t->second;
+ }
+ }
+ }
+
+ // build special tokens cache
+ {
+ for (llama_vocab::id id = 0; id < (llama_vocab::id)n_vocab; ++id) {
+ if (vocab.id_to_token[id].attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED | LLAMA_TOKEN_ATTR_UNKNOWN)) {
+ vocab.cache_special_tokens.push_back(id);
+ }
+ }
+
+ std::sort(vocab.cache_special_tokens.begin(), vocab.cache_special_tokens.end(),
+ [&] (const llama_vocab::id a, const llama_vocab::id b) {
+ return vocab.id_to_token[a].text.size() > vocab.id_to_token[b].text.size();
+ }
+ );
+
+ LLAMA_LOG_INFO("%s: special tokens cache size = %u\n", __func__, (uint32_t)vocab.cache_special_tokens.size());
+ }
+
+ // build token to piece cache
+ {
+ size_t size_cache = 0;
+
+ std::vector<llama_vocab::token> cache_token_to_piece(n_vocab);
+
+ for (uint32_t id = 0; id < n_vocab; ++id) {
+ cache_token_to_piece[id] = llama_token_to_piece(&model, id, true);
+
+ size_cache += cache_token_to_piece[id].size();
+ }
+
+ std::swap(vocab.cache_token_to_piece, cache_token_to_piece);
+
+ LLAMA_LOG_INFO("%s: token to piece cache size = %.4f MB\n", __func__, size_cache / 1024.0 / 1024.0);
+ }
+
+ // Handle per token attributes
+ //NOTE: Each model customizes per token attributes.
+ //NOTE: Per token attributes are missing from the GGUF file.
+ //TODO: Extract attributes from GGUF file.
+ {
+ auto _contains_any = [] (const std::string &str, const std::vector<std::string> &substrs) -> bool {
+ for (auto substr : substrs) {
+ if (str.find(substr) < std::string::npos) {
+ return true;
+ }
+ }
+ return false;
+ };
+
+ auto _set_tokenid_attr = [&] (const llama_vocab::id id, llama_token_attr attr, bool value) {
+ uint32_t current = vocab.id_to_token.at(id).attr;
+ current = value ? (current | attr) : (current & ~attr);
+ vocab.id_to_token[id].attr = (llama_token_attr) current;
+ };
+
+ auto _set_token_attr = [&] (const std::string & token, llama_token_attr attr, bool value) {
+ _set_tokenid_attr(vocab.token_to_id.at(token), attr, value);
+ };
+
+ std::string model_name;
+ std::string tokenizer_pre;
+
+ ml.get_key(LLM_KV_GENERAL_NAME, model_name, false);
+ ml.get_key(LLM_KV_TOKENIZER_PRE, tokenizer_pre, false);
+
+ // model name to lowercase
+ std::transform(model_name.begin(), model_name.end(), model_name.begin(),
+ [] (const std::string::value_type x) {
+ return std::tolower(x);
+ }
+ );
+
+ // set attributes by model/tokenizer name
+ if (_contains_any(tokenizer_pre, {"jina-v2-de", "jina-v2-es", "jina-v2-code"})) {
+ _set_token_attr("<mask>", LLAMA_TOKEN_ATTR_LSTRIP, true);
+ } else if (_contains_any(model_name, {"phi-3", "phi3"})) {
+ for (auto id : vocab.cache_special_tokens) {
+ _set_tokenid_attr(id, LLAMA_TOKEN_ATTR_RSTRIP, true);
+ }
+ for (auto token : {"</s>"}) {
+ _set_token_attr(token, LLAMA_TOKEN_ATTR_RSTRIP, true);
+ }
+ for (auto token : {"<unk>", "<s>", "<|endoftext|>"}) {
+ _set_token_attr(token, LLAMA_TOKEN_ATTR_RSTRIP, false);
+ }
+ }
+ }
+}
+
+static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {
+ const auto & hparams = model.hparams;
+ const auto & vocab = model.vocab;
+
+ const char * rope_scaling_type = LLAMA_ROPE_SCALING_TYPES.at(hparams.rope_scaling_type_train);
+
+ auto print_f = [](const std::function<uint32_t(uint32_t)> & f, uint32_t n) {
+ bool is_var = false;
+
+ std::vector<uint32_t> v;
+ for (uint32_t i = 0; i < n; ++i) {
+ v.push_back(f(i));
+ if (v[i] != v[0]) {
+ is_var = true;
+ }
+ }
+
+ std::stringstream ss;
+
+ if (is_var) {
+ ss << "[";
+ for (uint32_t i = 0; i < n; ++i) {
+ ss << v[i];
+ if (i < n - 1) {
+ ss << ", ";
+ }
+ }
+ ss << "]";
+ } else {
+ ss << v[0];
+ }
+
+ return ss.str();
+ };
+
+ // hparams
+ LLAMA_LOG_INFO("%s: format = %s\n", __func__, llama_file_version_name(ml.fver));
+ LLAMA_LOG_INFO("%s: arch = %s\n", __func__, LLM_ARCH_NAMES.at(model.arch));
+ LLAMA_LOG_INFO("%s: vocab type = %s\n", __func__, llama_model_vocab_type_name(vocab.type));
+ LLAMA_LOG_INFO("%s: n_vocab = %u\n", __func__, hparams.n_vocab);
+ LLAMA_LOG_INFO("%s: n_merges = %u\n", __func__, (int) vocab.bpe_ranks.size());
+ LLAMA_LOG_INFO("%s: vocab_only = %d\n", __func__, hparams.vocab_only);
+
+ if (!hparams.vocab_only) {
+ LLAMA_LOG_INFO("%s: n_ctx_train = %u\n", __func__, hparams.n_ctx_train);
+ LLAMA_LOG_INFO("%s: n_embd = %u\n", __func__, hparams.n_embd);
+ LLAMA_LOG_INFO("%s: n_layer = %u\n", __func__, hparams.n_layer);
+ LLAMA_LOG_INFO("%s: n_head = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_head(il); }, hparams.n_layer).c_str());
+ LLAMA_LOG_INFO("%s: n_head_kv = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_head_kv(il); }, hparams.n_layer).c_str());
+ LLAMA_LOG_INFO("%s: n_rot = %u\n", __func__, hparams.n_rot);
+ LLAMA_LOG_INFO("%s: n_swa = %u\n", __func__, hparams.n_swa);
+ LLAMA_LOG_INFO("%s: n_embd_head_k = %u\n", __func__, hparams.n_embd_head_k);
+ LLAMA_LOG_INFO("%s: n_embd_head_v = %u\n", __func__, hparams.n_embd_head_v);
+ LLAMA_LOG_INFO("%s: n_gqa = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_gqa(il); }, hparams.n_layer).c_str());
+ LLAMA_LOG_INFO("%s: n_embd_k_gqa = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_embd_k_gqa(il); }, hparams.n_layer).c_str());
+ LLAMA_LOG_INFO("%s: n_embd_v_gqa = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_embd_v_gqa(il); }, hparams.n_layer).c_str());
+ LLAMA_LOG_INFO("%s: f_norm_eps = %.1e\n", __func__, hparams.f_norm_eps);
+ LLAMA_LOG_INFO("%s: f_norm_rms_eps = %.1e\n", __func__, hparams.f_norm_rms_eps);
+ LLAMA_LOG_INFO("%s: f_clamp_kqv = %.1e\n", __func__, hparams.f_clamp_kqv);
+ LLAMA_LOG_INFO("%s: f_max_alibi_bias = %.1e\n", __func__, hparams.f_max_alibi_bias);
+ LLAMA_LOG_INFO("%s: f_logit_scale = %.1e\n", __func__, hparams.f_logit_scale);
+ LLAMA_LOG_INFO("%s: n_ff = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_ff(il); }, hparams.n_layer).c_str());
+ LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert);
+ LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used);
+ LLAMA_LOG_INFO("%s: causal attn = %d\n", __func__, hparams.causal_attn);
+ LLAMA_LOG_INFO("%s: pooling type = %d\n", __func__, hparams.pooling_type);
+ LLAMA_LOG_INFO("%s: rope type = %d\n", __func__, hparams.rope_type);
+ LLAMA_LOG_INFO("%s: rope scaling = %s\n", __func__, rope_scaling_type);
+ LLAMA_LOG_INFO("%s: freq_base_train = %.1f\n", __func__, hparams.rope_freq_base_train);
+ LLAMA_LOG_INFO("%s: freq_scale_train = %g\n", __func__, hparams.rope_freq_scale_train);
+ LLAMA_LOG_INFO("%s: n_ctx_orig_yarn = %u\n", __func__, hparams.n_ctx_orig_yarn);
+ LLAMA_LOG_INFO("%s: rope_finetuned = %s\n", __func__, hparams.rope_finetuned ? "yes" : "unknown");
+ LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv);
+ LLAMA_LOG_INFO("%s: ssm_d_inner = %u\n", __func__, hparams.ssm_d_inner);
+ LLAMA_LOG_INFO("%s: ssm_d_state = %u\n", __func__, hparams.ssm_d_state);
+ LLAMA_LOG_INFO("%s: ssm_dt_rank = %u\n", __func__, hparams.ssm_dt_rank);
+ LLAMA_LOG_INFO("%s: ssm_dt_b_c_rms = %d\n", __func__, hparams.ssm_dt_b_c_rms);
+ }
+
+ LLAMA_LOG_INFO("%s: model type = %s\n", __func__, llama_model_type_name(model.type));
+ LLAMA_LOG_INFO("%s: model ftype = %s\n", __func__, llama_model_ftype_name(model.ftype).c_str());
+ if (ml.n_elements >= 1e12) {
+ LLAMA_LOG_INFO("%s: model params = %.2f T\n", __func__, ml.n_elements*1e-12);
+ } else if (ml.n_elements >= 1e9) {
+ LLAMA_LOG_INFO("%s: model params = %.2f B\n", __func__, ml.n_elements*1e-9);
+ } else if (ml.n_elements >= 1e6) {
+ LLAMA_LOG_INFO("%s: model params = %.2f M\n", __func__, ml.n_elements*1e-6);
+ } else {
+ LLAMA_LOG_INFO("%s: model params = %.2f K\n", __func__, ml.n_elements*1e-3);
+ }
+ if (ml.n_bytes < GiB) {
+ LLAMA_LOG_INFO("%s: model size = %.2f MiB (%.2f BPW) \n", __func__, ml.n_bytes/1024.0/1024.0, ml.n_bytes*8.0/ml.n_elements);
+ } else {
+ LLAMA_LOG_INFO("%s: model size = %.2f GiB (%.2f BPW) \n", __func__, ml.n_bytes/1024.0/1024.0/1024.0, ml.n_bytes*8.0/ml.n_elements);
+ }
+
+ // general kv
+ LLAMA_LOG_INFO("%s: general.name = %s\n", __func__, model.name.c_str());
+
+ // special tokens
+ if (vocab.special_bos_id != -1) { LLAMA_LOG_INFO( "%s: BOS token = %d '%s'\n", __func__, vocab.special_bos_id, vocab.id_to_token[vocab.special_bos_id].text.c_str() ); }
+ if (vocab.special_eos_id != -1) { LLAMA_LOG_INFO( "%s: EOS token = %d '%s'\n", __func__, vocab.special_eos_id, vocab.id_to_token[vocab.special_eos_id].text.c_str() ); }
+ if (vocab.special_unk_id != -1) { LLAMA_LOG_INFO( "%s: UNK token = %d '%s'\n", __func__, vocab.special_unk_id, vocab.id_to_token[vocab.special_unk_id].text.c_str() ); }
+ if (vocab.special_sep_id != -1) { LLAMA_LOG_INFO( "%s: SEP token = %d '%s'\n", __func__, vocab.special_sep_id, vocab.id_to_token[vocab.special_sep_id].text.c_str() ); }
+ if (vocab.special_pad_id != -1) { LLAMA_LOG_INFO( "%s: PAD token = %d '%s'\n", __func__, vocab.special_pad_id, vocab.id_to_token[vocab.special_pad_id].text.c_str() ); }
+ if (vocab.special_cls_id != -1) { LLAMA_LOG_INFO( "%s: CLS token = %d '%s'\n", __func__, vocab.special_cls_id, vocab.id_to_token[vocab.special_cls_id].text.c_str() ); }
+ if (vocab.special_mask_id != -1) { LLAMA_LOG_INFO( "%s: MASK token = %d '%s'\n", __func__, vocab.special_mask_id, vocab.id_to_token[vocab.special_mask_id].text.c_str() ); }
+
+ if (vocab.linefeed_id != -1) { LLAMA_LOG_INFO( "%s: LF token = %d '%s'\n", __func__, vocab.linefeed_id, vocab.id_to_token[vocab.linefeed_id].text.c_str() ); }
+ if (vocab.special_prefix_id != -1) { LLAMA_LOG_INFO( "%s: PRE token = %d '%s'\n", __func__, vocab.special_prefix_id, vocab.id_to_token[vocab.special_prefix_id].text.c_str() ); }
+ if (vocab.special_suffix_id != -1) { LLAMA_LOG_INFO( "%s: SUF token = %d '%s'\n", __func__, vocab.special_suffix_id, vocab.id_to_token[vocab.special_suffix_id].text.c_str() ); }
+ if (vocab.special_middle_id != -1) { LLAMA_LOG_INFO( "%s: MID token = %d '%s'\n", __func__, vocab.special_middle_id, vocab.id_to_token[vocab.special_middle_id].text.c_str() ); }
+ if (vocab.special_eot_id != -1) { LLAMA_LOG_INFO( "%s: EOT token = %d '%s'\n", __func__, vocab.special_eot_id, vocab.id_to_token[vocab.special_eot_id].text.c_str() ); }
+
+ LLAMA_LOG_INFO("%s: max token length = %d\n", __func__, vocab.max_token_len);
+
+ if (model.arch == LLM_ARCH_DEEPSEEK2) {
+ LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
+ LLAMA_LOG_INFO("%s: n_lora_q = %d\n", __func__, hparams.n_lora_q);
+ LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv);
+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
+ LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
+ LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
+ LLAMA_LOG_INFO("%s: rope_yarn_log_mul = %.4f\n", __func__, hparams.rope_yarn_log_mul);
+ }
+
+ if (model.arch == LLM_ARCH_QWEN2MOE) {
+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
+ LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
+ }
+}
+
+// Returns false if cancelled by progress_callback
+static bool llm_load_tensors(
+ llama_model_loader & ml,
+ llama_model & model,
+ int n_gpu_layers,
+ enum llama_split_mode split_mode,
+ int main_gpu,
+ const float * tensor_split,
+ bool use_mlock,
+ llama_progress_callback progress_callback,
+ void * progress_callback_user_data) {
+ model.t_start_us = ggml_time_us();
+
+ auto & hparams = model.hparams;
+
+ model.split_mode = split_mode;
+ model.main_gpu = main_gpu;
+ model.n_gpu_layers = n_gpu_layers;
+
+ const int n_layer = hparams.n_layer;
+ const int i_gpu_start = std::max((int) hparams.n_layer - n_gpu_layers, (int) 0);
+ bool use_mmap_buffer = true;
+
+ // there is very little benefit to offloading the input layer, so always keep it on the CPU
+ model.buft_input = llama_default_buffer_type_cpu(true);
+ //model.buft_input = llama_default_buffer_type_offload(main_gpu);
+
+ model.buft_layer.resize(n_layer);
+
+ // assign cpu layers
+ for (int i = 0; i < i_gpu_start; ++i) {
+ model.buft_layer[i] = llama_default_buffer_type_cpu(true);
+ }
+
+ if (split_mode == LLAMA_SPLIT_MODE_LAYER) {
+ // calculate the split points
+ int device_count = llama_get_device_count(model);
+ bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + device_count, [](float x) { return x == 0.0f; });
+ std::vector<float> splits(device_count);
+ if (all_zero) {
+ // default split, by free memory
+ for (int i = 0; i < device_count; ++i) {
+ splits[i] = llama_get_device_memory(model, i);
+ }
+ } else {
+ std::copy(tensor_split, tensor_split + device_count, splits.begin());
+ }
+
+ // sum and normalize the splits to get the split points
+ float split_sum = 0.0f;
+ for (int i = 0; i < device_count; ++i) {
+ split_sum += splits[i];
+ splits[i] = split_sum;
+ }
+ for (int i = 0; i < device_count; ++i) {
+ splits[i] /= split_sum;
+ }
+
+ // assign the repeating layers to the devices according to the splits
+ int act_gpu_layers = std::min(n_gpu_layers, (int)n_layer + 1);
+ for (int i = i_gpu_start; i < n_layer; ++i) {
+ int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + device_count, float(i - i_gpu_start)/act_gpu_layers) - splits.begin();
+ model.buft_layer[i] = llama_default_buffer_type_offload(model, layer_gpu);
+ }
+ // assign the output layer
+ if (n_gpu_layers > n_layer) {
+ int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + device_count, float(act_gpu_layers - 1)/act_gpu_layers) - splits.begin();
+ model.buft_output = llama_default_buffer_type_offload(model, layer_gpu);
+ } else {
+ model.buft_output = llama_default_buffer_type_cpu(true);
+ }
+ } else {
+ ggml_backend_buffer_type_t split_buft;
+ if (split_mode == LLAMA_SPLIT_MODE_ROW) {
+ split_buft = llama_default_buffer_type_split(model, main_gpu, tensor_split);
+ } else {
+ // LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_LAYER in backends where it is not supported
+ split_buft = llama_default_buffer_type_offload(model, main_gpu);
+ }
+ // assign the repeating layers
+ for (int i = i_gpu_start; i < n_layer; ++i) {
+ model.buft_layer[i] = {
+ split_buft,
+ llama_default_buffer_type_offload(model, main_gpu)
+ };
+ }
+ // assign the output layer
+ if (n_gpu_layers > n_layer) {
+ model.buft_output = {
+ split_buft,
+ llama_default_buffer_type_offload(model, main_gpu)
+ };
+ } else {
+ model.buft_output = llama_default_buffer_type_cpu(true);
+ }
+ }
+
+ // count used buffer types
+ std::map<ggml_backend_buffer_type_t, int> buft_layer_count;
+ buft_layer_count[model.buft_input.buft]++;
+ buft_layer_count[model.buft_input.buft_matrix]++;
+ buft_layer_count[model.buft_output.buft]++;
+ buft_layer_count[model.buft_output.buft_matrix]++;
+ for (int i = 0; i < n_layer; ++i) {
+ buft_layer_count[model.buft_layer[i].buft]++;
+ buft_layer_count[model.buft_layer[i].buft_matrix]++;
+ }
+
+ // create one context per buffer type
+ size_t ctx_size = ggml_tensor_overhead()*(ml.n_tensors + 1); // +1 for models where tok_embd is duplicated as output
+
+ // for moe merged tensors
+ ctx_size += ggml_tensor_overhead()*n_layer*3;
+
+ std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;
+ for (auto & it : buft_layer_count) {
+ struct ggml_init_params params = {
+ /*.mem_size =*/ ctx_size,
+ /*.mem_buffer =*/ NULL,
+ /*.no_alloc =*/ true,
+ };
+ ggml_context * ctx = ggml_init(params);
+ if (!ctx) {
+ throw std::runtime_error(format("failed to create context"));
+ }
+ ctx_map[it.first] = ctx;
+ model.ctxs.push_back(ctx);
+ }
+
+ LLAMA_LOG_INFO("%s: ggml ctx size = %7.2f MiB\n", __func__, model.ctxs.size()*ctx_size/1024.0/1024.0);
+
+ // create tensors for the weights
+ {
+ // note: cast to int64_t since we will use these for the tensor dimensions
+ const int64_t n_head = hparams.n_head();
+ const int64_t n_head_kv = hparams.n_head_kv();
+ const int64_t n_embd = hparams.n_embd;
+ const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();
+ const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();
+ const int64_t n_embd_head_k = hparams.n_embd_head_k;
+ const int64_t n_embd_head_v = hparams.n_embd_head_v;
+ const int64_t n_ff = hparams.n_ff();
+ const int64_t n_embd_gqa = n_embd_v_gqa;
+ const int64_t n_vocab = hparams.n_vocab;
+ const int64_t n_vocab_type = hparams.n_vocab_type;
+ const int64_t n_rot = hparams.n_rot;
+ const int64_t n_expert = hparams.n_expert;
+ const int64_t n_expert_used = hparams.n_expert_used;
+ const int64_t n_ctx_train = hparams.n_ctx_train;
+
+ if (n_expert > 0 && hparams.n_expert_used == 0) {
+ throw std::runtime_error("model has expert layers but no expert layers are used");
+ }
+
+ ggml_context * ctx_input = ctx_map.at(model.buft_input.buft);
+ ggml_context * ctx_output = ctx_map.at(model.buft_output.buft);
+ ggml_context * ctx_output_split = ctx_map.at(model.buft_output.buft_matrix);
+
+ auto ctx_for_layer = [&](int i) { return ctx_map.at(model.buft_layer[i].buft); };
+ auto ctx_for_layer_split = [&](int i) { return ctx_map.at(model.buft_layer[i].buft_matrix); };
+
+ model.layers.resize(n_layer);
+
+ const auto tn = LLM_TN(model.arch);
+ switch (model.arch) {
+ case LLM_ARCH_LLAMA:
+ case LLM_ARCH_REFACT:
+ case LLM_ARCH_MINICPM:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ // if output is NULL, init from the input tok embed
+ if (model.output == NULL) {
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});
+
+ // optional bias tensors
+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+
+ layer.rope_freqs = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FREQS, "weight"), {n_rot/2}, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));
+
+ if (n_expert == 0) {
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+
+ // optional MLP bias
+ layer.ffn_gate_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE, "bias", i), {n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ } else {
+ layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert});
+
+ layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ if (layer.ffn_gate_exps) {
+ layer.ffn_down_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert});
+ layer.ffn_up_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert});
+ } else {
+ // merge split expert into a single tensor for compatibility with older models
+ // requires disabling mmap
+ use_mmap_buffer = false;
+
+ ggml_type type_gate = ml.require_tensor_meta(tn(LLM_TENSOR_FFN_GATE_EXP, "weight", i, 0).c_str())->type;
+ ggml_type type_down = ml.require_tensor_meta(tn(LLM_TENSOR_FFN_DOWN_EXP, "weight", i, 0).c_str())->type;
+ ggml_type type_up = ml.require_tensor_meta(tn(LLM_TENSOR_FFN_UP_EXP, "weight", i, 0).c_str())->type;
+
+ layer.ffn_gate_exps = ggml_new_tensor_3d(ctx_split, type_gate, n_embd, n_ff, n_expert);
+ layer.ffn_down_exps = ggml_new_tensor_3d(ctx_split, type_down, n_ff, n_embd, n_expert);
+ layer.ffn_up_exps = ggml_new_tensor_3d(ctx_split, type_up, n_embd, n_ff, n_expert);
+
+ ggml_set_name(layer.ffn_gate_exps, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i).c_str());
+ ggml_set_name(layer.ffn_down_exps, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i).c_str());
+ ggml_set_name(layer.ffn_up_exps, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i).c_str());
+
+ for (uint32_t x = 0; x < n_expert; ++x) {
+ // the individual experts are loaded into a view of the merged tensor
+ ml.create_tensor_as_view(ctx_split, layer.ffn_gate_exps, tn(LLM_TENSOR_FFN_GATE_EXP, "weight", i, x), { n_embd, n_ff }, layer.ffn_gate_exps->nb[2]*x);
+ ml.create_tensor_as_view(ctx_split, layer.ffn_down_exps, tn(LLM_TENSOR_FFN_DOWN_EXP, "weight", i, x), { n_ff, n_embd }, layer.ffn_down_exps->nb[2]*x);
+ ml.create_tensor_as_view(ctx_split, layer.ffn_up_exps, tn(LLM_TENSOR_FFN_UP_EXP, "weight", i, x), { n_embd, n_ff }, layer.ffn_up_exps->nb[2]*x);
+ }
+ }
+ }
+ }
+ } break;
+ case LLM_ARCH_GROK:
+ {
+ if (n_expert == 0) {
+ throw std::runtime_error("Grok model cannot have zero experts");
+ }
+
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ // if output is NULL, init from the input tok embed
+ if (model.output == NULL) {
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+
+ layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert});
+ layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ if (layer.ffn_gate_exps) {
+ layer.ffn_down_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert});
+ layer.ffn_up_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert});
+ } else {
+ // merge split expert into a single tensor for compatibility with older models
+ // requires disabling mmap
+ use_mmap_buffer = false;
+
+ ggml_type type_gate = ml.require_tensor_meta(tn(LLM_TENSOR_FFN_GATE_EXP, "weight", i, 0).c_str())->type;
+ ggml_type type_down = ml.require_tensor_meta(tn(LLM_TENSOR_FFN_DOWN_EXP, "weight", i, 0).c_str())->type;
+ ggml_type type_up = ml.require_tensor_meta(tn(LLM_TENSOR_FFN_UP_EXP, "weight", i, 0).c_str())->type;
+
+ layer.ffn_gate_exps = ggml_new_tensor_3d(ctx_split, type_gate, n_embd, n_ff, n_expert);
+ layer.ffn_down_exps = ggml_new_tensor_3d(ctx_split, type_down, n_ff, n_embd, n_expert);
+ layer.ffn_up_exps = ggml_new_tensor_3d(ctx_split, type_up, n_embd, n_ff, n_expert);
+
+ ggml_set_name(layer.ffn_gate_exps, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i).c_str());
+ ggml_set_name(layer.ffn_down_exps, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i).c_str());
+ ggml_set_name(layer.ffn_up_exps, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i).c_str());
+
+ for (uint32_t x = 0; x < n_expert; ++x) {
+ // the individual experts are loaded into a view of the merged tensor
+ ml.create_tensor_as_view(ctx_split, layer.ffn_gate_exps, tn(LLM_TENSOR_FFN_GATE_EXP, "weight", i, x), { n_embd, n_ff }, layer.ffn_gate_exps->nb[2]*x);
+ ml.create_tensor_as_view(ctx_split, layer.ffn_down_exps, tn(LLM_TENSOR_FFN_DOWN_EXP, "weight", i, x), { n_ff, n_embd }, layer.ffn_down_exps->nb[2]*x);
+ ml.create_tensor_as_view(ctx_split, layer.ffn_up_exps, tn(LLM_TENSOR_FFN_UP_EXP, "weight", i, x), { n_embd, n_ff }, layer.ffn_up_exps->nb[2]*x);
+ }
+ }
+
+ layer.layer_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd});
+ }
+ } break;
+ case LLM_ARCH_DBRX:
+ {
+ if (n_expert == 0) {
+ throw std::runtime_error("DBRX model cannot have zero experts");
+ }
+
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd});
+
+ layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert});
+ layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert});
+ layer.ffn_down_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff, n_embd, n_expert});
+ layer.ffn_up_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert});
+ }
+ } break;
+ case LLM_ARCH_BAICHUAN:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_FALCON:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ if (!model.output) {
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); // needs to be on GPU
+ }
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_STARCODER:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+ model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ if (!model.output) {
+ // needs to be on GPU
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
+
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});
+ }
+ } break;
+ case LLM_ARCH_BERT:
+ case LLM_ARCH_NOMIC_BERT:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+ model.type_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_vocab_type});
+
+ if (model.arch == LLM_ARCH_BERT) {
+ model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train});
+ }
+
+ model.tok_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd});
+ model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd});
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ if (model.arch == LLM_ARCH_BERT) {
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd});
+
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa});
+
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa});
+ } else {
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ }
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd});
+ layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd});
+
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+
+ if (model.arch == LLM_ARCH_BERT) {
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
+ } else {
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ }
+
+ layer.layer_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd});
+ layer.layer_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd});
+ }
+ } break;
+ case LLM_ARCH_JINA_BERT_V2:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // word_embeddings
+ model.type_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_vocab_type}); // token_type_embeddings
+
+ model.tok_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}); // LayerNorm
+ model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}); //LayerNorm bias
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i]; // JinaBertLayer
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd});
+
+ layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa});
+
+ layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa});
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); //output_dens
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); //output_dens
+
+ layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}); //output_norm
+ layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd});
+
+ layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
+
+ layer.layer_out_norm = ml.create_tensor(ctx_split, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd});
+ layer.layer_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd});
+ }
+ } break;
+ case LLM_ARCH_BLOOM:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+ model.tok_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd});
+ model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
+
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});
+ }
+ } break;
+ case LLM_ARCH_MPT:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+ model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ if (!model.output) {
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); // needs to be on GPU
+ }
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ // AWQ ScaleActivation layer
+ layer.ffn_act = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_ACT, "scales", i), {n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ }
+ } break;
+ case LLM_ARCH_STABLELM:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ // optional bias tensors, present in Stable LM 2 1.6B
+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ // optional q and k layernorms, present in StableLM 2 12B
+ layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k, n_head}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k, n_head_kv}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ // optional FFN norm, not present in StableLM 2 12B which uses parallel residual
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_QWEN:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd*3});
+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd*3});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff/2});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff/2, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff/2});
+ }
+ } break;
+ case LLM_ARCH_QWEN2:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ // if output is NULL, init from the input tok embed
+ if (model.output == NULL) {
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ // optional bias tensors
+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd});
+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa});
+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_QWEN2MOE:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ // optional bias tensors
+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd});
+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa});
+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+
+ layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert});
+
+ GGML_ASSERT(n_expert > 0);
+ GGML_ASSERT(n_expert_used > 0);
+
+ // MoE branch
+ const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;
+
+ layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert});
+ layer.ffn_down_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert});
+ layer.ffn_up_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert});
+
+ // Shared expert branch
+ const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff;
+
+ layer.ffn_gate_inp_shexp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", i), {n_embd});
+ layer.ffn_gate_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), { n_embd, n_ff_shexp});
+ layer.ffn_down_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd});
+ layer.ffn_up_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), { n_embd, n_ff_shexp});
+ }
+ } break;
+ case LLM_ARCH_PHI2:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ model.output_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT, "bias"), {n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ if (layer.wqkv == nullptr) {
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd});
+
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa});
+
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa});
+ }
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
+
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});
+ }
+ } break;
+ case LLM_ARCH_PHI3:
+ {
+ const int64_t n_embd_head = n_embd / n_head;
+
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab });
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd });
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab });
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd });
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), { n_embd, n_embd + 2 * n_embd_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd, n_embd });
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd });
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd });
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, 2 * n_ff });
+
+ layer.rope_long = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight"), { n_embd_head/2 }, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));
+ layer.rope_short = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight"), { n_embd_head/2 }, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));
+ }
+ } break;
+ case LLM_ARCH_PLAMO:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_GPT2:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+ model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
+
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});
+ }
+ } break;
+ case LLM_ARCH_CODESHELL:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
+
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});
+ }
+ } break;
+ case LLM_ARCH_ORION:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_INTERNLM2:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ // layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_GEMMA:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); // same as tok_embd, duplicated to allow offloading
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ }
+ } break;
+ case LLM_ARCH_GEMMA2:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); // same as tok_embd, duplicated to allow offloading
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});
+ layer.attn_post_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_post_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd});
+ }
+ } break;
+ case LLM_ARCH_STARCODER2:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ // if output is NULL, init from the input tok embed
+ if (model.output == NULL) {
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ // optional bias tensors
+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd});
+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa});
+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa});
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+
+ // optional bias tensors
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP , "bias", i), { n_ff});
+ }
+ } break;
+ case LLM_ARCH_MAMBA:
+ {
+ const int64_t d_conv = hparams.ssm_d_conv;
+ const int64_t d_inner = hparams.ssm_d_inner;
+ const int64_t d_state = hparams.ssm_d_state;
+ const int64_t dt_rank = hparams.ssm_dt_rank;
+
+ // only an expansion factor of 2 is supported for now
+ GGML_ASSERT(2 * n_embd == d_inner);
+
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ // if output is NULL, init from the input tok embed, duplicated to allow offloading
+ if (model.output == NULL) {
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ // norm
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.ssm_in = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_IN, "weight", i), {n_embd, 2*d_inner});
+
+ layer.ssm_conv1d = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {d_conv, d_inner});
+ layer.ssm_conv1d_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_SSM_CONV1D, "bias", i), {d_inner});
+
+ layer.ssm_x = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_X, "weight", i), {d_inner, dt_rank + 2*d_state});
+
+ layer.ssm_dt = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_DT, "weight", i), {dt_rank, d_inner});
+ layer.ssm_dt_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner});
+
+ // no "weight" suffix for these
+ layer.ssm_a = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_A, i), {d_state, d_inner});
+ layer.ssm_d = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_SSM_D, i), {d_inner});
+
+ // out_proj
+ layer.ssm_out = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_OUT, "weight", i), {d_inner, n_embd});
+ }
+ } break;
+ case LLM_ARCH_XVERSE:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_COMMAND_R:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ // init output from the input tok embed
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ if (n_layer >= 64){
+ layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k, n_head});
+ layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k, n_head_kv});
+ }
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_OLMO: // adapted from LLM_ARCH_LLAMA with norm params removed
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ // if output is NULL, init from the input tok embed
+ if (model.output == NULL) {
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_OPENELM:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ // init output from the input tok embed
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ const int64_t n_head = hparams.n_head(i);
+ const int64_t n_head_qkv = 2*hparams.n_head_kv(i) + n_head;
+ const int64_t n_ff = hparams.n_ff(i);
+
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_head_qkv*n_embd_head_k});
+ layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k});
+ layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head*n_embd_head_k, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_GPTNEOX:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
+
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});
+ }
+ } break;
+ case LLM_ARCH_ARCTIC:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ // if output is NULL, init from the input tok embed
+ if (model.output == NULL) {
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_embd});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_embd, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_embd});
+
+ layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert});
+ layer.ffn_norm_exps = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM_EXPS, "weight", i), {n_embd});
+ layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, false);
+ layer.ffn_down_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert});
+ layer.ffn_up_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert});
+ }
+ } break;
+ case LLM_ARCH_DEEPSEEK2:
+ {
+ const bool is_lite = (hparams.n_layer == 27);
+
+ const int64_t n_embd_head_qk_rope = hparams.n_rot;
+ const int64_t n_embd_head_qk_nope = hparams.n_embd_head_k - hparams.n_rot;
+
+ const int64_t q_lora_rank = hparams.n_lora_q;
+ const int64_t kv_lora_rank = hparams.n_lora_kv;
+
+ const int64_t n_ff_exp = hparams.n_ff_exp;
+ const int64_t n_expert_shared = hparams.n_expert_shared;
+
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ if (!is_lite) {
+ layer.attn_q_a_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank});
+ }
+
+ layer.attn_kv_a_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank});
+
+ if (!is_lite) {
+ layer.wq_a = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank});
+ layer.wq_b = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head_k});
+ } else {
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_k_gqa});
+ }
+
+ layer.wkv_a_mqa = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + (n_embd_head_qk_rope)});
+ layer.wkv_b = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_KV_B, "weight", i), {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v)});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_head * ( n_embd_head_v), n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+
+ if (i < (int) hparams.n_layer_dense_lead) {
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ } else {
+ layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert});
+
+ GGML_ASSERT(n_expert > 0);
+ GGML_ASSERT(n_expert_used > 0);
+
+ // MoE branch
+ layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert});
+ layer.ffn_down_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert});
+ layer.ffn_up_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert});
+
+ // Shared expert branch
+ layer.ffn_gate_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared});
+ layer.ffn_down_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_exp * n_expert_shared, n_embd});
+ layer.ffn_up_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared});
+ }
+ }
+ } break;
+ case LLM_ARCH_BITNET:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_sub_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_SUB_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wq_scale = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "scale", i), {1});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wk_scale = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "scale", i), {1});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv_scale = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "scale", i), {1});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+ layer.wo_scale = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "scale", i), {1});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_sub_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_SUB_NORM, "weight", i), {n_ff});
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_gate_scale = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE, "scale", i), {1});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ layer.ffn_down_scale = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "scale", i), {1});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_up_scale = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "scale", i), {1});
+ }
+ } break;
+ case LLM_ARCH_T5:
+ {
+ const auto n_rel_attn_bkts = hparams.n_rel_attn_bkts;
+
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm_enc = ml.create_tensor(ctx_output, tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_DEC_OUTPUT_NORM, "weight"), {n_embd});
+
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ // if output is NULL, init from the input tok embed
+ if (model.output == NULL) {
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_rel_b_enc = ml.create_tensor(ctx_input, tn(LLM_TENSOR_ENC_ATTN_REL_B, "weight", i), {n_head, n_rel_attn_bkts}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.wq_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_Q, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wk_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wv_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
+ layer.wo_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_OUT, "weight", i), {n_embd_v_gqa, n_embd});
+
+ layer.ffn_norm_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_gate_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_FFN_GATE, "weight", i), {n_embd, n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.ffn_down_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_FFN_UP, "weight", i), {n_embd, n_ff});
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_DEC_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_rel_b = ml.create_tensor(ctx_input, tn(LLM_TENSOR_DEC_ATTN_REL_B, "weight", i), {n_head, n_rel_attn_bkts}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_ATTN_Q, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_ATTN_OUT, "weight", i), {n_embd_v_gqa, n_embd});
+
+ layer.attn_norm_cross = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_DEC_CROSS_ATTN_NORM, "weight", i), {n_embd});
+ // this tensor seems to be unused in HF transformers implementation
+ layer.attn_rel_b_cross = ml.create_tensor(ctx_input, tn(LLM_TENSOR_DEC_CROSS_ATTN_REL_B, "weight", i), {n_head, n_rel_attn_bkts}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.wq_cross = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_CROSS_ATTN_Q, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wk_cross = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_CROSS_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wv_cross = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_CROSS_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
+ layer.wo_cross = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_CROSS_ATTN_OUT, "weight", i), {n_embd_v_gqa, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_DEC_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_gate = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_DEC_FFN_GATE, "weight", i), {n_embd, n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_T5ENCODER:
+ {
+ const auto n_rel_attn_bkts = hparams.n_rel_attn_bkts;
+
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm_enc = ml.create_tensor(ctx_output, tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ // if output is NULL, init from the input tok embed
+ if (model.output == NULL) {
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_rel_b_enc = ml.create_tensor(ctx_input, tn(LLM_TENSOR_ENC_ATTN_REL_B, "weight", i), {n_head, n_rel_attn_bkts}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.wq_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_Q, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wk_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wv_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
+ layer.wo_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_OUT, "weight", i), {n_embd_v_gqa, n_embd});
+
+ layer.ffn_norm_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_gate_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_FFN_GATE, "weight", i), {n_embd, n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.ffn_down_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_JAIS:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // Output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_gate_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE, "bias", i), {n_ff});
+
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});
+ }
+ } break;
+ case LLM_ARCH_CHATGLM:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});
+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});
+
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff * 2});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
+ }
+ } break;
+ case LLM_ARCH_NEMOTRON:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ // optional bias tensors
+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+
+ // optional MLP bias
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ }
+ } break;
+ case LLM_ARCH_EXAONE:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.rope_freqs = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FREQS, "weight"), {n_rot/2}, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ case LLM_ARCH_RWKV6:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // Block 0, LN0
+ model.tok_norm = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd});
+ model.tok_norm_b = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd});
+
+ // output
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+
+ const int time_mix_extra_dim = hparams.time_mix_extra_dim;
+ const int time_decay_extra_dim = hparams.time_decay_extra_dim;
+ const int head_size = hparams.wkv_head_size;
+ const int attn_hidden_size = n_embd;
+ const int ffn_size = hparams.n_ff_arr[0];
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd});
+ layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd});
+
+ layer.time_mix_w1 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_W1, "weight", i), {n_embd, time_mix_extra_dim * 5});
+ layer.time_mix_w2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_W2, "weight", i), {time_mix_extra_dim, n_embd, 5});
+
+ layer.time_mix_lerp_x = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_LERP_X, "weight", i), {n_embd, 1, 1});
+ layer.time_mix_lerp_w = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_LERP_W, "weight", i), {n_embd, 1, 1});
+ layer.time_mix_lerp_k = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_LERP_K, "weight", i), {n_embd, 1, 1});
+ layer.time_mix_lerp_v = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_LERP_V, "weight", i), {n_embd, 1, 1});
+ layer.time_mix_lerp_r = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_LERP_R, "weight", i), {n_embd, 1, 1});
+ layer.time_mix_lerp_g = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_LERP_G, "weight", i), {n_embd, 1, 1});
+
+ layer.time_mix_first = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_FIRST, "weight", i), {head_size, n_embd / head_size});
+ layer.time_mix_decay = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_DECAY, "weight", i), {n_embd});
+ layer.time_mix_decay_w1 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_DECAY_W1, "weight", i), {n_embd, time_decay_extra_dim});
+ layer.time_mix_decay_w2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_DECAY_W2, "weight", i), {time_decay_extra_dim, attn_hidden_size});
+ layer.time_mix_key = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_KEY, "weight", i), {attn_hidden_size, n_embd});
+ layer.time_mix_value = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_VALUE, "weight", i), {attn_hidden_size, n_embd});
+ layer.time_mix_receptance = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_RECEPTANCE, "weight", i), {attn_hidden_size, n_embd});
+ layer.time_mix_gate = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_GATE, "weight", i), {attn_hidden_size, n_embd});
+
+ layer.time_mix_ln = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_LN, "weight", i), {n_embd});
+ layer.time_mix_ln_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_LN, "bias", i), {n_embd});
+ layer.time_mix_output = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_TIME_MIX_OUTPUT, "weight", i), {n_embd, attn_hidden_size});
+
+ layer.channel_mix_lerp_k = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_CHANNEL_MIX_LERP_K, "weight", i), {n_embd, 1, 1});
+ layer.channel_mix_lerp_r = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_CHANNEL_MIX_LERP_R, "weight", i), {n_embd, 1, 1});
+
+ layer.channel_mix_key = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_CHANNEL_MIX_KEY, "weight", i), {n_embd, ffn_size});
+ layer.channel_mix_value = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_CHANNEL_MIX_VALUE, "weight", i), {ffn_size, n_embd});
+ layer.channel_mix_receptance = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_CHANNEL_MIX_RECEPTANCE, "weight", i), {n_embd, n_embd});
+ }
+
+ } break;
+ case LLM_ARCH_SOLAR:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+
+ layer.bskcn_tv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_BSKCN_TV, "weight"), {2}, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));
+
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
+ default:
+ throw std::runtime_error("unknown architecture");
+ }
+ }
+
+ ml.done_getting_tensors();
+
+ ml.init_mappings(true, use_mlock ? &model.mlock_mmaps : nullptr);
+ model.mappings.reserve(ml.mappings.size());
+
+ // create the backend buffers
+ std::vector<std::pair<ggml_context *, llama_buf_map>> ctx_bufs;
+ ctx_bufs.reserve(ctx_map.size());
+
+ // Ensure we have enough capacity for the maximum backend buffer we will potentially create
+ size_t n_max_backend_buffer = ctx_map.size() * ml.files.size();
+ model.bufs.reserve(n_max_backend_buffer);
+
+ for (auto & it : ctx_map) {
+ ggml_backend_buffer_type_t buft = it.first;
+ ggml_context * ctx = it.second;
+
+ llama_buf_map bufs;
+ bufs.reserve(n_max_backend_buffer);
+
+ // only the mmap region containing the tensors in the model is mapped to the backend buffer
+ // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer, then we could just use metal for all layers
+ // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size
+ if (ml.use_mmap && use_mmap_buffer && buft == llama_default_buffer_type_cpu(true)) {
+ for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
+ void * addr = nullptr;
+ size_t first, last;
+ ml.get_mapping_range(&first, &last, &addr, idx, ctx);
+ if (first >= last) {
+ continue;
+ }
+ ggml_backend_buffer_t buf = ggml_backend_cpu_buffer_from_ptr((char *) addr + first, last - first);
+ if (buf == nullptr) {
+ throw std::runtime_error("unable to allocate backend CPU buffer");
+ }
+ model.bufs.push_back(buf);
+ bufs.emplace(idx, buf);
+#ifdef GGML_USE_CUDA
+ if (n_layer >= n_gpu_layers) {
+ ggml_backend_cuda_register_host_buffer(
+ ggml_backend_buffer_get_base(buf),
+ ggml_backend_buffer_get_size(buf));
+ }
+#endif
+ }
+ }
+#ifdef GGML_USE_METAL
+ else if (ml.use_mmap && use_mmap_buffer && buft == ggml_backend_metal_buffer_type()) {
+ for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
+ const size_t max_size = ggml_get_max_tensor_size(ctx);
+ void * addr = nullptr;
+ size_t first, last;
+ ml.get_mapping_range(&first, &last, &addr, idx, ctx);
+ if (first >= last) {
+ continue;
+ }
+ ggml_backend_buffer_t buf = ggml_backend_metal_buffer_from_ptr((char *) addr + first, last - first, max_size);
+ if (buf == nullptr) {
+ throw std::runtime_error("unable to allocate backend metal buffer");
+ }
+ model.bufs.push_back(buf);
+ bufs.emplace(idx, buf);
+ }
+ }
+#endif
+ else {
+ ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);
+ if (buf == nullptr) {
+ throw std::runtime_error("unable to allocate backend buffer");
+ }
+ model.bufs.push_back(buf);
+ if (use_mlock && ggml_backend_buffer_is_host(buf)) {
+ model.mlock_bufs.emplace_back(new llama_mlock);
+ auto & mlock_buf = model.mlock_bufs.back();
+ mlock_buf->init (ggml_backend_buffer_get_base(buf));
+ mlock_buf->grow_to(ggml_backend_buffer_get_size(buf));
+ }
+ for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
+ bufs.emplace(idx, buf);
+ }
+ }
+
+ if (bufs.empty()) {
+ throw std::runtime_error("failed to allocate buffer");
+ }
+
+ for (auto & buf : bufs) {
+ // indicate that this buffer contains weights
+ // this is used by ggml_backend_sched to improve op scheduling -> ops that use a weight are preferably scheduled to the backend that contains the weight
+ ggml_backend_buffer_set_usage(buf.second, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+ }
+
+ ctx_bufs.emplace_back(ctx, bufs);
+ }
+
+ if (llama_supports_gpu_offload()) {
+ const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer));
+
+ LLAMA_LOG_INFO("%s: offloading %d repeating layers to GPU\n", __func__, n_gpu);
+ if (n_gpu_layers > (int) hparams.n_layer) {
+ LLAMA_LOG_INFO("%s: offloading non-repeating layers to GPU\n", __func__);
+ }
+
+ const int max_backend_supported_layers = hparams.n_layer + 1;
+ const int max_offloadable_layers = hparams.n_layer + 1;
+
+ LLAMA_LOG_INFO("%s: offloaded %d/%d layers to GPU\n", __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers);
+ }
+
+ // print memory requirements
+ for (ggml_backend_buffer_t buf : model.bufs) {
+ LLAMA_LOG_INFO("%s: %10s buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf) / 1024.0 / 1024.0);
+ }
+
+ // populate tensors_by_name
+ for (ggml_context * ctx : model.ctxs) {
+ for (auto * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) {
+ model.tensors_by_name.emplace_back(ggml_get_name(cur), cur);
+ }
+ }
+
+ // load tensor data
+ for (auto & it : ctx_bufs) {
+ ggml_context * ctx = it.first;
+ auto & bufs = it.second;
+ if (!ml.load_all_data(ctx, bufs, use_mlock ? &model.mlock_mmaps : NULL, progress_callback, progress_callback_user_data)) {
+ return false;
+ }
+ }
+
+ if (use_mmap_buffer) {
+ for (auto & mapping : ml.mappings) {
+ model.mappings.emplace_back(std::move(mapping));
+ }
+ }
+
+ // loading time will be recalculate after the first eval, so
+ // we take page faults deferred by mmap() into consideration
+ model.t_load_us = ggml_time_us() - model.t_start_us;
+ return true;
+}
+
+// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback
+static int llama_model_load(const std::string & fname, llama_model & model, llama_model_params & params) {
+ try {
+ llama_model_loader ml(fname, params.use_mmap, params.check_tensors, params.kv_overrides);
+
+ model.hparams.vocab_only = params.vocab_only;
+
+ try {
+ llm_load_arch(ml, model);
+ } catch(const std::exception & e) {
+ throw std::runtime_error("error loading model architecture: " + std::string(e.what()));
+ }
+ try {
+ llm_load_hparams(ml, model);
+ } catch(const std::exception & e) {
+ throw std::runtime_error("error loading model hyperparameters: " + std::string(e.what()));
+ }
+ try {
+ llm_load_vocab(ml, model);
+ } catch(const std::exception & e) {
+ throw std::runtime_error("error loading model vocabulary: " + std::string(e.what()));
+ }
+
+ llm_load_print_meta(ml, model);
+
+ if (model.vocab.type != LLAMA_VOCAB_TYPE_NONE &&
+ model.hparams.n_vocab != model.vocab.id_to_token.size()) {
+ throw std::runtime_error("vocab size mismatch");
+ }
+
+ if (params.vocab_only) {
+ LLAMA_LOG_INFO("%s: vocab only - skipping tensors\n", __func__);
+ return 0;
+ }
+
+#ifdef GGML_USE_KOMPUTE
+ if (params.n_gpu_layers > 0 && (
+ !(model.arch == LLM_ARCH_LLAMA || model.arch == LLM_ARCH_FALCON)
+ || !(
+ model.ftype == LLAMA_FTYPE_ALL_F32 ||
+ model.ftype == LLAMA_FTYPE_MOSTLY_F16 ||
+ model.ftype == LLAMA_FTYPE_MOSTLY_BF16 ||
+ model.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 ||
+ model.ftype == LLAMA_FTYPE_MOSTLY_Q4_1
+ )
+ )) {
+ // TODO(cebtenzzre): propagate this error outside of llama_load_model_from_file
+ LLAMA_LOG_WARN("%s: disabling Kompute due to unsupported model arch or quantization\n", __func__);
+ params.n_gpu_layers = 0;
+ }
+#endif
+
+ if (!llm_load_tensors(
+ ml, model, params.n_gpu_layers, params.split_mode, params.main_gpu, params.tensor_split, params.use_mlock,
+ params.progress_callback, params.progress_callback_user_data
+ )) {
+ return -2;
+ }
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what());
+ return -1;
+ }
+
+ return 0;
+}
+
+//
+// llm_build
+//
+
+using llm_build_cb = std::function<void(struct ggml_tensor * cur, const char * name, int nl)>;
+
+enum llm_ffn_op_type {
+ LLM_FFN_SILU,
+ LLM_FFN_GELU,
+ LLM_FFN_RELU,
+ LLM_FFN_RELU_SQR,
+ LLM_FFN_SWIGLU,
+};
+
+enum llm_ffn_gate_type {
+ LLM_FFN_SEQ,
+ LLM_FFN_PAR, // ffn_gate is parallel to ffn_up
+};
+
+enum llm_norm_type {
+ LLM_NORM,
+ LLM_NORM_RMS,
+};
+
+static struct ggml_tensor * llm_build_inp_embd(
+ struct ggml_context * ctx,
+ struct llama_context & lctx,
+ const llama_hparams & hparams,
+ const llama_ubatch & batch,
+ struct ggml_tensor * tok_embd,
+ const llm_build_cb & cb) {
+ const int64_t n_embd = hparams.n_embd;
+
+ struct ggml_tensor * inpL;
+
+ if (batch.token) {
+ lctx.inp_tokens = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, batch.n_tokens);
+ cb(lctx.inp_tokens, "inp_tokens", -1);
+ ggml_set_input(lctx.inp_tokens);
+
+ inpL = ggml_get_rows(ctx, tok_embd, lctx.inp_tokens);
+ } else {
+ lctx.inp_embd = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, batch.n_tokens);
+ inpL = lctx.inp_embd;
+ ggml_set_input(lctx.inp_embd);
+ }
+
+ cb(inpL, "inp_embd", -1);
+
+ return inpL;
+}
+
+static void llm_build_kv_store(
+ struct ggml_context * ctx,
+ const llama_hparams & hparams,
+ const llama_cparams & cparams,
+ const llama_kv_cache & kv,
+ struct ggml_cgraph * graph,
+ struct ggml_tensor * k_cur,
+ struct ggml_tensor * v_cur,
+ int32_t n_tokens,
+ int32_t kv_head,
+ const llm_build_cb & cb,
+ int64_t il) {
+ const int64_t n_ctx = cparams.n_ctx;
+
+ const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
+ const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);
+
+ GGML_ASSERT(kv.size == n_ctx);
+
+ struct ggml_tensor * k_cache_view = ggml_view_1d(ctx, kv.k_l[il], n_tokens*n_embd_k_gqa, ggml_row_size(kv.k_l[il]->type, n_embd_k_gqa)*kv_head);
+ cb(k_cache_view, "k_cache_view", il);
+
+ // note: storing RoPE-ed version of K in the KV cache
+ ggml_build_forward_expand(graph, ggml_cpy(ctx, k_cur, k_cache_view));
+
+ assert(v_cur->ne[0] == n_embd_v_gqa && v_cur->ne[1] == n_tokens);
+
+ struct ggml_tensor * v_cache_view = nullptr;
+
+ if (cparams.flash_attn) {
+ v_cache_view = ggml_view_1d(ctx, kv.v_l[il], n_tokens*n_embd_v_gqa, ggml_row_size(kv.v_l[il]->type, n_embd_v_gqa)*kv_head);
+ } else {
+ // note: the V cache is transposed when not using flash attention
+ v_cache_view = ggml_view_2d(ctx, kv.v_l[il], n_tokens, n_embd_v_gqa,
+ ( n_ctx)*ggml_element_size(kv.v_l[il]),
+ (kv_head)*ggml_element_size(kv.v_l[il]));
+
+ v_cur = ggml_transpose(ctx, v_cur);
+ }
+ cb(v_cache_view, "v_cache_view", il);
+
+ ggml_build_forward_expand(graph, ggml_cpy(ctx, v_cur, v_cache_view));
+}
+
+// do mat_mul, while optionally apply lora
+static struct ggml_tensor * llm_build_lora_mm(
+ struct llama_context & lctx,
+ struct ggml_context * ctx0,
+ struct ggml_tensor * w,
+ struct ggml_tensor * cur) {
+ struct ggml_tensor * res = ggml_mul_mat(ctx0, w, cur);
+ for (auto & it : lctx.lora_adapters) {
+ struct llama_lora_weight * lora = it.first->get_weight(w);
+ if (lora == nullptr) {
+ continue;
+ }
+ const float alpha = it.first->alpha;
+ const float rank = (float) lora->b->ne[0];
+ const float scale = alpha ? it.second * alpha / rank : it.second;
+ struct ggml_tensor * ab_cur = ggml_mul_mat(
+ ctx0, lora->b,
+ ggml_mul_mat(ctx0, lora->a, cur)
+ );
+ ab_cur = ggml_scale(ctx0, ab_cur, scale);
+ res = ggml_add(ctx0, res, ab_cur);
+ }
+ return res;
+}
+
+// do mat_mul_id, while optionally apply lora
+static struct ggml_tensor * llm_build_lora_mm_id(
+ struct llama_context & lctx,
+ struct ggml_context * ctx0,
+ struct ggml_tensor * w, // struct ggml_tensor * as
+ struct ggml_tensor * cur, // struct ggml_tensor * b
+ struct ggml_tensor * ids) {
+ struct ggml_tensor * res = ggml_mul_mat_id(ctx0, w, cur, ids);
+ for (auto & it : lctx.lora_adapters) {
+ struct llama_lora_weight * lora = it.first->get_weight(w);
+ if (lora == nullptr) {
+ continue;
+ }
+ const float alpha = it.first->alpha;
+ const float rank = (float) lora->b->ne[0];
+ const float scale = alpha ? it.second * alpha / rank : it.second;
+ struct ggml_tensor * ab_cur = ggml_mul_mat_id(
+ ctx0, lora->b,
+ ggml_mul_mat_id(ctx0, lora->a, cur, ids),
+ ids
+ );
+ ab_cur = ggml_scale(ctx0, ab_cur, scale);
+ res = ggml_add(ctx0, res, ab_cur);
+ }
+ return res;
+}
+
+static struct ggml_tensor * llm_build_norm(
+ struct ggml_context * ctx,
+ struct ggml_tensor * cur,
+ const llama_hparams & hparams,
+ struct ggml_tensor * mw,
+ struct ggml_tensor * mb,
+ llm_norm_type type,
+ const llm_build_cb & cb,
+ int il) {
+ switch (type) {
+ case LLM_NORM: cur = ggml_norm (ctx, cur, hparams.f_norm_eps); break;
+ case LLM_NORM_RMS: cur = ggml_rms_norm(ctx, cur, hparams.f_norm_rms_eps); break;
+ }
+
+ if (mw || mb) {
+ cb(cur, "norm", il);
+ }
+
+ if (mw) {
+ cur = ggml_mul(ctx, cur, mw);
+ if (mb) {
+ cb(cur, "norm_w", il);
+ }
+ }
+
+ if (mb) {
+ cur = ggml_add(ctx, cur, mb);
+ }
+
+ return cur;
+}
+
+static struct ggml_tensor * llm_build_ffn(
+ struct ggml_context * ctx,
+ struct llama_context & lctx,
+ struct ggml_tensor * cur,
+ struct ggml_tensor * up,
+ struct ggml_tensor * up_b,
+ struct ggml_tensor * up_s,
+ struct ggml_tensor * gate,
+ struct ggml_tensor * gate_b,
+ struct ggml_tensor * gate_s,
+ struct ggml_tensor * down,
+ struct ggml_tensor * down_b,
+ struct ggml_tensor * down_s,
+ struct ggml_tensor * act_scales,
+ llm_ffn_op_type type_op,
+ llm_ffn_gate_type type_gate,
+ const llm_build_cb & cb,
+ int il) {
+ struct ggml_tensor * tmp = up ? llm_build_lora_mm(lctx, ctx, up, cur) : cur;
+ cb(tmp, "ffn_up", il);
+
+ if (up_b) {
+ tmp = ggml_add(ctx, tmp, up_b);
+ cb(tmp, "ffn_up_b", il);
+ }
+
+ if (up_s) {
+ tmp = ggml_mul(ctx, tmp, up_s);
+ cb(tmp, "ffn_up_s", il);
+ }
+
+ if (gate) {
+ switch (type_gate) {
+ case LLM_FFN_SEQ:
+ {
+ cur = llm_build_lora_mm(lctx, ctx, gate, tmp);
+ cb(cur, "ffn_gate", il);
+ } break;
+ case LLM_FFN_PAR:
+ {
+ cur = llm_build_lora_mm(lctx, ctx, gate, cur);
+ cb(cur, "ffn_gate", il);
+ } break;
+ }
+
+ if (gate_b) {
+ cur = ggml_add(ctx, cur, gate_b);
+ cb(cur, "ffn_gate_b", il);
+ }
+
+ if (gate_s) {
+ cur = ggml_mul(ctx, cur, gate_s);
+ cb(cur, "ffn_gate_s", il);
+ }
+
+ } else {
+ cur = tmp;
+ }
+
+ switch (type_op) {
+ case LLM_FFN_SILU:
+ {
+ cur = ggml_silu(ctx, cur);
+ cb(cur, "ffn_silu", il);
+ } break;
+ case LLM_FFN_GELU:
+ {
+ cur = ggml_gelu(ctx, cur);
+ cb(cur, "ffn_gelu", il);
+ if (act_scales != NULL) {
+ cur = ggml_div(ctx, cur, act_scales);
+ cb(cur, "ffn_act", il);
+ }
+ } break;
+ case LLM_FFN_RELU:
+ {
+ cur = ggml_relu(ctx, cur);
+ cb(cur, "ffn_relu", il);
+ } break;
+ case LLM_FFN_RELU_SQR:
+ {
+ cur = ggml_relu(ctx, cur);
+ cb(cur, "ffn_relu", il);
+
+ cur = ggml_sqr(ctx, cur);
+ cb(cur, "ffn_sqr(relu)", il);
+ } break;
+ case LLM_FFN_SWIGLU:
+ {
+ // Project to 4h. If using swiglu double the output width, see https://arxiv.org/pdf/2002.05202.pdf
+ int64_t split_point = cur->ne[0] / 2;
+ struct ggml_tensor * x0 = ggml_cont(ctx, ggml_view_2d(ctx, cur, split_point, cur->ne[1], cur->nb[1], 0));
+ struct ggml_tensor * x1 = ggml_cont(ctx, ggml_view_2d(ctx, cur, split_point, cur->ne[1], cur->nb[1], split_point * ggml_element_size(cur)));
+
+ x0 = ggml_silu(ctx, x0);
+ cb(cur, "ffn_silu", il);
+
+ cur = ggml_mul(ctx, x0, x1);
+ cb(cur, "ffn_mul", il);
+ } break;
+ }
+
+ if (type_gate == LLM_FFN_PAR) {
+ cur = ggml_mul(ctx, cur, tmp);
+ cb(cur, "ffn_gate_par", il);
+ }
+
+ if (down) {
+ cur = llm_build_lora_mm(lctx, ctx, down, cur);
+ }
+
+ if (down_b) {
+ cb(cur, "ffn_down", il);
+ }
+
+ if (down_b) {
+ cur = ggml_add(ctx, cur, down_b);
+ }
+
+ if (down_s) {
+ cur = ggml_mul(ctx, cur, down_s);
+ cb(cur, "ffn_down_s", il);
+ }
+
+ return cur;
+}
+
+static struct ggml_tensor * llm_build_moe_ffn(
+ struct ggml_context * ctx,
+ struct llama_context & lctx,
+ struct ggml_tensor * cur,
+ struct ggml_tensor * gate_inp,
+ struct ggml_tensor * up_exps,
+ struct ggml_tensor * gate_exps,
+ struct ggml_tensor * down_exps,
+ int64_t n_expert,
+ int64_t n_expert_used,
+ llm_ffn_op_type type_op,
+ bool norm_w,
+ bool scale_w,
+ float w_scale,
+ const llm_build_cb & cb,
+ int il) {
+ int64_t n_embd = cur->ne[0];
+ int64_t n_tokens = cur->ne[1];
+
+ ggml_tensor * logits = llm_build_lora_mm(lctx, ctx, gate_inp, cur); // [n_expert, n_tokens]
+ cb(logits, "ffn_moe_logits", il);
+
+ ggml_tensor * probs = ggml_soft_max(ctx, logits); // [n_expert, n_tokens]
+ cb(probs, "ffn_moe_probs", il);
+
+ // select experts
+ ggml_tensor * selected_experts = ggml_top_k(ctx, probs, n_expert_used); // [n_expert_used, n_tokens]
+ cb(selected_experts->src[0], "ffn_moe_argsort", il);
+ cb(selected_experts, "ffn_moe_topk", il);
+
+ ggml_tensor * weights = ggml_get_rows(ctx,
+ ggml_reshape_3d(ctx, probs, 1, n_expert, n_tokens), selected_experts); // [1, n_expert_used, n_tokens]
+ cb(weights, "ffn_moe_weights", il);
+
+ if (norm_w) {
+ weights = ggml_reshape_2d(ctx, weights, n_expert_used, n_tokens);
+
+ ggml_tensor * weights_sum = ggml_sum_rows(ctx, weights); // [1, n_tokens]
+ cb(weights_sum, "ffn_moe_weights_sum", il);
+
+ weights = ggml_div(ctx, weights, weights_sum); // [n_expert_used, n_tokens]
+ cb(weights, "ffn_moe_weights_norm", il);
+
+ weights = ggml_reshape_3d(ctx, weights, 1, n_expert_used, n_tokens);
+ }
+ if (scale_w) {
+ weights = ggml_scale(ctx, weights, w_scale);
+ cb(weights, "ffn_moe_weights_scaled", il);
+ }
+
+ cur = ggml_reshape_3d(ctx, cur, n_embd, 1, n_tokens);
+ ggml_tensor * up = llm_build_lora_mm_id(lctx, ctx, up_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens]
+ cb(up, "ffn_moe_up", il);
+
+ ggml_tensor * gate = llm_build_lora_mm_id(lctx, ctx, gate_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens]
+ cb(gate, "ffn_moe_gate", il);
+
+ switch (type_op) {
+ case LLM_FFN_SILU:
+ {
+ gate = ggml_silu(ctx, gate);
+ cb(gate, "ffn_moe_silu", il);
+ } break;
+ case LLM_FFN_GELU:
+ {
+ gate = ggml_gelu(ctx, gate);
+ cb(gate, "ffn_moe_gelu", il);
+ } break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+
+ ggml_tensor * par = ggml_mul(ctx, up, gate); // [n_ff, n_expert_used, n_tokens]
+ cb(par, "ffn_moe_gate_par", il);
+
+ ggml_tensor * experts = llm_build_lora_mm_id(lctx, ctx, down_exps, par, selected_experts); // [n_embd, n_expert_used, n_tokens]
+ cb(experts, "ffn_moe_down", il);
+
+ experts = ggml_mul(ctx, experts, weights);
+
+ // aggregate experts
+ ggml_tensor * moe_out = nullptr;
+ for (int i = 0; i < n_expert_used; ++i) {
+ ggml_tensor * cur_expert = ggml_view_2d(ctx, experts, n_embd, n_tokens,
+ experts->nb[2], i*experts->nb[1]);
+
+ if (i == 0) {
+ moe_out = cur_expert;
+ } else {
+ moe_out = ggml_add(ctx, moe_out, cur_expert);
+ }
+ }
+
+ if (n_expert_used == 1) {
+ // avoid returning a non-contiguous tensor
+ moe_out = ggml_cont(ctx, moe_out);
+ }
+
+ return moe_out;
+}
+
+static struct ggml_tensor * llm_build_kqv(
+ struct ggml_context * ctx,
+ struct llama_context & lctx,
+ const llama_kv_cache & kv,
+ struct ggml_cgraph * graph,
+ struct ggml_tensor * wo,
+ struct ggml_tensor * wo_b,
+ struct ggml_tensor * q_cur,
+ struct ggml_tensor * kq_mask,
+ int32_t n_tokens,
+ int32_t n_kv,
+ float kq_scale,
+ const llm_build_cb & cb,
+ int il) {
+ const llama_model & model = lctx.model;
+ const llama_hparams & hparams = lctx.model.hparams;
+ const llama_cparams & cparams = lctx.cparams;
+
+ const int64_t n_ctx = cparams.n_ctx;
+ const int64_t n_head = hparams.n_head(il);
+ const int64_t n_head_kv = hparams.n_head_kv(il);
+ const int64_t n_embd_head_k = hparams.n_embd_head_k;
+ const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
+ const int64_t n_embd_head_v = hparams.n_embd_head_v;
+ const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);
+
+ struct ggml_tensor * q = ggml_permute(ctx, q_cur, 0, 2, 1, 3);
+ cb(q, "q", il);
+
+ struct ggml_tensor * k =
+ ggml_view_3d(ctx, kv.k_l[il],
+ n_embd_head_k, n_kv, n_head_kv,
+ ggml_row_size(kv.k_l[il]->type, n_embd_k_gqa),
+ ggml_row_size(kv.k_l[il]->type, n_embd_head_k),
+ 0);
+ cb(k, "k", il);
+
+ struct ggml_tensor * cur;
+
+ if (cparams.flash_attn) {
+ GGML_UNUSED(model);
+ GGML_UNUSED(n_ctx);
+
+ // split cached v into n_head heads (not transposed)
+ struct ggml_tensor * v =
+ ggml_view_3d(ctx, kv.v_l[il],
+ n_embd_head_v, n_kv, n_head_kv,
+ ggml_row_size(kv.v_l[il]->type, n_embd_v_gqa),
+ ggml_row_size(kv.v_l[il]->type, n_embd_head_v),
+ 0);
+ cb(v, "v", il);
+
+ cur = ggml_flash_attn_ext(ctx, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias,
+ hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f);
+
+ if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_GEMMA2) {
+ ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32);
+ }
+
+ cur = ggml_reshape_2d(ctx, cur, n_embd_head_v*n_head, n_tokens);
+ } else {
+ struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q);
+ cb(kq, "kq", il);
+
+ if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2 || model.arch == LLM_ARCH_NEMOTRON || model.arch == LLM_ARCH_CHATGLM) {
+ // for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs
+ // ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847
+ ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
+ }
+
+ if (model.arch == LLM_ARCH_GROK) {
+ // need to do the following:
+ // multiply by attn_output_multiplyer of 0.08838834764831845
+ // and then :
+ // kq = 30 * tanh(kq / 30)
+ // before the softmax below
+
+ //try from phi2
+ //ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
+
+ kq = ggml_tanh(ctx, ggml_scale(ctx, kq, 0.08838834764831845f/30.0f));
+ kq = ggml_scale(ctx, kq, 30);
+ }
+
+ if (hparams.attn_soft_cap) {
+ kq = ggml_scale(ctx, kq, 1.0f / hparams.f_attn_logit_softcapping);
+ kq = ggml_tanh(ctx, kq);
+ kq = ggml_scale(ctx, kq, hparams.f_attn_logit_softcapping);
+ }
+
+ kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias);
+ cb(kq, "kq_soft_max_ext", il);
+
+ GGML_ASSERT(kv.size == n_ctx);
+
+ // split cached v into n_head heads
+ struct ggml_tensor * v =
+ ggml_view_3d(ctx, kv.v_l[il],
+ n_kv, n_embd_head_v, n_head_kv,
+ ggml_element_size(kv.v_l[il])*n_ctx,
+ ggml_element_size(kv.v_l[il])*n_ctx*n_embd_head_v,
+ 0);
+ cb(v, "v", il);
+
+ struct ggml_tensor * kqv = ggml_mul_mat(ctx, v, kq);
+ cb(kqv, "kqv", il);
+
+ struct ggml_tensor * kqv_merged = ggml_permute(ctx, kqv, 0, 2, 1, 3);
+ cb(kqv_merged, "kqv_merged", il);
+
+ cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_v*n_head, n_tokens);
+ cb(cur, "kqv_merged_cont", il);
+ }
+
+ ggml_build_forward_expand(graph, cur);
+
+ if (wo) {
+ cur = llm_build_lora_mm(lctx, ctx, wo, cur);
+ }
+
+ if (wo_b) {
+ cb(cur, "kqv_wo", il);
+ }
+
+ if (wo_b) {
+ cur = ggml_add(ctx, cur, wo_b);
+ }
+
+ return cur;
+}
+
+static struct ggml_tensor * llm_build_kv(
+ struct ggml_context * ctx,
+ struct llama_context & lctx,
+ const llama_kv_cache & kv,
+ struct ggml_cgraph * graph,
+ struct ggml_tensor * wo,
+ struct ggml_tensor * wo_b,
+ struct ggml_tensor * k_cur,
+ struct ggml_tensor * v_cur,
+ struct ggml_tensor * q_cur,
+ struct ggml_tensor * kq_mask,
+ int32_t n_tokens,
+ int32_t kv_head,
+ int32_t n_kv,
+ float kq_scale,
+ const llm_build_cb & cb,
+ int il) {
+ const llama_hparams & hparams = lctx.model.hparams;
+ const llama_cparams & cparams = lctx.cparams;
+
+ // these nodes are added to the graph together so that they are not reordered
+ // by doing so, the number of splits in the graph is reduced
+ ggml_build_forward_expand(graph, q_cur);
+ ggml_build_forward_expand(graph, k_cur);
+ ggml_build_forward_expand(graph, v_cur);
+
+ llm_build_kv_store(ctx, hparams, cparams, kv, graph, k_cur, v_cur, n_tokens, kv_head, cb, il);
+
+ struct ggml_tensor * cur;
+
+ cur = llm_build_kqv(ctx, lctx, kv, graph, wo, wo_b, q_cur, kq_mask, n_tokens, n_kv, kq_scale, cb, il);
+ cb(cur, "kqv_out", il);
+
+ return cur;
+}
+
+static struct ggml_tensor * llm_build_copy_mask_state(
+ struct ggml_context * ctx,
+ struct ggml_cgraph * graph,
+ struct ggml_tensor * s,
+ struct ggml_tensor * state_copy,
+ struct ggml_tensor * state_mask,
+ int32_t n_state,
+ int32_t kv_size,
+ int32_t kv_head,
+ int32_t n_kv,
+ int32_t n_seqs) {
+ struct ggml_tensor * states = ggml_reshape_2d(ctx, s, n_state, kv_size);
+
+ // copy states
+ // NOTE: assuming the copy destinations are ALL contained between kv_head and kv_head + n_kv
+ // this shrinks the tensors's ne[1] to n_kv
+ states = ggml_get_rows(ctx, states, state_copy);
+
+ // clear states of sequences which are starting at the beginning of this batch
+ // FIXME: zero-out NANs?
+ states = ggml_mul(ctx, states, state_mask);
+
+ // copy states which won't be changed further (between n_seqs and n_rs)
+ ggml_build_forward_expand(graph,
+ ggml_cpy(ctx,
+ ggml_view_1d(ctx, states, n_state*(n_kv - n_seqs), n_seqs*n_state*ggml_element_size(states)),
+ ggml_view_1d(ctx, s, n_state*(n_kv - n_seqs), (kv_head + n_seqs)*n_state*ggml_element_size(s))));
+
+ // the part of the states that will be used and modified
+ return ggml_view_2d(ctx, states, n_state, n_seqs, states->nb[1], 0);
+}
+
+// TODO: split
+static struct ggml_tensor * llm_build_mamba(
+ struct ggml_context * ctx,
+ struct llama_context & lctx,
+ const llama_ubatch & batch,
+ struct ggml_cgraph * graph,
+ struct ggml_tensor * cur,
+ struct ggml_tensor * state_copy,
+ struct ggml_tensor * state_mask,
+ int32_t kv_head,
+ int32_t n_kv,
+ const llm_build_cb & cb,
+ int il) {
+ const llama_model & model = lctx.model;
+ const llama_hparams & hparams = model.hparams;
+ const llama_kv_cache & kv = lctx.kv_self;
+ const int64_t d_conv = hparams.ssm_d_conv;
+ const int64_t d_inner = hparams.ssm_d_inner;
+ const int64_t d_state = hparams.ssm_d_state;
+ const int64_t dt_rank = hparams.ssm_dt_rank;
+ const int64_t n_seqs = batch.n_seqs;
+ // Some variants of Mamba arch (e.g. FalconMamba do apply layer norm on B and Dt layers)
+ const bool ssm_dt_b_c_rms = hparams.ssm_dt_b_c_rms;
+ // Use the same RMS norm as the final layer norm
+ const float norm_rms_eps = hparams.f_norm_rms_eps;
+
+ const int64_t n_seq_tokens = batch.n_seq_tokens;
+
+ GGML_ASSERT(n_seqs != 0);
+ GGML_ASSERT(batch.equal_seqs);
+ GGML_ASSERT(batch.n_tokens == n_seq_tokens * n_seqs);
+
+ struct ggml_tensor * conv_states_all = kv.k_l[il];
+ struct ggml_tensor * ssm_states_all = kv.v_l[il];
+
+ // (ab)using the KV cache to store the states
+ struct ggml_tensor * conv = llm_build_copy_mask_state(ctx,
+ graph, conv_states_all, state_copy, state_mask,
+ hparams.n_embd_k_s(), kv.size, kv_head, n_kv, n_seqs);
+ conv = ggml_reshape_3d(ctx, conv, d_conv - 1, d_inner, n_seqs);
+ struct ggml_tensor * ssm = llm_build_copy_mask_state(ctx,
+ graph, ssm_states_all, state_copy, state_mask,
+ hparams.n_embd_v_s(), kv.size, kv_head, n_kv, n_seqs);
+ ssm = ggml_reshape_3d(ctx, ssm, d_state, d_inner, n_seqs);
+
+ // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs}
+ cur = ggml_reshape_3d(ctx, cur, cur->ne[0], n_seq_tokens, n_seqs);
+
+ // {n_embd, 2*d_inner} @ {n_embd, n_seq_tokens, n_seqs} => {2*d_inner, n_seq_tokens, n_seqs}
+ struct ggml_tensor * xz = llm_build_lora_mm(lctx, ctx, model.layers[il].ssm_in, cur);
+ // split the above in two
+ // => {d_inner, n_seq_tokens, n_seqs}
+ struct ggml_tensor * x = ggml_view_3d(ctx, xz, d_inner, xz->ne[1], xz->ne[2], xz->nb[1], xz->nb[2], 0);
+ struct ggml_tensor * z = ggml_view_3d(ctx, xz, d_inner, xz->ne[1], xz->ne[2], xz->nb[1], xz->nb[2], d_inner*ggml_element_size(xz));
+
+ // conv
+ {
+ // => {d_conv - 1 + n_seq_tokens, d_inner, n_seqs}
+ struct ggml_tensor * conv_x = ggml_concat(ctx, conv, ggml_transpose(ctx, x), 0);
+
+ // copy last (d_conv - 1) columns back into the state cache
+ struct ggml_tensor * last_conv = ggml_view_3d(ctx, conv_x, d_conv - 1, d_inner, n_seqs, conv_x->nb[1], conv_x->nb[2], n_seq_tokens*(conv_x->nb[0]));
+
+ ggml_build_forward_expand(graph,
+ ggml_cpy(ctx, last_conv,
+ ggml_view_1d(ctx, conv_states_all,
+ (d_conv - 1)*(d_inner)*(n_seqs),
+ kv_head*(d_conv - 1)*(d_inner)*ggml_element_size(conv_states_all))));
+
+ // 1D convolution
+ // The equivalent is to make a self-overlapping view of conv_x
+ // over d_conv columns at each stride in the 3rd dimension,
+ // then element-wise multiply that with the conv1d weight,
+ // then sum the elements of each row,
+ // (the last two steps are a dot product over rows (also doable with mul_mat))
+ // then permute away the ne[0] dimension,
+ // and then you're left with the resulting x tensor.
+ // For simultaneous sequences, all sequences need to have the same length.
+ x = ggml_ssm_conv(ctx, conv_x, model.layers[il].ssm_conv1d);
+
+ // bias
+ x = ggml_add(ctx, x, model.layers[il].ssm_conv1d_b);
+
+ x = ggml_silu(ctx, x);
+ }
+
+ // ssm
+ {
+ // {d_inner, dt_rank + 2*d_state} @ {d_inner, n_seq_tokens, n_seqs} => {dt_rank + 2*d_state, n_seq_tokens, n_seqs}
+ struct ggml_tensor * x_db = llm_build_lora_mm(lctx, ctx, model.layers[il].ssm_x, x);
+ // split
+ struct ggml_tensor * dt = ggml_view_3d(ctx, x_db, dt_rank, n_seq_tokens, n_seqs, x_db->nb[1], x_db->nb[2], 0);
+ struct ggml_tensor * B = ggml_view_3d(ctx, x_db, d_state, n_seq_tokens, n_seqs, x_db->nb[1], x_db->nb[2], ggml_element_size(x_db)*dt_rank);
+ struct ggml_tensor * C = ggml_view_3d(ctx, x_db, d_state, n_seq_tokens, n_seqs, x_db->nb[1], x_db->nb[2], ggml_element_size(x_db)*(dt_rank+d_state));
+
+ // Some Mamba variants (e.g. FalconMamba) apply RMS norm in B, C & Dt layers
+ if (ssm_dt_b_c_rms) {
+ dt = ggml_rms_norm(ctx, dt, norm_rms_eps);
+ B = ggml_rms_norm(ctx, B, norm_rms_eps);
+ C = ggml_rms_norm(ctx, C, norm_rms_eps);
+ }
+
+ // {dt_rank, d_inner} @ {dt_rank, n_seq_tokens, n_seqs} => {d_inner, n_seq_tokens, n_seqs}
+ dt = llm_build_lora_mm(lctx, ctx, model.layers[il].ssm_dt, dt);
+ dt = ggml_add(ctx, dt, model.layers[il].ssm_dt_b);
+
+ // Custom operator to optimize the parallel associative scan
+ // as described in the Annex D of the Mamba paper.
+ // => {d_inner, n_seq_tokens, n_seqs} and {d_state, d_inner, n_seqs}
+ struct ggml_tensor * y_ssm = ggml_ssm_scan(ctx, ssm, x, dt, model.layers[il].ssm_a, B, C);
+
+ // store last states
+ ggml_build_forward_expand(graph,
+ ggml_cpy(ctx,
+ ggml_view_1d(ctx, y_ssm, d_state*d_inner*n_seqs, x->nb[3]),
+ ggml_view_1d(ctx, ssm_states_all, d_state*d_inner*n_seqs, kv_head*d_state*d_inner*ggml_element_size(ssm_states_all))));
+
+ struct ggml_tensor * y = ggml_view_3d(ctx, y_ssm, d_inner, n_seq_tokens, n_seqs, x->nb[1], x->nb[2], 0);
+
+ // TODO: skip computing output earlier for unused tokens
+
+ // {d_inner, n_seq_tokens, n_seqs} * {d_inner} => {d_inner, n_seq_tokens, n_seqs}
+ y = ggml_add(ctx, y, ggml_mul(ctx, x, model.layers[il].ssm_d));
+ y = ggml_mul(ctx, y, ggml_silu(ctx, ggml_cont(ctx, z)));
+
+ // {d_inner, n_embd} @ {d_inner, n_seq_tokens, n_seqs} => {n_embd, n_seq_tokens, n_seqs}
+ cur = llm_build_lora_mm(lctx, ctx, model.layers[il].ssm_out, y);
+ }
+
+ // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens}
+ cur = ggml_reshape_2d(ctx, cur, cur->ne[0], n_seq_tokens * n_seqs);
+ cb(cur, "mamba_out", il);
+
+ return cur;
+}
+
+static struct ggml_tensor * llm_build_rwkv6_time_mix(
+ struct llama_context & lctx,
+ struct ggml_context * ctx,
+ const struct llama_layer * layer,
+ struct ggml_tensor * cur,
+ struct ggml_tensor * x_prev,
+ struct ggml_tensor ** wkv_state) {
+ size_t n_embed = cur->ne[0];
+ size_t n_seq_tokens = cur->ne[1];
+ size_t n_seqs = cur->ne[2];
+
+ size_t head_size = layer->time_mix_first->ne[0];
+ size_t head_count = layer->time_mix_first->ne[1];
+
+ size_t n_tokens = n_seqs * n_seq_tokens;
+
+ struct ggml_tensor * sx = ggml_sub(ctx, x_prev, cur);
+
+ sx = ggml_reshape_2d(ctx, sx, n_embed, n_tokens);
+ cur = ggml_reshape_2d(ctx, cur, n_embed, n_tokens);
+
+ struct ggml_tensor * xxx = ggml_add(ctx, ggml_mul(ctx, sx, layer->time_mix_lerp_x), cur);
+
+ xxx = ggml_reshape_4d(
+ ctx,
+ ggml_tanh(
+ ctx,
+ ggml_mul_mat(ctx, layer->time_mix_w1, xxx)
+ ),
+ layer->time_mix_w1->ne[1] / 5, 1, 5, n_tokens
+ );
+
+ xxx = ggml_cont(ctx, ggml_permute(ctx, xxx, 0, 1, 3, 2));
+
+ xxx = ggml_mul_mat(
+ ctx,
+ ggml_reshape_4d(
+ ctx,
+ layer->time_mix_w2,
+ layer->time_mix_w2->ne[0], layer->time_mix_w2->ne[1], 1, 5
+ ),
+ xxx
+ );
+
+ struct ggml_tensor *mw = ggml_view_2d(ctx, xxx, n_embed, n_tokens, xxx->nb[1], 0);
+ struct ggml_tensor *mk = ggml_view_2d(ctx, xxx, n_embed, n_tokens, xxx->nb[1], n_embed * n_tokens * sizeof(float));
+ struct ggml_tensor *mv = ggml_view_2d(ctx, xxx, n_embed, n_tokens, xxx->nb[1], n_embed * n_tokens * 2 * sizeof(float));
+ struct ggml_tensor *mr = ggml_view_2d(ctx, xxx, n_embed, n_tokens, xxx->nb[1], n_embed * n_tokens * 3 * sizeof(float));
+ struct ggml_tensor *mg = ggml_view_2d(ctx, xxx, n_embed, n_tokens, xxx->nb[1], n_embed * n_tokens * 4 * sizeof(float));
+
+ struct ggml_tensor * xw = ggml_add(
+ ctx,
+ ggml_mul(
+ ctx,
+ ggml_add(ctx, mw, layer->time_mix_lerp_w),
+ sx
+ ),
+ cur
+ );
+
+ struct ggml_tensor * xk = ggml_add(
+ ctx,
+ ggml_mul(
+ ctx,
+ ggml_add(ctx, mk, layer->time_mix_lerp_k),
+ sx
+ ),
+ cur
+ );
+
+ struct ggml_tensor * xv = ggml_add(
+ ctx,
+ ggml_mul(
+ ctx,
+ ggml_add(ctx, mv, layer->time_mix_lerp_v),
+ sx
+ ),
+ cur
+ );
+
+ struct ggml_tensor * xr = ggml_add(
+ ctx,
+ ggml_mul(
+ ctx,
+ ggml_add(ctx, mr, layer->time_mix_lerp_r),
+ sx
+ ),
+ cur
+ );
+
+ struct ggml_tensor * xg = ggml_add(
+ ctx,
+ ggml_mul(
+ ctx,
+ ggml_add(ctx, mg, layer->time_mix_lerp_g),
+ sx
+ ),
+ cur
+ );
+
+ struct ggml_tensor * r = ggml_reshape_4d(ctx, llm_build_lora_mm(lctx, ctx, layer->time_mix_receptance, xr), head_size, 1, head_count, n_tokens);
+ struct ggml_tensor * k = ggml_reshape_4d(ctx, llm_build_lora_mm(lctx, ctx, layer->time_mix_key, xk), 1, head_size, head_count, n_tokens);
+ struct ggml_tensor * v = ggml_reshape_4d(ctx, llm_build_lora_mm(lctx, ctx, layer->time_mix_value, xv), head_size, 1, head_count, n_tokens);
+ struct ggml_tensor * g = ggml_silu(
+ ctx,
+ llm_build_lora_mm(lctx, ctx, layer->time_mix_gate, xg)
+ );
+
+ struct ggml_tensor * w = ggml_mul_mat(
+ ctx,
+ layer->time_mix_decay_w2,
+ ggml_tanh(
+ ctx,
+ ggml_mul_mat(ctx, layer->time_mix_decay_w1, xw)
+ )
+ );
+
+ w = ggml_add(ctx, w, ggml_reshape_1d(ctx, layer->time_mix_decay, n_embed));
+ w = ggml_exp(ctx, ggml_neg(ctx, ggml_exp(ctx, w)));
+ w = ggml_reshape_4d(ctx, w, 1, head_size, head_count, n_tokens);
+
+ k = ggml_transpose(ctx, k);
+ v = ggml_transpose(ctx, v);
+ r = ggml_transpose(ctx, r);
+
+ struct ggml_tensor * wkv_output = ggml_rwkv_wkv(ctx, k, v, r, layer->time_mix_first, w, *wkv_state);
+ cur = ggml_view_1d(ctx, wkv_output, n_embed * n_tokens, 0);
+ *wkv_state = ggml_view_1d(ctx, wkv_output, n_embed * head_size * n_seqs, n_embed * n_tokens * sizeof(float));
+
+ // group norm with head_count groups
+ cur = ggml_reshape_3d(ctx, cur, n_embed / head_count, head_count, n_tokens);
+ cur = ggml_norm(ctx, cur, 64e-5f);
+
+ // Convert back to regular vectors.
+ cur = ggml_reshape_2d(ctx, cur, n_embed, n_tokens);
+ cur = ggml_add(ctx, ggml_mul(ctx, cur, layer->time_mix_ln), layer->time_mix_ln_b);
+
+ cur = ggml_mul(ctx, cur, g);
+ cur = llm_build_lora_mm(lctx, ctx, layer->time_mix_output, cur);
+
+ return ggml_reshape_3d(ctx, cur, n_embed, n_seq_tokens, n_seqs);
+}
+
+static struct ggml_tensor * llm_build_rwkv6_channel_mix(
+ struct llama_context & lctx,
+ struct ggml_context * ctx,
+ const struct llama_layer * layer,
+ struct ggml_tensor * cur,
+ struct ggml_tensor * x_prev) {
+ struct ggml_tensor * sx = ggml_sub(ctx, x_prev, cur);
+ struct ggml_tensor * xk = ggml_add(ctx, ggml_mul(ctx, sx, layer->channel_mix_lerp_k), cur);
+ struct ggml_tensor * xr = ggml_add(ctx, ggml_mul(ctx, sx, layer->channel_mix_lerp_r), cur);
+
+ struct ggml_tensor * r = ggml_sigmoid(ctx, llm_build_lora_mm(lctx, ctx, layer->channel_mix_receptance, xr));
+ struct ggml_tensor * k = ggml_sqr(
+ ctx,
+ ggml_relu(
+ ctx,
+ llm_build_lora_mm(lctx, ctx, layer->channel_mix_key, xk)
+ )
+ );
+
+ return ggml_mul(ctx, r, llm_build_lora_mm(lctx, ctx, layer->channel_mix_value, k));
+}
+
+struct llm_build_context {
+ const llama_model & model;
+ llama_context & lctx;
+ const llama_hparams & hparams;
+ const llama_cparams & cparams;
+ const llama_ubatch & batch;
+ const llama_kv_cache & kv_self;
+
+ const int64_t n_embd;
+ const int64_t n_layer;
+ const int64_t n_rot;
+ const int64_t n_ctx; // user-specified context size (can be different from n_ctx_train)
+ const int64_t n_head;
+ const int64_t n_head_kv;
+ const int64_t n_embd_head_k;
+ const int64_t n_embd_k_gqa;
+ const int64_t n_embd_head_v;
+ const int64_t n_embd_v_gqa;
+ const int64_t n_expert;
+ const int64_t n_expert_used;
+
+ const float freq_base;
+ const float freq_scale;
+ const float ext_factor;
+ const float attn_factor;
+ const float beta_fast;
+ const float beta_slow;
+ const float norm_eps;
+ const float norm_rms_eps;
+
+ const int32_t n_tokens;
+ const int32_t n_kv; // size of KV cache to consider (n_kv <= kv_self.size)
+ const int32_t n_outputs;
+ const int32_t n_outputs_enc;
+ const int32_t kv_head; // index of where we store new KV data in the cache
+ const int32_t n_ctx_orig;
+
+ const bool flash_attn;
+
+ const enum llama_pooling_type pooling_type;
+ const enum llama_rope_type rope_type;
+
+ const llm_build_cb & cb;
+
+ std::vector<uint8_t> & buf_compute_meta;
+
+ struct ggml_context * ctx0 = nullptr;
+
+ // TODO: consider making the entire interface noexcept
+ llm_build_context(
+ llama_context & lctx,
+ const llama_ubatch & batch,
+ const llm_build_cb & cb,
+ bool worst_case) :
+ model (lctx.model),
+ lctx (lctx),
+ hparams (model.hparams),
+ cparams (lctx.cparams),
+ batch (batch),
+ kv_self (lctx.kv_self),
+ n_embd (hparams.n_embd),
+ n_layer (hparams.n_layer),
+ n_rot (hparams.n_rot),
+ n_ctx (cparams.n_ctx),
+ n_head (hparams.n_head()),
+ n_head_kv (hparams.n_head_kv()),
+ n_embd_head_k (hparams.n_embd_head_k),
+ n_embd_k_gqa (hparams.n_embd_k_gqa()),
+ n_embd_head_v (hparams.n_embd_head_v),
+ n_embd_v_gqa (hparams.n_embd_v_gqa()),
+ n_expert (hparams.n_expert),
+ n_expert_used (hparams.n_expert_used),
+ freq_base (cparams.rope_freq_base),
+ freq_scale (cparams.rope_freq_scale),
+ ext_factor (cparams.yarn_ext_factor),
+ attn_factor (cparams.yarn_attn_factor),
+ beta_fast (cparams.yarn_beta_fast),
+ beta_slow (cparams.yarn_beta_slow),
+ norm_eps (hparams.f_norm_eps),
+ norm_rms_eps (hparams.f_norm_rms_eps),
+ n_tokens (batch.n_tokens),
+ n_kv (worst_case ? kv_self.size : kv_self.n),
+ n_outputs (worst_case ? n_tokens : lctx.n_outputs),
+ n_outputs_enc (worst_case ? n_tokens : lctx.embd_enc.size() / hparams.n_embd),
+ kv_head (worst_case ? (kv_self.recurrent ? 0 : kv_self.size - n_tokens) : kv_self.head),
+ n_ctx_orig (cparams.n_ctx_orig_yarn),
+ flash_attn (cparams.flash_attn),
+ pooling_type (cparams.pooling_type),
+ rope_type (hparams.rope_type),
+ cb (cb),
+ buf_compute_meta (lctx.buf_compute_meta) {
+ // all initializations should be done in init()
+ }
+
+ void init() {
+ struct ggml_init_params params = {
+ /*.mem_size =*/ buf_compute_meta.size(),
+ /*.mem_buffer =*/ buf_compute_meta.data(),
+ /*.no_alloc =*/ true,
+ };
+
+ ctx0 = ggml_init(params);
+
+ lctx.inp_tokens = nullptr;
+ lctx.inp_embd = nullptr;
+ lctx.inp_pos = nullptr;
+ lctx.inp_out_ids = nullptr;
+ lctx.inp_KQ_mask = nullptr;
+ lctx.inp_KQ_mask_swa = nullptr;
+ lctx.inp_K_shift = nullptr;
+ lctx.inp_mean = nullptr;
+ lctx.inp_cls = nullptr;
+ lctx.inp_s_copy = nullptr;
+ lctx.inp_s_mask = nullptr;
+ lctx.inp_s_seq = nullptr;
+ lctx.inp_pos_bucket = nullptr;
+ lctx.inp_embd_enc = nullptr;
+ lctx.inp_KQ_mask_cross = nullptr;
+ }
+
+ void free() {
+ if (ctx0) {
+ ggml_free(ctx0);
+ ctx0 = nullptr;
+ }
+ }
+
+ struct ggml_cgraph * build_k_shift() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ GGML_ASSERT(kv_self.size == n_ctx);
+
+ lctx.inp_K_shift = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_ctx);
+ cb(lctx.inp_K_shift, "K_shift", -1);
+ ggml_set_input(lctx.inp_K_shift);
+
+ for (int il = 0; il < n_layer; ++il) {
+ const int64_t n_head_kv = hparams.n_head_kv(il);
+ const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
+ struct ggml_tensor * rope_factors = build_rope_factors(il);
+ struct ggml_tensor * tmp =
+ // we rotate only the first n_rot dimensions
+ ggml_rope_ext_inplace(ctx0,
+ ggml_view_3d(ctx0, kv_self.k_l[il],
+ n_embd_head_k, n_head_kv, n_ctx,
+ ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k),
+ ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),
+ 0),
+ lctx.inp_K_shift, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+
+ cb(tmp, "K_shifted", il);
+ ggml_build_forward_expand(gf, tmp);
+ }
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_defrag(const std::vector<uint32_t> & ids) {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ for (uint32_t i = 0; i < ids.size(); ++i) {
+ const uint32_t id = ids[i];
+
+ if (i == id || id == ids.size()) {
+ continue;
+ }
+
+ uint32_t nm = 1;
+
+ while (i + nm < ids.size() && ids[i + nm] == id + nm) {
+ nm++;
+ }
+
+ for (int il = 0; il < n_layer; ++il) {
+ const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
+ const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);
+
+ ggml_tensor * view_k_src = ggml_view_2d(ctx0, kv_self.k_l[il],
+ n_embd_k_gqa, nm,
+ ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),
+ ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*i));
+
+ ggml_tensor * view_k_dst = ggml_view_2d(ctx0, kv_self.k_l[il],
+ n_embd_k_gqa, nm,
+ ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),
+ ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*id));
+
+ ggml_tensor * view_v_src;
+ ggml_tensor * view_v_dst;
+
+ if (flash_attn) {
+ // NOTE: the V cache is not transposed when using flash attention
+ view_v_src = ggml_view_2d(ctx0, kv_self.v_l[il],
+ n_embd_v_gqa, nm,
+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa),
+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*i));
+
+ view_v_dst = ggml_view_2d(ctx0, kv_self.v_l[il],
+ n_embd_v_gqa, nm,
+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa),
+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*id));
+ } else {
+ view_v_src = ggml_view_2d(ctx0, kv_self.v_l[il],
+ nm, n_embd_v_gqa,
+ ggml_row_size(kv_self.v_l[il]->type, kv_self.size),
+ ggml_row_size(kv_self.v_l[il]->type, i));
+
+ view_v_dst = ggml_view_2d(ctx0, kv_self.v_l[il],
+ nm, n_embd_v_gqa,
+ ggml_row_size(kv_self.v_l[il]->type, kv_self.size),
+ ggml_row_size(kv_self.v_l[il]->type, id));
+ }
+
+ ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_k_src, view_k_dst));
+ ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_v_src, view_v_dst));
+ }
+
+ i += nm - 1;
+ }
+
+ //LLAMA_LOG_INFO("gf->n_nodes = %d\n", gf->n_nodes);
+
+ return gf;
+ }
+
+ struct ggml_tensor * build_inp_pos() {
+ lctx.inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
+ cb(lctx.inp_pos, "inp_pos", -1);
+ ggml_set_input(lctx.inp_pos);
+ return lctx.inp_pos;
+ }
+
+ struct ggml_tensor * build_rope_factors(int il) {
+ // choose long/short freq factors based on the context size
+ const auto n_ctx_pre_seq = cparams.n_ctx / cparams.n_seq_max;
+
+ if (model.layers[il].rope_freqs != nullptr) {
+ return model.layers[il].rope_freqs;
+ }
+
+ if (n_ctx_pre_seq > hparams.n_ctx_orig_yarn) {
+ return model.layers[il].rope_long;
+ }
+
+ return model.layers[il].rope_short;
+ }
+
+ struct ggml_tensor * build_inp_out_ids() {
+ lctx.inp_out_ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_outputs);
+ cb(lctx.inp_out_ids, "inp_out_ids", -1);
+ ggml_set_input(lctx.inp_out_ids);
+ return lctx.inp_out_ids;
+ }
+
+ struct ggml_tensor * build_inp_KQ_mask(bool causal = true) {
+ lctx.inp_KQ_mask = causal
+ ? ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD))
+ : ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD));
+ cb(lctx.inp_KQ_mask, "KQ_mask", -1);
+ ggml_set_input(lctx.inp_KQ_mask);
+
+ return flash_attn ? ggml_cast(ctx0, lctx.inp_KQ_mask, GGML_TYPE_F16) : lctx.inp_KQ_mask;
+ }
+
+ struct ggml_tensor * build_inp_KQ_mask_swa(bool causal = true) {
+ GGML_ASSERT(hparams.n_swa > 0);
+
+ lctx.inp_KQ_mask_swa = causal
+ ? ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD))
+ : ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD));
+ cb(lctx.inp_KQ_mask_swa, "KQ_mask_swa", -1);
+ ggml_set_input(lctx.inp_KQ_mask_swa);
+
+ return flash_attn ? ggml_cast(ctx0, lctx.inp_KQ_mask_swa, GGML_TYPE_F16) : lctx.inp_KQ_mask_swa;
+ }
+
+ struct ggml_tensor * build_inp_mean() {
+ lctx.inp_mean = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens);
+ cb(lctx.inp_mean, "inp_mean", -1);
+ ggml_set_input(lctx.inp_mean);
+ return lctx.inp_mean;
+ }
+
+ struct ggml_tensor * build_inp_cls() {
+ lctx.inp_cls = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
+ cb(lctx.inp_cls, "inp_cls", -1);
+ ggml_set_input(lctx.inp_cls);
+ return lctx.inp_cls;
+ }
+
+ struct ggml_tensor * build_inp_s_copy() {
+ lctx.inp_s_copy = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_kv);
+ cb(lctx.inp_s_copy, "inp_s_copy", -1);
+ ggml_set_input(lctx.inp_s_copy);
+ return lctx.inp_s_copy;
+ }
+
+ struct ggml_tensor * build_inp_s_mask() {
+ lctx.inp_s_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_kv);
+ cb(lctx.inp_s_mask, "inp_s_mask", -1);
+ ggml_set_input(lctx.inp_s_mask);
+ return lctx.inp_s_mask;
+ }
+
+ struct ggml_cgraph * append_pooling(struct ggml_cgraph * gf) {
+ // find result_norm tensor for input
+ struct ggml_tensor * inp = nullptr;
+ for (int i = gf->n_nodes - 1; i >= 0; --i) {
+ inp = gf->nodes[i];
+ if (strcmp(inp->name, "result_norm") == 0 || strcmp(inp->name, "result_embd") == 0) {
+ break;
+ } else {
+ inp = nullptr;
+ }
+ }
+ GGML_ASSERT(inp != nullptr && "missing result_norm/result_embd tensor");
+
+ struct ggml_tensor * cur;
+
+ switch (pooling_type) {
+ case LLAMA_POOLING_TYPE_MEAN:
+ {
+ struct ggml_tensor * inp_mean = build_inp_mean();
+ cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, inp)), inp_mean);
+ } break;
+ case LLAMA_POOLING_TYPE_CLS:
+ case LLAMA_POOLING_TYPE_LAST:
+ {
+ struct ggml_tensor * inp_cls = build_inp_cls();
+ cur = ggml_get_rows(ctx0, inp, inp_cls);
+ } break;
+ case LLAMA_POOLING_TYPE_NONE:
+ {
+ cur = inp;
+ } break;
+ default:
+ {
+ GGML_ABORT("unknown pooling type");
+ }
+ }
+
+ cb(cur, "result_embd_pooled", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_tensor * llm_build_pos_bucket(bool causal) {
+ if (causal) {
+ lctx.inp_pos_bucket = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_kv, n_tokens);
+ } else {
+ lctx.inp_pos_bucket = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_tokens, n_tokens);
+ }
+
+ ggml_set_input(lctx.inp_pos_bucket);
+ cb(lctx.inp_pos_bucket, "pos_bucket", -1);
+
+ return lctx.inp_pos_bucket;
+ }
+
+ struct ggml_tensor * llm_build_pos_bias(struct ggml_tensor * pos_bucket, struct ggml_tensor * attn_rel_b) {
+ struct ggml_tensor * pos_bucket_1d = ggml_view_1d(ctx0, pos_bucket, pos_bucket->ne[0] * pos_bucket->ne[1], 0);
+ cb(pos_bucket_1d, "pos_bucket_1d", -1);
+
+ struct ggml_tensor * pos_bias = ggml_get_rows(ctx0, attn_rel_b, pos_bucket_1d);
+ cb(pos_bias, "pos_bias", -1);
+
+ pos_bias = ggml_view_3d(ctx0, pos_bias, pos_bias->ne[0], lctx.inp_pos_bucket->ne[0], lctx.inp_pos_bucket->ne[1], ggml_element_size(pos_bias) * pos_bias->ne[0], ggml_element_size(pos_bias) * pos_bias->ne[0] * lctx.inp_pos_bucket->ne[0], 0);
+ cb(pos_bias, "pos_bias", -1);
+
+ pos_bias = ggml_permute(ctx0, pos_bias, 2, 0, 1, 3);
+ cb(pos_bias, "pos_bias", -1);
+
+ pos_bias = ggml_cont(ctx0, pos_bias);
+ cb(pos_bias, "pos_bias", -1);
+
+ return pos_bias;
+ }
+
+ struct ggml_tensor * llm_build_inp_embd_enc() {
+ const int64_t n_embd = hparams.n_embd;
+ lctx.inp_embd_enc = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_outputs_enc);
+ ggml_set_input(lctx.inp_embd_enc);
+ cb(lctx.inp_embd_enc, "embd_enc", -1);
+ return lctx.inp_embd_enc;
+ }
+
+ struct ggml_tensor * llm_build_inp_KQ_mask_cross() {
+ lctx.inp_KQ_mask_cross = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_outputs_enc, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD));
+ ggml_set_input(lctx.inp_KQ_mask_cross);
+ cb(lctx.inp_KQ_mask_cross, "KQ_mask_cross", -1);
+ return lctx.inp_KQ_mask_cross;
+ }
+
+ struct ggml_cgraph * build_llama() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // rope freq factors for llama3; may return nullptr for llama2 and other models
+ struct ggml_tensor * rope_factors = build_rope_factors(il);
+
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ if (model.layers[il].ffn_gate_inp == nullptr) {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ } else {
+ // MoE branch
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_moe_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_gate_inp,
+ model.layers[il].ffn_up_exps,
+ model.layers[il].ffn_gate_exps,
+ model.layers[il].ffn_down_exps,
+ n_expert, n_expert_used,
+ LLM_FFN_SILU, true,
+ false, 0.0,
+ cb, il);
+ cb(cur, "ffn_moe_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_baichuan() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = model.type == MODEL_7B ? build_inp_pos() : nullptr;
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+
+ switch (model.type) {
+ case MODEL_7B:
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ break;
+ case MODEL_13B:
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd/n_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd/n_head, n_head, n_tokens);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_xverse() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams, model.output_norm, NULL, LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_falcon() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * attn_norm;
+
+ attn_norm = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(attn_norm, "attn_norm", il);
+
+ // self-attention
+ {
+ if (model.layers[il].attn_norm_2) {
+ // Falcon-40B
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm_2,
+ model.layers[il].attn_norm_2_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm_2", il);
+ } else {
+ cur = attn_norm;
+ }
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+
+ // using mode = 2 for neox mode
+ Qcur = ggml_rope_ext(
+ ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ attn_norm = ggml_get_rows(ctx0, attn_norm, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = cur;
+
+ // feed forward
+ {
+ cur = llm_build_ffn(ctx0, lctx, attn_norm, // !! use the attn norm, not the result
+ model.layers[il].ffn_up, NULL, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = ggml_add(ctx0, cur, inpL);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm,
+ model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_grok() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // multiply by embedding_multiplier_scale of 78.38367176906169
+ inpL = ggml_scale(ctx0, inpL, 78.38367176906169f);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ // Grok
+ // if attn_out_norm is present then apply it before adding the input
+ if (model.layers[il].attn_out_norm) {
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.layers[il].attn_out_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_out_norm", il);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ // MoE branch
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_moe_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_gate_inp,
+ model.layers[il].ffn_up_exps,
+ model.layers[il].ffn_gate_exps,
+ model.layers[il].ffn_down_exps,
+ n_expert, n_expert_used,
+ LLM_FFN_GELU, true,
+ false, 0.0,
+ cb, il);
+ cb(cur, "ffn_moe_out", il);
+
+ // Grok
+ // if layer_out_norm is present then apply it before adding the input
+ // Idea: maybe ffn_out_norm is a better name
+ if (model.layers[il].layer_out_norm) {
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.layers[il].layer_out_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "layer_out_norm", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+
+ // Grok
+ // multiply logits by output_multiplier_scale of 0.5773502691896257
+
+ cur = ggml_scale(ctx0, cur, 0.5773502691896257f);
+
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_dbrx() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ struct ggml_tensor * Qcur = nullptr;
+ struct ggml_tensor * Kcur = nullptr;
+ struct ggml_tensor * Vcur = nullptr;
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ cur = ggml_clamp(ctx0, cur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
+ cb(cur, "wqkv_clamped", il);
+
+ Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ // MoE branch
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].attn_out_norm, NULL,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_out_norm", il);
+
+ cur = llm_build_moe_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_gate_inp,
+ model.layers[il].ffn_up_exps,
+ model.layers[il].ffn_gate_exps,
+ model.layers[il].ffn_down_exps,
+ n_expert, n_expert_used,
+ LLM_FFN_SILU, true,
+ false, 0.0,
+ cb, il);
+ cb(cur, "ffn_moe_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_starcoder() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ struct ggml_tensor * pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);
+ cb(pos, "pos_embd", -1);
+
+ inpL = ggml_add(ctx0, inpL, pos);
+ cb(inpL, "inpL", -1);
+
+ for (int il = 0; il < n_layer; ++il) {
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
+ cb(cur, "bqkv", il);
+
+ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ // add the input
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // FF
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm,
+ model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.output_norm,
+ model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_refact() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+ cb(Kcur, "Kcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ cb(Qcur, "Qcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_bert() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+ struct ggml_tensor * inp_pos = nullptr;
+
+ if (model.arch != LLM_ARCH_JINA_BERT_V2) {
+ inp_pos = build_inp_pos();
+ }
+
+ // construct input embeddings (token, type, position)
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // token types are hardcoded to zero ("Sentence A")
+ struct ggml_tensor * type_row0 = ggml_view_1d(ctx0, model.type_embd, n_embd, 0);
+ inpL = ggml_add(ctx0, inpL, type_row0);
+ if (model.arch == LLM_ARCH_BERT) {
+ inpL = ggml_add(ctx0, ggml_get_rows(ctx0, model.pos_embd, inp_pos), inpL);
+ }
+ cb(inpL, "inp_embd", -1);
+
+ // embed layer norm
+ inpL = llm_build_norm(ctx0, inpL, hparams, model.tok_norm, model.tok_norm_b, LLM_NORM, cb, -1);
+ cb(inpL, "inp_norm", -1);
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask(false);
+
+ // iterate layers
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * cur = inpL;
+
+ struct ggml_tensor * Qcur;
+ struct ggml_tensor * Kcur;
+ struct ggml_tensor * Vcur;
+
+ // self-attention
+ if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT_V2) {
+ Qcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur), model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+
+ if (model.layers[il].attn_q_norm) {
+ Qcur = llm_build_norm(ctx0, Qcur, hparams,
+ model.layers[il].attn_q_norm,
+ model.layers[il].attn_q_norm_b,
+ LLM_NORM, cb, il);
+ }
+
+ Kcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur), model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+
+ if (model.layers[il].attn_k_norm) {
+ Kcur = llm_build_norm(ctx0, Kcur, hparams,
+ model.layers[il].attn_k_norm,
+ model.layers[il].attn_k_norm_b,
+ LLM_NORM, cb, il);
+ }
+ Vcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur), model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+ } else {
+ // compute Q and K and RoPE them
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+ struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));
+
+ struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
+ cb(kq, "kq", il);
+
+ kq = ggml_soft_max_ext(ctx0, kq, KQ_mask, 1.0f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias);
+ cb(kq, "kq_soft_max_ext", il);
+
+ struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens)));
+ cb(v, "v", il);
+
+ struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_tokens, n_embd_head, n_head_kv), kq);
+ cb(kqv, "kqv", il);
+
+ struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
+ cb(kqv_merged, "kqv_merged", il);
+
+ cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
+ cb(cur, "kqv_merged_cont", il);
+
+ ggml_build_forward_expand(gf, cur);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo, cur);
+ if (model.layers[il].bo) {
+ cb(cur, "kqv_wo", il);
+ }
+
+ if (model.layers[il].bo) {
+ cur = ggml_add(ctx0, cur, model.layers[il].bo);
+ }
+ cb(cur, "kqv_out", il);
+
+ if (il == n_layer - 1 && pooling_type == LLAMA_POOLING_TYPE_NONE) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ // re-add the layer input
+ cur = ggml_add(ctx0, cur, inpL);
+
+ // attention layer norm
+ cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_out_norm, model.layers[il].attn_out_norm_b, LLM_NORM, cb, il);
+
+ if (model.layers[il].attn_norm_2 != nullptr) {
+ cur = ggml_add(ctx0, cur, inpL); // re-add the layer input
+ cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_norm_2, model.layers[il].attn_norm_2_b, LLM_NORM, cb, il);
+ }
+
+ struct ggml_tensor * ffn_inp = cur;
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ if (model.arch == LLM_ARCH_BERT) {
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
+ } else if (model.arch == LLM_ARCH_JINA_BERT_V2) {
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_PAR, cb, il);
+ } else {
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ }
+ cb(cur, "ffn_out", il);
+
+ // attentions bypass the intermediate layer
+ cur = ggml_add(ctx0, cur, ffn_inp);
+
+ // output layer norm
+ cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].layer_out_norm, model.layers[il].layer_out_norm_b, LLM_NORM, cb, il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ // final output
+ cur = inpL;
+ cb(cur, "result_embd", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_bloom() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ inpL = llm_build_norm(ctx0, inpL, hparams,
+ model.tok_norm,
+ model.tok_norm_b,
+ LLM_NORM, cb, -1);
+ cb(inpL, "inp_norm", -1);
+
+ for (int il = 0; il < n_layer; ++il) {
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
+ cb(cur, "bqkv", il);
+
+ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ // Add the input
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // FF
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm,
+ model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.output_norm,
+ model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_mpt() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * pos;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ if (model.pos_embd) {
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+ pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);
+ cb(pos, "pos_embd", -1);
+
+ inpL = ggml_add(ctx0, inpL, pos);
+ cb(inpL, "inpL", -1);
+ }
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * attn_norm;
+
+ attn_norm = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(attn_norm, "attn_norm", il);
+
+ // self-attention
+ {
+ cur = attn_norm;
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ if (model.layers[il].bqkv){
+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
+ cb(cur, "bqkv", il);
+ }
+
+ if (hparams.f_clamp_kqv > 0.0f) {
+ cur = ggml_clamp(ctx0, cur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
+ cb(cur, "wqkv_clamped", il);
+ }
+
+ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ // Q/K Layernorm
+ if (model.layers[il].attn_q_norm) {
+ Qcur = llm_build_norm(ctx0, Qcur, hparams,
+ model.layers[il].attn_q_norm,
+ model.layers[il].attn_q_norm_b,
+ LLM_NORM, cb, il);
+ cb(Qcur, "Qcur", il);
+
+ Kcur = llm_build_norm(ctx0, Kcur, hparams,
+ model.layers[il].attn_k_norm,
+ model.layers[il].attn_k_norm_b,
+ LLM_NORM, cb, il);
+ cb(Kcur, "Kcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ } else {
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ // Add the input
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed forward
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm,
+ model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ model.layers[il].ffn_act,
+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm,
+ model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_stablelm() {
+ struct ggml_cgraph * gf = ggml_new_graph(ctx0);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ struct ggml_tensor * inpSA = cur;
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ cb(Qcur, "Qcur", il);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+ cb(Kcur, "Kcur", il);
+
+ if (model.layers[il].attn_q_norm) {
+ Qcur = llm_build_norm(ctx0, Qcur, hparams,
+ model.layers[il].attn_q_norm,
+ NULL,
+ LLM_NORM, cb, il);
+ cb(Qcur, "Qcur", il);
+ }
+ if (model.layers[il].attn_k_norm) {
+ Kcur = llm_build_norm(ctx0, Kcur, hparams,
+ model.layers[il].attn_k_norm,
+ NULL,
+ LLM_NORM, cb, il);
+ cb(Kcur, "Kcur", il);
+ }
+
+
+ Qcur = ggml_rope_ext(
+ ctx0, Qcur, inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, Kcur, inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ {
+ if (model.layers[il].ffn_norm) {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm,
+ model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+ } else {
+ // parallel residual
+ cur = inpSA;
+ }
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm,
+ model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_qwen() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
+ cb(cur, "bqkv", il);
+
+ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 2*sizeof(float)*(n_embd)));
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+
+ // using mode = 2 for neox mode
+ Qcur = ggml_rope_ext(
+ ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward forward
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_qwen2() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_qwen2moe() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self_attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // MoE branch
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ ggml_tensor * moe_out =
+ llm_build_moe_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_gate_inp,
+ model.layers[il].ffn_up_exps,
+ model.layers[il].ffn_gate_exps,
+ model.layers[il].ffn_down_exps,
+ n_expert, n_expert_used,
+ LLM_FFN_SILU, false,
+ false, 0.0,
+ cb, il);
+ cb(cur, "ffn_moe_out", il);
+
+ // FFN shared expert
+ {
+ ggml_tensor * cur_gate_inp = llm_build_lora_mm(lctx, ctx0, model.layers[il].ffn_gate_inp_shexp, cur);
+ cb(cur_gate_inp, "ffn_shexp_gate_inp", il);
+
+ // sigmoid
+ ggml_tensor * cur_gate = ggml_div(ctx0, ggml_silu(ctx0, cur_gate_inp), cur_gate_inp);
+ cb(cur_gate, "ffn_shexp_gate", il);
+
+ ggml_tensor * cur_ffn = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up_shexp, NULL, NULL,
+ model.layers[il].ffn_gate_shexp, NULL, NULL,
+ model.layers[il].ffn_down_shexp, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur_ffn, "ffn_shexp", il);
+
+ ggml_tensor * ffn_shexp_out = ggml_mul(ctx0, cur_ffn, cur_gate);
+ cb(ffn_shexp_out, "ffn_shexp_out", il);
+
+ moe_out = ggml_add(ctx0, moe_out, ffn_shexp_out);
+ cb(moe_out, "ffn_out", il);
+
+ cur = moe_out;
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_phi2() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * attn_norm_output;
+ struct ggml_tensor * ffn_output;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ attn_norm_output = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(attn_norm_output, "attn_norm", il);
+
+ // self-attention
+ {
+ struct ggml_tensor * Qcur = nullptr;
+ struct ggml_tensor * Kcur = nullptr;
+ struct ggml_tensor * Vcur = nullptr;
+
+ if (model.layers[il].wqkv) {
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, attn_norm_output);
+ cb(cur, "wqkv", il);
+
+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
+ cb(cur, "bqkv", il);
+
+ Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+ } else {
+ Qcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, attn_norm_output), model.layers[il].bq);
+ Kcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, attn_norm_output), model.layers[il].bk);
+ Vcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, attn_norm_output), model.layers[il].bv);
+ }
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+
+ Qcur = ggml_rope_ext(
+ ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ // with phi2, we scale the Q to avoid precision issues
+ // ref: https://github.com/ml-explore/mlx-examples/blob/08e862336ade809bc37d1035f94b359e7d1a5152/phi2/phi2.py#L64-L66
+ Qcur = ggml_scale(ctx0, Qcur, 1.0f/sqrtf(float(n_embd_head)));
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ attn_norm_output = ggml_get_rows(ctx0, attn_norm_output, inp_out_ids);
+ }
+
+ // FF
+ {
+ ffn_output = llm_build_ffn(ctx0, lctx, attn_norm_output,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
+ cb(ffn_output, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_output);
+ cur = ggml_add(ctx0, cur, inpL);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.output_norm,
+ model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output_no_bias", -1);
+
+ cur = ggml_add(ctx0, cur, model.output_b);
+ cb(cur, "result_output", -1);
+ ggml_build_forward_expand(gf, cur);
+ return gf;
+ }
+
+ struct ggml_cgraph * build_phi3() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask_swa = build_inp_KQ_mask_swa();
+
+ for (int il = 0; il < n_layer; ++il) {
+ auto residual = inpL;
+
+ // self-attention
+ {
+ // rope freq factors for 128k context
+ struct ggml_tensor * rope_factors = build_rope_factors(il);
+
+ struct ggml_tensor* attn_norm_output = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(attn_norm_output, "attn_norm", il);
+
+ struct ggml_tensor * Qcur = nullptr;
+ struct ggml_tensor * Kcur = nullptr;
+ struct ggml_tensor * Vcur = nullptr;
+
+ if (model.layers[il].wqkv) {
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, attn_norm_output);
+ cb(cur, "wqkv", il);
+
+ Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0 * sizeof(float) * (n_embd)));
+ Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1 * sizeof(float) * (n_embd)));
+ Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa)));
+ }
+ else {
+ Qcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, attn_norm_output), model.layers[il].bq);
+ Kcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, attn_norm_output), model.layers[il].bk);
+ Vcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, attn_norm_output), model.layers[il].bv);
+ }
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+
+ Qcur = ggml_rope_ext(
+ ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head)));
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask_swa, n_tokens, kv_head, n_kv, 1.0f, cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor* inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ residual = ggml_get_rows(ctx0, residual, inp_out_ids);
+ }
+
+ cur = ggml_add(ctx0, cur, residual);
+ residual = cur;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ // FF
+ // special-case: the up and gate tensors are merged into a single tensor
+ // TOOD: support into llm_build_ffn
+ {
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SWIGLU, LLM_FFN_SEQ, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, residual, cur);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.output_norm,
+ NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+
+ struct ggml_cgraph * build_plamo() {
+ struct ggml_cgraph * gf = ggml_new_graph(ctx0);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ struct ggml_tensor * attention_norm = cur;
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens), inp_pos, nullptr,
+ n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+ struct ggml_tensor * sa_out = cur;
+
+ cur = attention_norm;
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ sa_out = ggml_get_rows(ctx0, sa_out, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ // feed-forward network
+ {
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, sa_out);
+ cur = ggml_add(ctx0, cur, inpL);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_gpt2() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * pos;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);
+ cb(pos, "pos_embd", -1);
+
+ inpL = ggml_add(ctx0, inpL, pos);
+ cb(inpL, "inpL", -1);
+
+ for (int il = 0; il < n_layer; ++il) {
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
+ cb(cur, "bqkv", il);
+
+ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ // add the input
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // FF
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm,
+ model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.output_norm,
+ model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_codeshell() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
+ cb(cur, "bqkv", il);
+
+ struct ggml_tensor * tmpq = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ struct ggml_tensor * tmpk = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+
+ cb(tmpq, "tmpq", il);
+ cb(tmpk, "tmpk", il);
+ cb(Vcur, "Vcur", il);
+
+ struct ggml_tensor * Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ // add the input
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // FF
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm,
+ model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.output_norm,
+ model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_orion() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ // if (model.layers[il].bq) {
+ // Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ // cb(Qcur, "Qcur", il);
+ // }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ // if (model.layers[il].bk) {
+ // Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ // cb(Kcur, "Kcur", il);
+ // }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ // if (model.layers[il].bv) {
+ // Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ // cb(Vcur, "Vcur", il);
+ // }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_internlm2() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ // ref: https://arxiv.org/abs/2203.03466
+ // https://github.com/ggerganov/llama.cpp/issues/5276#issuecomment-1925774738
+ // based on the original build_llama() function
+ struct ggml_cgraph * build_minicpm() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ const int64_t n_embd = hparams.n_embd;
+ //TODO: if the model varies, these parameters need to be read from the model
+ const int64_t n_embd_base = 256;
+ const float scale_embd = 12.0f;
+ const float scale_depth = 1.4f;
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // scale the input embeddings
+ inpL = ggml_scale(ctx0, inpL, scale_embd);
+ cb(inpL, "inp_scaled", -1);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ // scale_res - scale the hidden states for residual connection
+ const float scale_res = scale_depth/sqrtf(float(n_layer));
+ cur = ggml_scale(ctx0, cur, scale_res);
+ cb(cur, "hidden_scaled", -1);
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ // scale the hidden states for residual connection
+ cur = ggml_scale(ctx0, cur, scale_res);
+ cb(cur, "hidden_scaled_ffn", -1);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head scaling
+ const float scale_lmhead = float(n_embd_base)/float(n_embd);
+ cur = ggml_scale(ctx0, cur, scale_lmhead);
+ cb(cur, "lmhead_scaling", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_gemma() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head_k = hparams.n_embd_head_k;
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd));
+ cb(inpL, "inp_scaled", -1);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(Qcur, "Qcur", il);
+
+ Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k)));
+ cb(Qcur, "Qcur_scaled", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL);
+ cb(sa_out, "sa_out", il);
+
+ cur = llm_build_norm(ctx0, sa_out, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ // feed-forward network
+ {
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, sa_out);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_gemma2() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head_k = hparams.n_embd_head_k;
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd));
+ cb(inpL, "inp_scaled", -1);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ // gemma 2 requires different mask for layers using sliding window (SWA)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask(true);
+ struct ggml_tensor * KQ_mask_swa = build_inp_KQ_mask_swa(true);
+
+ for (int il = 0; il < n_layer; ++il) {
+ // (il % 2) layers use SWA
+ struct ggml_tensor * KQ_mask_l = (il % 2 == 0) ? KQ_mask_swa : KQ_mask;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(Qcur, "Qcur", il);
+
+ // ref: https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e
+ switch (model.type) {
+ case e_model::MODEL_2B:
+ case e_model::MODEL_9B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); break;
+ case e_model::MODEL_27B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd / n_head))); break;
+ default: GGML_ABORT("fatal error");
+ };
+ cb(Qcur, "Qcur_scaled", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask_l, n_tokens, kv_head, n_kv, 1.0f, cb, il);
+ }
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.layers[il].attn_post_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_post_norm", il);
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL);
+ cb(sa_out, "sa_out", il);
+
+ cur = llm_build_norm(ctx0, sa_out, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ // feed-forward network
+ {
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.layers[il].ffn_post_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "ffn_post_norm", -1);
+
+ cur = ggml_add(ctx0, cur, sa_out);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+
+ // final logit soft-capping
+ cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);
+ cur = ggml_tanh(ctx0, cur);
+ cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);
+
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+
+ struct ggml_cgraph * build_starcoder2() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_mamba() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ // {n_embd, n_tokens}
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ struct ggml_tensor * state_copy = build_inp_s_copy();
+ struct ggml_tensor * state_mask = build_inp_s_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ cur = llm_build_mamba(ctx0, lctx, batch, gf, cur,
+ state_copy, state_mask,
+ kv_head, n_kv, cb, il);
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ // residual
+ cur = ggml_add(ctx0, cur, inpL);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ // final rmsnorm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_command_r() {
+
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ const float f_logit_scale = hparams.f_logit_scale;
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+ struct ggml_tensor * ffn_inp = cur;
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ if (model.layers[il].attn_q_norm) {
+ Qcur = ggml_view_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens,
+ ggml_element_size(Qcur) * n_embd_head,
+ ggml_element_size(Qcur) * n_embd_head * n_head,
+ 0);
+ cb(Qcur, "Qcur", il);
+ Kcur = ggml_view_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens,
+ ggml_element_size(Kcur) * n_embd_head,
+ ggml_element_size(Kcur) * n_embd_head * n_head_kv,
+ 0);
+ cb(Kcur, "Kcur", il);
+
+ Qcur = llm_build_norm(ctx0, Qcur, hparams,
+ model.layers[il].attn_q_norm,
+ NULL,
+ LLM_NORM, cb, il);
+ cb(Qcur, "Qcur", il);
+
+ Kcur = llm_build_norm(ctx0, Kcur, hparams,
+ model.layers[il].attn_k_norm,
+ NULL,
+ LLM_NORM, cb, il);
+ cb(Kcur, "Kcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids);
+ }
+
+ struct ggml_tensor * attn_out = cur;
+
+ // feed-forward network
+ {
+ cur = llm_build_ffn(ctx0, lctx, ffn_inp,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ // add together residual + FFN + self-attention
+ cur = ggml_add(ctx0, cur, inpL);
+ cur = ggml_add(ctx0, cur, attn_out);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+
+ if (f_logit_scale) {
+ cur = ggml_scale(ctx0, cur, f_logit_scale);
+ }
+
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+
+ }
+
+ // ref: https://allenai.org/olmo
+ // based on the original build_llama() function, changes:
+ // * non-parametric layer norm
+ // * clamp qkv
+ // * removed bias
+ // * removed MoE
+ struct ggml_cgraph * build_olmo() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ NULL, NULL,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (hparams.f_clamp_kqv > 0.0f) {
+ Qcur = ggml_clamp(ctx0, Qcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (hparams.f_clamp_kqv > 0.0f) {
+ Kcur = ggml_clamp(ctx0, Kcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (hparams.f_clamp_kqv > 0.0f) {
+ Vcur = ggml_clamp(ctx0, Vcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, nullptr,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ NULL, NULL,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ NULL, NULL,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_openelm() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ const int64_t n_head = hparams.n_head(il);
+ const int64_t n_head_kv = hparams.n_head_kv(il);
+ const int64_t n_head_qkv = 2*n_head_kv + n_head;
+
+ cur = inpL;
+ struct ggml_tensor * residual = cur;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ cur = ggml_reshape_3d(ctx0, cur, n_embd_head_k, n_head_qkv, n_tokens);
+
+ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, cur->nb[1], cur->nb[2], 0));
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, cur->nb[1], cur->nb[2], cur->nb[1]*n_head));
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, cur->nb[1], cur->nb[2], cur->nb[1]*(n_head+n_head_kv)));
+ cb(Vcur, "Vcur", il);
+
+ Qcur = llm_build_norm(ctx0, Qcur, hparams,
+ model.layers[il].attn_q_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(Qcur, "Qcur", il);
+
+ Kcur = llm_build_norm(ctx0, Kcur, hparams,
+ model.layers[il].attn_k_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(Kcur, "Kcur", il);
+
+ Qcur = ggml_rope_ext(
+ ctx0, Qcur, inp_pos, NULL, n_rot, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, Kcur, inp_pos, NULL, n_rot, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ Vcur = ggml_reshape_2d(ctx0, Vcur, n_embd_head * n_head_kv, n_tokens);
+ cb(Qcur, "Vcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ residual = ggml_get_rows(ctx0, residual, inp_out_ids);
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, residual, cur);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_gptneox() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
+ cb(cur, "bqkv", il);
+
+ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ // ffn
+ if (hparams.use_par_res) {
+ // attention and ffn are computed in parallel
+ // x = x + attn(ln1(x)) + ffn(ln2(x))
+
+ struct ggml_tensor * attn_out = cur;
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].ffn_norm,
+ model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, inpL);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, attn_out);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ } else {
+ // attention and ffn are computed sequentially
+ // x = x + attn(ln1(x))
+ // x = x + ffn(ln2(x))
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
+ cb(ffn_inp, "ffn_inp", il);
+
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm,
+ model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+ }
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.output_norm,
+ model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_arctic() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+
+ struct ggml_tensor * ffn_out = ggml_add(ctx0, cur, ffn_inp);
+ cb(ffn_out, "ffn_out", il);
+
+ // MoE
+ cur = llm_build_norm(ctx0, inpSA, hparams,
+ model.layers[il].ffn_norm_exps, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm_exps", il);
+
+ cur = llm_build_moe_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_gate_inp,
+ model.layers[il].ffn_up_exps,
+ model.layers[il].ffn_gate_exps,
+ model.layers[il].ffn_down_exps,
+ n_expert, n_expert_used,
+ LLM_FFN_SILU, true,
+ false, 0.0,
+ cb, il);
+ cb(cur, "ffn_moe_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_out);
+ cb(cur, "ffn_out", il);
+
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_deepseek2() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ bool is_lite = (hparams.n_layer == 27);
+
+ // We have to pre-scale kq_scale and attn_factor to make the YaRN RoPE work correctly.
+ // See https://github.com/ggerganov/llama.cpp/discussions/7416 for detailed explanation.
+ const float mscale = attn_factor * (1.0f + hparams.rope_yarn_log_mul * logf(1.0f / freq_scale));
+ const float kq_scale = 1.0f*mscale*mscale/sqrtf(float(hparams.n_embd_head_k));
+ const float attn_factor_scaled = 1.0f / (1.0f + 0.1f * logf(1.0f / freq_scale));
+
+ const uint32_t n_embd_head_qk_rope = hparams.n_rot;
+ const uint32_t n_embd_head_qk_nope = hparams.n_embd_head_k - hparams.n_rot;
+ const uint32_t kv_lora_rank = hparams.n_lora_kv;
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ // {n_embd, n_tokens}
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self_attention
+ {
+ struct ggml_tensor * q = NULL;
+ if (!is_lite) {
+ // {n_embd, q_lora_rank} * {n_embd, n_tokens} -> {q_lora_rank, n_tokens}
+ q = ggml_mul_mat(ctx0, model.layers[il].wq_a, cur);
+ cb(q, "q", il);
+
+ q = llm_build_norm(ctx0, q, hparams,
+ model.layers[il].attn_q_a_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(q, "q", il);
+
+ // {q_lora_rank, n_head * hparams.n_embd_head_k} * {q_lora_rank, n_tokens} -> {n_head * hparams.n_embd_head_k, n_tokens}
+ q = ggml_mul_mat(ctx0, model.layers[il].wq_b, q);
+ cb(q, "q", il);
+ } else {
+ q = ggml_mul_mat(ctx0, model.layers[il].wq, cur);
+ cb(q, "q", il);
+ }
+
+ // split into {n_head * n_embd_head_qk_nope, n_tokens}
+ struct ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
+ ggml_row_size(q->type, hparams.n_embd_head_k),
+ ggml_row_size(q->type, hparams.n_embd_head_k * n_head),
+ 0);
+ cb(q_nope, "q_nope", il);
+
+ // and {n_head * n_embd_head_qk_rope, n_tokens}
+ struct ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
+ ggml_row_size(q->type, hparams.n_embd_head_k),
+ ggml_row_size(q->type, hparams.n_embd_head_k * n_head),
+ ggml_row_size(q->type, n_embd_head_qk_nope));
+ cb(q_pe, "q_pe", il);
+
+ // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens}
+ struct ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
+ cb(kv_pe_compresseed, "kv_pe_compresseed", il);
+
+ // split into {kv_lora_rank, n_tokens}
+ struct ggml_tensor * kv_compressed = ggml_view_2d(ctx0, kv_pe_compresseed, kv_lora_rank, n_tokens,
+ kv_pe_compresseed->nb[1],
+ 0);
+ cb(kv_compressed, "kv_compressed", il);
+
+ // and {n_embd_head_qk_rope, n_tokens}
+ struct ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_pe_compresseed, n_embd_head_qk_rope, 1, n_tokens,
+ kv_pe_compresseed->nb[1],
+ kv_pe_compresseed->nb[1],
+ ggml_row_size(kv_pe_compresseed->type, kv_lora_rank));
+ cb(k_pe, "k_pe", il);
+
+ kv_compressed = ggml_cont(ctx0, kv_compressed); // TODO: the CUDA backend does not support non-contiguous norm
+ kv_compressed = llm_build_norm(ctx0, kv_compressed, hparams,
+ model.layers[il].attn_kv_a_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(kv_compressed, "kv_compressed", il);
+
+ // {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v)} * {kv_lora_rank, n_tokens} -> {n_head * (n_embd_head_qk_nope + n_embd_head_v), n_tokens}
+ struct ggml_tensor * kv = ggml_mul_mat(ctx0, model.layers[il].wkv_b, kv_compressed);
+ cb(kv, "kv", il);
+
+ // split into {n_head * n_embd_head_qk_nope, n_tokens}
+ struct ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens,
+ ggml_row_size(kv->type, n_embd_head_qk_nope + hparams.n_embd_head_v),
+ ggml_row_size(kv->type, n_head * (n_embd_head_qk_nope + hparams.n_embd_head_v)),
+ 0);
+ cb(k_nope, "k_nope", il);
+
+ // and {n_head * n_embd_head_v, n_tokens}
+ struct ggml_tensor * v_states = ggml_view_3d(ctx0, kv, hparams.n_embd_head_v, n_head, n_tokens,
+ ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)),
+ ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)*n_head),
+ ggml_row_size(kv->type, (n_embd_head_qk_nope)));
+ cb(v_states, "v_states", il);
+
+ v_states = ggml_cont(ctx0, v_states);
+ cb(v_states, "v_states", il);
+
+ v_states = ggml_view_2d(ctx0, v_states, hparams.n_embd_head_v * n_head, n_tokens,
+ ggml_row_size(kv->type, hparams.n_embd_head_v * n_head),
+ 0);
+ cb(v_states, "v_states", il);
+
+ q_pe = ggml_cont(ctx0, q_pe); // TODO: the CUDA backend does not support non-contiguous RoPE
+ q_pe = ggml_rope_ext(
+ ctx0, q_pe, inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor_scaled, beta_fast, beta_slow
+ );
+ cb(q_pe, "q_pe", il);
+
+ // shared RoPE key
+ k_pe = ggml_cont(ctx0, k_pe); // TODO: the CUDA backend does not support non-contiguous RoPE
+ k_pe = ggml_rope_ext(
+ ctx0, k_pe, inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor_scaled, beta_fast, beta_slow
+ );
+ cb(k_pe, "k_pe", il);
+
+ struct ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0);
+ cb(q_states, "q_states", il);
+
+ struct ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);
+ cb(k_states, "k_states", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ k_states, v_states, q_states, KQ_mask, n_tokens, kv_head, n_kv, kq_scale, cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ if ((uint32_t) il < hparams.n_layer_dense_lead) {
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ } else {
+ // MoE branch
+ ggml_tensor * moe_out =
+ llm_build_moe_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_gate_inp,
+ model.layers[il].ffn_up_exps,
+ model.layers[il].ffn_gate_exps,
+ model.layers[il].ffn_down_exps,
+ n_expert, n_expert_used,
+ LLM_FFN_SILU, false,
+ true, hparams.expert_weights_scale,
+ cb, il);
+ cb(moe_out, "ffn_moe_out", il);
+
+ // FFN shared expert
+ {
+ ggml_tensor * ffn_shexp = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up_shexp, NULL, NULL,
+ model.layers[il].ffn_gate_shexp, NULL, NULL,
+ model.layers[il].ffn_down_shexp, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(ffn_shexp, "ffn_shexp", il);
+
+ cur = ggml_add(ctx0, moe_out, ffn_shexp);
+ cb(cur, "ffn_out", il);
+ }
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = ggml_mul_mat(ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_bitnet() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ Qcur = ggml_mul(ctx0, Qcur, model.layers[il].wq_scale);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ // B1.K
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ Kcur = ggml_mul(ctx0, Kcur, model.layers[il].wk_scale);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ // B1.V
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ Vcur = ggml_mul(ctx0, Vcur, model.layers[il].wv_scale);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ NULL, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.layers[il].attn_sub_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_sub_norm", il);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo, cur);
+ cur = ggml_mul(ctx0, cur, model.layers[il].wo_scale);
+ if (model.layers[il].bo) {
+ cur = ggml_add(ctx0, cur, model.layers[il].bo);
+ }
+ cb(cur, "attn_o_out", il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward forward
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_scale,
+ model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_scale,
+ NULL, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_sub_out", il);
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.layers[il].ffn_sub_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_sub_norm", il);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].ffn_down, cur);
+ cur = ggml_mul(ctx0, cur, model.layers[il].ffn_down_scale);
+ cb(cur, "ffn_down", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.tok_embd, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+ return gf;
+ }
+
+ struct ggml_cgraph * build_t5_encoder() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ GGML_ASSERT(lctx.is_encoding);
+ struct ggml_tensor * pos_bucket_enc = llm_build_pos_bucket(false);
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask_enc = build_inp_KQ_mask(false);
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm_enc, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq_enc, cur);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk_enc, cur);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv_enc, cur);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+
+ struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+ struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));
+
+ struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
+ cb(kq, "kq", il);
+
+ struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b_enc ? model.layers[il].attn_rel_b_enc : model.layers[0].attn_rel_b_enc;
+ struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_enc, attn_rel_b);
+ struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias);
+ cb(kq_b, "kq_b", il);
+
+ kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_enc, 1.0f, hparams.f_max_alibi_bias);
+ cb(kq, "kq_soft_max_ext", il);
+
+ struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens)));
+ cb(v, "v", il);
+
+ struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_tokens, n_embd_head, n_head_kv), kq);
+ cb(kqv, "kqv", il);
+
+ struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
+ cb(kqv_merged, "kqv_merged", il);
+
+ cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
+ cb(cur, "kqv_merged_cont", il);
+
+ ggml_build_forward_expand(gf, cur);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo_enc, cur);
+ cb(cur, "kqv_out", il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm_enc, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ // T5 uses relu, flan-T5 uses gelu-gated
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up_enc, NULL, NULL,
+ model.layers[il].ffn_gate_enc, NULL, NULL,
+ model.layers[il].ffn_down_enc, NULL, NULL,
+ NULL,
+ model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,
+ model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,
+ cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
+ if (layer_dir != nullptr) {
+ cur = ggml_add(ctx0, cur, layer_dir);
+ }
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+ cb(cur, "result_embd", -1);
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm_enc, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_t5_decoder() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ GGML_ASSERT(!lctx.is_encoding);
+ GGML_ASSERT(n_outputs_enc > 0 && "call llama_encode() first");
+
+ struct ggml_tensor * embd_enc = llm_build_inp_embd_enc();
+ struct ggml_tensor * pos_bucket_dec = llm_build_pos_bucket(true);
+
+ struct ggml_tensor * KQ_mask_dec = build_inp_KQ_mask();
+ struct ggml_tensor * KQ_mask_cross = llm_build_inp_KQ_mask_cross();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+
+ llm_build_kv_store(ctx0, hparams, cparams, kv_self, gf, Kcur, Vcur, n_tokens, kv_head, cb, il);
+
+ struct ggml_tensor * k =
+ ggml_view_3d(ctx0, kv_self.k_l[il],
+ n_embd_head_k, n_kv, n_head_kv,
+ ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),
+ ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k),
+ 0);
+ cb(k, "k", il);
+
+ struct ggml_tensor * v =
+ ggml_view_3d(ctx0, kv_self.v_l[il],
+ n_kv, n_embd_head_v, n_head_kv,
+ ggml_element_size(kv_self.v_l[il])*n_ctx,
+ ggml_element_size(kv_self.v_l[il])*n_ctx*n_embd_head_v,
+ 0);
+ cb(v, "v", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+
+ struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+
+ struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
+ cb(kq, "kq", il);
+
+ struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b;
+ struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_dec, attn_rel_b);
+ struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias);
+ cb(kq_b, "kq_b", il);
+
+ kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_dec, 1.0f, hparams.f_max_alibi_bias);
+ cb(kq, "kq_soft_max_ext", il);
+
+ struct ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);
+ cb(kqv, "kqv", il);
+
+ struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
+ cb(kqv_merged, "kqv_merged", il);
+
+ cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
+ cb(cur, "kqv_merged_cont", il);
+
+ ggml_build_forward_expand(gf, cur);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo, cur);
+ cb(cur, "kqv_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, inpSA);
+ cb(cur, "cross_inp", il);
+
+ struct ggml_tensor * inpCA = cur;
+
+ // norm
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.layers[il].attn_norm_cross, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm_cross", il);
+
+ // cross-attention
+ {
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq_cross, cur);
+ cb(Qcur, "Qcur", il);
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk_cross, embd_enc);
+ cb(Kcur, "Kcur", il);
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv_cross, embd_enc);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_outputs_enc);
+
+ struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+ struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));
+
+ struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
+ cb(kq, "kq", il);
+
+ kq = ggml_soft_max_ext(ctx0, kq, KQ_mask_cross, 1.0f, hparams.f_max_alibi_bias);
+ cb(kq, "kq_soft_max_ext", il);
+
+ struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_outputs_enc)));
+ cb(v, "v", il);
+
+ struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_outputs_enc, n_embd_head, n_head_kv), kq);
+ cb(kqv, "kqv", il);
+
+ struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
+ cb(kqv_merged, "kqv_merged", il);
+
+ cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
+ cb(cur, "kqv_merged_cont", il);
+
+ ggml_build_forward_expand(gf, cur);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo_cross, cur);
+ cb(cur, "kqv_out", il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpCA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ // T5 uses relu, flan-T5 uses gelu-gated
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,
+ model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,
+ cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
+ if (layer_dir != nullptr) {
+ cur = ggml_add(ctx0, cur, layer_dir);
+ }
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+ cb(cur, "result_embd", -1);
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_jais() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
+ cb(cur, "bqkv", il);
+
+ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*cur->nb[0]*(n_embd)));
+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*cur->nb[0]*(n_embd)));
+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*cur->nb[0]*(n_embd + n_embd_gqa)));
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/float(n_embd_head), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ // add the input
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // FF
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm,
+ model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ inpL = ggml_add(ctx0, cur, ffn_inp);
+ cb(inpL, "l_out", il);
+ }
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.output_norm,
+ model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_chatglm() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ struct ggml_tensor * Qcur = nullptr;
+ struct ggml_tensor * Kcur = nullptr;
+ struct ggml_tensor * Vcur = nullptr;
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+
+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
+ cb(cur, "bqkv", il);
+
+ Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+ //printf("freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\n", freq_base, freq_scale, ext_factor, attn_factor);
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur_rope", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur_rope", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ // Add the input
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // FF
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm,
+ NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SWIGLU, LLM_FFN_SEQ, cb, il);
+ cb(cur, "ffn_out", il);
+
+ }
+
+ inpL = ggml_add(ctx0, cur, ffn_inp);
+ cb(inpL, "l_out", il);
+ }
+
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.output_norm,
+ NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_nemotron() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ //GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm,
+ model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_RELU_SQR, LLM_FFN_SEQ, cb, il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_exaone() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // rope freq factors for llama3; may return nullptr for llama2 and other models
+ struct ggml_tensor * rope_factors = build_rope_factors(il);
+
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ ggml_cgraph * build_rwkv6() {
+ ggml_cgraph *gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // Token shift state dimensions should be 2 * n_emb
+ GGML_ASSERT(n_embd == hparams.n_embd_k_s() / 2);
+
+ const int64_t n_seqs = batch.n_seqs;
+ const int64_t n_seq_tokens = batch.n_seq_tokens;
+ const int64_t n_tokens = batch.n_tokens;
+ GGML_ASSERT(n_seqs != 0);
+ GGML_ASSERT(batch.equal_seqs);
+ GGML_ASSERT(n_tokens == n_seq_tokens * n_seqs);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+ struct ggml_tensor * state_copy = build_inp_s_copy();
+ struct ggml_tensor * state_mask = build_inp_s_mask();
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+ inpL = llm_build_norm(ctx0, inpL, hparams, model.tok_norm, model.tok_norm_b, LLM_NORM, cb, -1);
+
+ for (int il = 0; il < n_layer; ++il) {
+ const llama_layer * layer = &model.layers[il];
+
+ // (ab)using the KV cache to store the states
+ struct ggml_tensor * token_shift = llm_build_copy_mask_state(ctx0,
+ gf, kv_self.k_l[il], state_copy, state_mask,
+ hparams.n_embd_k_s(), kv_self.size, kv_head, n_kv, n_seqs);
+ struct ggml_tensor * wkv_states = llm_build_copy_mask_state(ctx0,
+ gf, kv_self.v_l[il], state_copy, state_mask,
+ hparams.n_embd_v_s(), kv_self.size, kv_head, n_kv, n_seqs);
+
+ cur = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs);
+ token_shift = ggml_reshape_3d(ctx0, token_shift, n_embd, 2, n_seqs);
+
+ struct ggml_tensor * att_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], 0);
+ struct ggml_tensor * ffn_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], n_embd * ggml_element_size(token_shift));
+
+ struct ggml_tensor * x_norm_att = llm_build_norm(ctx0, cur, hparams, layer->attn_norm, layer->attn_norm_b, LLM_NORM, cb, il);
+ struct ggml_tensor * x_prev = ggml_concat(
+ ctx0,
+ att_shift,
+ ggml_view_3d(ctx0, x_norm_att, n_embd, n_seq_tokens - 1, n_seqs, x_norm_att->nb[1], x_norm_att->nb[2], 0),
+ 1
+ );
+
+ cur = ggml_add(ctx0, cur, llm_build_rwkv6_time_mix(lctx, ctx0, layer, x_norm_att, x_prev, &wkv_states));
+ ggml_build_forward_expand(gf, cur);
+ ggml_build_forward_expand(
+ gf,
+ ggml_cpy(
+ ctx0,
+ wkv_states,
+ ggml_view_1d(
+ ctx0,
+ kv_self.v_l[il],
+ hparams.n_embd_v_s() * n_seqs,
+ hparams.n_embd_v_s() * kv_head * ggml_element_size(kv_self.v_l[il])
+ )
+ )
+ );
+
+ struct ggml_tensor * x_norm_ffn = llm_build_norm(ctx0, cur, hparams, layer->attn_norm_2, layer->attn_norm_2_b, LLM_NORM, cb, il);
+ x_prev = ggml_concat(
+ ctx0,
+ ffn_shift,
+ ggml_view_3d(ctx0, x_norm_ffn, n_embd, n_seq_tokens - 1, n_seqs, x_norm_ffn->nb[1], x_norm_ffn->nb[2], 0),
+ 1
+ );
+ cur = ggml_add(ctx0, cur, llm_build_rwkv6_channel_mix(lctx, ctx0, layer, x_norm_ffn, x_prev));
+ ggml_build_forward_expand(gf, cur);
+
+ struct ggml_tensor * last_norm_att = ggml_view_3d(ctx0, x_norm_att, n_embd, 1, n_seqs, x_norm_att->nb[1], x_norm_att->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(x_norm_att));
+ struct ggml_tensor * last_norm_ffn = ggml_view_3d(ctx0, x_norm_ffn, n_embd, 1, n_seqs, x_norm_ffn->nb[1], x_norm_ffn->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(x_norm_ffn));
+
+ token_shift = ggml_concat(ctx0, last_norm_att, last_norm_ffn, 1);
+
+ ggml_build_forward_expand(
+ gf,
+ ggml_cpy(
+ ctx0,
+ ggml_view_1d(ctx0, token_shift, n_embd * n_seqs * 2, 0),
+ ggml_view_1d(ctx0, kv_self.k_l[il], hparams.n_embd_k_s() * n_seqs, hparams.n_embd_k_s() * kv_head * ggml_element_size(kv_self.k_l[il]))
+ )
+ );
+
+ if (hparams.rescale_every_n_layers != 0 && (il + 1) % hparams.rescale_every_n_layers == 0) {
+ cur = ggml_scale(ctx0, cur, 0.5F);
+ }
+
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens);
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+
+ cur = llm_build_norm(ctx0, cur, hparams, model.output_norm, model.output_norm_b, LLM_NORM, cb, -1);
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+
+ cb(cur, "result_output", -1);
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ ggml_cgraph * build_solar() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ struct ggml_tensor * bskcn_1;
+ struct ggml_tensor * bskcn_2;
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ if (hparams.n_bskcn(0, il)) {
+ bskcn_1 = inpSA;
+ }
+
+ if (hparams.n_bskcn(1, il)) {
+ bskcn_2 = inpSA;
+ }
+
+ if (hparams.n_bskcn(2, il)) {
+ inpSA = ggml_add(
+ ctx0,
+ ggml_mul(ctx0, bskcn_1, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, 0)),
+ ggml_mul(ctx0, inpSA, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, ggml_element_size(model.layers[il].bskcn_tv))));
+ }
+
+ if (hparams.n_bskcn(3, il)) {
+ inpSA = ggml_add(
+ ctx0,
+ ggml_mul(ctx0, bskcn_2, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, 0)),
+ ggml_mul(ctx0, inpSA, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, ggml_element_size(model.layers[il].bskcn_tv))));
+ }
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // rope freq factors for llama3; may return nullptr for llama2 and other models
+ struct ggml_tensor * rope_factors = build_rope_factors(il);
+
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+};
+
+static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector<uint32_t> & ids) {
+ llama_ubatch dummy = {};
+ dummy.equal_seqs = true;
+
+ llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { };
+
+ struct llm_build_context llm(lctx, dummy, cb, false);
+
+ llm.init();
+
+ struct ggml_cgraph * result = llm.build_defrag(ids);
+
+ llm.free();
+
+ return result;
+}
+
+static struct ggml_cgraph * llama_build_graph_k_shift(llama_context & lctx) {
+ llama_ubatch dummy = {};
+ dummy.equal_seqs = true;
+
+ llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { };
+
+ struct llm_build_context llm(lctx, dummy, cb, false);
+
+ llm.init();
+
+ struct ggml_cgraph * result = llm.build_k_shift();
+
+ llm.free();
+
+ return result;
+}
+
+static struct ggml_cgraph * llama_build_graph(
+ llama_context & lctx,
+ const llama_ubatch & batch,
+ bool worst_case) {
+ const auto & model = lctx.model;
+
+ // this callback allows us to apply custom logic to each tensor (e.g. ggml-alloc, offloading, etc.)
+ llm_build_cb cb = [&](struct ggml_tensor * cur, const char * name, int il) {
+ if (il >= 0) {
+ ggml_format_name(cur, "%s-%d", name, il);
+ } else {
+ ggml_set_name(cur, name);
+ }
+
+ if (!lctx.cparams.offload_kqv) {
+ if (strcmp(name, "kqv_merged_cont") == 0) {
+ // all nodes between the KV store and the attention output are run on the CPU
+ ggml_backend_sched_set_tensor_backend(lctx.sched, cur, lctx.backend_cpu);
+ }
+ }
+
+ // norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends
+ // FIXME: fix in ggml_backend_sched
+ const bool full_offload = lctx.model.n_gpu_layers > (int)lctx.model.hparams.n_layer;
+ if (batch.n_tokens < 32 || full_offload) {
+ if (il != -1 && strcmp(name, "norm") == 0) {
+ for (auto * backend : lctx.backends) {
+ if (ggml_backend_supports_buft(backend, lctx.model.buft_layer[il].buft) &&
+ (ggml_backend_supports_op(backend, cur) || ggml_backend_offload_op(backend, cur))) {
+ ggml_backend_sched_set_tensor_backend(lctx.sched, cur, backend);
+ break;
+ }
+ }
+ }
+ }
+ };
+
+ struct ggml_cgraph * result = NULL;
+
+ struct llm_build_context llm(lctx, batch, cb, worst_case);
+
+ llm.init();
+
+ switch (model.arch) {
+ case LLM_ARCH_LLAMA:
+ {
+ result = llm.build_llama();
+ } break;
+ case LLM_ARCH_BAICHUAN:
+ {
+ result = llm.build_baichuan();
+ } break;
+ case LLM_ARCH_FALCON:
+ {
+ result = llm.build_falcon();
+ } break;
+ case LLM_ARCH_GROK:
+ {
+ result = llm.build_grok();
+ } break;
+ case LLM_ARCH_STARCODER:
+ {
+ result = llm.build_starcoder();
+ } break;
+ case LLM_ARCH_REFACT:
+ {
+ result = llm.build_refact();
+ } break;
+ case LLM_ARCH_BERT:
+ case LLM_ARCH_JINA_BERT_V2:
+ case LLM_ARCH_NOMIC_BERT:
+ {
+ result = llm.build_bert();
+ } break;
+ case LLM_ARCH_BLOOM:
+ {
+ result = llm.build_bloom();
+ } break;
+ case LLM_ARCH_MPT:
+ {
+ result = llm.build_mpt();
+ } break;
+ case LLM_ARCH_STABLELM:
+ {
+ result = llm.build_stablelm();
+ } break;
+ case LLM_ARCH_QWEN:
+ {
+ result = llm.build_qwen();
+ } break;
+ case LLM_ARCH_QWEN2:
+ {
+ result = llm.build_qwen2();
+ } break;
+ case LLM_ARCH_QWEN2MOE:
+ {
+ result = llm.build_qwen2moe();
+ } break;
+ case LLM_ARCH_PHI2:
+ {
+ result = llm.build_phi2();
+ } break;
+ case LLM_ARCH_PHI3:
+ {
+ result = llm.build_phi3();
+ } break;
+ case LLM_ARCH_PLAMO:
+ {
+ result = llm.build_plamo();
+ } break;
+ case LLM_ARCH_GPT2:
+ {
+ result = llm.build_gpt2();
+ } break;
+ case LLM_ARCH_CODESHELL:
+ {
+ result = llm.build_codeshell();
+ } break;
+ case LLM_ARCH_ORION:
+ {
+ result = llm.build_orion();
+ } break;
+ case LLM_ARCH_INTERNLM2:
+ {
+ result = llm.build_internlm2();
+ } break;
+ case LLM_ARCH_MINICPM:
+ {
+ result = llm.build_minicpm();
+ } break;
+ case LLM_ARCH_GEMMA:
+ {
+ result = llm.build_gemma();
+ } break;
+ case LLM_ARCH_GEMMA2:
+ {
+ result = llm.build_gemma2();
+ } break;
+ case LLM_ARCH_STARCODER2:
+ {
+ result = llm.build_starcoder2();
+ } break;
+ case LLM_ARCH_MAMBA:
+ {
+ result = llm.build_mamba();
+ } break;
+ case LLM_ARCH_XVERSE:
+ {
+ result = llm.build_xverse();
+ } break;
+ case LLM_ARCH_COMMAND_R:
+ {
+ result = llm.build_command_r();
+ } break;
+ case LLM_ARCH_DBRX:
+ {
+ result = llm.build_dbrx();
+ } break;
+ case LLM_ARCH_OLMO:
+ {
+ result = llm.build_olmo();
+ } break;
+ case LLM_ARCH_OPENELM:
+ {
+ result = llm.build_openelm();
+ } break;
+ case LLM_ARCH_GPTNEOX:
+ {
+ result = llm.build_gptneox();
+ } break;
+ case LLM_ARCH_ARCTIC:
+ {
+ result = llm.build_arctic();
+ } break;
+ case LLM_ARCH_DEEPSEEK2:
+ {
+ result = llm.build_deepseek2();
+ } break;
+ case LLM_ARCH_CHATGLM:
+ {
+ result = llm.build_chatglm();
+ } break;
+ case LLM_ARCH_BITNET:
+ {
+ result = llm.build_bitnet();
+ } break;
+ case LLM_ARCH_T5:
+ {
+ if (lctx.is_encoding) {
+ result = llm.build_t5_encoder();
+ } else {
+ result = llm.build_t5_decoder();
+ }
+ } break;
+ case LLM_ARCH_T5ENCODER:
+ {
+ result = llm.build_t5_encoder();
+ } break;
+ case LLM_ARCH_JAIS:
+ {
+ result = llm.build_jais();
+ } break;
+ case LLM_ARCH_NEMOTRON:
+ {
+ result = llm.build_nemotron();
+ } break;
+ case LLM_ARCH_EXAONE:
+ {
+ result = llm.build_exaone();
+ } break;
+ case LLM_ARCH_RWKV6:
+ {
+ result = llm.build_rwkv6();
+ } break;
+ case LLM_ARCH_SOLAR:
+ {
+ result = llm.build_solar();
+ } break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+
+ // add on pooling layer
+ if (lctx.cparams.embeddings) {
+ result = llm.append_pooling(result);
+ }
+
+ llm.free();
+
+ return result;
+}
+
+static void llama_set_k_shift(llama_context & lctx) {
+ const int64_t kv_size = lctx.kv_self.size;
+
+ assert(ggml_backend_buffer_is_host(lctx.inp_K_shift->buffer));
+
+ int32_t * data = (int32_t *) lctx.inp_K_shift->data;
+
+ for (int i = 0; i < kv_size; ++i) {
+ data[i] = lctx.kv_self.cells[i].delta;
+ }
+}
+
+static void llama_set_s_copy(llama_context & lctx) {
+ const int64_t kv_size = lctx.kv_self.size;
+
+ assert(ggml_backend_buffer_is_host(lctx.inp_s_copy->buffer));
+
+ int32_t * data = (int32_t *) lctx.inp_s_copy->data;
+
+ for (int i = 0; i < kv_size; ++i) {
+ data[i] = lctx.kv_self.cells[i].src;
+ }
+}
+
+static int32_t llama_relative_position_bucket(llama_pos x, llama_pos y, uint64_t n_buckets, bool bidirectional) {
+ // TODO move to hparams if a T5 variant appears that uses a different value
+ const int64_t max_distance = 128;
+
+ if (bidirectional) {
+ n_buckets >>= 1;
+ }
+
+ const int64_t max_exact = n_buckets >> 1;
+
+ int32_t relative_position = x - y;
+ int32_t relative_bucket = 0;
+ if (bidirectional) {
+ relative_bucket += (relative_position > 0) * n_buckets;
+ relative_position = abs(relative_position);
+ } else {
+ relative_position = -std::min<int32_t>(relative_position, 0);
+ }
+ int32_t relative_position_if_large = floorf(max_exact + logf(1.0 * relative_position / max_exact) * (n_buckets - max_exact) / log(1.0 * max_distance / max_exact));
+ relative_position_if_large = std::min<int32_t>(relative_position_if_large, n_buckets - 1);
+ relative_bucket += (relative_position < max_exact ? relative_position : relative_position_if_large);
+ return relative_bucket;
+}
+
+static void llama_set_inputs(llama_context & lctx, const llama_ubatch & batch) {
+ //
+ // set input data
+ //
+
+ const auto & hparams = lctx.model.hparams;
+ const auto & cparams = lctx.cparams;
+ const auto & kv_self = lctx.kv_self;
+
+ if (batch.token) {
+ const int64_t n_tokens = batch.n_tokens;
+
+ ggml_backend_tensor_set(lctx.inp_tokens, batch.token, 0, n_tokens*ggml_element_size(lctx.inp_tokens));
+ }
+
+ if (batch.embd) {
+ const int64_t n_embd = hparams.n_embd;
+ const int64_t n_tokens = batch.n_tokens;
+
+ ggml_backend_tensor_set(lctx.inp_embd, batch.embd, 0, n_tokens*n_embd*ggml_element_size(lctx.inp_embd));
+ }
+
+ if (batch.pos && lctx.inp_pos) {
+ const int64_t n_tokens = batch.n_tokens;
+
+ ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos));
+ }
+
+ if (hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE) {
+ GGML_ASSERT(lctx.inp_out_ids && "every model that can must skip unused outputs");
+ const int64_t n_tokens = batch.n_tokens;
+
+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_out_ids->buffer));
+ int32_t * data = (int32_t *) lctx.inp_out_ids->data;
+
+ if (lctx.n_outputs == n_tokens) {
+ for (int i = 0; i < n_tokens; ++i) {
+ data[i] = i;
+ }
+ } else if (batch.output) {
+ int32_t n_outputs = 0;
+ for (int i = 0; i < n_tokens; ++i) {
+ if (batch.output[i]) {
+ data[n_outputs++] = i;
+ }
+ }
+ // the graph needs to have been passed the correct number of outputs
+ GGML_ASSERT(lctx.n_outputs == n_outputs);
+ } else if (lctx.n_outputs == 1) {
+ // only keep last output
+ data[0] = n_tokens - 1;
+ } else {
+ GGML_ASSERT(lctx.n_outputs == 0);
+ }
+ }
+
+ GGML_ASSERT(
+ // (!a || b) is a logical implication (a -> b)
+ // !hparams.causal_attn -> !cparams.causal_attn
+ (hparams.causal_attn || !cparams.causal_attn) &&
+ "causal attention is not supported by this model"
+ );
+
+ if (lctx.inp_KQ_mask || lctx.inp_KQ_mask_swa) {
+ // NOTE: hparams.causal_attn indicates the model is capable of generation and uses the kv cache.
+ if (cparams.causal_attn && !lctx.is_encoding) {
+ const int64_t n_kv = kv_self.n;
+ const int64_t n_tokens = batch.n_tokens;
+ const int64_t n_seq_tokens = batch.n_seq_tokens;
+ const int64_t n_seqs = batch.n_seqs;
+
+
+ float * data = nullptr;
+ float * data_swa = nullptr;
+
+ if (lctx.inp_KQ_mask) {
+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer));
+ data = (float *) lctx.inp_KQ_mask->data;
+ }
+
+ if (lctx.inp_KQ_mask_swa) {
+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask_swa->buffer));
+ data_swa = (float *) lctx.inp_KQ_mask_swa->data;
+ }
+
+ // For causal attention, use only the previous KV cells
+ // of the correct sequence for each token of the batch.
+ // It's assumed that if a token in the batch has multiple sequences, they are equivalent.
+ for (int h = 0; h < 1; ++h) {
+ for (int s = 0; s < n_seqs; ++s) {
+ const llama_seq_id seq_id = batch.seq_id[s][0];
+
+ for (int j = 0; j < n_seq_tokens; ++j) {
+ const llama_pos pos = batch.pos[s*n_seq_tokens + j];
+
+ for (int i = 0; i < n_kv; ++i) {
+ float f;
+ if (!kv_self.cells[i].has_seq_id(seq_id) || kv_self.cells[i].pos > pos) {
+ f = -INFINITY;
+ } else {
+ if (hparams.use_alibi) {
+ f = -std::abs(kv_self.cells[i].pos - pos);
+ } else {
+ f = 0.0f;
+ }
+ }
+
+ if (data) {
+ data[h*(n_kv*n_tokens) + s*(n_kv*n_seq_tokens) + j*n_kv + i] = f;
+ }
+
+ // may need to cut off old tokens for sliding window
+ if (data_swa) {
+ if (pos - kv_self.cells[i].pos >= (int32_t)hparams.n_swa) {
+ f = -INFINITY;
+ }
+ data_swa[h*(n_kv*n_tokens) + s*(n_kv*n_seq_tokens) + j*n_kv + i] = f;
+ }
+ }
+ }
+ }
+
+ if (data) {
+ for (int i = n_tokens; i < GGML_PAD(n_tokens, GGML_KQ_MASK_PAD); ++i) {
+ for (int j = 0; j < n_kv; ++j) {
+ data[h*(n_kv*n_tokens) + i*n_kv + j] = -INFINITY;
+ }
+ }
+ }
+
+ if (data_swa) {
+ for (int i = n_tokens; i < GGML_PAD(n_tokens, GGML_KQ_MASK_PAD); ++i) {
+ for (int j = 0; j < n_kv; ++j) {
+ data_swa[h*(n_kv*n_tokens) + i*n_kv + j] = -INFINITY;
+ }
+ }
+ }
+ }
+ } else {
+ const int64_t n_tokens = batch.n_tokens;
+ const int64_t n_seq_tokens = batch.n_seq_tokens;
+ const int64_t n_seqs = batch.n_seqs;
+ // when using kv cache, the mask needs to match the kv cache size
+ const int64_t n_stride = hparams.causal_attn && !lctx.is_encoding ? kv_self.n : n_tokens;
+
+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer));
+
+ float * data = (float *) lctx.inp_KQ_mask->data;
+
+ for (int h = 0; h < 1; ++h) {
+ for (int s1 = 0; s1 < n_seqs; ++s1) {
+ const llama_seq_id seq_id = batch.seq_id[s1][0];
+
+ for (int j = 0; j < n_seq_tokens; ++j) {
+ const int32_t tj = s1*n_seq_tokens + j;
+
+ for (int s0 = 0; s0 < n_seqs; ++s0) {
+ for (int i = 0; i < n_seq_tokens; ++i) {
+ const int32_t ti = s0*n_seq_tokens + i;
+ float f = -INFINITY;
+
+ for (int s = 0; s < batch.n_seq_id[s0]; ++s) {
+ if (batch.seq_id[s0][s] == seq_id) {
+ if (hparams.use_alibi) {
+ f = -std::abs(batch.pos[ti] - batch.pos[tj]);
+ } else {
+ f = 0.0f;
+ }
+ break;
+ }
+ }
+
+ data[h*(n_tokens*n_tokens) + tj*n_stride + ti] = f;
+ }
+ }
+
+ for (int i = n_tokens; i < n_stride; ++i) {
+ data[h*(n_tokens*n_tokens) + tj*n_stride + i] = -INFINITY;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (cparams.embeddings && cparams.pooling_type == LLAMA_POOLING_TYPE_MEAN) {
+ const int64_t n_tokens = batch.n_tokens;
+ const int64_t n_seq_tokens = batch.n_seq_tokens;
+ const int64_t n_seqs = batch.n_seqs;
+
+ GGML_ASSERT(lctx.inp_mean);
+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_mean->buffer));
+
+ float * data = (float *) lctx.inp_mean->data;
+ memset(lctx.inp_mean->data, 0, n_tokens * n_tokens * ggml_element_size(lctx.inp_mean));
+
+ std::vector<uint64_t> sum(n_tokens, 0);
+
+ for (int s = 0; s < n_seqs; ++s) {
+ const llama_seq_id seq_id = batch.seq_id[s][0];
+
+ // TODO: adapt limits to n_seqs when batch.equal_seqs is true
+ GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == MEAN");
+
+ sum[seq_id] += batch.n_seq_tokens;
+ }
+
+ std::vector<float> div(n_tokens, 0.0f);
+ for (int i = 0; i < n_tokens; ++i) {
+ const uint64_t s = sum[i];
+ if (s > 0) {
+ div[i] = 1.0f/float(s);
+ }
+ }
+
+ for (int s = 0; s < n_seqs; ++s) {
+ const llama_seq_id seq_id = batch.seq_id[s][0];
+
+ for (int i = 0; i < n_seq_tokens; ++i) {
+ data[seq_id*n_tokens + s*n_seq_tokens + i] = div[seq_id];
+ }
+ }
+ }
+
+ if (cparams.embeddings && cparams.pooling_type == LLAMA_POOLING_TYPE_CLS) {
+ const int64_t n_tokens = batch.n_tokens;
+ const int64_t n_seq_tokens = batch.n_seq_tokens;
+ const int64_t n_seqs = batch.n_seqs;
+
+ GGML_ASSERT(lctx.inp_cls);
+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer));
+
+ uint32_t * data = (uint32_t *) lctx.inp_cls->data;
+ memset(lctx.inp_cls->data, 0, n_tokens * ggml_element_size(lctx.inp_cls));
+
+ for (int s = 0; s < n_seqs; ++s) {
+ const llama_seq_id seq_id = batch.seq_id[s][0];
+
+ // TODO: adapt limits to n_seqs when batch.equal_seqs is true
+ GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == CLS");
+
+ for (int i = 0; i < n_seq_tokens; ++i) {
+ const llama_pos pos = batch.pos[s*n_seq_tokens + i];
+
+ if (pos == 0) {
+ data[seq_id] = s*n_seq_tokens + i;
+ }
+ }
+ }
+ }
+
+ if (cparams.embeddings && cparams.pooling_type == LLAMA_POOLING_TYPE_LAST) {
+ const int64_t n_tokens = batch.n_tokens;
+ const int64_t n_seq_tokens = batch.n_seq_tokens;
+ const int64_t n_seqs = batch.n_seqs;
+
+ GGML_ASSERT(lctx.inp_cls);
+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer));
+
+ uint32_t * data = (uint32_t *) lctx.inp_cls->data;
+ memset(lctx.inp_cls->data, 0, n_tokens * ggml_element_size(lctx.inp_cls));
+
+ std::vector<int> last_pos(n_tokens, -1);
+ std::vector<int> last_row(n_tokens, -1);
+
+ for (int s = 0; s < n_seqs; ++s) {
+ const llama_seq_id seq_id = batch.seq_id[s][0];
+
+ // TODO: adapt limits to n_seqs when batch.equal_seqs is true
+ GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == LAST");
+
+ for (int i = 0; i < n_seq_tokens; ++i) {
+ const llama_pos pos = batch.pos[s*n_seq_tokens + i];
+
+ if (pos >= last_pos[seq_id]) {
+ last_pos[seq_id] = pos;
+ last_row[seq_id] = s*n_seq_tokens + i;
+ }
+ }
+ }
+
+ for (int i = 0; i < n_tokens; ++i) {
+ if (last_row[i] >= 0) {
+ data[i] = last_row[i];
+ }
+ }
+ }
+
+ if (kv_self.recurrent) {
+ const int64_t n_kv = kv_self.n;
+
+ if (lctx.inp_s_mask) {
+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_s_mask->buffer));
+ float * data = (float *) lctx.inp_s_mask->data;
+
+ // clear unused states
+ for (int i = 0; i < n_kv; ++i) {
+ uint32_t cell_id = i + kv_self.head;
+ llama_kv_cell & kv_cell = lctx.kv_self.cells[cell_id];
+
+ data[i] = (float) (kv_cell.src >= 0);
+
+ // only clear once
+ if (kv_cell.src < 0) {
+ kv_cell.src = cell_id;
+ }
+ }
+ }
+
+ if (lctx.inp_s_copy) {
+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_s_copy->buffer));
+ int32_t * data = (int32_t *) lctx.inp_s_copy->data;
+
+ // assuming copy destinations ALWAYS happen ONLY on the cells between head and head+n
+ for (uint32_t i = 0; i < n_kv; ++i) {
+ const uint32_t cell_id = i + kv_self.head;
+ llama_kv_cell & kv_cell = lctx.kv_self.cells[cell_id];
+
+ // prevent out-of-bound sources
+ if (kv_cell.src < 0 || (uint32_t) kv_cell.src >= kv_self.size) {
+ kv_cell.src = cell_id;
+ }
+
+ data[i] = kv_cell.src;
+
+ // ensure copy only happens once
+ if (kv_cell.src != (int32_t) cell_id) {
+ kv_cell.src = cell_id;
+ }
+ }
+ }
+ }
+
+ if (lctx.inp_pos_bucket) {
+ const int64_t n_tokens = batch.n_tokens;
+
+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_pos_bucket->buffer));
+ GGML_ASSERT(!batch.equal_seqs); // TODO: use batch.n_seqs instead of failing
+
+ int32_t * data = (int32_t *) lctx.inp_pos_bucket->data;
+
+ if (!lctx.is_encoding) {
+ const int64_t n_kv = kv_self.n;
+ for (int h = 0; h < 1; ++h) {
+ for (int j = 0; j < n_tokens; ++j) {
+ for (int i = 0; i < n_kv; ++i) {
+ data[h*(n_kv*n_tokens) + j*n_kv + i] = llama_relative_position_bucket(lctx.kv_self.cells[i].pos, batch.pos[j], hparams.n_rel_attn_bkts, lctx.is_encoding);
+ }
+ }
+ }
+ } else {
+ for (int h = 0; h < 1; ++h) {
+ for (int j = 0; j < n_tokens; ++j) {
+ for (int i = 0; i < n_tokens; ++i) {
+ data[h*(n_tokens*n_tokens) + j*n_tokens + i] = llama_relative_position_bucket(batch.pos[i], batch.pos[j], hparams.n_rel_attn_bkts, lctx.is_encoding);
+ }
+ }
+ }
+ }
+ }
+
+ if (!lctx.is_encoding && lctx.inp_embd_enc) {
+ assert(lctx.inp_embd_enc->type == GGML_TYPE_F32);
+ assert((size_t) ggml_nelements(lctx.inp_embd_enc) == lctx.embd_enc.size());
+
+ ggml_backend_tensor_set(lctx.inp_embd_enc, lctx.embd_enc.data(), 0, ggml_nbytes(lctx.inp_embd_enc));
+ }
+
+ if (!lctx.is_encoding && lctx.inp_KQ_mask_cross) {
+ const int64_t n_output_enc = lctx.embd_enc.size() / hparams.n_embd;
+ const int64_t n_tokens = batch.n_tokens;
+
+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask_cross->buffer));
+ GGML_ASSERT(!batch.equal_seqs); // TODO: use batch.n_seqs instead of failing
+
+ float * data = (float *) lctx.inp_KQ_mask_cross->data;
+
+ for (int h = 0; h < 1; ++h) {
+ for (int j = 0; j < n_tokens; ++j) {
+ for (int i = 0; i < n_output_enc; ++i) {
+ float f = -INFINITY;
+ for (int s = 0; s < batch.n_seq_id[j]; ++s) {
+ const llama_seq_id seq_id = batch.seq_id[j][s];
+ if (lctx.seq_ids_enc[i].find(seq_id) != lctx.seq_ids_enc[i].end()) {
+ f = 0.0f;
+ }
+ }
+ data[h*(n_output_enc*n_tokens) + j*n_output_enc + i] = f;
+ }
+ }
+
+ for (int i = n_tokens; i < GGML_PAD(n_tokens, GGML_KQ_MASK_PAD); ++i) {
+ for (int j = 0; j < n_output_enc; ++j) {
+ data[h*(n_output_enc*n_tokens) + i*n_output_enc + j] = -INFINITY;
+ }
+ }
+ }
+ }
+}
+
+// Make sure enough space is available for outputs.
+// Returns max number of outputs for which space was reserved.
+static size_t llama_output_reserve(llama_context & lctx, size_t n_outputs) {
+ const auto & cparams = lctx.cparams;
+ const auto & hparams = lctx.model.hparams;
+
+ const size_t n_outputs_max = std::max(n_outputs, (size_t) cparams.n_seq_max);
+
+ const auto n_batch = cparams.n_batch;
+ const auto n_vocab = hparams.n_vocab;
+ const auto n_embd = hparams.n_embd;
+
+ // TODO: use a per-batch flag for logits presence instead
+ const bool has_logits = cparams.causal_attn;
+ const bool has_embd = cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE);
+
+ const size_t logits_size = has_logits ? n_vocab*n_outputs_max : 0;
+ const size_t embd_size = has_embd ? n_embd*n_outputs_max : 0;
+
+ if (lctx.output_ids.empty()) {
+ // init, never resized afterwards
+ lctx.output_ids.resize(n_batch);
+ }
+
+ const size_t prev_size = lctx.buf_output ? ggml_backend_buffer_get_size(lctx.buf_output) : 0;
+ const size_t new_size = (logits_size + embd_size) * sizeof(float);
+
+ // alloc only when more than the current capacity is required
+ // TODO: also consider shrinking the buffer
+ if (!lctx.buf_output || prev_size < new_size) {
+ if (lctx.buf_output) {
+#ifndef NDEBUG
+ // This doesn't happen often, but may be annoying in some cases (like the HellaSwag benchmark)
+ LLAMA_LOG_INFO("%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);
+#endif
+ ggml_backend_buffer_free(lctx.buf_output);
+ lctx.buf_output = nullptr;
+ lctx.logits = nullptr;
+ lctx.embd = nullptr;
+ }
+
+ lctx.buf_output = ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), new_size);
+ if (lctx.buf_output == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to allocate output buffer of size %.2f MiB\n", __func__, new_size / (1024.0 * 1024.0));
+ return 0;
+ }
+ }
+
+ float * output_base = (float *) ggml_backend_buffer_get_base(lctx.buf_output);
+
+ lctx.logits = has_logits ? output_base : nullptr;
+ lctx.embd = has_embd ? output_base + logits_size : nullptr;
+
+ lctx.output_size = n_outputs_max;
+ lctx.logits_size = logits_size;
+ lctx.embd_size = embd_size;
+
+ // set all ids as invalid (negative)
+ std::fill(lctx.output_ids.begin(), lctx.output_ids.end(), -1);
+
+ ggml_backend_buffer_clear(lctx.buf_output, 0);
+
+ lctx.n_outputs = 0;
+
+ return n_outputs_max;
+}
+
+// make the outputs have the same order they had in the user-provided batch
+static void llama_output_reorder(struct llama_context * ctx) {
+ std::vector<size_t> & out_ids = ctx->sbatch.out_ids;
+ if (!out_ids.empty()) {
+ uint32_t n_vocab = ctx->model.hparams.n_vocab;
+ uint32_t n_embd = ctx->model.hparams.n_embd;
+ int32_t n_outputs = ctx->n_outputs;
+ GGML_ASSERT((size_t) n_outputs == out_ids.size());
+ // TODO: is there something more efficient which also minimizes swaps?
+ // selection sort, to minimize swaps (from https://en.wikipedia.org/wiki/Selection_sort)
+ for (int32_t i = 0; i < n_outputs - 1; ++i) {
+ int32_t j_min = i;
+ for (int32_t j = i + 1; j < n_outputs; ++j) {
+ if (out_ids[j] < out_ids[j_min]) {
+ j_min = j;
+ }
+ }
+ if (j_min == i) { continue; }
+ std::swap(out_ids[i], out_ids[j_min]);
+ if (ctx->logits_size > 0) {
+ for (uint32_t k = 0; k < n_vocab; k++) {
+ std::swap(ctx->logits[i*n_vocab + k], ctx->logits[j_min*n_vocab + k]);
+ }
+ }
+ if (ctx->embd_size > 0) {
+ for (uint32_t k = 0; k < n_embd; k++) {
+ std::swap(ctx->embd[i*n_embd + k], ctx->embd[j_min*n_embd + k]);
+ }
+ }
+ }
+ std::fill(ctx->output_ids.begin(), ctx->output_ids.end(), -1);
+ for (int32_t i = 0; i < n_outputs; ++i) {
+ ctx->output_ids[out_ids[i]] = i;
+ }
+ out_ids.clear();
+ }
+}
+
+static void llama_graph_compute(
+ llama_context & lctx,
+ ggml_cgraph * gf,
+ int n_threads,
+ ggml_threadpool * threadpool) {
+#ifdef GGML_USE_METAL
+ if (ggml_backend_is_metal(lctx.backend_metal)) {
+ ggml_backend_metal_set_n_cb(lctx.backend_metal, n_threads);
+ }
+#endif
+
+ if (lctx.backend_cpu != nullptr) {
+ ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads);
+ ggml_backend_cpu_set_threadpool(lctx.backend_cpu, threadpool);
+ ggml_backend_cpu_set_abort_callback(lctx.backend_cpu, lctx.abort_callback, lctx.abort_callback_data);
+ }
+#ifdef GGML_USE_BLAS
+ if (lctx.backend_blas != nullptr) {
+ ggml_backend_blas_set_n_threads(lctx.backend_blas, n_threads);
+ }
+#endif
+
+ ggml_backend_sched_graph_compute_async(lctx.sched, gf);
+
+ // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched));
+}
+
+// decode a batch of tokens by evaluating the transformer
+//
+// - lctx: llama context
+// - batch: batch to evaluate
+//
+// return 0 on success
+// return positive int on warning
+// return negative int on error
+//
+static int llama_decode_internal(
+ llama_context & lctx,
+ llama_batch batch_all) { // TODO: rename back to batch
+
+ lctx.is_encoding = false;
+ const uint32_t n_tokens_all = batch_all.n_tokens;
+
+ if (n_tokens_all == 0) {
+ LLAMA_LOG_ERROR("%s: n_tokens == 0", __func__);
+ return -1;
+ }
+
+ const auto & model = lctx.model;
+ const auto & hparams = model.hparams;
+ const auto & cparams = lctx.cparams;
+
+ GGML_ASSERT((!batch_all.token && batch_all.embd) || (batch_all.token && !batch_all.embd)); // NOLINT
+
+ GGML_ASSERT(n_tokens_all <= cparams.n_batch);
+
+ GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens");
+
+ if (lctx.t_compute_start_us == 0) {
+ lctx.t_compute_start_us = ggml_time_us();
+ }
+ lctx.n_queued_tokens += n_tokens_all;
+
+ auto & kv_self = lctx.kv_self;
+
+ const int64_t n_embd = hparams.n_embd;
+ const int64_t n_vocab = hparams.n_vocab;
+
+ uint32_t n_outputs = 0;
+ uint32_t n_outputs_prev = 0;
+
+ const auto n_ubatch = cparams.n_ubatch;
+
+ // this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens
+ const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE;
+
+ lctx.embd_seq.clear();
+
+ // count outputs
+ if (batch_all.logits && !embd_pooled) {
+ for (uint32_t i = 0; i < n_tokens_all; ++i) {
+ n_outputs += batch_all.logits[i] != 0;
+ }
+ } else if (lctx.logits_all || embd_pooled) {
+ n_outputs = n_tokens_all;
+ } else {
+ // keep last output only
+ n_outputs = 1;
+ }
+
+ lctx.sbatch.from_batch(batch_all, n_embd,
+ /* simple_split */ !kv_self.recurrent,
+ /* logits_all */ n_outputs == n_tokens_all);
+
+ // reserve output buffer
+ if (llama_output_reserve(lctx, n_outputs) < n_outputs) {
+ LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_outputs);
+ return -2;
+ };
+
+ while (lctx.sbatch.n_tokens > 0) {
+ llama_ubatch ubatch;
+ if (kv_self.recurrent) {
+ if (embd_pooled) {
+ // Pooled embeddings cannot be split across ubatches (yet)
+ ubatch = lctx.sbatch.split_seq(n_ubatch);
+ } else {
+ // recurrent model architectures are easier to implement
+ // with equal-length sequences
+ ubatch = lctx.sbatch.split_equal(n_ubatch);
+ }
+ } else {
+ ubatch = lctx.sbatch.split_simple(n_ubatch);
+ }
+ const uint32_t n_tokens = ubatch.n_tokens;
+
+ // count the outputs in this u_batch
+ {
+ int32_t n_outputs_new = 0;
+
+ if (n_outputs == n_tokens_all) {
+ n_outputs_new = n_tokens;
+ } else {
+ GGML_ASSERT(ubatch.output);
+ for (uint32_t i = 0; i < n_tokens; i++) {
+ n_outputs_new += (int32_t) (ubatch.output[i] != 0);
+ }
+ }
+
+ // needs to happen before the graph is built
+ lctx.n_outputs = n_outputs_new;
+ }
+
+ int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch;
+ ggml_threadpool_t threadpool = n_tokens == 1 ? lctx.threadpool : lctx.threadpool_batch;
+
+ GGML_ASSERT(n_threads > 0);
+
+ // non-causal masks do not use the KV cache
+ if (hparams.causal_attn) {
+ llama_kv_cache_update(&lctx);
+
+ // if we have enough unused cells before the current head ->
+ // better to start searching from the beginning of the cache, hoping to fill it
+ if (kv_self.head > kv_self.used + 2*n_tokens) {
+ kv_self.head = 0;
+ }
+
+ if (!llama_kv_cache_find_slot(kv_self, ubatch)) {
+ return 1;
+ }
+
+ if (!kv_self.recurrent) {
+ // a heuristic, to avoid attending the full cache if it is not yet utilized
+ // after enough generations, the benefit from this heuristic disappears
+ // if we start defragmenting the cache, the benefit from this will be more important
+ const uint32_t pad = llama_kv_cache_get_padding(cparams);
+ kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(llama_kv_cache_cell_max(kv_self), pad)));
+ //kv_self.n = llama_kv_cache_cell_max(kv_self);
+ }
+ }
+
+ //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head);
+
+ ggml_backend_sched_reset(lctx.sched);
+ ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data);
+
+ ggml_cgraph * gf = llama_build_graph(lctx, ubatch, false);
+
+ // the output is always the last tensor in the graph
+ struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1];
+ struct ggml_tensor * embd = gf->nodes[gf->n_nodes - 2];
+
+ if (lctx.n_outputs == 0) {
+ // no output
+ res = nullptr;
+ embd = nullptr;
+ }
+
+ if (cparams.embeddings) {
+ for (int i = gf->n_nodes - 1; i >= 0; --i) {
+ embd = gf->nodes[i];
+ if (strcmp(embd->name, "result_embd_pooled") == 0) {
+ break;
+ }
+ }
+ } else {
+ embd = nullptr; // do not extract embeddings when not needed
+ GGML_ASSERT(strcmp(res->name, "result_output") == 0 && "missing result_output tensor");
+ }
+
+ if (!cparams.causal_attn) {
+ res = nullptr; // do not extract logits when not needed
+ }
+ // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);
+
+ ggml_backend_sched_alloc_graph(lctx.sched, gf);
+
+ llama_set_inputs(lctx, ubatch);
+
+ llama_graph_compute(lctx, gf, n_threads, threadpool);
+
+ // update the kv ring buffer
+ {
+ kv_self.head += n_tokens;
+
+ // Ensure kv cache head points to a valid index.
+ if (kv_self.head >= kv_self.size) {
+ kv_self.head = 0;
+ }
+ }
+
+ // plot the computation graph in dot format (for debugging purposes)
+ //if (n_past%100 == 0) {
+ // ggml_graph_dump_dot(gf, NULL, "llama.dot");
+ //}
+
+ // extract logits
+ if (res) {
+ ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(lctx.sched, res);
+ GGML_ASSERT(backend_res != nullptr);
+ GGML_ASSERT(lctx.logits != nullptr);
+
+ float * logits_out = lctx.logits + n_outputs_prev*n_vocab;
+ const int32_t n_outputs_new = lctx.n_outputs;
+
+ if (n_outputs_new) {
+ GGML_ASSERT( n_outputs_prev + n_outputs_new <= n_outputs);
+ GGML_ASSERT((n_outputs_prev + n_outputs_new)*n_vocab <= (int64_t) lctx.logits_size);
+ ggml_backend_tensor_get_async(backend_res, res, logits_out, 0, n_outputs_new*n_vocab*sizeof(float));
+ }
+ }
+
+ // extract embeddings
+ if (embd) {
+ ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(lctx.sched, embd);
+ GGML_ASSERT(backend_embd != nullptr);
+
+ switch (cparams.pooling_type) {
+ case LLAMA_POOLING_TYPE_NONE:
+ {
+ // extract token embeddings
+ GGML_ASSERT(lctx.embd != nullptr);
+ float * embd_out = lctx.embd + n_outputs_prev*n_embd;
+ const int32_t n_outputs_new = lctx.n_outputs;
+
+ if (n_outputs_new) {
+ GGML_ASSERT( n_outputs_prev + n_outputs_new <= n_outputs);
+ GGML_ASSERT((n_outputs_prev + n_outputs_new)*n_embd <= (int64_t) lctx.embd_size);
+ ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_outputs_new*n_embd*sizeof(float));
+ }
+ } break;
+ case LLAMA_POOLING_TYPE_MEAN:
+ case LLAMA_POOLING_TYPE_CLS:
+ case LLAMA_POOLING_TYPE_LAST:
+ {
+ // extract sequence embeddings (cleared before processing each batch)
+ auto & embd_seq_out = lctx.embd_seq;
+
+ for (uint32_t s = 0; s < ubatch.n_seqs; ++s) {
+ const llama_seq_id seq_id = ubatch.seq_id[s][0];
+ if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {
+ continue;
+ }
+ embd_seq_out[seq_id].resize(n_embd);
+ ggml_backend_tensor_get_async(backend_embd, embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float));
+ }
+ } break;
+ case LLAMA_POOLING_TYPE_UNSPECIFIED:
+ {
+ GGML_ABORT("unknown pooling type");
+ }
+ }
+ }
+ n_outputs_prev += lctx.n_outputs;
+ }
+
+ // set output mappings
+ {
+ bool sorted_output = true;
+
+ GGML_ASSERT(lctx.sbatch.out_ids.size() == n_outputs);
+
+ for (size_t i = 0; i < n_outputs; ++i) {
+ size_t out_id = lctx.sbatch.out_ids[i];
+ lctx.output_ids[out_id] = i;
+ if (out_id != i) {
+ sorted_output = false;
+ }
+ }
+
+ if (sorted_output) {
+ lctx.sbatch.out_ids.clear();
+ }
+ }
+
+ // set to total number of outputs in the batch, for use in llama_get_logits_ith
+ lctx.n_outputs = n_outputs;
+
+ // wait for the computation to finish (automatically done when obtaining the model output)
+ //llama_synchronize(&lctx);
+
+ // decide if we need to defrag the kv cache
+ if (cparams.causal_attn && cparams.defrag_thold >= 0.0f) {
+ const float fragmentation = kv_self.n >= 128 ? 1.0f - float(kv_self.used)/float(kv_self.n) : 0.0f;
+
+ // queue defragmentation for next llama_kv_cache_update
+ if (fragmentation > cparams.defrag_thold) {
+ //LLAMA_LOG_INFO("fragmentation: %.2f\n", fragmentation);
+
+ llama_kv_cache_defrag(kv_self);
+ }
+ }
+
+ // Reset state for the next token before backend sync, to allow the CPU activities in the reset to
+ // overlap with device computation.
+ ggml_backend_sched_reset(lctx.sched);
+
+ return 0;
+}
+
+// encode a batch of tokens by evaluating the encoder part of the transformer
+//
+// - lctx: llama context
+// - batch: batch to evaluate
+//
+// return 0 on success
+// return positive int on warning
+// return negative int on error
+//
+static int llama_encode_internal(
+ llama_context & lctx,
+ llama_batch batch) {
+
+ lctx.is_encoding = true;
+
+ const uint32_t n_tokens = batch.n_tokens;
+
+ if (n_tokens == 0) {
+ LLAMA_LOG_ERROR("%s: n_tokens == 0", __func__);
+ return -1;
+ }
+
+ const auto & model = lctx.model;
+ const auto & hparams = model.hparams;
+ const auto & cparams = lctx.cparams;
+
+ GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT
+
+ // micro-batching is not possible for non-causal encoding, so we process the batch in a single shot
+ GGML_ASSERT(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens");
+
+ if (lctx.t_compute_start_us == 0) {
+ lctx.t_compute_start_us = ggml_time_us();
+ }
+
+ lctx.n_queued_tokens += n_tokens;
+
+ const int64_t n_embd = hparams.n_embd;
+
+ lctx.sbatch.from_batch(batch, n_embd, /* simple_split */ true, /* logits_all */ true);
+
+ const llama_ubatch ubatch = lctx.sbatch.split_simple(n_tokens);
+
+ // reserve output buffer
+ if (llama_output_reserve(lctx, n_tokens) < n_tokens) {
+ LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens);
+ return -2;
+ };
+
+ for (uint32_t i = 0; i < n_tokens; ++i) {
+ lctx.output_ids[i] = i;
+ }
+
+ lctx.inp_embd_enc = NULL;
+ lctx.n_outputs = n_tokens;
+
+ int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch;
+ ggml_threadpool_t threadpool = n_tokens == 1 ? lctx.threadpool : lctx.threadpool_batch;
+
+ GGML_ASSERT(n_threads > 0);
+
+ ggml_backend_sched_reset(lctx.sched);
+ ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data);
+
+ ggml_cgraph * gf = llama_build_graph(lctx, ubatch, false);
+
+ // the output embeddings after the final encoder normalization
+ struct ggml_tensor * embd = nullptr;
+
+ // there are two cases here
+ if (llama_model_has_decoder(&lctx.model)) {
+ // first case is an encoder-decoder T5 model where embeddings are passed to decoder
+ embd = gf->nodes[gf->n_nodes - 1];
+ GGML_ASSERT(strcmp(embd->name, "result_norm") == 0 && "missing result_output tensor");
+ } else {
+ // second case is an encoder-only T5 model
+ if (cparams.embeddings) {
+ // only output embeddings if required
+ embd = gf->nodes[gf->n_nodes - 1];
+ if (strcmp(embd->name, "result_embd_pooled") != 0) {
+ embd = gf->nodes[gf->n_nodes - 2];
+ }
+ GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0 && "missing embeddings tensor");
+ }
+ }
+
+ ggml_backend_sched_alloc_graph(lctx.sched, gf);
+
+ llama_set_inputs(lctx, ubatch);
+
+ llama_graph_compute(lctx, gf, n_threads, threadpool);
+
+ // extract embeddings
+ if (embd) {
+ ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(lctx.sched, embd);
+ GGML_ASSERT(backend_embd != nullptr);
+
+ if (llama_model_has_decoder(&lctx.model)) {
+ lctx.embd_enc.resize(n_tokens*n_embd);
+ float * embd_out = lctx.embd_enc.data();
+
+ ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_tokens*n_embd*sizeof(float));
+ GGML_ASSERT(!ubatch.equal_seqs); // TODO: handle equal splits
+
+ // remember the sequence ids used during the encoding - needed for cross attention later
+ lctx.seq_ids_enc.resize(n_tokens);
+ for (uint32_t i = 0; i < n_tokens; i++) {
+ for (int s = 0; s < ubatch.n_seq_id[i]; s++) {
+ llama_seq_id seq_id = ubatch.seq_id[i][s];
+ lctx.seq_ids_enc[i].insert(seq_id);
+ }
+ }
+ } else {
+ GGML_ASSERT(lctx.embd != nullptr);
+
+ switch (cparams.pooling_type) {
+ case LLAMA_POOLING_TYPE_NONE:
+ {
+ // extract token embeddings
+ GGML_ASSERT(lctx.embd != nullptr);
+ float * embd_out = lctx.embd;
+
+ GGML_ASSERT(n_tokens*n_embd <= (int64_t) lctx.embd_size);
+ ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_tokens*n_embd*sizeof(float));
+ } break;
+ case LLAMA_POOLING_TYPE_MEAN:
+ case LLAMA_POOLING_TYPE_CLS:
+ case LLAMA_POOLING_TYPE_LAST:
+ {
+ // extract sequence embeddings
+ auto & embd_seq_out = lctx.embd_seq;
+ embd_seq_out.clear();
+
+ GGML_ASSERT(!ubatch.equal_seqs); // TODO: handle equal splits
+
+ for (uint32_t i = 0; i < n_tokens; i++) {
+ const llama_seq_id seq_id = ubatch.seq_id[i][0];
+ if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {
+ continue;
+ }
+ embd_seq_out[seq_id].resize(n_embd);
+ ggml_backend_tensor_get_async(backend_embd, embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float));
+ }
+ } break;
+ case LLAMA_POOLING_TYPE_UNSPECIFIED:
+ {
+ GGML_ABORT("unknown pooling type");
+ }
+ }
+ }
+ }
+
+ // Reset state for the next token before backend sync, to allow the CPU activities in the reset to
+ // overlap with device computation.
+ ggml_backend_sched_reset(lctx.sched);
+
+ return 0;
+}
+
+// find holes from the beginning of the KV cache and fill them by moving data from the end of the cache
+static void llama_kv_cache_defrag_internal(struct llama_context & lctx) {
+ auto & kv_self = lctx.kv_self;
+
+ const auto & hparams = lctx.model.hparams;
+
+ const uint32_t n_layer = hparams.n_layer;
+
+ const uint32_t n_kv = llama_kv_cache_cell_max(kv_self);
+ const uint32_t n_used = kv_self.used;
+
+ assert(n_used <= n_kv);
+
+ //const int64_t t_start = ggml_time_us();
+
+ // number of cells moved
+ uint32_t n_moves = 0;
+
+ // each move requires 6*n_layer tensors (see build_defrag)
+ // - source view, destination view, copy operation
+ // - x2 for keys and values
+ //const uint32_t max_moves = llama_model_max_nodes(model)/(6*n_layer);
+ // TODO: tmp fix https://github.com/ggerganov/llama.cpp/issues/6685#issuecomment-2057579516
+ const uint32_t max_moves = (llama_model_max_nodes(lctx.model) - 2*n_layer)/(6*n_layer);
+
+ // determine which KV cells to move where
+ //
+ // cell i moves to ids[i]
+ //
+ // if ids[i] == i || ids[i] == n_kv, then cell i is not moved
+ //
+ std::vector<uint32_t> ids(n_kv, n_kv);
+
+ for (uint32_t i0 = 0; i0 < n_used; ++i0) {
+ const auto & cell0 = kv_self.cells[i0];
+
+ if (!cell0.is_empty()) {
+ ids[i0] = i0;
+
+ continue;
+ }
+
+ // found a hole - fill it with data from the end of the cache
+
+ uint32_t nh = 1;
+
+ // determine the size of the hole
+ while (i0 + nh < n_used && kv_self.cells[i0 + nh].is_empty()) {
+ nh++;
+ }
+
+ uint32_t nf = 0;
+ uint32_t is = n_kv - 1;
+
+ // starting from the end, find nh non-empty cells
+ for (; is > i0; --is) {
+ const auto & cell1 = kv_self.cells[is];
+
+ if (cell1.is_empty() || ids[is] != n_kv) {
+ continue;
+ }
+
+ // non-empty cell which is not yet moved
+ nf++;
+
+ if (nf == nh) {
+ break;
+ }
+ }
+
+ // this can only happen if `n_used` is not accurate, which would be a bug
+ GGML_ASSERT(nf == nh && "KV defrag bug: nf != nh");
+
+ nf = 0;
+
+ uint32_t i1 = is;
+
+ // are we moving a continuous block of memory?
+ bool cont = false;
+
+ // should we stop searching for the next move?
+ bool stop = false;
+
+ // go back and move the nf cells to the hole
+ for (; i1 < n_kv; ++i1) {
+ auto & cell1 = kv_self.cells[i1];
+
+ if (cell1.is_empty() || ids[i1] != n_kv) {
+ if (n_moves == max_moves) {
+ stop = true;
+ break;
+ }
+
+ cont = false;
+ continue;
+ }
+
+ // this cell goes to (i0 + nf)
+ ids[i1] = i0 + nf;
+
+ // move the cell meta data
+ kv_self.cells[i0 + nf] = cell1;
+
+ // clear the old cell and move the head there
+ cell1 = llama_kv_cell();
+ kv_self.head = n_used;
+
+ if (!cont) {
+ n_moves++;
+ cont = true;
+ }
+
+ nf++;
+
+ if (nf == nh) {
+ break;
+ }
+ }
+
+ if (stop || n_moves == max_moves) {
+ break;
+ }
+
+ //LLAMA_LOG_INFO("(tmp log) KV defrag: move [%u, %u) to [%u, %u)\n", is, i1 + 1, i0, i0 + nh);
+
+ i0 += nh - 1;
+ }
+
+ if (n_moves == 0) {
+ return;
+ }
+
+ //LLAMA_LOG_INFO("(tmp log) KV defrag cell moves: %u\n", n_moves);
+
+ //LLAMA_LOG_INFO("expected gf nodes: %u\n", 6*n_moves*n_layer);
+
+#if 0
+ // CPU defrag
+ //
+ // TODO: optimizations are possible:
+ // - multiple threads
+ // - avoid copying to the host memory when already there
+ //
+ // likely not worth the effort, as we have ggml_graph based defrag
+ //
+
+ const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa();
+ const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa();
+
+ const uint32_t kv_size = kv_self.size;
+
+ std::vector<uint8_t> buf_k;
+ std::vector<uint8_t> buf_v;
+
+ for (uint32_t il = 0; il < n_layer; ++il) {
+ const size_t k_size_row = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa);
+ const size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_size);
+
+ const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type);
+ const size_t v_size = ggml_row_size (kv_self.v_l[il]->type, n_embd_v_gqa*kv_size);
+
+ buf_k.resize(k_size);
+ buf_v.resize(v_size);
+
+ ggml_backend_tensor_get(kv_self.k_l[il], buf_k.data(), 0, buf_k.size());
+ ggml_backend_tensor_get(kv_self.v_l[il], buf_v.data(), 0, buf_v.size());
+
+ // batch move [i, i+nm) to [id, id+nm)
+ // note: cells can move only to a lower index
+ for (uint32_t i = 0; i < n_kv; ++i) {
+ const uint32_t id = ids[i];
+
+ if (i == id || id == n_kv) {
+ continue;
+ }
+
+ uint32_t nm = 1;
+
+ while (i + nm < n_kv && ids[i + nm] == id + nm) {
+ nm++;
+ }
+
+ // move keys
+ {
+ const int64_t os = i*k_size_row;
+ const int64_t od = id*k_size_row;
+
+ memcpy(buf_k.data() + od, buf_k.data() + os, nm*k_size_row);
+ }
+
+ // move values (note: they are transposed)
+ {
+ const int64_t os = i;
+ const int64_t od = id;
+
+ for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
+ memcpy(buf_v.data() + (od + j*kv_size)*v_size_el, buf_v.data() + (os + j*kv_size)*v_size_el, nm*v_size_el);
+ }
+ }
+
+ i += nm - 1;
+ }
+
+ ggml_backend_tensor_set(kv_self.k_l[il], buf_k.data(), 0, buf_k.size());
+ ggml_backend_tensor_set(kv_self.v_l[il], buf_v.data(), 0, buf_v.size());
+ }
+#else
+ // ggml_graph defrag
+
+ ggml_backend_sched_reset(lctx.sched);
+
+ ggml_cgraph * gf = llama_build_graph_defrag(lctx, ids);
+
+ llama_graph_compute(lctx, gf, lctx.cparams.n_threads, lctx.threadpool);
+#endif
+
+ //const int64_t t_end = ggml_time_us();
+
+ //LLAMA_LOG_INFO("(tmp log) KV defrag time: %.3f ms\n", (t_end - t_start)/1000.0);
+}
+
+static void llama_kv_cache_update_internal(struct llama_context & lctx) {
+ bool need_reserve = false;
+
+ // apply K-shift if needed
+ if (lctx.model.hparams.rope_type != LLAMA_ROPE_TYPE_NONE && lctx.kv_self.has_shift) {
+ if (lctx.model.arch == LLM_ARCH_DEEPSEEK2) { // not supported due to MLA
+ GGML_ABORT("Deepseek2 does not support K-shift");
+ }
+
+ {
+ ggml_backend_sched_reset(lctx.sched);
+
+ ggml_cgraph * gf = llama_build_graph_k_shift(lctx);
+
+ ggml_backend_sched_alloc_graph(lctx.sched, gf);
+
+ llama_set_k_shift(lctx);
+
+ llama_graph_compute(lctx, gf, lctx.cparams.n_threads, lctx.threadpool);
+
+ need_reserve = true;
+ }
+
+ {
+ auto & kv_self = lctx.kv_self;
+
+ kv_self.has_shift = false;
+
+ for (uint32_t i = 0; i < kv_self.size; ++i) {
+ kv_self.cells[i].delta = 0;
+ }
+ }
+ }
+
+ // defragment the KV cache if needed
+ if (lctx.kv_self.do_defrag) {
+ llama_kv_cache_defrag_internal(lctx);
+
+ need_reserve = true;
+
+ lctx.kv_self.do_defrag = false;
+ }
+
+ // reserve a worst case graph again
+ if (need_reserve) {
+ // TODO: extract to a function
+ // build worst-case graph
+ uint32_t n_seqs = 1; // TODO: worst-case number of sequences
+ uint32_t n_tokens = std::min(lctx.cparams.n_ctx, lctx.cparams.n_ubatch);
+ llama_token token = llama_token_bos(&lctx.model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph
+ llama_ubatch ubatch = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr};
+ ggml_cgraph * gf = llama_build_graph(lctx, ubatch, true);
+
+ // initialize scheduler with the worst-case graph
+ ggml_backend_sched_reset(lctx.sched);
+ if (!ggml_backend_sched_reserve(lctx.sched, gf)) {
+ LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__);
+ }
+ }
+}
+
+//
+// quantization
+//
+
+struct quantize_state_internal {
+ const llama_model & model;
+ const llama_model_quantize_params * params;
+
+ int n_attention_wv = 0;
+ int n_ffn_down = 0;
+ int n_ffn_gate = 0;
+ int n_ffn_up = 0;
+ int i_attention_wv = 0;
+ int i_ffn_down = 0;
+ int i_ffn_gate = 0;
+ int i_ffn_up = 0;
+
+ int n_k_quantized = 0;
+ int n_fallback = 0;
+
+ bool has_imatrix = false;
+
+ // used to figure out if a model shares tok_embd with the output weight
+ bool has_output = false;
+
+ quantize_state_internal(const llama_model & model, const llama_model_quantize_params * params)
+ : model(model)
+ , params(params)
+ {}
+};
+
+static void llama_tensor_dequantize_internal(
+ struct ggml_tensor * tensor, std::vector<no_init<float>> & output, std::vector<std::thread> & workers,
+ const size_t nelements, const int nthread
+) {
+ if (output.size() < nelements) {
+ output.resize(nelements);
+ }
+ float * f32_output = (float *) output.data();
+
+ ggml_type_traits_t qtype;
+ if (ggml_is_quantized(tensor->type)) {
+ qtype = ggml_internal_get_type_traits(tensor->type);
+ if (qtype.to_float == NULL) {
+ throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(tensor->type)));
+ }
+ } else if (tensor->type != GGML_TYPE_F16 &&
+ tensor->type != GGML_TYPE_BF16) {
+ throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor->type)));
+ }
+
+ if (nthread < 2) {
+ if (tensor->type == GGML_TYPE_F16) {
+ ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor->data, f32_output, nelements);
+ } else if (tensor->type == GGML_TYPE_BF16) {
+ ggml_bf16_to_fp32_row((ggml_bf16_t *)tensor->data, f32_output, nelements);
+ } else if (ggml_is_quantized(tensor->type)) {
+ qtype.to_float(tensor->data, f32_output, nelements);
+ } else {
+ GGML_ABORT("fatal error"); // unreachable
+ }
+ return;
+ }
+
+ size_t block_size;
+ if (tensor->type == GGML_TYPE_F16 ||
+ tensor->type == GGML_TYPE_BF16) {
+ block_size = 1;
+ } else {
+ block_size = (size_t)ggml_blck_size(tensor->type);
+ }
+
+ size_t block_size_bytes = ggml_type_size(tensor->type);
+
+ GGML_ASSERT(nelements % block_size == 0);
+ size_t nblocks = nelements / block_size;
+ size_t blocks_per_thread = nblocks / nthread;
+ size_t spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count
+
+ size_t in_buff_offs = 0;
+ size_t out_buff_offs = 0;
+
+ for (int tnum = 0; tnum < nthread; tnum++) {
+ size_t thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread
+ size_t thr_elems = thr_blocks * block_size; // number of elements for this thread
+ size_t thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread
+
+ auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) {
+ if (typ == GGML_TYPE_F16) {
+ ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels);
+ } else if (typ == GGML_TYPE_BF16) {
+ ggml_bf16_to_fp32_row((ggml_bf16_t *)inbuf, outbuf, nels);
+ } else {
+ qtype.to_float(inbuf, outbuf, nels);
+ }
+ };
+ workers.emplace_back(compute, tensor->type, (uint8_t *) tensor->data + in_buff_offs, f32_output + out_buff_offs, thr_elems);
+ in_buff_offs += thr_block_bytes;
+ out_buff_offs += thr_elems;
+ }
+ for (auto & w : workers) { w.join(); }
+ workers.clear();
+}
+
+static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) {
+ const std::string name = ggml_get_name(tensor);
+
+ // TODO: avoid hardcoded tensor names - use the TN_* constants
+ const llm_arch arch = qs.model.arch;
+ const auto tn = LLM_TN(arch);
+
+ auto use_more_bits = [](int i_layer, int n_layers) -> bool {
+ return i_layer < n_layers/8 || i_layer >= 7*n_layers/8 || (i_layer - n_layers/8)%3 == 2;
+ };
+ const int n_expert = std::max(1, (int)qs.model.hparams.n_expert);
+ auto layer_info = [n_expert] (int i_layer, int n_layer, const char * name) {
+ if (n_expert > 1) {
+ // Believe it or not, "experts" in the FFN of Mixtral-8x7B are not consecutive, but occasionally randomly
+ // sprinkled in the model. Hence, simply dividing i_ffn_down by n_expert does not work
+ // for getting the current layer as I initially thought, and we need to resort to parsing the
+ // tensor name.
+ if (sscanf(name, "blk.%d.", &i_layer) != 1) {
+ throw std::runtime_error(format("Failed to determine layer for tensor %s", name));
+ }
+ if (i_layer < 0 || i_layer >= n_layer) {
+ throw std::runtime_error(format("Bad layer %d for tensor %s. Must be in [0, %d)", i_layer, name, n_layer));
+ }
+ }
+ return std::make_pair(i_layer, n_layer);
+ };
+
+ // for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings
+ // with the quantization of the output tensor
+ if (name == tn(LLM_TENSOR_OUTPUT, "weight") || (!qs.has_output && name == tn(LLM_TENSOR_TOKEN_EMBD, "weight"))) {
+ if (qs.params->output_tensor_type < GGML_TYPE_COUNT) {
+ new_type = qs.params->output_tensor_type;
+ } else {
+ int nx = tensor->ne[0];
+ if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) {
+ new_type = GGML_TYPE_Q8_0;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS ||
+ ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ||
+ ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {
+ new_type = GGML_TYPE_Q5_K;
+ }
+ else if (new_type != GGML_TYPE_Q8_0) {
+ new_type = GGML_TYPE_Q6_K;
+ }
+ }
+ } else if (name == "token_embd.weight") {
+ if (qs.params->token_embedding_type < GGML_TYPE_COUNT) {
+ new_type = qs.params->token_embedding_type;
+ } else {
+ if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS ||
+ ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {
+ new_type = GGML_TYPE_Q2_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) {
+ new_type = GGML_TYPE_IQ3_S;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {
+ new_type = GGML_TYPE_IQ3_S;
+ }
+ else if (new_type == GGML_TYPE_Q4_0_4_4 || new_type == GGML_TYPE_Q4_0_4_8 ||
+ new_type == GGML_TYPE_Q4_0_8_8) {
+ new_type = GGML_TYPE_Q4_0;
+ }
+ }
+ } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S ||
+ ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {
+ if (name.find("attn_v.weight") != std::string::npos) {
+ if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K;
+ else new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;
+ ++qs.i_attention_wv;
+ }
+ else if (qs.model.hparams.n_expert == 8 && name.find("attn_k.weight") != std::string::npos) {
+ new_type = GGML_TYPE_Q4_K;
+ }
+ else if (name.find("ffn_down") != std::string::npos) {
+ if (qs.i_ffn_down < qs.n_ffn_down/8) {
+ new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;
+ }
+ ++qs.i_ffn_down;
+ }
+ else if (name.find("attn_output.weight") != std::string::npos) {
+ if (qs.model.hparams.n_expert == 8) {
+ new_type = GGML_TYPE_Q5_K;
+ } else {
+ if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) new_type = GGML_TYPE_IQ2_XXS;
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) new_type = GGML_TYPE_IQ3_S;
+ }
+ }
+ } else if (name.find("attn_v.weight") != std::string::npos) {
+ if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) {
+ new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && qs.model.hparams.n_gqa() >= 4) {
+ new_type = GGML_TYPE_Q4_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {
+ new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : !qs.has_imatrix ? GGML_TYPE_IQ3_S : GGML_TYPE_IQ3_XXS;
+ }
+ else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S) && qs.model.hparams.n_gqa() >= 4) {
+ new_type = GGML_TYPE_Q4_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) {
+ new_type = GGML_TYPE_Q4_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {
+ new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;
+ else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) && qs.model.hparams.n_gqa() >= 4) {
+ new_type = GGML_TYPE_Q5_K;
+ }
+ else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) &&
+ use_more_bits(qs.i_attention_wv, qs.n_attention_wv)) new_type = GGML_TYPE_Q6_K;
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && qs.i_attention_wv < 4) new_type = GGML_TYPE_Q5_K;
+ if (qs.model.type == MODEL_70B) {
+ // In the 70B model we have 8 heads sharing the same attn_v weights. As a result, the attn_v.weight tensor is
+ // 8x smaller compared to attn_q.weight. Hence, we can get a nice boost in quantization accuracy with
+ // nearly negligible increase in model size by quantizing this tensor with more bits:
+ if (new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K) new_type = GGML_TYPE_Q5_K;
+ }
+ if (qs.model.hparams.n_expert == 8) {
+ // for the 8-expert model, bumping this to Q8_0 trades just ~128MB
+ // TODO: explore better strategies
+ new_type = GGML_TYPE_Q8_0;
+ }
+ ++qs.i_attention_wv;
+ } else if (name.find("attn_k.weight") != std::string::npos) {
+ if (qs.model.hparams.n_expert == 8) {
+ // for the 8-expert model, bumping this to Q8_0 trades just ~128MB
+ // TODO: explore better strategies
+ new_type = GGML_TYPE_Q8_0;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS) {
+ new_type = GGML_TYPE_IQ3_XXS;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {
+ new_type = GGML_TYPE_IQ2_S;
+ }
+ } else if (name.find("attn_q.weight") != std::string::npos) {
+ if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS) {
+ new_type = GGML_TYPE_IQ3_XXS;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {
+ new_type = GGML_TYPE_IQ2_S;
+ }
+ } else if (name.find("ffn_down") != std::string::npos) {
+ auto info = layer_info(qs.i_ffn_down, qs.n_ffn_down, name.c_str());
+ int i_layer = info.first, n_layer = info.second;
+ if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) {
+ if (i_layer < n_layer/8) new_type = GGML_TYPE_Q4_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS && !qs.has_imatrix) {
+ new_type = i_layer < n_layer/8 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {
+ new_type = i_layer < n_layer/16 ? GGML_TYPE_Q5_K
+ : arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q4_K
+ : GGML_TYPE_Q3_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M && (i_layer < n_layer/8 ||
+ (qs.model.hparams.n_expert == 8 && use_more_bits(i_layer, n_layer)))) {
+ new_type = GGML_TYPE_Q4_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) {
+ new_type = arch == LLM_ARCH_FALCON ? GGML_TYPE_Q4_K : GGML_TYPE_Q5_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) {
+ if (arch == LLM_ARCH_FALCON) {
+ new_type = i_layer < n_layer/16 ? GGML_TYPE_Q6_K :
+ use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;
+ } else {
+ if (use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;
+ }
+ }
+ else if (i_layer < n_layer/8 && (ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) && !qs.has_imatrix) {
+ new_type = GGML_TYPE_Q5_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && i_layer < n_layer/8) {
+ new_type = GGML_TYPE_Q5_K;
+ }
+ else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_0 || ftype == LLAMA_FTYPE_MOSTLY_Q5_0)
+ && qs.has_imatrix && i_layer < n_layer/8) {
+ // Guard against craziness in the first few ffn_down layers that can happen even with imatrix for Q4_0/Q5_0.
+ // We only do it when an imatrix is provided because a) we want to make sure that one can always get the
+ // same quantization as before imatrix stuff, and b) Q4_1/Q5_1 do go crazy on ffn_down without an imatrix.
+ new_type = ftype == LLAMA_FTYPE_MOSTLY_Q4_0 ? GGML_TYPE_Q4_1 : GGML_TYPE_Q5_1;
+ }
+ ++qs.i_ffn_down;
+ } else if (name.find("attn_output.weight") != std::string::npos) {
+ if (arch != LLM_ARCH_FALCON) {
+ if (qs.model.hparams.n_expert == 8) {
+ if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS ||
+ ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL ||
+ ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S ||
+ ftype == LLAMA_FTYPE_MOSTLY_IQ3_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) {
+ new_type = GGML_TYPE_Q5_K;
+ }
+ } else {
+ if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K;
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) new_type = GGML_TYPE_IQ3_S;
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M ) new_type = GGML_TYPE_Q4_K;
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L ) new_type = GGML_TYPE_Q5_K;
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M ) new_type = GGML_TYPE_Q4_K;
+ }
+ } else {
+ if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q4_K;
+ }
+ }
+ else if (name.find("attn_qkv.weight") != std::string::npos) {
+ if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L || ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) {
+ new_type = GGML_TYPE_Q4_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) new_type = GGML_TYPE_Q5_K;
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) new_type = GGML_TYPE_Q6_K;
+ }
+ else if (name.find("ffn_gate") != std::string::npos) {
+ auto info = layer_info(qs.i_ffn_gate, qs.n_ffn_gate, name.c_str());
+ int i_layer = info.first, n_layer = info.second;
+ if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) {
+ new_type = GGML_TYPE_IQ3_XXS;
+ }
+ ++qs.i_ffn_gate;
+ }
+ else if (name.find("ffn_up") != std::string::npos) {
+ auto info = layer_info(qs.i_ffn_up, qs.n_ffn_up, name.c_str());
+ int i_layer = info.first, n_layer = info.second;
+ if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) {
+ new_type = GGML_TYPE_IQ3_XXS;
+ }
+ ++qs.i_ffn_up;
+ }
+
+ // if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;
+ //}
+ // IK: let's remove this, else Q2_K is almost the same as Q3_K_S
+ //else if (name.find("ffn_gate") != std::string::npos || name.find("ffn_up") != std::string::npos) {
+ // if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;
+ //}
+ // This can be used to reduce the size of the Q5_K_S model.
+ // The associated PPL increase is fully in line with the size reduction
+ //else {
+ // if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_S) new_type = GGML_TYPE_Q4_K;
+ //}
+ bool convert_incompatible_tensor = false;
+ if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K ||
+ new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K || new_type == GGML_TYPE_IQ4_XS ||
+ new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_XXS || new_type == GGML_TYPE_IQ2_S ||
+ new_type == GGML_TYPE_IQ3_XXS || new_type == GGML_TYPE_IQ1_S || new_type == GGML_TYPE_IQ3_S ||
+ new_type == GGML_TYPE_IQ1_M) {
+ int nx = tensor->ne[0];
+ int ny = tensor->ne[1];
+ if (nx % QK_K != 0) {
+ LLAMA_LOG_WARN("\n\n%s : tensor cols %d x %d are not divisible by %d, required for %s", __func__, nx, ny, QK_K, ggml_type_name(new_type));
+ convert_incompatible_tensor = true;
+ } else {
+ ++qs.n_k_quantized;
+ }
+ }
+ if (convert_incompatible_tensor) {
+ switch (new_type) {
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_IQ4_XS: new_type = GGML_TYPE_IQ4_NL; break;
+ case GGML_TYPE_Q4_K: new_type = GGML_TYPE_Q5_0; break;
+ case GGML_TYPE_Q5_K: new_type = GGML_TYPE_Q5_1; break;
+ case GGML_TYPE_Q6_K: new_type = GGML_TYPE_Q8_0; break;
+ default: throw std::runtime_error("\nUnsupported tensor size encountered\n");
+ }
+ if (tensor->ne[0] % ggml_blck_size(new_type) != 0) {
+ new_type = GGML_TYPE_F16;
+ }
+ LLAMA_LOG_WARN(" - using fallback quantization %s\n", ggml_type_name(new_type));
+ ++qs.n_fallback;
+ }
+
+ return new_type;
+}
+
+static size_t llama_tensor_quantize_internal(enum ggml_type new_type, const float * f32_data, void * new_data, const int64_t chunk_size, int64_t nrows, int64_t n_per_row, const float * imatrix, std::vector<std::thread> & workers, const int nthread) {
+ if (nthread < 2) {
+ // single-thread
+ size_t new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, imatrix);
+ if (!ggml_validate_row_data(new_type, new_data, new_size)) {
+ throw std::runtime_error("quantized data validation failed");
+ }
+ return new_size;
+ }
+
+ std::mutex mutex;
+ int64_t counter = 0;
+ size_t new_size = 0;
+ bool valid = true;
+ auto compute = [&mutex, &counter, &new_size, &valid, new_type, f32_data, new_data, chunk_size,
+ nrows, n_per_row, imatrix]() {
+ const int64_t nrows_per_chunk = chunk_size / n_per_row;
+ size_t local_size = 0;
+ while (true) {
+ std::unique_lock<std::mutex> lock(mutex);
+ int64_t first_row = counter; counter += nrows_per_chunk;
+ if (first_row >= nrows) {
+ if (local_size > 0) {
+ new_size += local_size;
+ }
+ break;
+ }
+ lock.unlock();
+ const int64_t this_nrow = std::min(nrows - first_row, nrows_per_chunk);
+ size_t this_size = ggml_quantize_chunk(new_type, f32_data, new_data, first_row * n_per_row, this_nrow, n_per_row, imatrix);
+ local_size += this_size;
+
+ // validate the quantized data
+ const size_t row_size = ggml_row_size(new_type, n_per_row);
+ void * this_data = (char *) new_data + first_row * row_size;
+ if (!ggml_validate_row_data(new_type, this_data, this_size)) {
+ std::unique_lock<std::mutex> lock(mutex);
+ valid = false;
+ break;
+ }
+ }
+ };
+ for (int it = 0; it < nthread - 1; ++it) {
+ workers.emplace_back(compute);
+ }
+ compute();
+ for (auto & w : workers) { w.join(); }
+ workers.clear();
+ if (!valid) {
+ throw std::runtime_error("quantized data validation failed");
+ }
+ return new_size;
+}
+
+static void llama_model_quantize_internal(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) {
+ ggml_type default_type;
+ llama_ftype ftype = params->ftype;
+
+ switch (params->ftype) {
+ case LLAMA_FTYPE_MOSTLY_Q4_0: default_type = GGML_TYPE_Q4_0; break;
+ case LLAMA_FTYPE_MOSTLY_Q4_1: default_type = GGML_TYPE_Q4_1; break;
+ case LLAMA_FTYPE_MOSTLY_Q5_0: default_type = GGML_TYPE_Q5_0; break;
+ case LLAMA_FTYPE_MOSTLY_Q5_1: default_type = GGML_TYPE_Q5_1; break;
+ case LLAMA_FTYPE_MOSTLY_Q8_0: default_type = GGML_TYPE_Q8_0; break;
+ case LLAMA_FTYPE_MOSTLY_F16: default_type = GGML_TYPE_F16; break;
+ case LLAMA_FTYPE_MOSTLY_BF16: default_type = GGML_TYPE_BF16; break;
+ case LLAMA_FTYPE_ALL_F32: default_type = GGML_TYPE_F32; break;
+
+ // K-quants
+ case LLAMA_FTYPE_MOSTLY_Q2_K_S:
+ case LLAMA_FTYPE_MOSTLY_Q2_K: default_type = GGML_TYPE_Q2_K; break;
+ case LLAMA_FTYPE_MOSTLY_IQ3_XS: default_type = GGML_TYPE_IQ3_S; break;
+ case LLAMA_FTYPE_MOSTLY_Q3_K_S:
+ case LLAMA_FTYPE_MOSTLY_Q3_K_M:
+ case LLAMA_FTYPE_MOSTLY_Q3_K_L: default_type = GGML_TYPE_Q3_K; break;
+ case LLAMA_FTYPE_MOSTLY_Q4_K_S:
+ case LLAMA_FTYPE_MOSTLY_Q4_K_M: default_type = GGML_TYPE_Q4_K; break;
+ case LLAMA_FTYPE_MOSTLY_Q5_K_S:
+ case LLAMA_FTYPE_MOSTLY_Q5_K_M: default_type = GGML_TYPE_Q5_K; break;
+ case LLAMA_FTYPE_MOSTLY_Q6_K: default_type = GGML_TYPE_Q6_K; break;
+ case LLAMA_FTYPE_MOSTLY_IQ2_XXS: default_type = GGML_TYPE_IQ2_XXS; break;
+ case LLAMA_FTYPE_MOSTLY_IQ2_XS: default_type = GGML_TYPE_IQ2_XS; break;
+ case LLAMA_FTYPE_MOSTLY_IQ2_S: default_type = GGML_TYPE_IQ2_XS; break;
+ case LLAMA_FTYPE_MOSTLY_IQ2_M: default_type = GGML_TYPE_IQ2_S; break;
+ case LLAMA_FTYPE_MOSTLY_IQ3_XXS: default_type = GGML_TYPE_IQ3_XXS; break;
+ case LLAMA_FTYPE_MOSTLY_IQ1_S: default_type = GGML_TYPE_IQ1_S; break;
+ case LLAMA_FTYPE_MOSTLY_IQ1_M: default_type = GGML_TYPE_IQ1_M; break;
+ case LLAMA_FTYPE_MOSTLY_IQ4_NL: default_type = GGML_TYPE_IQ4_NL; break;
+ case LLAMA_FTYPE_MOSTLY_IQ4_XS: default_type = GGML_TYPE_IQ4_XS; break;
+ case LLAMA_FTYPE_MOSTLY_IQ3_S: default_type = GGML_TYPE_IQ3_S; break;
+ case LLAMA_FTYPE_MOSTLY_IQ3_M: default_type = GGML_TYPE_IQ3_S; break;
+ case LLAMA_FTYPE_MOSTLY_Q4_0_4_4: default_type = GGML_TYPE_Q4_0_4_4; break;
+ case LLAMA_FTYPE_MOSTLY_Q4_0_4_8: default_type = GGML_TYPE_Q4_0_4_8; break;
+ case LLAMA_FTYPE_MOSTLY_Q4_0_8_8: default_type = GGML_TYPE_Q4_0_8_8; break;
+
+ default: throw std::runtime_error(format("invalid output file type %d\n", ftype));
+ }
+
+ int nthread = params->nthread;
+
+ if (nthread <= 0) {
+ nthread = std::thread::hardware_concurrency();
+ }
+
+ // mmap consistently increases speed Linux, and also increases speed on Windows with
+ // hot cache. It may cause a slowdown on macOS, possibly related to free memory.
+#if defined(__linux__) || defined(_WIN32)
+ constexpr bool use_mmap = true;
+#else
+ constexpr bool use_mmap = false;
+#endif
+
+ llama_model_kv_override * kv_overrides = nullptr;
+ if (params->kv_overrides) {
+ auto v = (std::vector<llama_model_kv_override>*)params->kv_overrides;
+ kv_overrides = v->data();
+ }
+ llama_model_loader ml(fname_inp, use_mmap, /*check_tensors*/ true, kv_overrides);
+ ml.init_mappings(false); // no prefetching
+
+ llama_model model;
+ llm_load_arch(ml, model);
+ llm_load_hparams(ml, model);
+
+ struct quantize_state_internal qs(model, params);
+
+ if (params->only_copy) {
+ ftype = model.ftype;
+ }
+ const std::unordered_map<std::string, std::vector<float>> * imatrix_data = nullptr;
+ if (params->imatrix) {
+ imatrix_data = static_cast<const std::unordered_map<std::string, std::vector<float>>*>(params->imatrix);
+ if (imatrix_data) {
+ LLAMA_LOG_INFO("================================ Have weights data with %d entries\n",int(imatrix_data->size()));
+ qs.has_imatrix = true;
+ // check imatrix for nans or infs
+ for (const auto & kv : *imatrix_data) {
+ for (float f : kv.second) {
+ if (!std::isfinite(f)) {
+ throw std::runtime_error(format("imatrix contains non-finite value %f\n", f));
+ }
+ }
+ }
+ }
+ }
+
+ const size_t align = GGUF_DEFAULT_ALIGNMENT;
+ struct gguf_context * ctx_out = gguf_init_empty();
+
+ // copy the KV pairs from the input file
+ gguf_set_kv (ctx_out, ml.meta);
+ gguf_set_val_u32(ctx_out, "general.quantization_version", GGML_QNT_VERSION); // TODO: use LLM_KV
+ gguf_set_val_u32(ctx_out, "general.file_type", ftype); // TODO: use LLM_KV
+
+ // Remove split metadata
+ gguf_remove_key(ctx_out, ml.llm_kv(LLM_KV_SPLIT_NO).c_str());
+ gguf_remove_key(ctx_out, ml.llm_kv(LLM_KV_SPLIT_COUNT).c_str());
+ gguf_remove_key(ctx_out, ml.llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str());
+
+ if (params->kv_overrides) {
+ const std::vector<llama_model_kv_override> & overrides = *(const std::vector<llama_model_kv_override> *)params->kv_overrides;
+ for (auto & o : overrides) {
+ if (o.key[0] == 0) break;
+ if (o.tag == LLAMA_KV_OVERRIDE_TYPE_FLOAT) {
+ gguf_set_val_f32(ctx_out, o.key, o.val_f64);
+ } else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_INT) {
+ gguf_set_val_i32(ctx_out, o.key, o.val_i64);
+ } else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_BOOL) {
+ gguf_set_val_bool(ctx_out, o.key, o.val_bool);
+ } else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_STR) {
+ gguf_set_val_str(ctx_out, o.key, o.val_str);
+ } else {
+ LLAMA_LOG_WARN("%s: unknown KV override type for key %s\n", __func__, o.key);
+ }
+ }
+ }
+
+ for (int i = 0; i < ml.n_tensors; ++i) {
+ const struct ggml_tensor * meta = ml.get_tensor_meta(i);
+
+ const std::string name = ggml_get_name(meta);
+
+ // TODO: avoid hardcoded tensor names - use the TN_* constants
+ if (name.find("attn_v.weight") != std::string::npos ||
+ name.find("attn_qkv.weight") != std::string::npos ||
+ name.find("attn_kv_b.weight")!= std::string::npos) {
+ ++qs.n_attention_wv;
+ } else if (name == LLM_TN(model.arch)(LLM_TENSOR_OUTPUT, "weight")) {
+ qs.has_output = true;
+ }
+ }
+
+ qs.n_ffn_down = qs.n_ffn_gate = qs.n_ffn_up = (int)model.hparams.n_layer;
+
+ // sanity checks
+ {
+ const auto & n_head_kv_iter = model.hparams.n_head_kv_arr.begin();
+ // attention layers have a non-zero number of kv heads
+ int32_t n_attn_layer = model.hparams.n_layer - std::count(n_head_kv_iter, n_head_kv_iter + model.hparams.n_layer, 0);
+ if (llama_model_has_encoder(&model)) {
+ n_attn_layer *= 3;
+ }
+ GGML_ASSERT((qs.n_attention_wv == n_attn_layer) && "n_attention_wv is unexpected");
+ }
+
+ size_t total_size_org = 0;
+ size_t total_size_new = 0;
+
+ std::vector<std::thread> workers;
+ workers.reserve(nthread);
+
+ int idx = 0;
+
+ std::vector<no_init<uint8_t>> read_data;
+ std::vector<no_init<uint8_t>> work;
+ std::vector<no_init<float>> f32_conv_buf;
+
+ uint16_t n_split = 1;
+ // Assume split index is continuous
+ if (params->keep_split) {
+ for (int i = 0; i < ml.n_tensors; ++i) {
+ n_split = std::max(uint16_t(ml.get_weight(i)->idx+1), n_split);
+ }
+ }
+ std::vector<gguf_context*> ctx_outs(n_split, NULL);
+ ctx_outs[0] = ctx_out;
+
+ // populate the original tensors so we get an initial meta data
+ for (int i = 0; i < ml.n_tensors; ++i) {
+ auto weight = ml.get_weight(i);
+ uint16_t i_split = params->keep_split ? weight->idx : 0;
+ struct ggml_tensor * tensor = weight->tensor;
+ if (ctx_outs[i_split] == NULL) {
+ ctx_outs[i_split] = gguf_init_empty();
+ }
+ gguf_add_tensor(ctx_outs[i_split], tensor);
+ }
+
+ // Set split info if needed
+ if (n_split > 1) {
+ for (size_t i = 0; i < ctx_outs.size(); ++i) {
+ gguf_set_val_u16(ctx_outs[i], ml.llm_kv(LLM_KV_SPLIT_NO).c_str(), i);
+ gguf_set_val_u16(ctx_outs[i], ml.llm_kv(LLM_KV_SPLIT_COUNT).c_str(), n_split);
+ gguf_set_val_i32(ctx_outs[i], ml.llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str(), ml.n_tensors);
+ }
+ }
+
+ int cur_split = -1;
+ std::ofstream fout;
+ auto close_ofstream = [&]() {
+ // Write metadata and close file handler
+ if (fout.is_open()) {
+ fout.seekp(0);
+ std::vector<uint8_t> data(gguf_get_meta_size(ctx_outs[cur_split]));
+ gguf_get_meta_data(ctx_outs[cur_split], data.data());
+ fout.write((const char *) data.data(), data.size());
+ fout.close();
+ }
+ };
+ auto new_ofstream = [&](int index) {
+ cur_split = index;
+ GGML_ASSERT(ctx_outs[cur_split] && "Find uninitialized gguf_context");
+ std::string fname = fname_out;
+ if (params->keep_split) {
+ char split_path[PATH_MAX] = {0};
+ llama_split_path(split_path, sizeof(split_path), fname_out.c_str(), cur_split, n_split);
+ fname = std::string(split_path);
+ }
+
+ fout = std::ofstream(fname, std::ios::binary);
+ fout.exceptions(std::ofstream::failbit); // fail fast on write errors
+ const size_t meta_size = gguf_get_meta_size(ctx_outs[cur_split]);
+ // placeholder for the meta data
+ ::zeros(fout, meta_size);
+ };
+
+ const auto tn = LLM_TN(model.arch);
+ new_ofstream(0);
+ for (int i = 0; i < ml.n_tensors; ++i) {
+ auto weight = ml.get_weight(i);
+ struct ggml_tensor * tensor = weight->tensor;
+ if (weight->idx != cur_split && params->keep_split) {
+ close_ofstream();
+ new_ofstream(weight->idx);
+ }
+
+ const std::string name = ggml_get_name(tensor);
+
+ if (!ml.use_mmap) {
+ if (read_data.size() < ggml_nbytes(tensor)) {
+ read_data.resize(ggml_nbytes(tensor));
+ }
+ tensor->data = read_data.data();
+ }
+ ml.load_data_for(tensor);
+
+ LLAMA_LOG_INFO("[%4d/%4d] %36s - [%s], type = %6s, ",
+ ++idx, ml.n_tensors,
+ ggml_get_name(tensor),
+ llama_format_tensor_shape(tensor).c_str(),
+ ggml_type_name(tensor->type));
+
+ // This used to be a regex, but <regex> has an extreme cost to compile times.
+ bool quantize = name.rfind("weight") == name.size() - 6; // ends with 'weight'?
+
+ // quantize only 2D and 3D tensors (experts)
+ quantize &= (ggml_n_dims(tensor) >= 2);
+
+ // do not quantize norm tensors
+ quantize &= name.find("_norm.weight") == std::string::npos;
+
+ quantize &= params->quantize_output_tensor || name != "output.weight";
+ quantize &= !params->only_copy;
+
+ // do not quantize expert gating tensors
+ // NOTE: can't use LLM_TN here because the layer number is not known
+ quantize &= name.find("ffn_gate_inp.weight") == std::string::npos;
+
+ // do not quantize positional embeddings and token types (BERT)
+ quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_POS_EMBD, "weight");
+ quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_TOKEN_TYPES, "weight");
+
+ // do not quantize Mamba's small yet 2D weights
+ // NOTE: can't use LLM_TN here because the layer number is not known
+ quantize &= name.find("ssm_conv1d.weight") == std::string::npos;
+
+ // do not quantize RWKV's time_mix_first tensors
+ quantize &= name.find("time_mix_first.weight") == std::string::npos;
+ quantize &= name.find("time_mix_w1.weight") == std::string::npos;
+ quantize &= name.find("time_mix_w2.weight") == std::string::npos;
+
+ // do not quantize relative position bias (T5)
+ quantize &= name.find("attn_rel_b.weight") == std::string::npos;
+
+ enum ggml_type new_type;
+ void * new_data;
+ size_t new_size;
+
+ if (quantize) {
+ new_type = default_type;
+
+ // get more optimal quantization type based on the tensor shape, layer, etc.
+ if (!params->pure && ggml_is_quantized(default_type)) {
+ new_type = llama_tensor_get_type(qs, new_type, tensor, ftype);
+ }
+ if (params->token_embedding_type < GGML_TYPE_COUNT && strcmp(tensor->name, "token_embd.weight") == 0) {
+ new_type = params->token_embedding_type;
+ }
+ if (params->output_tensor_type < GGML_TYPE_COUNT && strcmp(tensor->name, "output.weight") == 0) {
+ new_type = params->output_tensor_type;
+ }
+
+ // If we've decided to quantize to the same type the tensor is already
+ // in then there's nothing to do.
+ quantize = tensor->type != new_type;
+ }
+
+ if (!quantize) {
+ new_type = tensor->type;
+ new_data = tensor->data;
+ new_size = ggml_nbytes(tensor);
+ LLAMA_LOG_INFO("size = %8.3f MB\n", ggml_nbytes(tensor)/1024.0/1024.0);
+ } else {
+ const int64_t nelements = ggml_nelements(tensor);
+
+ const float * imatrix = nullptr;
+ if (imatrix_data) {
+ auto it = imatrix_data->find(tensor->name);
+ if (it == imatrix_data->end()) {
+ LLAMA_LOG_INFO("\n====== %s: did not find weights for %s\n", __func__, tensor->name);
+ } else {
+ if (it->second.size() == (size_t)tensor->ne[0]*tensor->ne[2]) {
+ imatrix = it->second.data();
+ } else {
+ LLAMA_LOG_INFO("\n====== %s: imatrix size %d is different from tensor size %d for %s\n", __func__,
+ int(it->second.size()), int(tensor->ne[0]*tensor->ne[2]), tensor->name);
+
+ // this can happen when quantizing an old mixtral model with split tensors with a new incompatible imatrix
+ // this is a significant error and it may be good idea to abort the process if this happens,
+ // since many people will miss the error and not realize that most of the model is being quantized without an imatrix
+ // tok_embd should be ignored in this case, since it always causes this warning
+ if (name != tn(LLM_TENSOR_TOKEN_EMBD, "weight")) {
+ throw std::runtime_error(format("imatrix size %d is different from tensor size %d for %s",
+ int(it->second.size()), int(tensor->ne[0]*tensor->ne[2]), tensor->name));
+ }
+ }
+ }
+ }
+ if ((new_type == GGML_TYPE_IQ2_XXS ||
+ new_type == GGML_TYPE_IQ2_XS ||
+ new_type == GGML_TYPE_IQ2_S ||
+ new_type == GGML_TYPE_IQ1_S ||
+ (new_type == GGML_TYPE_IQ1_M && strcmp(tensor->name, "token_embd.weight") && strcmp(tensor->name, "output.weight")) ||
+ (new_type == GGML_TYPE_Q2_K && params->ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && strcmp(tensor->name, "token_embd.weight") != 0)) && !imatrix) {
+ LLAMA_LOG_ERROR("\n\n============================================================\n");
+ LLAMA_LOG_ERROR("Missing importance matrix for tensor %s in a very low-bit quantization\n", tensor->name);
+ LLAMA_LOG_ERROR("The result will be garbage, so bailing out\n");
+ LLAMA_LOG_ERROR("============================================================\n\n");
+ throw std::runtime_error(format("Missing importance matrix for tensor %s in a very low-bit quantization", tensor->name));
+ }
+
+ float * f32_data;
+
+ if (tensor->type == GGML_TYPE_F32) {
+ f32_data = (float *) tensor->data;
+ } else if (ggml_is_quantized(tensor->type) && !params->allow_requantize) {
+ throw std::runtime_error(format("requantizing from type %s is disabled", ggml_type_name(tensor->type)));
+ } else {
+ llama_tensor_dequantize_internal(tensor, f32_conv_buf, workers, nelements, nthread);
+ f32_data = (float *) f32_conv_buf.data();
+ }
+
+ int chunk_size_multiplier = 1;
+ if (new_type == GGML_TYPE_Q4_0_4_4 || new_type == GGML_TYPE_Q4_0_4_8 || new_type == GGML_TYPE_Q4_0_8_8) {
+ if ((new_type == GGML_TYPE_Q4_0_8_8) && (tensor->ne[1] % 8 != 0)) new_type = GGML_TYPE_Q4_0;
+ else if (tensor->ne[1] % 4 != 0) new_type = GGML_TYPE_Q4_0;
+ if (new_type == GGML_TYPE_Q4_0_8_8) chunk_size_multiplier = 8;
+ else if (new_type == GGML_TYPE_Q4_0_4_4 || new_type == GGML_TYPE_Q4_0_4_8) chunk_size_multiplier = 4;
+ }
+
+ LLAMA_LOG_INFO("converting to %s .. ", ggml_type_name(new_type));
+ fflush(stdout);
+
+ if (work.size() < (size_t)nelements * 4) {
+ work.resize(nelements * 4); // upper bound on size
+ }
+ new_data = work.data();
+
+ const int64_t n_per_row = tensor->ne[0];
+ const int64_t nrows = tensor->ne[1];
+
+ static const int64_t min_chunk_size = 32 * 512;
+ const int64_t chunk_size = (n_per_row >= min_chunk_size ? n_per_row : n_per_row * ((min_chunk_size + n_per_row - 1)/n_per_row)) *
+ chunk_size_multiplier;
+
+ const int64_t nelements_matrix = tensor->ne[0] * tensor->ne[1];
+ const int64_t nchunk = (nelements_matrix + chunk_size - 1)/chunk_size;
+ const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1;
+
+ // quantize each expert separately since they have different importance matrices
+ new_size = 0;
+ for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) {
+ const float * f32_data_03 = f32_data + i03 * nelements_matrix;
+ void * new_data_03 = (char *)new_data + ggml_row_size(new_type, n_per_row) * i03 * nrows;
+ const float * imatrix_03 = imatrix ? imatrix + i03 * n_per_row : nullptr;
+
+ new_size += llama_tensor_quantize_internal(new_type, f32_data_03, new_data_03, chunk_size, nrows, n_per_row, imatrix_03, workers, nthread_use);
+ }
+ LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0);
+ }
+ total_size_org += ggml_nbytes(tensor);
+ total_size_new += new_size;
+
+ // update the gguf meta data as we go
+ gguf_set_tensor_type(ctx_outs[cur_split], name.c_str(), new_type);
+ gguf_set_tensor_data(ctx_outs[cur_split], name.c_str(), new_data, new_size);
+
+ // write tensor data + padding
+ fout.write((const char *) new_data, new_size);
+ zeros(fout, GGML_PAD(new_size, align) - new_size);
+ }
+ close_ofstream();
+ for (auto & c:ctx_outs) {
+ gguf_free(c);
+ }
+
+ LLAMA_LOG_INFO("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0);
+ LLAMA_LOG_INFO("%s: quant size = %8.2f MB\n", __func__, total_size_new/1024.0/1024.0);
+
+ if (qs.n_fallback > 0) {
+ LLAMA_LOG_WARN("%s: WARNING: %d of %d tensor(s) required fallback quantization\n",
+ __func__, qs.n_fallback, qs.n_k_quantized + qs.n_fallback);
+ }
+}
+
+static void llama_lora_adapter_init_internal(struct llama_model * model, const char * path_lora, struct llama_lora_adapter & adapter) {
+ LLAMA_LOG_INFO("%s: loading lora adapter from '%s' ...\n", __func__, path_lora);
+
+ ggml_context * ctx = nullptr;
+ struct gguf_init_params meta_gguf_params = {
+ /* .no_alloc = */ true,
+ /* .ctx = */ &ctx,
+ };
+ struct gguf_context * ctx_gguf = gguf_init_from_file(path_lora, meta_gguf_params);
+ if (!ctx_gguf) {
+ throw std::runtime_error("failed to load lora adapter file from " + std::string(path_lora));
+ }
+
+ // check metadata
+ {
+ auto get_kv_str = [&](const std::string & key) -> std::string {
+ int id = gguf_find_key(ctx_gguf, key.c_str());
+ return id < 0 ? "" : std::string(gguf_get_val_str(ctx_gguf, id));
+ };
+ auto get_kv_f32 = [&](const std::string & key) -> float {
+ int id = gguf_find_key(ctx_gguf, key.c_str());
+ return id < 0 ? 0.0f : gguf_get_val_f32(ctx_gguf, id);
+ };
+ LLM_KV llm_kv = LLM_KV(LLM_ARCH_UNKNOWN);
+
+ auto general_type = get_kv_str(llm_kv(LLM_KV_GENERAL_TYPE));
+ if (general_type != "adapter") {
+ gguf_free(ctx_gguf);
+ throw std::runtime_error("expect general.type to be 'adapter', but got: " + general_type);
+ }
+
+ auto general_arch_str = get_kv_str(llm_kv(LLM_KV_GENERAL_ARCHITECTURE));
+ auto general_arch = llm_arch_from_string(general_arch_str);
+ if (general_arch != model->arch) {
+ gguf_free(ctx_gguf);
+ throw std::runtime_error("model arch and LoRA arch mismatch");
+ }
+
+ auto adapter_type = get_kv_str(llm_kv(LLM_KV_ADAPTER_TYPE));
+ if (adapter_type != "lora") {
+ gguf_free(ctx_gguf);
+ throw std::runtime_error("expect adapter.type to be 'lora', but got: " + adapter_type);
+ }
+
+ adapter.alpha = get_kv_f32(llm_kv(LLM_KV_ADAPTER_LORA_ALPHA));
+ }
+
+ int n_tensors = gguf_get_n_tensors(ctx_gguf);
+
+ // contexts for each buffer type
+ std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;
+ auto get_ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {
+ auto it = ctx_map.find(buft);
+ if (it == ctx_map.end()) {
+ // add a new context
+ struct ggml_init_params params = {
+ /*.mem_size =*/ n_tensors*ggml_tensor_overhead(),
+ /*.mem_buffer =*/ NULL,
+ /*.no_alloc =*/ true,
+ };
+ ggml_context * buft_ctx = ggml_init(params);
+ ctx_map[buft] = buft_ctx;
+ return buft_ctx;
+ };
+ return it->second;
+ };
+
+ // bundle lora_a and lora_b into pairs
+ std::map<std::string, llama_lora_weight> ab_map;
+ auto str_endswith = [](const std::string & str, const std::string & suffix) {
+ return str.size() >= suffix.size() && str.compare(str.size()-suffix.size(), suffix.size(), suffix) == 0;
+ };
+ for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
+ std::string name(cur->name);
+ if (str_endswith(name, ".lora_a")) {
+ replace_all(name, ".lora_a", "");
+ if (ab_map.find(name) == ab_map.end()) {
+ ab_map[name] = llama_lora_weight(cur, nullptr);
+ } else {
+ ab_map[name].a = cur;
+ }
+ } else if (str_endswith(name, ".lora_b")) {
+ replace_all(name, ".lora_b", "");
+ if (ab_map.find(name) == ab_map.end()) {
+ ab_map[name] = llama_lora_weight(nullptr, cur);
+ } else {
+ ab_map[name].b = cur;
+ }
+ } else {
+ gguf_free(ctx_gguf);
+ ggml_free(ctx);
+ throw std::runtime_error("LoRA tensor '" + name + "' has unexpected suffix");
+ }
+ }
+
+ // add tensors
+ for (auto & it : ab_map) {
+ const std::string & name = it.first;
+ llama_lora_weight & w = it.second;
+
+ if (!w.a || !w.b) {
+ gguf_free(ctx_gguf);
+ ggml_free(ctx);
+ throw std::runtime_error("LoRA tensor pair for '" + name + "' is missing one component");
+ }
+
+ // device buft and device ctx
+ auto * model_tensor = llama_get_model_tensor(model, name.c_str());
+ if (!model_tensor) {
+ gguf_free(ctx_gguf);
+ ggml_free(ctx);
+ throw std::runtime_error("LoRA tensor '" + name + "' does not exist in base model");
+ }
+ struct ggml_context * dev_ctx = get_ctx_for_buft(ggml_backend_buffer_get_type(model_tensor->buffer));
+ // validate tensor shape
+ if (model_tensor->ne[0] != w.a->ne[0] || model_tensor->ne[1] != w.b->ne[1]) {
+ gguf_free(ctx_gguf);
+ ggml_free(ctx);
+ throw std::runtime_error("tensor '" + name + "' has incorrect shape");
+ }
+ if (w.a->ne[1] != w.b->ne[0]) {
+ gguf_free(ctx_gguf);
+ ggml_free(ctx);
+ throw std::runtime_error("lora_a tensor is not transposed (hint: adapter from \"finetune\" example is no longer supported)");
+ }
+ // save tensor to adapter
+ struct ggml_tensor * tensor_a = ggml_dup_tensor(dev_ctx, w.a);
+ struct ggml_tensor * tensor_b = ggml_dup_tensor(dev_ctx, w.b);
+ ggml_set_name(tensor_a, w.a->name);
+ ggml_set_name(tensor_b, w.b->name);
+ adapter.ab_map[name] = llama_lora_weight(tensor_a, tensor_b);
+ }
+
+ // allocate tensors / buffers and zero
+ {
+ adapter.ctxs.reserve(ctx_map.size());
+ adapter.bufs.reserve(ctx_map.size());
+ for (auto it : ctx_map) {
+ ggml_backend_buffer_type_t buft = it.first;
+ ggml_context * ctx_dev = it.second;
+ ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx_dev, buft);
+ if (!buf) {
+ gguf_free(ctx_gguf);
+ ggml_free(ctx);
+ throw std::runtime_error("failed to allocate buffer for lora adapter\n");
+ }
+ LLAMA_LOG_INFO("%s: %10s LoRA buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
+ adapter.ctxs.push_back(ctx_dev);
+ adapter.bufs.push_back(buf);
+ }
+ }
+
+ // set tensor data
+ {
+ llama_file gguf_file(path_lora, "rb");
+ std::vector<uint8_t> read_buf;
+ auto set_tensor = [&](struct ggml_tensor * orig, struct ggml_tensor * dev) {
+ size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, gguf_find_tensor(ctx_gguf, orig->name));
+ size_t size = ggml_nbytes(orig);
+ read_buf.resize(size);
+ gguf_file.seek(offs, SEEK_SET);
+ gguf_file.read_raw(read_buf.data(), size);
+ ggml_backend_tensor_set(dev, read_buf.data(), 0, size);
+ };
+ for (auto & it : adapter.ab_map) {
+ auto orig = ab_map[it.first];
+ auto dev = it.second;
+ set_tensor(orig.a, dev.a);
+ set_tensor(orig.b, dev.b);
+ }
+ }
+
+ LLAMA_LOG_INFO("%s: loaded %ld tensors from lora file\n", __func__, adapter.ab_map.size()*2);
+
+ // free ctx for reading gguf
+ gguf_free(ctx_gguf);
+ ggml_free(ctx);
+}
+
+int32_t llama_lora_adapter_set(
+ struct llama_context * ctx,
+ struct llama_lora_adapter * adapter,
+ float scale) {
+ if (ctx->cparams.flash_attn) {
+ LLAMA_LOG_ERROR("%s: flash_attn is not compatible with LoRA\n", __func__);
+ return -1;
+ }
+ ctx->lora_adapters[adapter] = scale;
+ return 0;
+}
+
+int32_t llama_lora_adapter_remove(
+ struct llama_context * ctx,
+ struct llama_lora_adapter * adapter) {
+ auto pos = ctx->lora_adapters.find(adapter);
+ if (pos != ctx->lora_adapters.end()) {
+ ctx->lora_adapters.erase(pos);
+ return 0;
+ }
+ return -1;
+}
+
+void llama_lora_adapter_clear(struct llama_context * ctx) {
+ ctx->lora_adapters.clear();
+}
+
+void llama_lora_adapter_free(struct llama_lora_adapter * adapter) {
+ delete adapter;
+}
+
+//
+// interface implementation
+//
+struct llama_model_params llama_model_default_params() {
+ struct llama_model_params result = {
+ /*.n_gpu_layers =*/ 0,
+ /*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER,
+ /*.main_gpu =*/ 0,
+ /*.tensor_split =*/ nullptr,
+ /*.rpc_servers =*/ nullptr,
+ /*.progress_callback =*/ nullptr,
+ /*.progress_callback_user_data =*/ nullptr,
+ /*.kv_overrides =*/ nullptr,
+ /*.vocab_only =*/ false,
+ /*.use_mmap =*/ true,
+ /*.use_mlock =*/ false,
+ /*.check_tensors =*/ false,
+ };
+
+#ifdef GGML_USE_METAL
+ // note: we usually have plenty of VRAM, so by default offload all layers to the GPU
+ result.n_gpu_layers = 999;
+#endif
+
+ return result;
+}
+
+struct llama_context_params llama_context_default_params() {
+ struct llama_context_params result = {
+ /*.seed =*/ LLAMA_DEFAULT_SEED,
+ /*.n_ctx =*/ 512,
+ /*.n_batch =*/ 2048,
+ /*.n_ubatch =*/ 512,
+ /*.n_seq_max =*/ 1,
+ /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default
+ /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS,
+ /*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED,
+ /*.pooling_type =*/ LLAMA_POOLING_TYPE_UNSPECIFIED,
+ /*.attention_type =*/ LLAMA_ATTENTION_TYPE_UNSPECIFIED,
+ /*.rope_freq_base =*/ 0.0f,
+ /*.rope_freq_scale =*/ 0.0f,
+ /*.yarn_ext_factor =*/ -1.0f,
+ /*.yarn_attn_factor =*/ 1.0f,
+ /*.yarn_beta_fast =*/ 32.0f,
+ /*.yarn_beta_slow =*/ 1.0f,
+ /*.yarn_orig_ctx =*/ 0,
+ /*.defrag_thold =*/ -1.0f,
+ /*.cb_eval =*/ nullptr,
+ /*.cb_eval_user_data =*/ nullptr,
+ /*.type_k =*/ GGML_TYPE_F16,
+ /*.type_v =*/ GGML_TYPE_F16,
+ /*.logits_all =*/ false,
+ /*.embeddings =*/ false,
+ /*.offload_kqv =*/ true,
+ /*.flash_attn =*/ false,
+ /*.abort_callback =*/ nullptr,
+ /*.abort_callback_data =*/ nullptr,
+ };
+
+ return result;
+}
+
+struct llama_model_quantize_params llama_model_quantize_default_params() {
+ struct llama_model_quantize_params result = {
+ /*.nthread =*/ 0,
+ /*.ftype =*/ LLAMA_FTYPE_MOSTLY_Q5_1,
+ /*.output_tensor_type =*/ GGML_TYPE_COUNT,
+ /*.token_embedding_type =*/ GGML_TYPE_COUNT,
+ /*.allow_requantize =*/ false,
+ /*.quantize_output_tensor =*/ true,
+ /*.only_copy =*/ false,
+ /*.pure =*/ false,
+ /*.keep_split =*/ false,
+ /*.imatrix =*/ nullptr,
+ /*.kv_overrides =*/ nullptr,
+ };
+
+ return result;
+}
+
+size_t llama_max_devices(void) {
+#if defined(GGML_USE_RPC)
+ return GGML_RPC_MAX_SERVERS;
+#elif defined(GGML_USE_METAL)
+ return 1;
+#elif defined(GGML_USE_CUDA)
+ return GGML_CUDA_MAX_DEVICES;
+#elif defined(GGML_USE_SYCL)
+ return GGML_SYCL_MAX_DEVICES;
+#elif defined(GGML_USE_VULKAN)
+ return GGML_VK_MAX_DEVICES;
+#elif defined(GGML_USE_CANN)
+ return GGML_CANN_MAX_DEVICES;
+#else
+ return 1;
+#endif
+}
+
+bool llama_supports_mmap(void) {
+ return llama_mmap::SUPPORTED;
+}
+
+bool llama_supports_mlock(void) {
+ return llama_mlock::SUPPORTED;
+}
+
+bool llama_supports_gpu_offload(void) {
+#if defined(GGML_USE_CUDA) || defined(GGML_USE_METAL) || defined(GGML_USE_VULKAN) || \
+ defined(GGML_USE_SYCL) || defined(GGML_USE_KOMPUTE) || defined(GGML_USE_RPC)
+ // Defined when llama.cpp is compiled with support for offloading model layers to GPU.
+ return true;
+#else
+ return false;
+#endif
+}
+
+void llama_backend_init(void) {
+ ggml_time_init();
+
+ // needed to initialize f16 tables
+ {
+ struct ggml_init_params params = { 0, NULL, false };
+ struct ggml_context * ctx = ggml_init(params);
+ ggml_free(ctx);
+ }
+}
+
+void llama_numa_init(enum ggml_numa_strategy numa) {
+ if (numa != GGML_NUMA_STRATEGY_DISABLED) {
+ ggml_numa_init(numa);
+ }
+}
+
+void llama_attach_threadpool(
+ struct llama_context * ctx,
+ ggml_threadpool_t threadpool,
+ ggml_threadpool_t threadpool_batch) {
+ ctx->threadpool = threadpool;
+ ctx->threadpool_batch = threadpool_batch ? threadpool_batch : threadpool;
+}
+
+void llama_detach_threadpool(struct llama_context * ctx) {
+ ctx->threadpool = nullptr;
+ ctx->threadpool_batch = nullptr;
+}
+
+void llama_backend_free(void) {
+ ggml_quantize_free();
+}
+
+int64_t llama_time_us(void) {
+ return ggml_time_us();
+}
+
+struct llama_model * llama_load_model_from_file(
+ const char * path_model,
+ struct llama_model_params params) {
+ ggml_time_init();
+
+ llama_model * model = new llama_model;
+
+ unsigned cur_percentage = 0;
+ if (params.progress_callback == NULL) {
+ params.progress_callback_user_data = &cur_percentage;
+ params.progress_callback = [](float progress, void * ctx) {
+ unsigned * cur_percentage_p = (unsigned *) ctx;
+ unsigned percentage = (unsigned) (100 * progress);
+ while (percentage > *cur_percentage_p) {
+ *cur_percentage_p = percentage;
+ LLAMA_LOG_INFO(".");
+ if (percentage >= 100) {
+ LLAMA_LOG_INFO("\n");
+ }
+ }
+ return true;
+ };
+ }
+ if (params.rpc_servers != nullptr && params.rpc_servers[0] != '\0') {
+ // split the servers set them into model->rpc_servers
+ std::string servers(params.rpc_servers);
+ size_t pos = 0;
+ while ((pos = servers.find(",")) != std::string::npos) {
+ std::string server = servers.substr(0, pos);
+ model->rpc_servers.push_back(server);
+ servers.erase(0, pos + 1);
+ }
+ model->rpc_servers.push_back(servers);
+ }
+ int status = llama_model_load(path_model, *model, params);
+ GGML_ASSERT(status <= 0);
+ if (status < 0) {
+ if (status == -1) {
+ LLAMA_LOG_ERROR("%s: failed to load model\n", __func__);
+ } else if (status == -2) {
+ LLAMA_LOG_INFO("%s: cancelled model load\n", __func__);
+ }
+ delete model;
+ return nullptr;
+ }
+
+ return model;
+}
+
+void llama_free_model(struct llama_model * model) {
+ delete model;
+}
+
+struct llama_context * llama_new_context_with_model(
+ struct llama_model * model,
+ struct llama_context_params params) {
+
+ if (!model) {
+ LLAMA_LOG_ERROR("%s: model cannot be NULL\n", __func__);
+ return nullptr;
+ }
+
+ if (params.n_batch == 0 && params.n_ubatch == 0) {
+ LLAMA_LOG_ERROR("%s: n_batch and n_ubatch cannot both be zero\n", __func__);
+ return nullptr;
+ }
+
+ if (params.n_ctx == 0 && model->hparams.n_ctx_train == 0) {
+ LLAMA_LOG_ERROR("%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\n", __func__);
+ return nullptr;
+ }
+
+ if (params.flash_attn && model->arch == LLM_ARCH_GROK) {
+ LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__);
+ params.flash_attn = false;
+ }
+
+ if (params.flash_attn && model->hparams.n_embd_head_k != model->hparams.n_embd_head_v) {
+ LLAMA_LOG_WARN("%s: flash_attn requires n_embd_head_k == n_embd_head_v - forcing off\n", __func__);
+ params.flash_attn = false;
+ }
+
+ if (params.type_v != GGML_TYPE_F16 && !params.flash_attn) {
+ LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__);
+ return nullptr;
+ }
+
+ llama_context * ctx = new llama_context(*model);
+
+ const auto & hparams = model->hparams;
+ auto & cparams = ctx->cparams;
+
+ cparams.n_seq_max = std::max(1u, params.n_seq_max);
+ cparams.n_threads = params.n_threads;
+ cparams.n_threads_batch = params.n_threads_batch;
+ cparams.yarn_ext_factor = params.yarn_ext_factor;
+ cparams.yarn_attn_factor = params.yarn_attn_factor;
+ cparams.yarn_beta_fast = params.yarn_beta_fast;
+ cparams.yarn_beta_slow = params.yarn_beta_slow;
+ cparams.defrag_thold = params.defrag_thold;
+ cparams.embeddings = params.embeddings;
+ cparams.offload_kqv = params.offload_kqv;
+ cparams.flash_attn = params.flash_attn;
+ cparams.pooling_type = params.pooling_type;
+
+ cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;
+ cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;
+ cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;
+
+ // this is necessary due to kv_self.n being padded later during inference
+ cparams.n_ctx = GGML_PAD(cparams.n_ctx, llama_kv_cache_get_padding(cparams));
+
+ // with causal attention, the batch size is limited by the context size
+ cparams.n_batch = hparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;
+
+ // the batch has to be at least GGML_KQ_MASK_PAD because we will be padding the KQ_mask
+ // this is required by GPU kernels in order to avoid out-of-bounds accesses (e.g. ggml_flash_attn_ext)
+ // ref: https://github.com/ggerganov/llama.cpp/pull/5021
+ if (cparams.n_batch < GGML_KQ_MASK_PAD) {
+ LLAMA_LOG_WARN("%s: n_batch is less than GGML_KQ_MASK_PAD - increasing to %d\n", __func__, GGML_KQ_MASK_PAD);
+ cparams.n_batch = GGML_KQ_MASK_PAD;
+ }
+
+ cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);
+
+ cparams.n_ctx_orig_yarn = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx :
+ hparams.n_ctx_orig_yarn != 0 ? hparams.n_ctx_orig_yarn :
+ hparams.n_ctx_train;
+
+ cparams.cb_eval = params.cb_eval;
+ cparams.cb_eval_user_data = params.cb_eval_user_data;
+
+ auto rope_scaling_type = params.rope_scaling_type;
+ if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {
+ rope_scaling_type = hparams.rope_scaling_type_train;
+ }
+
+ if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {
+ cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none
+ }
+
+ if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'
+ cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;
+ }
+
+ cparams.yarn_attn_factor *= hparams.rope_attn_factor;
+
+ if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {
+ if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {
+ cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;
+ } else {
+ cparams.pooling_type = hparams.pooling_type;
+ }
+ }
+
+ if (params.attention_type == LLAMA_ATTENTION_TYPE_UNSPECIFIED) {
+ cparams.causal_attn = hparams.causal_attn;
+ } else {
+ cparams.causal_attn = params.attention_type == LLAMA_ATTENTION_TYPE_CAUSAL;
+ }
+
+ if (params.seed == LLAMA_DEFAULT_SEED) {
+ params.seed = time(NULL);
+ }
+
+ LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx);
+ LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch);
+ LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch);
+ LLAMA_LOG_INFO("%s: flash_attn = %d\n", __func__, cparams.flash_attn);
+ LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);
+ LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);
+
+ ctx->abort_callback = params.abort_callback;
+ ctx->abort_callback_data = params.abort_callback_data;
+
+ ctx->sampling.rng = std::mt19937(params.seed);
+ ctx->logits_all = params.logits_all;
+ // build worst-case graph for encoder if a model contains encoder
+ ctx->is_encoding = llama_model_has_encoder(model);
+
+ uint32_t kv_size = cparams.n_ctx;
+ ggml_type type_k = params.type_k;
+ ggml_type type_v = params.type_v;
+
+ // Mamba only needs a constant number of KV cache cells per sequence
+ if (llama_model_is_recurrent(model)) {
+ // Mamba needs at least as many KV cells as there are sequences kept at any time
+ kv_size = std::max((uint32_t) 1, params.n_seq_max);
+ // it's probably best to keep as much precision as possible for the states
+ type_k = GGML_TYPE_F32; // required by ggml_ssm_conv for Mamba's conv_states
+ type_v = GGML_TYPE_F32; // required by ggml_ssm_scan for Mamba's ssm_states
+ }
+
+ GGML_ASSERT(hparams.n_embd_head_k % ggml_blck_size(type_k) == 0);
+ GGML_ASSERT(hparams.n_embd_head_v % ggml_blck_size(type_v) == 0);
+
+ if (!hparams.vocab_only) {
+ // initialize backends
+#if defined(GGML_USE_METAL)
+ if (model->n_gpu_layers > 0) {
+ ctx->backend_metal = ggml_backend_metal_init();
+ if (ctx->backend_metal == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize Metal backend\n", __func__);
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(ctx->backend_metal);
+ }
+#elif defined(GGML_USE_CUDA)
+ if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) {
+ // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used
+ ggml_backend_t backend = ggml_backend_cuda_init(model->main_gpu);
+ if (backend == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, model->main_gpu);
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(backend);
+ } else {
+ // LLAMA_SPLIT_MODE_LAYER requires a backend for each GPU
+ for (int device = 0; device < ggml_backend_cuda_get_device_count(); ++device) {
+ ggml_backend_t backend = ggml_backend_cuda_init(device);
+ if (backend == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, device);
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(backend);
+ }
+ }
+#elif defined(GGML_USE_VULKAN)
+ if (model->split_mode == LLAMA_SPLIT_MODE_ROW) {
+ LLAMA_LOG_ERROR("%s: Row split not supported. Failed to initialize Vulkan backend\n", __func__);
+ llama_free(ctx);
+ return nullptr;
+ }
+ if (model->split_mode == LLAMA_SPLIT_MODE_NONE) {
+ ggml_backend_t backend = ggml_backend_vk_init(model->main_gpu);
+ if (backend == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize Vulkan backend\n", __func__);
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(backend);
+ } else {
+ for (int device = 0; device < ggml_backend_vk_get_device_count(); ++device) {
+ ggml_backend_t backend = ggml_backend_vk_init(device);
+ if (backend == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize Vulkan%d backend\n", __func__, device);
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(backend);
+ }
+ }
+#elif defined(GGML_USE_SYCL)
+ // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used
+ if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) {
+ ggml_backend_t backend = ggml_backend_sycl_init(model->main_gpu);
+ if (backend == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d backend\n", __func__, model->main_gpu);
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(backend);
+ } else {
+ // LLAMA_SPLIT_LAYER requires a backend for each GPU
+ for (int i = 0; i < ggml_backend_sycl_get_device_count(); ++i) {
+ ggml_backend_t backend = ggml_backend_sycl_init(i);
+ if (backend == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d for No.%d backend\n", __func__, i, i);
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(backend);
+ }
+ }
+#elif defined(GGML_USE_KOMPUTE)
+ if (model->n_gpu_layers > 0) {
+ auto * backend = ggml_backend_kompute_init(model->main_gpu);
+ if (backend == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize Kompute backend\n", __func__);
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(backend);
+ }
+#elif defined(GGML_USE_CANN)
+ // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used
+ // TODO: ggml_backend_cann is not support split tensor now, just leave code here.
+ if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) {
+ ggml_backend_t backend = ggml_backend_cann_init(model->main_gpu);
+ if (backend == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize CANN%d backend\n", __func__, model->main_gpu);
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(backend);
+ } else {
+ // LLAMA_SPLIT_MODE_LAYER requires a backend for each GPU
+ // TODO: currently, CANN can't use multi-gpus, just leave code here for further cann version.
+ for (int32_t device = 0; device < ggml_backend_cann_get_device_count(); ++device) {
+ ggml_backend_t backend = ggml_backend_cann_init(device);
+ if (backend == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize CANN%d backend\n", __func__, device);
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(backend);
+ }
+ }
+#endif
+
+#ifdef GGML_USE_BLAS
+ ctx->backend_blas = ggml_backend_blas_init();
+ if (ctx->backend_blas == nullptr) {
+ LLAMA_LOG_WARN("%s: failed to initialize BLAS backend\n", __func__);
+ } else {
+ ctx->backends.push_back(ctx->backend_blas);
+ }
+#endif
+
+#if defined(GGML_USE_RPC)
+ if (model->n_gpu_layers > 0) {
+ for (const auto & endpoint : model->rpc_servers) {
+ ggml_backend_t backend = ggml_backend_rpc_init(endpoint.c_str());
+ if (backend == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize RPC to '%s'\n", __func__, endpoint.c_str());
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(backend);
+ }
+ }
+#endif
+ ctx->backend_cpu = ggml_backend_cpu_init();
+ if (ctx->backend_cpu == nullptr) {
+ LLAMA_LOG_ERROR("%s: failed to initialize CPU backend\n", __func__);
+ llama_free(ctx);
+ return nullptr;
+ }
+ ctx->backends.push_back(ctx->backend_cpu);
+
+ if (!llama_kv_cache_init(ctx->kv_self, ctx, type_k, type_v, kv_size, cparams.offload_kqv)) {
+ LLAMA_LOG_ERROR("%s: llama_kv_cache_init() failed for self-attention cache\n", __func__);
+ llama_free(ctx);
+ return nullptr;
+ }
+
+ {
+ size_t memory_size_k = 0;
+ size_t memory_size_v = 0;
+
+ for (auto & k : ctx->kv_self.k_l) {
+ memory_size_k += ggml_nbytes(k);
+ }
+
+ for (auto & v : ctx->kv_self.v_l) {
+ memory_size_v += ggml_nbytes(v);
+ }
+
+ LLAMA_LOG_INFO("%s: KV self size = %7.2f MiB, K (%s): %7.2f MiB, V (%s): %7.2f MiB\n", __func__,
+ (float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f),
+ ggml_type_name(type_k), (float)memory_size_k / (1024.0f * 1024.0f),
+ ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f));
+ }
+
+ // graph outputs buffer
+ {
+ // resized during inference when a batch uses more outputs
+ if (llama_output_reserve(*ctx, params.n_seq_max) < params.n_seq_max) {
+ LLAMA_LOG_ERROR("%s: failed to reserve initial output buffer\n", __func__);
+ llama_free(ctx);
+ return nullptr;
+ }
+
+ LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__,
+ ggml_backend_buffer_name(ctx->buf_output),
+ ggml_backend_buffer_get_size(ctx->buf_output) / 1024.0 / 1024.0);
+ }
+
+ // scheduler and compute buffers
+ {
+ // buffer types used for the compute buffer of each backend
+ std::vector<ggml_backend_buffer_type_t> backend_buft;
+ for (auto * backend : ctx->backends) {
+ if (ggml_backend_is_cpu(backend)) {
+ // use host buffers for the CPU backend compute buffer
+ backend_buft.push_back(llama_default_buffer_type_cpu(true));
+ } else {
+ backend_buft.push_back(ggml_backend_get_default_buffer_type(backend));
+ }
+ }
+
+ const size_t max_nodes = llama_model_max_nodes(*model);
+
+ // buffer used to store the computation graph and the tensor meta data
+ ctx->buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false));
+
+ // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary
+ bool pipeline_parallel =
+ llama_get_device_count(*model) > 1 &&
+ model->n_gpu_layers > (int)model->hparams.n_layer &&
+ model->split_mode == LLAMA_SPLIT_MODE_LAYER &&
+ params.offload_kqv;
+#ifndef GGML_USE_CUDA
+ // pipeline parallelism requires support for async compute and events
+ // currently this is only implemented in the CUDA backend
+ pipeline_parallel = false;
+#endif
+ ctx->sched = ggml_backend_sched_new(ctx->backends.data(), backend_buft.data(), ctx->backends.size(), max_nodes, pipeline_parallel);
+
+ if (pipeline_parallel) {
+ LLAMA_LOG_INFO("%s: pipeline parallelism enabled (n_copies=%d)\n", __func__, ggml_backend_sched_get_n_copies(ctx->sched));
+ }
+
+ // build worst-case graph
+ uint32_t n_seqs = 1; // TODO: worst-case number of sequences
+ uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);
+ llama_token token = llama_token_bos(&ctx->model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph
+ llama_ubatch ubatch = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr};
+ ggml_cgraph * gf = llama_build_graph(*ctx, ubatch, true);
+
+ // initialize scheduler with the worst-case graph
+ if (!ggml_backend_sched_reserve(ctx->sched, gf)) {
+ LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__);
+ llama_free(ctx);
+ return nullptr;
+ }
+
+ for (size_t i = 0; i < ctx->backends.size(); i++) {
+ ggml_backend_t backend = ctx->backends[i];
+ ggml_backend_buffer_type_t buft = backend_buft[i];
+ size_t size = ggml_backend_sched_get_buffer_size(ctx->sched, backend);
+ if (size > 1) {
+ LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__,
+ ggml_backend_buft_name(buft),
+ size / 1024.0 / 1024.0);
+ }
+ }
+
+ // note: the number of splits during measure is higher than during inference due to the kv shift
+ int n_splits = ggml_backend_sched_get_n_splits(ctx->sched);
+ LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, gf->n_nodes);
+ LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits);
+ }
+ }
+
+ return ctx;
+}
+
+void llama_free(struct llama_context * ctx) {
+ delete ctx;
+}
+
+const struct llama_model * llama_get_model(const struct llama_context * ctx) {
+ return &ctx->model;
+}
+
+const struct llama_vocab * llama_get_vocab(const struct llama_context * ctx) {
+ return &ctx->model.vocab;
+}
+
+uint32_t llama_n_ctx(const struct llama_context * ctx) {
+ return ctx->cparams.n_ctx;
+}
+
+uint32_t llama_n_batch(const struct llama_context * ctx) {
+ return ctx->cparams.n_batch;
+}
+
+uint32_t llama_n_ubatch(const struct llama_context * ctx) {
+ return ctx->cparams.n_ubatch;
+}
+
+uint32_t llama_n_seq_max(const struct llama_context * ctx) {
+ return ctx->kv_self.size;
+}
+
+enum llama_vocab_type llama_vocab_type(const struct llama_model * model) {
+ return model->vocab.type;
+}
+
+enum llama_rope_type llama_rope_type(const struct llama_model * model) {
+ switch (model->arch) {
+ // these models do not use RoPE
+ case LLM_ARCH_GPT2:
+ case LLM_ARCH_GPTJ:
+ case LLM_ARCH_MPT:
+ case LLM_ARCH_REFACT:
+ case LLM_ARCH_BLOOM:
+ case LLM_ARCH_MAMBA:
+ case LLM_ARCH_JINA_BERT_V2:
+ case LLM_ARCH_T5:
+ case LLM_ARCH_T5ENCODER:
+ case LLM_ARCH_JAIS:
+ case LLM_ARCH_RWKV6:
+ return LLAMA_ROPE_TYPE_NONE;
+
+ // use what we call a normal RoPE, operating on pairs of consecutive head values
+ case LLM_ARCH_LLAMA:
+ case LLM_ARCH_BAICHUAN:
+ case LLM_ARCH_STARCODER:
+ case LLM_ARCH_PLAMO:
+ case LLM_ARCH_ORION:
+ case LLM_ARCH_INTERNLM2:
+ case LLM_ARCH_MINICPM:
+ case LLM_ARCH_XVERSE:
+ case LLM_ARCH_COMMAND_R:
+ case LLM_ARCH_OLMO:
+ case LLM_ARCH_ARCTIC:
+ case LLM_ARCH_DEEPSEEK2:
+ case LLM_ARCH_CHATGLM:
+ case LLM_ARCH_SOLAR:
+ return LLAMA_ROPE_TYPE_NORM;
+
+ // the pairs of head values are offset by n_rot/2
+ case LLM_ARCH_FALCON:
+ case LLM_ARCH_GROK:
+ case LLM_ARCH_DBRX:
+ case LLM_ARCH_BERT:
+ case LLM_ARCH_NOMIC_BERT:
+ case LLM_ARCH_STABLELM:
+ case LLM_ARCH_BITNET:
+ case LLM_ARCH_QWEN:
+ case LLM_ARCH_QWEN2:
+ case LLM_ARCH_QWEN2MOE:
+ case LLM_ARCH_PHI2:
+ case LLM_ARCH_PHI3:
+ case LLM_ARCH_GEMMA:
+ case LLM_ARCH_GEMMA2:
+ case LLM_ARCH_STARCODER2:
+ case LLM_ARCH_OPENELM:
+ case LLM_ARCH_GPTNEOX:
+ case LLM_ARCH_CODESHELL:
+ case LLM_ARCH_NEMOTRON:
+ case LLM_ARCH_EXAONE:
+ return LLAMA_ROPE_TYPE_NEOX;
+
+ // all model arches should be listed explicitly here
+ case LLM_ARCH_UNKNOWN:
+ GGML_ABORT("unknown architecture");
+ }
+
+ return LLAMA_ROPE_TYPE_NONE;
+}
+
+enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx) {
+ return ctx->cparams.pooling_type;
+}
+
+int32_t llama_n_vocab(const struct llama_model * model) {
+ return model->hparams.n_vocab;
+}
+
+int32_t llama_n_ctx_train(const struct llama_model * model) {
+ return model->hparams.n_ctx_train;
+}
+
+int32_t llama_n_embd(const struct llama_model * model) {
+ return model->hparams.n_embd;
+}
+
+int32_t llama_n_layer(const struct llama_model * model) {
+ return model->hparams.n_layer;
+}
+
+float llama_rope_freq_scale_train(const struct llama_model * model) {
+ return model->hparams.rope_freq_scale_train;
+}
+
+int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size) {
+ const auto & it = model->gguf_kv.find(key);
+ if (it == model->gguf_kv.end()) {
+ if (buf_size > 0) {
+ buf[0] = '\0';
+ }
+ return -1;
+ }
+ return snprintf(buf, buf_size, "%s", it->second.c_str());
+}
+
+int32_t llama_model_meta_count(const struct llama_model * model) {
+ return (int)model->gguf_kv.size();
+}
+
+int32_t llama_model_meta_key_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size) {
+ if (i < 0 || i >= (int)model->gguf_kv.size()) {
+ if (buf_size > 0) {
+ buf[0] = '\0';
+ }
+ return -1;
+ }
+ auto it = model->gguf_kv.begin();
+ std::advance(it, i);
+ return snprintf(buf, buf_size, "%s", it->first.c_str());
+}
+
+int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size) {
+ if (i < 0 || i >= (int)model->gguf_kv.size()) {
+ if (buf_size > 0) {
+ buf[0] = '\0';
+ }
+ return -1;
+ }
+ auto it = model->gguf_kv.begin();
+ std::advance(it, i);
+ return snprintf(buf, buf_size, "%s", it->second.c_str());
+}
+
+int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size) {
+ return snprintf(buf, buf_size, "%s %s %s",
+ llama_model_arch_name(model->arch),
+ llama_model_type_name(model->type),
+ llama_model_ftype_name(model->ftype).c_str());
+}
+
+uint64_t llama_model_size(const struct llama_model * model) {
+ uint64_t size = 0;
+ for (const auto & it : model->tensors_by_name) {
+ size += ggml_nbytes(it.second);
+ }
+ return size;
+}
+
+uint64_t llama_model_n_params(const struct llama_model * model) {
+ uint64_t nparams = 0;
+ for (const auto & it : model->tensors_by_name) {
+ nparams += ggml_nelements(it.second);
+ }
+ return nparams;
+}
+
+struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const char * name) {
+ auto it = std::find_if(model->tensors_by_name.begin(), model->tensors_by_name.end(),
+ [name](const std::pair<std::string, struct ggml_tensor *> & it) {
+ return it.first == name;
+ });
+ if (it == model->tensors_by_name.end()) {
+ return nullptr;
+ }
+ return it->second;
+}
+
+bool llama_model_has_encoder(const struct llama_model * model) {
+ switch (model->arch) {
+ case LLM_ARCH_T5: return true;
+ case LLM_ARCH_T5ENCODER: return true;
+ default: return false;
+ }
+}
+
+bool llama_model_has_decoder(const struct llama_model * model) {
+ switch (model->arch) {
+ case LLM_ARCH_T5ENCODER: return false;
+ default: return true;
+ }
+}
+
+llama_token llama_model_decoder_start_token(const struct llama_model * model) {
+ return model->hparams.dec_start_token_id;
+}
+
+bool llama_model_is_recurrent(const struct llama_model * model) {
+ switch (model->arch) {
+ case LLM_ARCH_MAMBA: return true;
+ case LLM_ARCH_RWKV6: return true;
+ default: return false;
+ }
+}
+
+uint32_t llama_model_quantize(
+ const char * fname_inp,
+ const char * fname_out,
+ const llama_model_quantize_params * params) {
+ try {
+ llama_model_quantize_internal(fname_inp, fname_out, params);
+ return 0;
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: failed to quantize: %s\n", __func__, err.what());
+ return 1;
+ }
+}
+
+struct llama_lora_adapter * llama_lora_adapter_init(struct llama_model * model, const char * path_lora) {
+ try {
+ struct llama_lora_adapter * adapter = new llama_lora_adapter(model);
+ llama_lora_adapter_init_internal(model, path_lora, *adapter);
+ return adapter;
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: failed to apply lora adapter: %s\n", __func__, err.what());
+ return nullptr;
+ }
+}
+
+static bool llama_control_vector_init(struct llama_control_vector & cvec, const llama_model & model) {
+ GGML_ASSERT(cvec.tensors.empty());
+ GGML_ASSERT(cvec.ctxs.empty());
+ GGML_ASSERT(cvec.bufs.empty());
+
+ // count layer buffer types
+ std::map<ggml_backend_buffer_type_t, int> buft_layer_count;
+ for (int64_t i = 0; i < model.hparams.n_layer; i++) {
+ buft_layer_count[model.buft_layer[i].buft]++;
+ }
+
+ // allocate contexts
+ std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;
+ for (auto & it : buft_layer_count) {
+ int n_layers = it.second;
+ struct ggml_init_params params = {
+ /*.mem_size =*/ n_layers * ggml_tensor_overhead(),
+ /*.mem_buffer =*/ NULL,
+ /*.no_alloc =*/ true,
+ };
+ ggml_context * ctx = ggml_init(params);
+ if (!ctx) {
+ LLAMA_LOG_ERROR("%s: failed to allocate context for control vector\n", __func__);
+ return 1;
+ }
+ ctx_map[it.first] = ctx;
+ }
+
+ // make tensors
+ cvec.tensors.reserve(model.hparams.n_layer);
+ cvec.tensors.push_back(nullptr); // there's never a tensor for layer 0
+ for (size_t il = 1; il < model.hparams.n_layer; il++) {
+ struct ggml_context * ctx = ctx_map.at(model.buft_layer[il].buft);
+ ggml_tensor * tensor = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, model.hparams.n_embd);
+ cvec.tensors.push_back(tensor);
+ }
+
+ // allocate tensors / buffers and zero
+ cvec.ctxs.reserve(ctx_map.size());
+ cvec.bufs.reserve(ctx_map.size());
+ for (auto it : ctx_map) {
+ ggml_backend_buffer_type_t buft = it.first;
+ ggml_context * ctx = it.second;
+ ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);
+ if (!buf) {
+ LLAMA_LOG_ERROR("%s: failed to allocate buffer for control vector\n", __func__);
+ return false;
+ }
+ ggml_backend_buffer_clear(buf, 0);
+ cvec.ctxs.push_back(ctx);
+ cvec.bufs.push_back(buf);
+ }
+
+ return true;
+}
+
+int32_t llama_control_vector_apply(struct llama_context * lctx, const float * data, size_t len, int32_t n_embd, int32_t il_start, int32_t il_end) {
+ const llama_model & model = lctx->model;
+ llama_control_vector & cvec = lctx->cvec;
+
+ if (data == nullptr) {
+ // disable the current control vector (but leave allocated for later)
+ cvec.layer_start = -1;
+ cvec.layer_end = -1;
+ return 0;
+ }
+
+ if (n_embd != (int) model.hparams.n_embd) {
+ LLAMA_LOG_ERROR("%s: control vector n_embd does not match model\n", __func__);
+ return 1;
+ }
+
+ if (cvec.tensors.empty()) {
+ if (!llama_control_vector_init(cvec, model)) {
+ return 1;
+ }
+ }
+
+ cvec.layer_start = il_start;
+ cvec.layer_end = il_end;
+
+ for (size_t il = 1; il < model.hparams.n_layer; il++) {
+ assert(cvec.tensors[il] != nullptr);
+
+ const size_t off = n_embd * (il - 1); // buffer doesn't have data for layer 0, since it's never present
+ if (off + n_embd <= len) {
+ ggml_backend_tensor_set(cvec.tensors[il], data + off, 0, n_embd * ggml_element_size(cvec.tensors[il]));
+ }
+ }
+
+ return 0;
+}
+
+struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_seq_max) {
+ struct llama_kv_cache_view result = {
+ /*.n_cells = */ 0,
+ /*.n_seq_max = */ n_seq_max,
+ /*.token_count = */ 0,
+ /*.used_cells = */ llama_get_kv_cache_used_cells(ctx),
+ /*.max_contiguous = */ 0,
+ /*.max_contiguous_idx = */ -1,
+ /*.cells = */ nullptr,
+ /*.cells_sequences = */ nullptr,
+ };
+ return result;
+}
+
+void llama_kv_cache_view_free(struct llama_kv_cache_view * view) {
+ if (view->cells != nullptr) {
+ free(view->cells);
+ view->cells = nullptr;
+ }
+ if (view->cells_sequences != nullptr) {
+ free(view->cells_sequences);
+ view->cells_sequences = nullptr;
+ }
+}
+
+void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kv_cache_view * view) {
+ if (uint32_t(view->n_cells) < ctx->kv_self.size || view->cells == nullptr) {
+ view->n_cells = int32_t(ctx->kv_self.size);
+ void * p = realloc(view->cells, sizeof(struct llama_kv_cache_view_cell) * view->n_cells);
+ GGML_ASSERT(p != nullptr && "Failed to alloc kv_cache_view cells");
+ view->cells = (struct llama_kv_cache_view_cell *)p;
+ p = realloc(view->cells_sequences, sizeof(llama_seq_id) * view->n_seq_max * view->n_cells);
+ GGML_ASSERT(p != nullptr && "Failed to alloc kv_cache_view cells sequences");
+ view->cells_sequences = (llama_seq_id *)p;
+ }
+
+ const std::vector<llama_kv_cell> & kv_cells = ctx->kv_self.cells;
+ llama_kv_cache_view_cell * c_curr = view->cells;
+ llama_seq_id * cs_curr = view->cells_sequences;
+ int32_t used_cells = 0;
+ int32_t token_count = 0;
+ int32_t curr_contig_idx = -1;
+ uint32_t max_contig = 0;
+ int32_t max_contig_idx = -1;
+
+ for (int32_t i = 0; i < int32_t(ctx->kv_self.size); i++, c_curr++, cs_curr += view->n_seq_max) {
+ const size_t curr_size = kv_cells[i].seq_id.size();
+ token_count += curr_size;
+ c_curr->pos = kv_cells[i].pos + kv_cells[i].delta;
+
+ if (curr_size > 0) {
+ if (curr_contig_idx >= 0 && uint32_t(i - curr_contig_idx) > max_contig) {
+ max_contig = i - curr_contig_idx;
+ max_contig_idx = curr_contig_idx;
+ }
+ curr_contig_idx = -1;
+ } else if (curr_contig_idx < 0) {
+ curr_contig_idx = i;
+ }
+
+ int seq_idx = 0;
+ for (const llama_seq_id it : kv_cells[i].seq_id) {
+ if (seq_idx >= view->n_seq_max) {
+ break;
+ }
+ cs_curr[seq_idx] = it;
+ seq_idx++;
+ }
+ if (seq_idx != 0) {
+ used_cells++;
+ }
+ for (; seq_idx < view->n_seq_max; seq_idx++) {
+ cs_curr[seq_idx] = -1;
+ }
+ }
+ if (curr_contig_idx >= 0 && kv_cells.size() - curr_contig_idx > max_contig) {
+ max_contig_idx = curr_contig_idx;
+ max_contig = kv_cells.size() - curr_contig_idx;
+ }
+ view->max_contiguous = max_contig;
+ view->max_contiguous_idx = max_contig_idx;
+ view->token_count = token_count;
+ view->used_cells = used_cells;
+ if (uint32_t(used_cells) != ctx->kv_self.used) {
+ LLAMA_LOG_ERROR("%s: used cells mismatch. kv_cache says %d but we calculated %d\n",
+ __func__, ctx->kv_self.used, used_cells);
+ }
+}
+
+int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx) {
+ int result = 0;
+
+ for (uint32_t i = 0; i < ctx->kv_self.size; i++) {
+ result += ctx->kv_self.cells[i].seq_id.size();
+ }
+
+ return result;
+}
+
+int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx) {
+ return ctx->kv_self.used;
+}
+
+void llama_kv_cache_clear(struct llama_context * ctx) {
+ llama_kv_cache_clear(ctx->kv_self);
+}
+
+bool llama_kv_cache_seq_rm(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
+ return llama_kv_cache_seq_rm(ctx->kv_self, seq_id, p0, p1);
+}
+
+void llama_kv_cache_seq_cp(struct llama_context * ctx, llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {
+ if (seq_id_src == seq_id_dst) {
+ return;
+ }
+ llama_kv_cache_seq_cp(ctx->kv_self, seq_id_src, seq_id_dst, p0, p1);
+}
+
+void llama_kv_cache_seq_keep(struct llama_context * ctx, llama_seq_id seq_id) {
+ llama_kv_cache_seq_keep(ctx->kv_self, seq_id);
+}
+
+void llama_kv_cache_seq_add(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) {
+ if (delta == 0) {
+ return;
+ }
+
+ llama_kv_cache_seq_add(ctx->kv_self, seq_id, p0, p1, delta);
+}
+
+void llama_kv_cache_seq_div(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {
+ if (d == 1) {
+ return;
+ }
+
+ llama_kv_cache_seq_div(ctx->kv_self, seq_id, p0, p1, d);
+}
+
+llama_pos llama_kv_cache_seq_pos_max(struct llama_context * ctx, llama_seq_id seq_id) {
+ return llama_kv_cache_seq_pos_max(ctx->kv_self, seq_id);
+}
+
+void llama_kv_cache_defrag(struct llama_context * ctx) {
+ llama_kv_cache_defrag(ctx->kv_self);
+}
+
+void llama_kv_cache_update(struct llama_context * ctx) {
+ llama_kv_cache_update_internal(*ctx);
+}
+
+// deprecated
+size_t llama_get_state_size(struct llama_context * ctx) {
+ return llama_state_get_size(ctx);
+}
+
+// deprecated
+size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) {
+ return llama_state_get_data(ctx, dst, -1);
+}
+
+// deprecated
+size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) {
+ return llama_state_set_data(ctx, src, -1);
+}
+
+// deprecated
+bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
+ return llama_state_load_file(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out);
+}
+
+// deprecated
+bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {
+ return llama_state_save_file(ctx, path_session, tokens, n_token_count);
+}
+
+// TODO: replace all non-fatal assertions with returned errors or exceptions
+struct llama_data_write {
+ virtual void write(const void * src, size_t size) = 0;
+ virtual void write_tensor_data(const struct ggml_tensor * tensor, size_t offset, size_t size) = 0;
+ virtual size_t get_size_written() = 0;
+ virtual ~llama_data_write() = default;
+
+ void write_string(const std::string & str) {
+ uint32_t str_size = str.size();
+
+ write(&str_size, sizeof(str_size));
+ write(str.data(), str_size);
+ }
+
+ void write_model_info(const struct llama_context * ctx) {
+ std::string arch_str = LLM_ARCH_NAMES.at(ctx->model.arch);
+ write_string(arch_str);
+ // TODO: add more model-specific info which should prevent loading the session file if not identical
+ }
+
+ void write_rng(const std::mt19937 & rng) {
+ std::ostringstream rng_ss;
+ rng_ss << rng;
+
+ const std::string & rng_str = rng_ss.str();
+
+ write_string(rng_str);
+ }
+
+ void write_output_ids(struct llama_context * ctx) {
+ llama_output_reorder(ctx);
+
+ const uint32_t n_outputs = ctx->n_outputs;
+
+ std::vector<int32_t> output_pos;
+
+ const size_t n_batch = ctx->cparams.n_batch;
+ const auto & output_ids = ctx->output_ids;
+
+ GGML_ASSERT(n_outputs <= ctx->output_size);
+
+ output_pos.resize(n_outputs);
+
+ // build a more compact representation of the output ids
+ for (size_t i = 0; i < n_batch; ++i) {
+ // map an output id to a position in the batch
+ int32_t pos = output_ids[i];
+ if (pos >= 0) {
+ GGML_ASSERT((uint32_t) pos < n_outputs);
+ output_pos[pos] = i;
+ }
+ }
+
+ write(&n_outputs, sizeof(n_outputs));
+
+ if (n_outputs) {
+ write(output_pos.data(), n_outputs * sizeof(int32_t));
+ }
+ }
+
+ void write_logits(const struct llama_context * ctx) {
+ const uint64_t logits_size = std::min((uint64_t) ctx->logits_size, (uint64_t) ctx->n_outputs * ctx->model.hparams.n_vocab);
+
+ write(&logits_size, sizeof(logits_size));
+
+ if (logits_size) {
+ write(ctx->logits, logits_size * sizeof(float));
+ }
+ }
+
+ void write_embeddings(const struct llama_context * ctx) {
+ const uint64_t embeddings_size = std::min((uint64_t) ctx->embd_size, (uint64_t) ctx->n_outputs * ctx->model.hparams.n_embd);
+
+ write(&embeddings_size, sizeof(embeddings_size));
+
+ if (embeddings_size) {
+ write(ctx->embd, embeddings_size * sizeof(float));
+ }
+ }
+
+ void write_kv_cache_meta(const llama_kv_cache & kv_self, const std::vector<std::pair<uint32_t, uint32_t>> & cell_ranges, llama_seq_id seq_id = -1) {
+
+ for (const auto & range : cell_ranges) {
+ for (uint32_t i = range.first; i < range.second; ++i) {
+ const auto & cell = kv_self.cells[i];
+ const llama_pos pos = cell.pos;
+ const uint32_t n_seq_id = seq_id == -1 ? cell.seq_id.size() : 0;
+
+ write(&pos, sizeof(pos));
+ write(&n_seq_id, sizeof(n_seq_id));
+
+ if (n_seq_id) {
+ for (auto seq_id : cell.seq_id) {
+ write(&seq_id, sizeof(seq_id));
+ }
+ }
+ }
+ }
+ }
+
+ void write_kv_cache_data(const struct llama_context * ctx, const std::vector<std::pair<uint32_t, uint32_t>> & cell_ranges) {
+ const struct llama_kv_cache & kv_self = ctx->kv_self;
+ const struct llama_hparams & hparams = ctx->model.hparams;
+
+ const uint32_t v_trans = kv_self.v_trans ? 1 : 0;
+ const uint32_t n_layer = hparams.n_layer;
+
+ write(&v_trans, sizeof(v_trans));
+ write(&n_layer, sizeof(n_layer));
+
+ std::vector<uint8_t> tmp_buf;
+
+ // Iterate and write all the keys first, each row is a cell
+ // Get whole range at a time
+ for (uint32_t il = 0; il < n_layer; ++il) {
+ const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il) + hparams.n_embd_k_s();
+
+ // Write key type
+ const int32_t k_type_i = (int32_t)kv_self.k_l[il]->type;
+ write(&k_type_i, sizeof(k_type_i));
+
+ // Write row size of key
+ const uint64_t k_size_row = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa);
+ write(&k_size_row, sizeof(k_size_row));
+
+ // Read each range of cells of k_size length each into tmp_buf and write out
+ for (const auto & range : cell_ranges) {
+ const size_t range_size = range.second - range.first;
+ const size_t buf_size = range_size * k_size_row;
+ write_tensor_data(kv_self.k_l[il], range.first * k_size_row, buf_size);
+ }
+ }
+
+ if (!kv_self.v_trans) {
+ for (uint32_t il = 0; il < n_layer; ++il) {
+ const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s();
+
+ // Write value type
+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;
+ write(&v_type_i, sizeof(v_type_i));
+
+ // Write row size of value
+ const uint64_t v_size_row = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa);
+ write(&v_size_row, sizeof(v_size_row));
+
+ // Read each range of cells of v_size length each into tmp_buf and write out
+ for (const auto & range : cell_ranges) {
+ const size_t range_size = range.second - range.first;
+ const size_t buf_size = range_size * v_size_row;
+ write_tensor_data(kv_self.v_l[il], range.first * v_size_row, buf_size);
+ }
+ }
+ } else {
+ // When v is transposed, we also need the element size and get the element ranges from each row
+ const uint32_t kv_size = kv_self.size;
+ for (uint32_t il = 0; il < n_layer; ++il) {
+ const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s();
+
+ // Write value type
+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;
+ write(&v_type_i, sizeof(v_type_i));
+
+ // Write element size
+ const uint32_t v_size_el = ggml_type_size(kv_self.v_l[il]->type);
+ write(&v_size_el, sizeof(v_size_el));
+
+ // Write GQA embedding size
+ write(&n_embd_v_gqa, sizeof(n_embd_v_gqa));
+
+ // For each row, we get the element values of each cell
+ for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
+ // Read each range of cells of v_size_el length each into tmp_buf and write out
+ for (const auto & range : cell_ranges) {
+ const size_t range_size = range.second - range.first;
+ const size_t src_offset = (range.first + j * kv_size) * v_size_el;
+ const size_t buf_size = range_size * v_size_el;
+ write_tensor_data(kv_self.v_l[il], src_offset, buf_size);
+ }
+ }
+ }
+ }
+ }
+
+ void write_kv_cache(const struct llama_context * ctx, llama_seq_id seq_id = -1) {
+ const struct llama_kv_cache & kv_self = ctx->kv_self;
+ std::vector<std::pair<uint32_t, uint32_t>> cell_ranges; // ranges, from inclusive, to exclusive
+ uint32_t cell_count = 0;
+
+ // Count the number of cells with the specified seq_id
+ // Find all the ranges of cells with this seq id (or all, when -1)
+ uint32_t cell_range_begin = kv_self.size;
+ for (uint32_t i = 0; i < kv_self.size; ++i) {
+ const auto & cell = kv_self.cells[i];
+ if ((seq_id == -1 && !cell.is_empty()) || cell.has_seq_id(seq_id)) {
+ ++cell_count;
+ if (cell_range_begin == kv_self.size) {
+ cell_range_begin = i;
+ }
+ } else {
+ if (cell_range_begin != kv_self.size) {
+ cell_ranges.emplace_back(cell_range_begin, i);
+ cell_range_begin = kv_self.size;
+ }
+ }
+ }
+ if (cell_range_begin != kv_self.size) {
+ cell_ranges.emplace_back(cell_range_begin, kv_self.size);
+ }
+
+ // DEBUG CHECK: Sum of cell counts in ranges should equal the total cell count
+ uint32_t cell_count_check = 0;
+ for (const auto & range : cell_ranges) {
+ cell_count_check += range.second - range.first;
+ }
+ GGML_ASSERT(cell_count == cell_count_check);
+
+ write(&cell_count, sizeof(cell_count));
+
+ write_kv_cache_meta(kv_self, cell_ranges, seq_id);
+ write_kv_cache_data(ctx, cell_ranges);
+ }
+};
+
+struct llama_data_read {
+ virtual const uint8_t * read(size_t size) = 0;
+ virtual void read_to(void * dst, size_t size) = 0;
+ virtual size_t get_size_read() = 0;
+ virtual ~llama_data_read() = default;
+
+ void read_string(std::string & str) {
+ uint32_t str_size;
+ read_to(&str_size, sizeof(str_size));
+
+ str.assign((const char *) read(str_size), str_size);
+ }
+
+ // validate model information
+ void read_model_info(const struct llama_context * ctx) {
+ std::string cur_arch_str = LLM_ARCH_NAMES.at(ctx->model.arch);
+ std::string arch_str;
+ read_string(arch_str);
+ if (cur_arch_str != arch_str) {
+ throw std::runtime_error(format("wrong model arch: '%s' instead of '%s'", arch_str.c_str(), cur_arch_str.c_str()));
+ }
+ // TODO: add more info which needs to be identical but which is not verified otherwise
+ }
+
+ void read_rng(std::mt19937 & rng) {
+ std::string rng_str;
+ read_string(rng_str);
+
+ std::istringstream rng_ss(rng_str);
+ rng_ss >> rng;
+
+ if (rng_ss.fail()) {
+ throw std::runtime_error("failed to load RNG state");
+ }
+ }
+
+ void read_output_ids(struct llama_context * ctx) {
+ std::vector<int32_t> output_pos;
+
+ uint32_t n_outputs;
+ read_to(&n_outputs, sizeof(n_outputs));
+
+ if (n_outputs > llama_output_reserve(*ctx, n_outputs)) {
+ throw std::runtime_error("could not reserve outputs");
+ }
+
+ if (n_outputs) {
+ output_pos.resize(n_outputs);
+ read_to(output_pos.data(), n_outputs * sizeof(int32_t));
+
+ for (int32_t i = 0; i < (int32_t) output_pos.size(); ++i) {
+ int32_t id = output_pos[i];
+ if ((uint32_t) id >= ctx->cparams.n_batch) {
+ throw std::runtime_error(format("invalid output id, %d does not fit in batch size of %u", id, ctx->cparams.n_batch));
+ }
+ ctx->output_ids[id] = i;
+ }
+
+ ctx->n_outputs = n_outputs;
+ }
+ }
+
+ void read_logits(struct llama_context * ctx) {
+ uint64_t logits_size;
+ read_to(&logits_size, sizeof(logits_size));
+
+ if (ctx->logits_size < logits_size) {
+ throw std::runtime_error("logits buffer too small");
+ }
+
+ if (logits_size) {
+ read_to(ctx->logits, logits_size * sizeof(float));
+ }
+ }
+
+ void read_embeddings(struct llama_context * ctx) {
+ uint64_t embeddings_size;
+ read_to(&embeddings_size, sizeof(embeddings_size));
+
+ if (ctx->embd_size < embeddings_size) {
+ throw std::runtime_error("embeddings buffer too small");
+ }
+
+ if (embeddings_size) {
+ read_to(ctx->embd, embeddings_size * sizeof(float));
+ }
+ }
+
+ bool read_kv_cache_meta(struct llama_context * ctx, uint32_t cell_count, llama_seq_id dest_seq_id = -1) {
+ struct llama_kv_cache & kv_self = ctx->kv_self;
+
+ if (dest_seq_id != -1) {
+ // single sequence
+
+ llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);
+
+ llama_ubatch batch = ctx->sbatch.reserve_ubatch(cell_count, /* has_embd */ false);
+ batch.n_tokens = cell_count;
+ batch.n_seq_tokens = cell_count;
+ batch.n_seqs = 1;
+
+ for (uint32_t i = 0; i < cell_count; ++i) {
+ llama_pos pos;
+ uint32_t n_seq_id;
+
+ read_to(&pos, sizeof(pos));
+ read_to(&n_seq_id, sizeof(n_seq_id));
+
+ if (n_seq_id != 0) {
+ LLAMA_LOG_ERROR("%s: invalid seq_id-agnostic kv cell\n", __func__);
+ return false;
+ }
+
+ batch.pos[i] = pos;
+ }
+ batch.n_seq_id[0] = 1;
+ batch.seq_id[0] = &dest_seq_id;
+ if (!llama_kv_cache_find_slot(kv_self, batch)) {
+ LLAMA_LOG_ERROR("%s: failed to find available cells in kv cache\n", __func__);
+ return false;
+ }
+
+ // DEBUG CHECK: kv_self.head should be our first cell, kv_self.head + cell_count - 1 should be our last cell (verify seq_id and pos values)
+ // Assume that this is one contiguous block of cells
+ GGML_ASSERT(kv_self.head + cell_count <= kv_self.size);
+ GGML_ASSERT(kv_self.cells[kv_self.head].pos == batch.pos[0]);
+ GGML_ASSERT(kv_self.cells[kv_self.head + cell_count - 1].pos == batch.pos[cell_count - 1]);
+ GGML_ASSERT(kv_self.cells[kv_self.head].has_seq_id(dest_seq_id));
+ GGML_ASSERT(kv_self.cells[kv_self.head + cell_count - 1].has_seq_id(dest_seq_id));
+ } else {
+ // whole KV cache restore
+
+ if (cell_count > kv_self.size) {
+ LLAMA_LOG_ERROR("%s: not enough cells in kv cache\n", __func__);
+ return false;
+ }
+
+ llama_kv_cache_clear(kv_self);
+
+ for (uint32_t i = 0; i < cell_count; ++i) {
+ llama_kv_cell & cell = kv_self.cells[i];
+
+ llama_pos pos;
+ uint32_t n_seq_id;
+
+ read_to(&pos, sizeof(pos));
+ read_to(&n_seq_id, sizeof(n_seq_id));
+
+ cell.pos = pos;
+
+ for (uint32_t j = 0; j < n_seq_id; ++j) {
+ llama_seq_id seq_id;
+ read_to(&seq_id, sizeof(seq_id));
+
+ if (seq_id < 0 || (uint32_t) seq_id >= llama_n_seq_max(ctx)) {
+ LLAMA_LOG_ERROR("%s: invalid seq_id, %d is out of range [0, %u)\n", __func__, seq_id, llama_n_seq_max(ctx));
+ return false;
+ }
+
+ cell.seq_id.insert(seq_id);
+
+ if (kv_self.recurrent) {
+ int32_t & tail = kv_self.cells[seq_id].tail;
+ if (tail != -1) {
+ LLAMA_LOG_ERROR("%s: duplicate tail for seq_id %d in cell %d and %d\n", __func__, seq_id, i, tail);
+ return false;
+ }
+ tail = i;
+ }
+ }
+ }
+
+ kv_self.head = 0;
+ kv_self.used = cell_count;
+ }
+
+ if (kv_self.recurrent) {
+ for (uint32_t i = 0; i < cell_count; ++i) {
+ uint32_t cell_id = kv_self.head + i;
+ // make sure the recurrent states will keep their restored state
+ kv_self.cells[cell_id].src = cell_id;
+ }
+ }
+
+ return true;
+ }
+
+ bool read_kv_cache_data(struct llama_context * ctx, uint32_t cell_count) {
+ const struct llama_hparams & hparams = ctx->model.hparams;
+ struct llama_kv_cache & kv_self = ctx->kv_self;
+ uint32_t v_trans;
+ uint32_t n_layer;
+ read_to(&v_trans, sizeof(v_trans));
+ read_to(&n_layer, sizeof(n_layer));
+
+ if (n_layer != hparams.n_layer) {
+ LLAMA_LOG_ERROR("%s: mismatched layer count (%u instead of %u)\n", __func__, n_layer, hparams.n_layer);
+ return false;
+ }
+ if (cell_count > kv_self.size) {
+ LLAMA_LOG_ERROR("%s: not enough cells in kv cache to restore state (%u > %u)\n", __func__, cell_count, kv_self.size);
+ return false;
+ }
+ if (kv_self.v_trans != (bool) v_trans) {
+ LLAMA_LOG_ERROR("%s: incompatible V transposition\n", __func__);
+ return false;
+ }
+
+ // For each layer, read the keys for each cell, one row is one cell, read as one contiguous block
+ for (uint32_t il = 0; il < n_layer; ++il) {
+ const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il) + hparams.n_embd_k_s();
+
+ // Read type of key
+ int32_t k_type_i_ref;
+ read_to(&k_type_i_ref, sizeof(k_type_i_ref));
+ const int32_t k_type_i = (int32_t)kv_self.k_l[il]->type;
+ if (k_type_i != k_type_i_ref) {
+ LLAMA_LOG_ERROR("%s: mismatched key type (%d != %d, layer %d)\n", __func__, k_type_i, k_type_i_ref, il);
+ return false;
+ }
+
+ // Read row size of key
+ uint64_t k_size_row_ref;
+ read_to(&k_size_row_ref, sizeof(k_size_row_ref));
+ const size_t k_size_row = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa);
+ if (k_size_row != k_size_row_ref) {
+ LLAMA_LOG_ERROR("%s: mismatched key row size (%zu != %zu, layer %d)\n", __func__, k_size_row, (size_t) k_size_row_ref, il);
+ return false;
+ }
+
+ if (cell_count) {
+ // Read and set the keys for the whole cell range
+ ggml_backend_tensor_set(kv_self.k_l[il], read(cell_count * k_size_row), kv_self.head * k_size_row, cell_count * k_size_row);
+ }
+ }
+
+ if (!kv_self.v_trans) {
+ for (uint32_t il = 0; il < n_layer; ++il) {
+ const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s();
+
+ // Read type of value
+ int32_t v_type_i_ref;
+ read_to(&v_type_i_ref, sizeof(v_type_i_ref));
+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;
+ if (v_type_i != v_type_i_ref) {
+ LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il);
+ return false;
+ }
+
+ // Read row size of value
+ uint64_t v_size_row_ref;
+ read_to(&v_size_row_ref, sizeof(v_size_row_ref));
+ const size_t v_size_row = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa);
+ if (v_size_row != v_size_row_ref) {
+ LLAMA_LOG_ERROR("%s: mismatched value row size (%zu != %zu, layer %d)\n", __func__, v_size_row, (size_t) v_size_row_ref, il);
+ return false;
+ }
+
+ if (cell_count) {
+ // Read and set the values for the whole cell range
+ ggml_backend_tensor_set(kv_self.v_l[il], read(cell_count * v_size_row), kv_self.head * v_size_row, cell_count * v_size_row);
+ }
+ }
+ } else {
+ // For each layer, read the values for each cell (transposed)
+ for (uint32_t il = 0; il < n_layer; ++il) {
+ const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s();
+
+ // Read type of value
+ int32_t v_type_i_ref;
+ read_to(&v_type_i_ref, sizeof(v_type_i_ref));
+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;
+ if (v_type_i != v_type_i_ref) {
+ LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il);
+ return false;
+ }
+
+ // Read element size of value
+ uint32_t v_size_el_ref;
+ read_to(&v_size_el_ref, sizeof(v_size_el_ref));
+ const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type);
+ if (v_size_el != v_size_el_ref) {
+ LLAMA_LOG_ERROR("%s: mismatched value element size (%zu != %zu, layer %d)\n", __func__, v_size_el, (size_t) v_size_el_ref, il);
+ return false;
+ }
+
+ // Read GQA embedding size
+ uint32_t n_embd_v_gqa_ref;
+ read_to(&n_embd_v_gqa_ref, sizeof(n_embd_v_gqa_ref));
+ if (n_embd_v_gqa != n_embd_v_gqa_ref) {
+ LLAMA_LOG_ERROR("%s: mismatched GQA embedding size (%u != %u, layer %d)\n", __func__, n_embd_v_gqa, n_embd_v_gqa_ref, il);
+ return false;
+ }
+
+ if (cell_count) {
+ // For each row in the transposed matrix, read the values for the whole cell range
+ for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
+ const size_t dst_offset = (kv_self.head + j * kv_self.size) * v_size_el;
+ ggml_backend_tensor_set(kv_self.v_l[il], read(cell_count * v_size_el), dst_offset, cell_count * v_size_el);
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ void read_kv_cache(struct llama_context * ctx, llama_seq_id seq_id = -1) {
+ uint32_t cell_count;
+ read_to(&cell_count, sizeof(cell_count));
+
+ bool res = read_kv_cache_meta(ctx, cell_count, seq_id) && read_kv_cache_data(ctx, cell_count);
+
+ if (!res) {
+ if (seq_id == -1) {
+ llama_kv_cache_clear(ctx);
+ } else {
+ llama_kv_cache_seq_rm(ctx, seq_id, -1, -1);
+ }
+ throw std::runtime_error("failed to restore kv cache");
+ }
+ }
+};
+
+struct llama_data_write_dummy : llama_data_write {
+ size_t size_written = 0;
+
+ llama_data_write_dummy() {}
+
+ void write(const void * /* src */, size_t size) override {
+ size_written += size;
+ }
+
+ void write_tensor_data(const struct ggml_tensor * /* tensor */, size_t /* offset */, size_t size) override {
+ size_written += size;
+ }
+
+ size_t get_size_written() override {
+ return size_written;
+ }
+};
+
+struct llama_data_write_buffer : llama_data_write {
+ uint8_t * ptr;
+ size_t buf_size = 0;
+ size_t size_written = 0;
+
+ llama_data_write_buffer(uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
+
+ void write(const void * src, size_t size) override {
+ if (size > buf_size) {
+ throw std::runtime_error("unexpectedly reached end of buffer");
+ }
+ memcpy(ptr, src, size);
+ ptr += size;
+ size_written += size;
+ buf_size -= size;
+ }
+
+ void write_tensor_data(const struct ggml_tensor * tensor, size_t offset, size_t size) override {
+ if (size > buf_size) {
+ throw std::runtime_error("unexpectedly reached end of buffer");
+ }
+ ggml_backend_tensor_get(tensor, ptr, offset, size);
+ ptr += size;
+ size_written += size;
+ buf_size -= size;
+ }
+
+ size_t get_size_written() override {
+ return size_written;
+ }
+};
+
+struct llama_data_read_buffer : llama_data_read {
+ const uint8_t * ptr;
+ size_t buf_size = 0;
+ size_t size_read = 0;
+
+ llama_data_read_buffer(const uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
+
+ const uint8_t * read(size_t size) override {
+ const uint8_t * base_ptr = ptr;
+ if (size > buf_size) {
+ throw std::runtime_error("unexpectedly reached end of buffer");
+ }
+ ptr += size;
+ size_read += size;
+ buf_size -= size;
+ return base_ptr;
+ }
+
+ void read_to(void * dst, size_t size) override {
+ memcpy(dst, read(size), size);
+ }
+
+ size_t get_size_read() override {
+ return size_read;
+ }
+};
+
+struct llama_data_write_file : llama_data_write {
+ llama_file * file;
+ size_t size_written = 0;
+ std::vector<uint8_t> temp_buffer;
+
+ llama_data_write_file(llama_file * f) : file(f) {}
+
+ void write(const void * src, size_t size) override {
+ file->write_raw(src, size);
+ size_written += size;
+ }
+
+ void write_tensor_data(const struct ggml_tensor * tensor, size_t offset, size_t size) override {
+ temp_buffer.resize(size);
+ ggml_backend_tensor_get(tensor, temp_buffer.data(), offset, size);
+ write(temp_buffer.data(), temp_buffer.size());
+ }
+
+ size_t get_size_written() override {
+ return size_written;
+ }
+};
+
+struct llama_data_read_file : llama_data_read {
+ llama_file * file;
+ size_t size_read = 0;
+ std::vector<uint8_t> temp_buffer;
+
+ llama_data_read_file(llama_file * f) : file(f) {}
+
+ void read_to(void * dst, size_t size) override {
+ file->read_raw(dst, size);
+ size_read += size;
+ }
+
+ const uint8_t * read(size_t size) override {
+ temp_buffer.resize(size);
+ read_to(temp_buffer.data(), size);
+ return temp_buffer.data();
+ }
+
+ size_t get_size_read() override {
+ return size_read;
+ }
+};
+
+/** copy state data into either a buffer or file depending on the passed in context
+ *
+ * file context:
+ * llama_file file("/path", "wb");
+ * llama_data_write_file data_ctx(&file);
+ * llama_state_get_data_internal(ctx, data_ctx);
+ *
+ * buffer context:
+ * std::vector<uint8_t> buf(max_size, 0);
+ * llama_data_write_buffer data_ctx(buf.data(), max_size);
+ * llama_state_get_data_internal(ctx, data_ctx);
+ *
+*/
+static size_t llama_state_get_data_internal(struct llama_context * ctx, llama_data_write & data_ctx) {
+ llama_synchronize(ctx);
+
+ data_ctx.write_model_info(ctx);
+
+ data_ctx.write_rng(ctx->sampling.rng);
+
+ // copy outputs
+ data_ctx.write_output_ids(ctx);
+ data_ctx.write_logits(ctx);
+ data_ctx.write_embeddings(ctx);
+
+ data_ctx.write_kv_cache(ctx);
+
+ return data_ctx.get_size_written();
+}
+
+size_t llama_state_get_data(struct llama_context * ctx, uint8_t * dst, size_t size) {
+ llama_data_write_buffer data_ctx(dst, size);
+ try {
+ return llama_state_get_data_internal(ctx, data_ctx);
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what());
+ return 0;
+ }
+}
+
+// Returns the *actual* size of the state.
+// Intended to be used when saving to state to a buffer.
+size_t llama_state_get_size(struct llama_context * ctx) {
+ llama_data_write_dummy data_ctx;
+ try {
+ return llama_state_get_data_internal(ctx, data_ctx);
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: error getting state size: %s\n", __func__, err.what());
+ return 0;
+ }
+}
+
+static size_t llama_state_set_data_internal(struct llama_context * ctx, llama_data_read & data_ctx) {
+ llama_synchronize(ctx);
+
+ data_ctx.read_model_info(ctx);
+
+ // set rng
+ data_ctx.read_rng(ctx->sampling.rng);
+
+ // set outputs
+ data_ctx.read_output_ids(ctx);
+ data_ctx.read_logits(ctx);
+ data_ctx.read_embeddings(ctx);
+
+ data_ctx.read_kv_cache(ctx);
+
+ return data_ctx.get_size_read();
+}
+
+// Sets the state reading from the specified source address
+size_t llama_state_set_data(struct llama_context * ctx, const uint8_t * src, size_t size) {
+ llama_data_read_buffer data_ctx(src, size);
+ try {
+ return llama_state_set_data_internal(ctx, data_ctx);
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what());
+ return 0;
+ }
+}
+
+static bool llama_state_load_file_internal(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
+ llama_file file(path_session, "rb");
+
+ // sanity checks
+ {
+ const uint32_t magic = file.read_u32();
+ const uint32_t version = file.read_u32();
+
+ if (magic != LLAMA_SESSION_MAGIC || version != LLAMA_SESSION_VERSION) {
+ LLAMA_LOG_ERROR("%s: unknown (magic, version) for session file: %08x, %08x\n", __func__, magic, version);
+ return false;
+ }
+ }
+
+ // load the prompt
+ {
+ const uint32_t n_token_count = file.read_u32();
+
+ if (n_token_count > n_token_capacity) {
+ LLAMA_LOG_ERROR("%s: token count in session file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity);
+ return false;
+ }
+
+ file.read_raw(tokens_out, sizeof(llama_token) * n_token_count);
+ *n_token_count_out = n_token_count;
+ }
+
+ // restore the context state
+ {
+ const size_t n_state_size_cur = file.size - file.tell();
+
+ llama_data_read_file data_ctx(&file);
+ const size_t n_read = llama_state_set_data_internal(ctx, data_ctx);
+
+ if (n_read != n_state_size_cur) {
+ LLAMA_LOG_ERROR("%s: did not read all of the session file data! size %zu, got %zu\n", __func__, n_state_size_cur, n_read);
+ return false;
+ }
+ }
+ return true;
+}
+
+bool llama_state_load_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
+ try {
+ return llama_state_load_file_internal(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out);
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: error loading session file: %s\n", __func__, err.what());
+ return false;
+ }
+}
+
+static bool llama_state_save_file_internal(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {
+ llama_file file(path_session, "wb");
+
+ file.write_u32(LLAMA_SESSION_MAGIC);
+ file.write_u32(LLAMA_SESSION_VERSION);
+
+ // save the prompt
+ file.write_u32((uint32_t) n_token_count);
+ file.write_raw(tokens, sizeof(llama_token) * n_token_count);
+
+ // save the context state using stream saving
+ llama_data_write_file data_ctx(&file);
+ llama_state_get_data_internal(ctx, data_ctx);
+
+ return true;
+}
+
+bool llama_state_save_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {
+ try {
+ return llama_state_save_file_internal(ctx, path_session, tokens, n_token_count);
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: error saving session file: %s\n", __func__, err.what());
+ return false;
+ }
+}
+
+static size_t llama_state_seq_get_data_internal(struct llama_context * ctx, llama_data_write & data_ctx, llama_seq_id seq_id) {
+ llama_synchronize(ctx);
+
+ data_ctx.write_kv_cache(ctx, seq_id);
+
+ return data_ctx.get_size_written();
+}
+
+size_t llama_state_seq_get_size(struct llama_context * ctx, llama_seq_id seq_id) {
+ llama_data_write_dummy data_ctx;
+ return llama_state_seq_get_data_internal(ctx, data_ctx, seq_id);
+}
+
+size_t llama_state_seq_get_data(struct llama_context * ctx, uint8_t * dst, size_t size, llama_seq_id seq_id) {
+ llama_data_write_buffer data_ctx(dst, size);
+ try {
+ return llama_state_seq_get_data_internal(ctx, data_ctx, seq_id);
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: error saving sequence state: %s\n", __func__, err.what());
+ return 0;
+ }
+}
+
+static size_t llama_state_seq_set_data_internal(struct llama_context * ctx, llama_data_read & data_ctx, llama_seq_id dest_seq_id) {
+ llama_synchronize(ctx);
+
+ data_ctx.read_kv_cache(ctx, dest_seq_id);
+
+ return data_ctx.get_size_read();
+}
+
+size_t llama_state_seq_set_data(struct llama_context * ctx, const uint8_t * src, size_t size, llama_seq_id dest_seq_id) {
+ llama_data_read_buffer data_ctx(src, size);
+ try {
+ return llama_state_seq_set_data_internal(ctx, data_ctx, dest_seq_id);
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: error loading sequence state: %s\n", __func__, err.what());
+ return 0;
+ }
+}
+
+static size_t llama_state_seq_save_file_internal(struct llama_context * ctx, const char * filepath, llama_seq_id seq_id, const llama_token * tokens, size_t n_token_count) {
+ llama_file file(filepath, "wb");
+
+ file.write_u32(LLAMA_STATE_SEQ_MAGIC);
+ file.write_u32(LLAMA_STATE_SEQ_VERSION);
+
+ // save the prompt
+ file.write_u32((uint32_t) n_token_count);
+ file.write_raw(tokens, sizeof(llama_token) * n_token_count);
+
+ // save the context state using stream saving
+ llama_data_write_file data_ctx(&file);
+ llama_state_seq_get_data_internal(ctx, data_ctx, seq_id);
+
+ const size_t res = file.tell();
+ GGML_ASSERT(res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + data_ctx.get_size_written());
+ return res;
+}
+
+static size_t llama_state_seq_load_file_internal(struct llama_context * ctx, const char * filepath, llama_seq_id dest_seq_id, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
+ llama_file file(filepath, "rb");
+
+ // version checks
+ {
+ const uint32_t magic = file.read_u32();
+ const uint32_t version = file.read_u32();
+
+ if (magic != LLAMA_STATE_SEQ_MAGIC || version != LLAMA_STATE_SEQ_VERSION) {
+ LLAMA_LOG_ERROR("%s: unknown (magic, version) for sequence state file: %08x, %08x\n", __func__, magic, version);
+ return 0;
+ }
+ }
+
+ // load the prompt
+ {
+ const uint32_t n_token_count = file.read_u32();
+
+ if (n_token_count > n_token_capacity) {
+ LLAMA_LOG_ERROR("%s: token count in sequence state file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity);
+ return 0;
+ }
+
+ file.read_raw(tokens_out, sizeof(llama_token) * n_token_count);
+ *n_token_count_out = n_token_count;
+ }
+
+ // restore the context state
+ {
+ const size_t state_size = file.size - file.tell();
+ llama_data_read_file data_ctx(&file);
+ const size_t nread = llama_state_seq_set_data_internal(ctx, data_ctx, dest_seq_id);
+ if (!nread) {
+ LLAMA_LOG_ERROR("%s: failed to restore sequence state\n", __func__);
+ return 0;
+ }
+ GGML_ASSERT(nread <= state_size);
+ GGML_ASSERT(nread + sizeof(uint32_t) * 3 + sizeof(llama_token) * *n_token_count_out == file.tell());
+ }
+
+ return file.tell();
+}
+
+size_t llama_state_seq_save_file(struct llama_context * ctx, const char * filepath, llama_seq_id seq_id, const llama_token * tokens, size_t n_token_count) {
+ try {
+ return llama_state_seq_save_file_internal(ctx, filepath, seq_id, tokens, n_token_count);
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: error saving sequence state file: %s\n", __func__, err.what());
+ return 0;
+ }
+}
+
+size_t llama_state_seq_load_file(struct llama_context * ctx, const char * filepath, llama_seq_id dest_seq_id, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
+ try {
+ return llama_state_seq_load_file_internal(ctx, filepath, dest_seq_id, tokens_out, n_token_capacity, n_token_count_out);
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: error loading sequence state file: %s\n", __func__, err.what());
+ return 0;
+ }
+}
+
+void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch) {
+ ctx->cparams.n_threads = n_threads;
+ ctx->cparams.n_threads_batch = n_threads_batch;
+}
+
+int32_t llama_n_threads(struct llama_context * ctx) {
+ return ctx->cparams.n_threads;
+}
+
+int32_t llama_n_threads_batch(struct llama_context * ctx) {
+ return ctx->cparams.n_threads_batch;
+}
+
+void llama_set_abort_callback(struct llama_context * ctx, bool (*abort_callback)(void * data), void * abort_callback_data) {
+ ctx->abort_callback = abort_callback;
+ ctx->abort_callback_data = abort_callback_data;
+}
+
+void llama_set_embeddings(struct llama_context * ctx, bool embeddings) {
+ ctx->cparams.embeddings = embeddings;
+}
+
+void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn) {
+ ctx->cparams.causal_attn = causal_attn;
+}
+
+struct llama_batch llama_batch_get_one(
+ llama_token * tokens,
+ int32_t n_tokens,
+ llama_pos pos_0,
+ llama_seq_id seq_id) {
+ return {
+ /*n_tokens =*/ n_tokens,
+ /*tokens =*/ tokens,
+ /*embd =*/ nullptr,
+ /*pos =*/ nullptr,
+ /*n_seq_id =*/ nullptr,
+ /*seq_id =*/ nullptr,
+ /*logits =*/ nullptr,
+ /*all_pos_0 =*/ pos_0,
+ /*all_pos_1 =*/ 1,
+ /*all_seq_id =*/ seq_id,
+ };
+}
+
+struct llama_batch llama_batch_init(int32_t n_tokens_alloc, int32_t embd, int32_t n_seq_max) {
+ llama_batch batch = {
+ /*n_tokens =*/ 0,
+ /*tokens =*/ nullptr,
+ /*embd =*/ nullptr,
+ /*pos =*/ nullptr,
+ /*n_seq_id =*/ nullptr,
+ /*seq_id =*/ nullptr,
+ /*logits =*/ nullptr,
+ /*all_pos_0 =*/ 0,
+ /*all_pos_1 =*/ 0,
+ /*all_seq_id =*/ 0,
+ };
+
+ if (embd) {
+ batch.embd = (float *) malloc(sizeof(float) * n_tokens_alloc * embd);
+ } else {
+ batch.token = (llama_token *) malloc(sizeof(llama_token) * n_tokens_alloc);
+ }
+
+ batch.pos = (llama_pos *) malloc(sizeof(llama_pos) * n_tokens_alloc);
+ batch.n_seq_id = (int32_t *) malloc(sizeof(int32_t) * n_tokens_alloc);
+ batch.seq_id = (llama_seq_id **) malloc(sizeof(llama_seq_id *) * (n_tokens_alloc + 1));
+ for (int i = 0; i < n_tokens_alloc; ++i) {
+ batch.seq_id[i] = (llama_seq_id *) malloc(sizeof(llama_seq_id) * n_seq_max);
+ }
+ batch.seq_id[n_tokens_alloc] = nullptr;
+
+ batch.logits = (int8_t *) malloc(sizeof(int8_t) * n_tokens_alloc);
+
+ return batch;
+}
+
+void llama_batch_free(struct llama_batch batch) {
+ if (batch.token) free(batch.token);
+ if (batch.embd) free(batch.embd);
+ if (batch.pos) free(batch.pos);
+ if (batch.n_seq_id) free(batch.n_seq_id);
+ if (batch.seq_id) {
+ for (int i = 0; batch.seq_id[i] != nullptr; ++i) {
+ free(batch.seq_id[i]);
+ }
+ free(batch.seq_id);
+ }
+ if (batch.logits) free(batch.logits);
+}
+
+int32_t llama_encode(
+ struct llama_context * ctx,
+ struct llama_batch batch) {
+ const int ret = llama_encode_internal(*ctx, batch);
+ if (ret < 0) {
+ LLAMA_LOG_ERROR("%s: failed to encode, ret = %d\n", __func__, ret);
+ }
+
+ return ret;
+}
+
+int32_t llama_decode(
+ struct llama_context * ctx,
+ struct llama_batch batch) {
+ const int ret = llama_decode_internal(*ctx, batch);
+ if (ret < 0) {
+ LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret);
+ }
+
+ return ret;
+}
+
+void llama_synchronize(struct llama_context * ctx) {
+ ggml_backend_sched_synchronize(ctx->sched);
+
+ // FIXME: if multiple single tokens are evaluated without a synchronization,
+ // the stats will be added to the prompt evaluation stats
+ // this should only happen when using batch size 1 to evaluate a batch
+
+ // add the evaluation to the stats
+ if (ctx->n_queued_tokens == 1) {
+ ctx->t_eval_us += ggml_time_us() - ctx->t_compute_start_us;
+ ctx->n_eval++;
+ } else if (ctx->n_queued_tokens > 1) {
+ ctx->t_p_eval_us += ggml_time_us() - ctx->t_compute_start_us;
+ ctx->n_p_eval += ctx->n_queued_tokens;
+ }
+
+ // get a more accurate load time, upon first eval
+ if (ctx->n_queued_tokens > 0 && !ctx->has_evaluated_once) {
+ ctx->t_load_us = ggml_time_us() - ctx->t_start_us;
+ ctx->has_evaluated_once = true;
+ }
+
+ ctx->n_queued_tokens = 0;
+ ctx->t_compute_start_us = 0;
+}
+
+float * llama_get_logits(struct llama_context * ctx) {
+ llama_synchronize(ctx);
+
+ // reorder logits for backward compatibility
+ // TODO: maybe deprecate this
+ llama_output_reorder(ctx);
+
+ return ctx->logits;
+}
+
+float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) {
+ int32_t j = -1;
+ llama_synchronize(ctx);
+
+ try {
+ if (ctx->logits == nullptr) {
+ throw std::runtime_error("no logits");
+ }
+
+ if (i < 0) {
+ j = ctx->n_outputs + i;
+ if (j < 0) {
+ throw std::runtime_error(format("negative index out of range [0, %d)", ctx->n_outputs));
+ }
+ } else if ((size_t) i >= ctx->output_ids.size()) {
+ throw std::runtime_error(format("out of range [0, %lu)", ctx->output_ids.size()));
+ } else {
+ j = ctx->output_ids[i];
+ }
+
+ if (j < 0) {
+ throw std::runtime_error(format("batch.logits[%d] != true", i));
+ }
+ if (j >= ctx->n_outputs) {
+ // This should not happen
+ throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, ctx->n_outputs));
+ }
+
+ return ctx->logits + j*ctx->model.hparams.n_vocab;
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, err.what());
+#ifndef NDEBUG
+ GGML_ABORT("fatal error");
+#endif
+ return nullptr;
+ }
+}
+
+float * llama_get_embeddings(struct llama_context * ctx) {
+ llama_synchronize(ctx);
+
+ // reorder embeddings for backward compatibility
+ // TODO: maybe deprecate this
+ llama_output_reorder(ctx);
+
+ return ctx->embd;
+}
+
+float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) {
+ int32_t j = -1;
+
+ llama_synchronize(ctx);
+
+ try {
+ if (ctx->embd == nullptr) {
+ throw std::runtime_error("no embeddings");
+ }
+
+ if (i < 0) {
+ j = ctx->n_outputs + i;
+ if (j < 0) {
+ throw std::runtime_error(format("negative index out of range [0, %d)", ctx->n_outputs));
+ }
+ } else if ((size_t) i >= ctx->output_ids.size()) {
+ throw std::runtime_error(format("out of range [0, %lu)", ctx->output_ids.size()));
+ } else {
+ j = ctx->output_ids[i];
+ }
+
+ if (j < 0) {
+ throw std::runtime_error(format("batch.logits[%d] != true", i));
+ }
+ if (j >= ctx->n_outputs) {
+ // This should not happen
+ throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, ctx->n_outputs));
+ }
+
+ return ctx->embd + j*ctx->model.hparams.n_embd;
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, err.what());
+#ifndef NDEBUG
+ GGML_ABORT("fatal error");
+#endif
+ return nullptr;
+ }
+}
+
+float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id) {
+ llama_synchronize(ctx);
+
+ auto it = ctx->embd_seq.find(seq_id);
+ if (it == ctx->embd_seq.end()) {
+ return nullptr;
+ }
+
+ return it->second.data();
+}
+
+//
+// vocab
+//
+
+const char * llama_token_get_text(const struct llama_model * model, llama_token token) {
+ return llama_token_get_text_impl(model->vocab, token);
+}
+
+float llama_token_get_score(const struct llama_model * model, llama_token token) {
+ return llama_token_get_score_impl(model->vocab, token);
+}
+
+enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token) {
+ return llama_token_get_attr_impl(model->vocab, token);
+}
+
+bool llama_token_is_eog(const struct llama_model * model, llama_token token) {
+ return llama_token_is_eog_impl(model->vocab, token);
+}
+
+bool llama_token_is_control(const struct llama_model * model, llama_token token) {
+ return llama_token_is_control_impl(model->vocab, token);
+}
+
+llama_token llama_token_bos(const struct llama_model * model) {
+ return llama_token_bos_impl(model->vocab);
+}
+
+llama_token llama_token_eos(const struct llama_model * model) {
+ return llama_token_eos_impl(model->vocab);
+}
+
+llama_token llama_token_cls(const struct llama_model * model) {
+ return llama_token_cls_impl(model->vocab);
+}
+
+llama_token llama_token_sep(const struct llama_model * model) {
+ return llama_token_sep_impl(model->vocab);
+}
+
+llama_token llama_token_nl (const struct llama_model * model) {
+ return llama_token_nl_impl(model->vocab);
+}
+
+llama_token llama_token_pad(const struct llama_model * model) {
+ return llama_token_pad_impl(model->vocab);
+}
+
+bool llama_add_bos_token(const struct llama_model * model) {
+ return llama_add_bos_token_impl(model->vocab);
+}
+
+bool llama_add_eos_token(const struct llama_model * model) {
+ return llama_add_eos_token_impl(model->vocab);
+}
+
+llama_token llama_token_prefix(const struct llama_model * model) {
+ return llama_token_prefix_impl(model->vocab);
+}
+
+llama_token llama_token_middle(const struct llama_model * model) {
+ return llama_token_middle_impl(model->vocab);
+}
+
+llama_token llama_token_suffix(const struct llama_model * model) {
+ return llama_token_suffix_impl(model->vocab);
+}
+
+llama_token llama_token_eot(const struct llama_model * model) {
+ return llama_token_eot_impl(model->vocab);
+}
+
+//
+// tokenization
+//
+
+int32_t llama_tokenize(
+ const struct llama_model * model,
+ const char * text,
+ int32_t text_len,
+ llama_token * tokens,
+ int32_t n_tokens_max,
+ bool add_special,
+ bool parse_special) {
+ return llama_tokenize_impl(model->vocab, text, text_len, tokens, n_tokens_max, add_special, parse_special);
+}
+
+int32_t llama_token_to_piece(
+ const struct llama_model * model,
+ llama_token token,
+ char * buf,
+ int32_t length,
+ int32_t lstrip,
+ bool special) {
+ return llama_token_to_piece_impl(model->vocab, token, buf, length, lstrip, special);
+}
+
+int32_t llama_detokenize(
+ const struct llama_model * model,
+ const llama_token * tokens,
+ int32_t n_tokens,
+ char * text,
+ int32_t text_len_max,
+ bool remove_special,
+ bool unparse_special) {
+ return llama_detokenize_impl(model->vocab, tokens, n_tokens, text, text_len_max, remove_special, unparse_special);
+}
+
+//
+// chat templates
+//
+
+// Simple version of "llama_apply_chat_template" that only works with strings
+// This function uses heuristic checks to determine commonly used template. It is not a jinja parser.
+static int32_t llama_chat_apply_template_internal(
+ const std::string & tmpl,
+ const std::vector<const llama_chat_message *> & chat,
+ std::string & dest, bool add_ass) {
+ // Taken from the research: https://github.com/ggerganov/llama.cpp/issues/5527
+ std::stringstream ss;
+ auto tmpl_contains = [&tmpl](std::string haystack) -> bool {
+ return tmpl.find(haystack) != std::string::npos;
+ };
+ if (tmpl == "chatml" || tmpl_contains("<|im_start|>")) {
+ // chatml template
+ for (auto message : chat) {
+ ss << "<|im_start|>" << message->role << "\n" << message->content << "<|im_end|>\n";
+ }
+ if (add_ass) {
+ ss << "<|im_start|>assistant\n";
+ }
+ } else if (tmpl == "llama2" || tmpl == "mistral" || tmpl_contains("[INST]")) {
+ // llama2 template and its variants
+ // [variant] support system message
+ bool support_system_message = tmpl_contains("<<SYS>>") || tmpl == "mistral";
+ // [variant] space before + after response
+ bool space_around_response = tmpl_contains("' ' + eos_token");
+ // [variant] add BOS inside history
+ bool add_bos_inside_history = tmpl_contains("bos_token + '[INST]");
+ // [variant] trim spaces from the input message
+ bool strip_message = tmpl_contains("content.strip()");
+ // construct the prompt
+ bool is_inside_turn = true; // skip BOS at the beginning
+ ss << "[INST] ";
+ for (auto message : chat) {
+ std::string content = strip_message ? trim(message->content) : message->content;
+ std::string role(message->role);
+ if (!is_inside_turn) {
+ is_inside_turn = true;
+ ss << (add_bos_inside_history ? "<s>[INST] " : "[INST] ");
+ }
+ if (role == "system") {
+ if (support_system_message) {
+ ss << "<<SYS>>\n" << content << "\n<</SYS>>\n\n";
+ } else {
+ // if the model does not support system message, we still include it in the first message, but without <<SYS>>
+ ss << content << "\n";
+ }
+ } else if (role == "user") {
+ ss << content << " [/INST]";
+ } else {
+ ss << (space_around_response ? " " : "") << content << (space_around_response ? " " : "") << "</s>";
+ is_inside_turn = false;
+ }
+ }
+ // llama2 templates seem to not care about "add_generation_prompt"
+ } else if (tmpl == "phi3" || (tmpl_contains("<|assistant|>") && tmpl_contains("<|end|>"))) {
+ // Phi 3
+ for (auto message : chat) {
+ std::string role(message->role);
+ ss << "<|" << role << "|>\n" << message->content << "<|end|>\n";
+ }
+ if (add_ass) {
+ ss << "<|assistant|>\n";
+ }
+ } else if (tmpl == "zephyr" || tmpl_contains("<|user|>")) {
+ // zephyr template
+ for (auto message : chat) {
+ ss << "<|" << message->role << "|>" << "\n" << message->content << "<|endoftext|>\n";
+ }
+ if (add_ass) {
+ ss << "<|assistant|>\n";
+ }
+ } else if (tmpl == "monarch" || tmpl_contains("bos_token + message['role']")) {
+ // mlabonne/AlphaMonarch-7B template (the <s> is included inside history)
+ for (auto message : chat) {
+ std::string bos = (message == chat.front()) ? "" : "<s>"; // skip BOS for first message
+ ss << bos << message->role << "\n" << message->content << "</s>\n";
+ }
+ if (add_ass) {
+ ss << "<s>assistant\n";
+ }
+ } else if (tmpl == "gemma" || tmpl == "gemma2" || tmpl_contains("<start_of_turn>")) {
+ // google/gemma-7b-it
+ std::string system_prompt = "";
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "system") {
+ // there is no system message for gemma, but we will merge it with user prompt, so nothing is broken
+ system_prompt = trim(message->content);
+ continue;
+ }
+ // in gemma, "assistant" is "model"
+ role = role == "assistant" ? "model" : message->role;
+ ss << "<start_of_turn>" << role << "\n";
+ if (!system_prompt.empty() && role != "model") {
+ ss << system_prompt << "\n\n";
+ system_prompt = "";
+ }
+ ss << trim(message->content) << "<end_of_turn>\n";
+ }
+ if (add_ass) {
+ ss << "<start_of_turn>model\n";
+ }
+ } else if (tmpl == "orion" || tmpl_contains("'\\n\\nAssistant: ' + eos_token")) {
+ // OrionStarAI/Orion-14B-Chat
+ std::string system_prompt = "";
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "system") {
+ // there is no system message support, we will merge it with user prompt
+ system_prompt = message->content;
+ continue;
+ } else if (role == "user") {
+ ss << "Human: ";
+ if (!system_prompt.empty()) {
+ ss << system_prompt << "\n\n";
+ system_prompt = "";
+ }
+ ss << message->content << "\n\nAssistant: </s>";
+ } else {
+ ss << message->content << "</s>";
+ }
+ }
+ } else if (tmpl == "openchat" || tmpl_contains("GPT4 Correct ")) {
+ // openchat/openchat-3.5-0106,
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "system") {
+ ss << message->content << "<|end_of_turn|>";
+ } else {
+ role[0] = toupper(role[0]);
+ ss << "GPT4 Correct " << role << ": " << message->content << "<|end_of_turn|>";
+ }
+ }
+ if (add_ass) {
+ ss << "GPT4 Correct Assistant:";
+ }
+ } else if (tmpl == "vicuna" || tmpl == "vicuna-orca" || (tmpl_contains("USER: ") && tmpl_contains("ASSISTANT: "))) {
+ // eachadea/vicuna-13b-1.1 (and Orca variant)
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "system") {
+ // Orca-Vicuna variant uses a system prefix
+ if (tmpl == "vicuna-orca" || tmpl_contains("SYSTEM: ")) {
+ ss << "SYSTEM: " << message->content << "\n";
+ } else {
+ ss << message->content << "\n\n";
+ }
+ } else if (role == "user") {
+ ss << "USER: " << message->content << "\n";
+ } else if (role == "assistant") {
+ ss << "ASSISTANT: " << message->content << "</s>\n";
+ }
+ }
+ if (add_ass) {
+ ss << "ASSISTANT:";
+ }
+ } else if (tmpl == "deepseek" || (tmpl_contains("### Instruction:") && tmpl_contains("<|EOT|>"))) {
+ // deepseek-ai/deepseek-coder-33b-instruct
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "system") {
+ ss << message->content;
+ } else if (role == "user") {
+ ss << "### Instruction:\n" << message->content << "\n";
+ } else if (role == "assistant") {
+ ss << "### Response:\n" << message->content << "\n<|EOT|>\n";
+ }
+ }
+ if (add_ass) {
+ ss << "### Response:\n";
+ }
+ } else if (tmpl == "command-r" || (tmpl_contains("<|START_OF_TURN_TOKEN|>") && tmpl_contains("<|USER_TOKEN|>"))) {
+ // CohereForAI/c4ai-command-r-plus
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "system") {
+ ss << "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>" << trim(message->content) << "<|END_OF_TURN_TOKEN|>";
+ } else if (role == "user") {
+ ss << "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>" << trim(message->content) << "<|END_OF_TURN_TOKEN|>";
+ } else if (role == "assistant") {
+ ss << "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>" << trim(message->content) << "<|END_OF_TURN_TOKEN|>";
+ }
+ }
+ if (add_ass) {
+ ss << "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>";
+ }
+ } else if (tmpl == "llama3" || (tmpl_contains("<|start_header_id|>") && tmpl_contains("<|end_header_id|>"))) {
+ // Llama 3
+ for (auto message : chat) {
+ std::string role(message->role);
+ ss << "<|start_header_id|>" << role << "<|end_header_id|>\n\n" << trim(message->content) << "<|eot_id|>";
+ }
+ if (add_ass) {
+ ss << "<|start_header_id|>assistant<|end_header_id|>\n\n";
+ }
+ } else if (tmpl == "chatglm3" || tmpl_contains("[gMASK]sop")) {
+ // chatglm3-6b
+ ss << "[gMASK]" << "sop";
+ for (auto message : chat) {
+ std::string role(message->role);
+ ss << "<|" << role << "|>" << "\n " << message->content;
+ }
+ if (add_ass) {
+ ss << "<|assistant|>";
+ }
+ } else if (tmpl == "chatglm4" || tmpl_contains("[gMASK]<sop>")) {
+ ss << "[gMASK]" << "<sop>";
+ for (auto message : chat) {
+ std::string role(message->role);
+ ss << "<|" << role << "|>" << "\n" << message->content;
+ }
+ if (add_ass) {
+ ss << "<|assistant|>";
+ }
+ } else if (tmpl == "minicpm" || tmpl_contains(LU8("<用户>"))) {
+ // MiniCPM-3B-OpenHermes-2.5-v2-GGUF
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "user") {
+ ss << LU8("<用户>");
+ ss << trim(message->content);
+ ss << "<AI>";
+ } else {
+ ss << trim(message->content);
+ }
+ }
+ } else if (tmpl == "deepseek2" || tmpl_contains("'Assistant: ' + message['content'] + eos_token")) {
+ // DeepSeek-V2
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "system") {
+ ss << message->content << "\n\n";
+ } else if (role == "user") {
+ ss << "User: " << message->content << "\n\n";
+ } else if (role == "assistant") {
+ ss << "Assistant: " << message->content << LU8("<｜end▁of▁sentence｜>");
+ }
+ }
+ if (add_ass) {
+ ss << "Assistant:";
+ }
+ } else if (tmpl == "exaone3" || (tmpl_contains("[|system|]") && tmpl_contains("[|assistant|]") && tmpl_contains("[|endofturn|]"))) {
+ // ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/discussions/8#66bae61b1893d14ee8ed85bb
+ // EXAONE-3.0-7.8B-Instruct
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "system") {
+ ss << "[|system|]" << trim(message->content) << "[|endofturn|]\n";
+ } else if (role == "user") {
+ ss << "[|user|]" << trim(message->content) << "\n";
+ } else if (role == "assistant") {
+ ss << "[|assistant|]" << trim(message->content) << "[|endofturn|]\n";
+ }
+ }
+ if (add_ass) {
+ ss << "[|assistant|]";
+ }
+ } else {
+ // template not supported
+ return -1;
+ }
+ dest = ss.str();
+ return dest.size();
+}
+
+int32_t llama_chat_apply_template(
+ const struct llama_model * model,
+ const char * tmpl,
+ const struct llama_chat_message * chat,
+ size_t n_msg,
+ bool add_ass,
+ char * buf,
+ int32_t length) {
+ std::string curr_tmpl(tmpl == nullptr ? "" : tmpl);
+ if (tmpl == nullptr) {
+ GGML_ASSERT(model != nullptr);
+ // load template from model
+ std::vector<char> model_template(2048, 0); // longest known template is about 1200 bytes
+ std::string template_key = "tokenizer.chat_template";
+ int32_t res = llama_model_meta_val_str(model, template_key.c_str(), model_template.data(), model_template.size());
+ if (res < 0) {
+ // worst case: there is no information about template, we will use chatml by default
+ curr_tmpl = "chatml"; // see llama_chat_apply_template_internal
+ } else {
+ curr_tmpl = std::string(model_template.data(), model_template.size());
+ }
+ }
+
+ // format the chat to string
+ std::vector<const llama_chat_message *> chat_vec;
+ chat_vec.resize(n_msg);
+ for (size_t i = 0; i < n_msg; i++) {
+ chat_vec[i] = &chat[i];
+ }
+
+ std::string formatted_chat;
+ int32_t res = llama_chat_apply_template_internal(curr_tmpl, chat_vec, formatted_chat, add_ass);
+ if (res < 0) {
+ return res;
+ }
+ if (buf && length > 0) {
+ strncpy(buf, formatted_chat.c_str(), length);
+ }
+ return res;
+}
+
+//
+// grammar
+//
+
+struct llama_grammar * llama_grammar_init(
+ const llama_grammar_element ** rules,
+ size_t n_rules,
+ size_t start_rule_index) {
+ return llama_grammar_init_impl(rules, n_rules, start_rule_index);
+}
+
+void llama_grammar_free(struct llama_grammar * grammar) {
+ llama_grammar_free_impl(grammar);
+}
+
+struct llama_grammar * llama_grammar_copy(const struct llama_grammar * grammar) {
+ return llama_grammar_copy_impl(grammar);
+}
+
+void llama_grammar_sample(
+ const struct llama_grammar * grammar,
+ const struct llama_context * ctx,
+ llama_token_data_array * candidates) {
+ llama_grammar_sample_impl(grammar, &ctx->model.vocab, &ctx->sampling, candidates);
+}
+
+void llama_sample_grammar(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ const struct llama_grammar * grammar) {
+ llama_grammar_sample(grammar, ctx, candidates);
+}
+
+void llama_grammar_accept_token(
+ struct llama_grammar * grammar,
+ struct llama_context * ctx,
+ llama_token token) {
+ llama_grammar_accept_token_impl(grammar, &ctx->model.vocab, &ctx->sampling, token);
+}
+
+//
+// sampling
+//
+
+void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed) {
+ llama_set_rng_seed_impl(&ctx->sampling, seed);
+}
+
+void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates) {
+ llama_sample_softmax_impl(ctx ? &ctx->sampling : nullptr, candidates);
+}
+
+void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int32_t k, size_t min_keep) {
+ llama_sample_top_k_impl(ctx ? &ctx->sampling : nullptr, candidates, k, min_keep);
+}
+
+void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
+ llama_sample_top_p_impl(ctx ? &ctx->sampling : nullptr, candidates, p, min_keep);
+}
+
+void llama_sample_min_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
+ llama_sample_min_p_impl(ctx ? &ctx->sampling : nullptr, candidates, p, min_keep);
+}
+
+void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep) {
+ llama_sample_tail_free_impl(ctx ? &ctx->sampling : nullptr, candidates, z, min_keep);
+}
+
+void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
+ llama_sample_typical_impl(ctx ? &ctx->sampling : nullptr, candidates, p, min_keep);
+}
+
+void llama_sample_entropy(struct llama_context * ctx, llama_token_data_array * candidates_p, float min_temp, float max_temp, float exponent_val) {
+ llama_sample_entropy_impl(ctx ? &ctx->sampling : nullptr, candidates_p, min_temp, max_temp, exponent_val);
+}
+
+void llama_sample_temp(struct llama_context * ctx, llama_token_data_array * candidates_p, float temp) {
+ llama_sample_temp_impl(ctx ? &ctx->sampling : nullptr, candidates_p, temp);
+}
+
+void llama_sample_repetition_penalties(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ const llama_token * last_tokens,
+ size_t penalty_last_n,
+ float penalty_repeat,
+ float penalty_freq,
+ float penalty_present) {
+ llama_sample_repetition_penalties_impl(ctx ? &ctx->sampling : nullptr, candidates, last_tokens, penalty_last_n, penalty_repeat, penalty_freq, penalty_present);
+}
+
+void llama_sample_apply_guidance(
+ struct llama_context * ctx,
+ float * logits,
+ float * logits_guidance,
+ float scale) {
+ llama_sample_apply_guidance_impl(&ctx->sampling, logits, logits_guidance, scale);
+}
+
+llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int32_t m, float * mu) {
+ return llama_sample_token_mirostat_impl(&ctx->sampling, candidates, tau, eta, m, mu);
+}
+
+llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu) {
+ return llama_sample_token_mirostat_v2_impl(ctx ? &ctx->sampling : nullptr, candidates, tau, eta, mu);
+}
+
+llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates) {
+ return llama_sample_token_greedy_impl(ctx ? &ctx->sampling : nullptr, candidates);
+}
+
+llama_token llama_sample_token_with_rng(struct llama_context * ctx, llama_token_data_array * candidates, std::mt19937 & rng) {
+ return llama_sample_token_with_rng_impl(&ctx->sampling, candidates, rng);
+}
+
+llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates) {
+ return llama_sample_token_with_rng_impl(&ctx->sampling, candidates, ctx->sampling.rng);
+}
+
+int llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int split_no, int split_count) {
+ static const char * const SPLIT_PATH_FORMAT = "%s-%05d-of-%05d.gguf";
+ if (snprintf(split_path, maxlen, SPLIT_PATH_FORMAT, path_prefix, split_no + 1, split_count)) {
+ return strlen(split_path);
+ }
+ return 0;
+}
+
+int llama_split_prefix(char * dest, size_t maxlen, const char * split_path, int split_no, int split_count) {
+ std::string str_split_path(split_path);
+ char postfix[32];
+ snprintf(postfix, 32, "-%05d-of-%05d.gguf", split_no + 1, split_count);
+ std::string str_postfix(postfix);
+
+ // check if dest ends with postfix
+ int size_prefix = str_split_path.size() - str_postfix.size();
+ if (size_prefix > 0 && str_split_path.find(str_postfix, size_prefix) != std::string::npos) {
+ snprintf(dest, std::min((size_t) size_prefix + 1, maxlen), "%s", split_path);
+ return size_prefix;
+ }
+
+ return 0;
+}
+
+struct llama_timings llama_get_timings(struct llama_context * ctx) {
+ struct llama_timings result = {
+ /*.t_start_ms =*/ 1e-3 * ctx->t_start_us,
+ /*.t_end_ms =*/ 1.00 * ggml_time_ms(),
+ /*.t_load_ms =*/ 1e-3 * ctx->t_load_us,
+ /*.t_sample_ms =*/ 1e-3 * ctx->sampling.t_sample_us,
+ /*.t_p_eval_ms =*/ 1e-3 * ctx->t_p_eval_us,
+ /*.t_eval_ms =*/ 1e-3 * ctx->t_eval_us,
+
+ /*.n_sample =*/ std::max(1, ctx->sampling.n_sample),
+ /*.n_p_eval =*/ std::max(0, ctx->n_p_eval),
+ /*.n_eval =*/ std::max(1, ctx->n_eval),
+ };
+
+ return result;
+}
+
+void llama_print_timings(struct llama_context * ctx) {
+ const llama_timings timings = llama_get_timings(ctx);
+
+ LLAMA_LOG_INFO("\n");
+ LLAMA_LOG_INFO("%s: load time = %10.2f ms\n", __func__, timings.t_load_ms);
+ LLAMA_LOG_INFO("%s: sample time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",
+ __func__, timings.t_sample_ms, timings.n_sample, timings.t_sample_ms / timings.n_sample, 1e3 / timings.t_sample_ms * timings.n_sample);
+ LLAMA_LOG_INFO("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n",
+ __func__, timings.t_p_eval_ms, timings.n_p_eval, timings.t_p_eval_ms / timings.n_p_eval, 1e3 / timings.t_p_eval_ms * timings.n_p_eval);
+ LLAMA_LOG_INFO("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",
+ __func__, timings.t_eval_ms, timings.n_eval, timings.t_eval_ms / timings.n_eval, 1e3 / timings.t_eval_ms * timings.n_eval);
+ LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (timings.t_end_ms - timings.t_start_ms), (timings.n_p_eval + timings.n_eval));
+}
+
+void llama_reset_timings(struct llama_context * ctx) {
+ ctx->t_start_us = ggml_time_us();
+ ctx->t_eval_us = ctx->n_eval = 0;
+ ctx->t_p_eval_us = ctx->n_p_eval = 0;
+
+ ctx->sampling.reset_timings();
+}
+
+const char * llama_print_system_info(void) {
+ static std::string s;
+
+ s = "";
+ s += "AVX = " + std::to_string(ggml_cpu_has_avx()) + " | ";
+ s += "AVX_VNNI = " + std::to_string(ggml_cpu_has_avx_vnni()) + " | ";
+ s += "AVX2 = " + std::to_string(ggml_cpu_has_avx2()) + " | ";
+ s += "AVX512 = " + std::to_string(ggml_cpu_has_avx512()) + " | ";
+ s += "AVX512_VBMI = " + std::to_string(ggml_cpu_has_avx512_vbmi()) + " | ";
+ s += "AVX512_VNNI = " + std::to_string(ggml_cpu_has_avx512_vnni()) + " | ";
+ s += "AVX512_BF16 = " + std::to_string(ggml_cpu_has_avx512_bf16()) + " | ";
+ s += "FMA = " + std::to_string(ggml_cpu_has_fma()) + " | ";
+ s += "NEON = " + std::to_string(ggml_cpu_has_neon()) + " | ";
+ s += "SVE = " + std::to_string(ggml_cpu_has_sve()) + " | ";
+ s += "ARM_FMA = " + std::to_string(ggml_cpu_has_arm_fma()) + " | ";
+ s += "F16C = " + std::to_string(ggml_cpu_has_f16c()) + " | ";
+ s += "FP16_VA = " + std::to_string(ggml_cpu_has_fp16_va()) + " | ";
+ s += "WASM_SIMD = " + std::to_string(ggml_cpu_has_wasm_simd()) + " | ";
+ s += "BLAS = " + std::to_string(ggml_cpu_has_blas()) + " | ";
+ s += "SSE3 = " + std::to_string(ggml_cpu_has_sse3()) + " | ";
+ s += "SSSE3 = " + std::to_string(ggml_cpu_has_ssse3()) + " | ";
+ s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | ";
+ s += "MATMUL_INT8 = " + std::to_string(ggml_cpu_has_matmul_int8()) + " | ";
+ s += "LLAMAFILE = " + std::to_string(ggml_cpu_has_llamafile()) + " | ";
+
+ return s.c_str();
+}
+
+void llama_dump_timing_info_yaml(FILE * stream, const llama_context * ctx) {
+ fprintf(stream, "\n");
+ fprintf(stream, "###########\n");
+ fprintf(stream, "# Timings #\n");
+ fprintf(stream, "###########\n");
+ fprintf(stream, "\n");
+
+ fprintf(stream, "mst_eval: %.2f # ms / token during generation\n",
+ 1.0e-3 * ctx->t_eval_us / ctx->n_eval);
+ fprintf(stream, "mst_p_eval: %.2f # ms / token during prompt processing\n",
+ 1.0e-3 * ctx->t_p_eval_us / ctx->n_p_eval);
+ fprintf(stream, "mst_sample: %.2f # ms / token during sampling\n",
+ 1.0e-3 * ctx->sampling.t_sample_us / ctx->sampling.n_sample);
+ fprintf(stream, "n_eval: %d # number of tokens generated (excluding the first one)\n", ctx->n_eval);
+ fprintf(stream, "n_p_eval: %d # number of tokens processed in batches at the beginning\n", ctx->n_p_eval);
+ fprintf(stream, "n_sample: %d # number of sampled tokens\n", ctx->sampling.n_sample);
+ fprintf(stream, "t_eval_us: %" PRId64 " # total microseconds spent generating tokens\n", ctx->t_eval_us);
+ fprintf(stream, "t_load_us: %" PRId64 " # total microseconds spent loading the model\n", ctx->t_load_us);
+ fprintf(stream, "t_p_eval_us: %" PRId64 " # total microseconds spent prompt processing\n", ctx->t_p_eval_us);
+ fprintf(stream, "t_sample_us: %" PRId64 " # total microseconds spent sampling\n", ctx->sampling.t_sample_us);
+ fprintf(stream, "ts_eval: %.2f # tokens / second during generation\n",
+ 1.0e6 * ctx->n_eval / ctx->t_eval_us);
+ fprintf(stream, "ts_p_eval: %.2f # tokens / second during prompt processing\n",
+ 1.0e6 * ctx->n_p_eval / ctx->t_p_eval_us);
+ fprintf(stream, "ts_sample: %.2f # tokens / second during sampling\n",
+ 1.0e6 * ctx->sampling.n_sample / ctx->sampling.t_sample_us);
+}
+
+// For internal test use
+const std::vector<std::pair<std::string, struct ggml_tensor *>> & llama_internal_get_tensor_map(
+ struct llama_context * ctx
+) {
+ return ctx->model.tensors_by_name;
+}
+
+void llama_log_set(ggml_log_callback log_callback, void * user_data) {
+ g_state.log_callback = log_callback ? log_callback : llama_log_callback_default;
+ g_state.log_callback_user_data = user_data;
+#ifdef GGML_USE_METAL
+ ggml_backend_metal_log_set_callback(g_state.log_callback, g_state.log_callback_user_data);
+#elif defined(GGML_USE_CUDA)
+ ggml_backend_cuda_log_set_callback(g_state.log_callback, g_state.log_callback_user_data);
+#elif defined(GGML_USE_CANN)
+ ggml_backend_cann_log_set_callback(g_state.log_callback, g_state.log_callback_user_data);
+#endif
+}
+
+static void llama_log_internal_v(ggml_log_level level, const char * format, va_list args) {
+ va_list args_copy;
+ va_copy(args_copy, args);
+ char buffer[128];
+ int len = vsnprintf(buffer, 128, format, args);
+ if (len < 128) {
+ g_state.log_callback(level, buffer, g_state.log_callback_user_data);
+ } else {
+ char* buffer2 = new char[len+1];
+ vsnprintf(buffer2, len+1, format, args_copy);
+ buffer2[len] = 0;
+ g_state.log_callback(level, buffer2, g_state.log_callback_user_data);
+ delete[] buffer2;
+ }
+ va_end(args_copy);
+}
+
+void llama_log_internal(ggml_log_level level, const char * format, ...) {
+ va_list args;
+ va_start(args, format);
+ llama_log_internal_v(level, format, args);
+ va_end(args);
+}
+
+void llama_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
+ (void) level;
+ (void) user_data;
+ fputs(text, stderr);
+ fflush(stderr);
+}
diff --git a/llama/llama.go b/llama/llama.go
new file mode 100644
index 00000000..4918a46d
--- /dev/null
+++ b/llama/llama.go
@@ -0,0 +1,513 @@
+package llama
+
+/*
+#cgo CFLAGS: -O2 -std=c11 -DGGML_BUILD=1 -DNDEBUG -DLOG_DISABLE_LOGS -DGGML_USE_LLAMAFILE
+#cgo CXXFLAGS: -O2 -std=c++11 -DGGML_BUILD=1 -DNDEBUG -DLOG_DISABLE_LOGS -DGGML_USE_LLAMAFILE
+#cgo darwin,arm64 CFLAGS: -DGGML_USE_METAL -DGGML_USE_ACCELERATE -DGGML_METAL_EMBED_LIBRARY -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_BLAS -mmacosx-version-min=11.3
+#cgo darwin,arm64 CXXFLAGS: -DGGML_USE_METAL -DGGML_USE_ACCELERATE -DGGML_METAL_EMBED_LIBRARY -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_BLAS -mmacosx-version-min=11.3
+#cgo darwin,arm64 LDFLAGS: -framework Foundation -framework Metal -framework MetalKit -framework Accelerate -mmacosx-version-min=11.3
+#cgo darwin,amd64 CFLAGS: -Wno-incompatible-pointer-types-discards-qualifiers -mmacosx-version-min=11.3
+#cgo darwin,amd64 CXXFLAGS: -Wno-incompatible-pointer-types-discards-qualifiers -mmacosx-version-min=11.3
+#cgo darwin,amd64 LDFLAGS: -framework Foundation -mmacosx-version-min=11.3
+#cgo darwin,amd64,avx2 CFLAGS: -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64
+#cgo darwin,amd64,avx2 CXXFLAGS: -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64
+#cgo darwin,amd64,avx2 LDFLAGS: -framework Accelerate
+#cgo linux CFLAGS: -D_GNU_SOURCE
+#cgo linux CXXFLAGS: -D_GNU_SOURCE
+#cgo linux,arm64 LDFLAGS: -L${SRCDIR}/build/Linux/arm64
+#cgo linux,amd64 LDFLAGS: -L${SRCDIR}/build/Linux/amd64
+#cgo windows CFLAGS: -Wno-discarded-qualifiers
+#cgo windows LDFLAGS: -lmsvcrt -static-libstdc++ -static-libgcc -static
+#cgo windows,arm64 LDFLAGS: -L${SRCDIR}/build/Windows/arm64
+#cgo windows,amd64 LDFLAGS: -L${SRCDIR}/build/Windows/amd64
+#cgo avx CFLAGS: -mavx
+#cgo avx CXXFLAGS: -mavx
+#cgo avx2 CFLAGS: -mavx2 -mfma -mf16c
+#cgo avx2 CXXFLAGS: -mavx2 -mfma -mf16c
+#cgo cuda CFLAGS: -fPIE -DGGML_USE_CUDA -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_CUDA_MMV_Y=1
+#cgo cuda CXXFLAGS: -fPIE -DGGML_USE_CUDA -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_CUDA_MMV_Y=1
+#cgo rocm CFLAGS: -DGGML_USE_CUDA -DGGML_USE_HIPBLAS -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_CUDA_MMV_Y=1 -D__HIP_PLATFORM_AMD__=1 -D__HIP_ROCclr__=1
+#cgo rocm CXXFLAGS: -DGGML_USE_CUDA -DGGML_USE_HIPBLAS -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_CUDA_MMV_Y=1 -D__HIP_PLATFORM_AMD__=1 -D__HIP_ROCclr__=1
+#cgo rocm LDFLAGS: -L${SRCDIR} -lggml_rocm -lhipblas -lamdhip64 -lrocblas
+#cgo cuda_v11 LDFLAGS: -lggml_cuda_v11 -L/usr/local/cuda-11/lib64
+#cgo cuda_v12 LDFLAGS: -lggml_cuda_v12 -L/usr/local/cuda-12/lib64
+#cgo windows,cuda LDFLAGS: -lcuda -lcudart -lcublas -lcublasLt
+#cgo windows,rocm LDFLAGS: -lggml_rocm -lhipblas -lamdhip64 -lrocblas
+#cgo linux,cuda LDFLAGS: -lcuda -lcudart -lcublas -lcublasLt -lpthread -ldl -lrt -lresolv
+#cgo linux,rocm LDFLAGS: -L/opt/rocm/lib -lpthread -ldl -lrt -lresolv
+
+#include <stdlib.h>
+#include "llama.h"
+#include "clip.h"
+#include "llava.h"
+#include "sampling_ext.h"
+
+bool llamaProgressCallback(float progress, void *user_data);
+*/
+import "C"
+
+import (
+ _ "embed"
+ "errors"
+ "fmt"
+ "runtime"
+ "runtime/cgo"
+ "strings"
+ "unsafe"
+)
+
+var CpuFeatures = ""
+
+func BackendInit() {
+ C.llama_backend_init()
+}
+
+func PrintSystemInfo() string {
+ return C.GoString(C.llama_print_system_info())
+}
+
+type ContextParams struct {
+ c C.struct_llama_context_params
+}
+
+func NewContextParams(numCtx int, batchSize int, numSeqMax int, threads int, flashAttention bool) ContextParams {
+ params := C.llama_context_default_params()
+ params.n_ctx = C.uint(numCtx)
+ params.n_batch = C.uint(batchSize)
+ params.n_seq_max = C.uint(numSeqMax)
+ params.n_threads = C.int(threads)
+ params.n_threads_batch = params.n_threads
+ params.embeddings = C.bool(true)
+ params.flash_attn = C.bool(flashAttention)
+ return ContextParams{c: params}
+}
+
+type Context struct {
+ c *C.struct_llama_context
+ numThreads int
+}
+
+func (c *Context) KvCacheClear() {
+ C.llama_kv_cache_clear(c.c)
+}
+
+func (c *Context) Decode(batch *Batch) error {
+ // Positive return values does not mean a fatal error, but rather a warning.
+ // 0 - success
+ // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
+ // < 0 - error
+ code := int(C.llama_decode(c.c, batch.c))
+
+ if code < 0 {
+ return fmt.Errorf("llama_decode failed with code %d", code)
+ }
+
+ if code > 0 {
+ return fmt.Errorf("could not find a KV slot for the batch - try reducing the size of the batch or increase the context. code: %d", code)
+ }
+
+ return nil
+}
+
+func (c *Context) Model() *Model {
+ return &Model{c: C.llama_get_model(c.c)}
+}
+
+func (c *Context) GetLogitsIth(i int) []float32 {
+ return unsafe.Slice((*float32)(unsafe.Pointer(C.llama_get_logits_ith(c.c, C.int(i)))), c.Model().NumVocab())
+}
+
+func (c *Context) SampleTokenGreedy(logits []float32) int {
+ candidates := (*C.struct_llama_token_data)(C.malloc(C.size_t(len(logits)) * C.size_t(unsafe.Sizeof(C.struct_llama_token_data{}))))
+ defer C.free(unsafe.Pointer(candidates))
+
+ for i, logit := range logits {
+ ptr := (*C.struct_llama_token_data)(unsafe.Pointer(uintptr(unsafe.Pointer(candidates)) + uintptr(i)*unsafe.Sizeof(C.struct_llama_token_data{})))
+ ptr.id = C.int(i)
+ ptr.logit = C.float(logit)
+ ptr.p = 0.0
+ }
+
+ return int(C.llama_sample_token_greedy(c.c, &C.llama_token_data_array{
+ data: candidates,
+ size: C.size_t(len(logits)),
+ sorted: C.bool(false),
+ }))
+}
+
+func (c *Context) KvCacheSeqAdd(seqId int, p0 int, p1 int, delta int) {
+ C.llama_kv_cache_seq_add(c.c, C.int(seqId), C.int(p0), C.int(p1), C.int(delta))
+}
+
+func (c *Context) KvCacheSeqRm(seqId int, p0 int, p1 int) bool {
+ return bool(C.llama_kv_cache_seq_rm(c.c, C.int(seqId), C.int(p0), C.int(p1)))
+}
+
+func (c *Context) KvCacheSeqCp(srcSeqId int, dstSeqId int, p0 int, p1 int) {
+ C.llama_kv_cache_seq_cp(c.c, C.int(srcSeqId), C.int(dstSeqId), C.int(p0), C.int(p1))
+}
+
+// Get the embeddings for a sequence id
+func (c *Context) GetEmbeddingsSeq(seqId int) []float32 {
+ embeddings := unsafe.Pointer(C.llama_get_embeddings_seq(c.c, C.int(seqId)))
+ if embeddings == nil {
+ return nil
+ }
+
+ return unsafe.Slice((*float32)(embeddings), c.Model().NEmbd())
+}
+
+func (c *Context) GetEmbeddingsIth(i int) []float32 {
+ return unsafe.Slice((*float32)(unsafe.Pointer(C.llama_get_embeddings_ith(c.c, C.int32_t(i)))), c.Model().NEmbd())
+}
+
+type ModelParams struct {
+ NumGpuLayers int
+ MainGpu int
+ UseMmap bool
+ UseMlock bool
+ TensorSplit []float32
+ Progress func(float32)
+ VocabOnly bool
+}
+
+//export llamaProgressCallback
+func llamaProgressCallback(progress C.float, userData unsafe.Pointer) C.bool {
+ handle := *(*cgo.Handle)(userData)
+ callback := handle.Value().(func(float32))
+ callback(float32(progress))
+ return true
+}
+
+func LoadModelFromFile(modelPath string, params ModelParams) *Model {
+ cparams := C.llama_model_default_params()
+ cparams.n_gpu_layers = C.int(params.NumGpuLayers)
+ cparams.main_gpu = C.int32_t(params.MainGpu)
+ cparams.use_mmap = C.bool(params.UseMmap)
+ cparams.use_mlock = C.bool(params.UseMlock)
+ cparams.vocab_only = C.bool(params.VocabOnly)
+
+ if len(params.TensorSplit) > 0 {
+ tensorSplitData := ¶ms.TensorSplit[0]
+
+ var tensorSplitPin runtime.Pinner
+ tensorSplitPin.Pin(tensorSplitData)
+ defer tensorSplitPin.Unpin()
+
+ cparams.tensor_split = (*C.float)(unsafe.Pointer(tensorSplitData))
+ }
+
+ if params.Progress != nil {
+ handle := cgo.NewHandle(params.Progress)
+ defer handle.Delete()
+
+ var handlePin runtime.Pinner
+ handlePin.Pin(&handle)
+ defer handlePin.Unpin()
+
+ cparams.progress_callback = C.llama_progress_callback(C.llamaProgressCallback)
+ cparams.progress_callback_user_data = unsafe.Pointer(&handle)
+ }
+
+ return &Model{c: C.llama_load_model_from_file(C.CString(modelPath), cparams)}
+}
+
+func FreeModel(model *Model) {
+ C.llama_free_model(model.c)
+}
+
+func NewContextWithModel(model *Model, params ContextParams) *Context {
+ return &Context{
+ c: C.llama_new_context_with_model(model.c, params.c),
+ numThreads: int(params.c.n_threads),
+ }
+}
+
+func (m *Model) NumVocab() int {
+ return int(C.llama_n_vocab(m.c))
+}
+
+func (m *Model) TokenIsEog(token int) bool {
+ return bool(C.llama_token_is_eog(m.c, C.llama_token(token)))
+}
+
+func (m *Model) AddBOSToken() bool {
+ return bool(C.llama_add_bos_token(m.c))
+}
+
+func (m *Model) ApplyLoraFromFile(context *Context, loraPath string, scale float32, threads int) error {
+ cLoraPath := C.CString(loraPath)
+ defer C.free(unsafe.Pointer(cLoraPath))
+
+ loraAdapter := C.llama_lora_adapter_init(m.c, cLoraPath)
+
+ err := -1
+ if loraAdapter != nil {
+ err = int(C.llama_lora_adapter_set(context.c, loraAdapter, C.float(scale)))
+ }
+ if err != 0 {
+ return errors.New("error applying lora from file")
+ }
+
+ return nil
+}
+
+type Batch struct {
+ c C.struct_llama_batch
+ batchSize int
+ embedSize int
+}
+
+// Creates a new batch for either word tokens if embed is 0 or
+// image embeddings if embed is specified. Batches cannot contain
+// both types at the same time
+func NewBatch(nTokens int, embed int, maxSeq int) *Batch {
+ return &Batch{
+ c: C.llama_batch_init(C.int(nTokens), C.int(embed), C.int(maxSeq)),
+ batchSize: nTokens,
+ embedSize: embed,
+ }
+}
+
+func (b *Batch) NumTokens() int {
+ return int(b.c.n_tokens)
+}
+
+func (b *Batch) IsEmbedding() bool {
+ return b.embedSize != 0
+}
+
+// Add adds either a token or an image embedding to the batch depending on the type
+// when the batch was initialized. The other argument will be ignored. Adds to the
+// batch with the given position for the given sequence ids, and optionally instructs
+// to include logits.
+func (b *Batch) Add(token int, embed []float32, pos int, seqIds []int, logits bool) {
+ if !b.IsEmbedding() {
+ unsafe.Slice(b.c.token, b.batchSize)[b.c.n_tokens] = C.llama_token(token)
+ } else {
+ copy(unsafe.Slice((*float32)(b.c.embd), b.batchSize*b.embedSize)[int(b.c.n_tokens)*b.embedSize:], embed)
+ }
+ unsafe.Slice(b.c.pos, b.batchSize)[b.c.n_tokens] = C.llama_pos(pos)
+ unsafe.Slice(b.c.n_seq_id, b.batchSize)[b.c.n_tokens] = C.int(len(seqIds))
+
+ for i, s := range seqIds {
+ unsafe.Slice((unsafe.Slice(b.c.seq_id, b.batchSize)[b.c.n_tokens]), C.int(len(seqIds)))[i] = C.int32_t(s)
+ }
+
+ if logits {
+ unsafe.Slice(b.c.logits, b.batchSize)[b.c.n_tokens] = 1
+ }
+
+ b.c.n_tokens += 1
+}
+
+func (b *Batch) Clear() {
+ b.c.n_tokens = 0
+}
+
+func (b *Batch) Free() {
+ b.batchSize = 0
+ C.llama_batch_free(b.c)
+}
+
+type Model struct {
+ c *C.struct_llama_model
+}
+
+func (m *Model) TokenToPiece(token int) string {
+ tokenLen := 12
+ buf := make([]byte, tokenLen)
+ tokenLen = int(C.llama_token_to_piece(
+ m.c,
+ C.int32_t(token),
+ (*C.char)(unsafe.Pointer(&buf[0])),
+ C.int32_t(tokenLen),
+ C.int32_t(0),
+ C.bool(true),
+ ))
+ if tokenLen < 0 {
+ tokenLen = -tokenLen
+
+ buf = make([]byte, tokenLen)
+ C.llama_token_to_piece(
+ m.c,
+ C.int32_t(token),
+ (*C.char)(unsafe.Pointer(&buf[0])),
+ C.int32_t(tokenLen),
+ C.int32_t(0),
+ C.bool(true),
+ )
+ }
+ return strings.TrimRight(string(buf), "\x00")
+}
+
+func (m *Model) Tokenize(text string, addSpecial bool, parseSpecial bool) ([]int, error) {
+ maxTokens := len(text) + 2
+ cTokens := make([]C.llama_token, maxTokens)
+ cText := C.CString(text)
+ defer C.free(unsafe.Pointer(cText))
+
+ result := C.llama_tokenize(
+ m.c,
+ cText,
+ C.int32_t(len(text)),
+ &cTokens[0],
+ C.int32_t(maxTokens),
+ C.bool(addSpecial),
+ C.bool(parseSpecial),
+ )
+
+ // if the result is negative, reallocate and retry with the correct buffer size
+ if result < 0 {
+ maxTokens = int(-result)
+ cTokens = make([]C.llama_token, maxTokens)
+ result = C.llama_tokenize(
+ m.c,
+ cText,
+ C.int32_t(len(text)),
+ &cTokens[0],
+ C.int32_t(maxTokens),
+ C.bool(addSpecial),
+ C.bool(parseSpecial),
+ )
+ if result < 0 {
+ return nil, fmt.Errorf("tokenization failed, required %d tokens", -result)
+ }
+ }
+
+ tokens := make([]int, result)
+ for i := range result {
+ tokens[i] = int(cTokens[i])
+ }
+
+ return tokens, nil
+}
+
+func (m *Model) NEmbd() int {
+ return int(C.llama_n_embd(m.c))
+}
+
+func Quantize(infile, outfile string, ftype uint32) error {
+ cinfile := C.CString(infile)
+ defer C.free(unsafe.Pointer(cinfile))
+
+ coutfile := C.CString(outfile)
+ defer C.free(unsafe.Pointer(coutfile))
+
+ params := C.llama_model_quantize_default_params()
+ params.nthread = -1
+ params.ftype = ftype
+
+ if rc := C.llama_model_quantize(cinfile, coutfile, ¶ms); rc != 0 {
+ return fmt.Errorf("llama_model_quantize: %d", rc)
+ }
+
+ return nil
+}
+
+// llava
+type ClipContext struct {
+ c *C.struct_clip_ctx
+}
+
+func NewClipContext(modelPath string) *ClipContext {
+ mp := C.CString(modelPath)
+ defer C.free(unsafe.Pointer(mp))
+ cc := C.clip_model_load(mp, 1)
+ return &ClipContext{c: cc}
+}
+
+func (c *ClipContext) Free() {
+ C.clip_free(c.c)
+}
+
+func NewLlavaImageEmbed(llamaContext *Context, clipContext *ClipContext, data []byte) [][]float32 {
+ c := C.llava_image_embed_make_with_bytes(clipContext.c, C.int(llamaContext.numThreads), (*C.uchar)(unsafe.Pointer(&data[0])), C.int(len(data)))
+
+ numTokens := int(c.n_image_pos)
+ numEmbed := llamaContext.Model().NEmbd()
+
+ s := unsafe.Slice((*float32)(c.embed), numEmbed*numTokens)
+
+ embed := make([][]float32, numTokens)
+ rows := make([]float32, len(s))
+ copy(rows, s)
+
+ for i := range embed {
+ embed[i] = rows[i*numEmbed : (i+1)*numEmbed]
+ }
+
+ C.llava_image_embed_free(c)
+
+ return embed
+}
+
+// sampling
+// TODO: this is a temporary wrapper to allow calling C++ code from CGo
+type SamplingContext struct {
+ c *C.struct_llama_sampling_context
+}
+
+type SamplingParams struct {
+ TopK int
+ TopP float32
+ MinP float32
+ TfsZ float32
+ TypicalP float32
+ Temp float32
+ RepeatLastN int
+ PenaltyRepeat float32
+ PenaltyFreq float32
+ PenaltyPresent float32
+ Mirostat int
+ MirostatTau float32
+ MirostatEta float32
+ PenalizeNl bool
+ Seed uint32
+ Grammar string
+}
+
+func NewSamplingContext(params SamplingParams) *SamplingContext {
+ var cparams C.struct_llama_sampling_cparams
+ cparams.top_k = C.int32_t(params.TopK)
+ cparams.top_p = C.float(params.TopP)
+ cparams.min_p = C.float(params.MinP)
+ cparams.tfs_z = C.float(params.TfsZ)
+ cparams.typical_p = C.float(params.TypicalP)
+ cparams.temp = C.float(params.Temp)
+ cparams.penalty_last_n = C.int32_t(params.RepeatLastN)
+ cparams.penalty_repeat = C.float(params.PenaltyRepeat)
+ cparams.penalty_freq = C.float(params.PenaltyFreq)
+ cparams.penalty_present = C.float(params.PenaltyFreq)
+ cparams.mirostat = C.int32_t(params.Mirostat)
+ cparams.mirostat_tau = C.float(params.MirostatTau)
+ cparams.mirostat_eta = C.float(params.MirostatEta)
+ cparams.penalize_nl = C.bool(params.PenalizeNl)
+ cparams.seed = C.uint32_t(params.Seed)
+
+ grammar := C.CString(params.Grammar)
+ defer C.free(unsafe.Pointer(grammar))
+
+ cparams.grammar = grammar
+ context := &SamplingContext{c: C.llama_sampling_cinit(&cparams)}
+ runtime.SetFinalizer(context, func(s *SamplingContext) { C.llama_sampling_cfree(s.c) })
+
+ return context
+}
+
+func (s *SamplingContext) Reset() {
+ C.llama_sampling_creset(s.c)
+}
+
+func (s *SamplingContext) Sample(ctxMain *Context, ctxConfig *Context, idx int) int {
+ // TODO (jmorganca): handle nil for all args
+ if ctxConfig == nil {
+ return int(C.llama_sampling_csample(s.c, ctxMain.c, nil, C.int(idx)))
+ }
+
+ return int(C.llama_sampling_csample(s.c, ctxMain.c, ctxConfig.c, C.int(idx)))
+}
+
+func (s *SamplingContext) Accept(ctxMain *Context, id int, applyGrammar bool) {
+ C.llama_sampling_caccept(s.c, ctxMain.c, C.llama_token(id), C.bool(applyGrammar))
+}
diff --git a/llama/llama.h b/llama/llama.h
new file mode 100644
index 00000000..b0787fa2
--- /dev/null
+++ b/llama/llama.h
@@ -0,0 +1,1277 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef LLAMA_H
+#define LLAMA_H
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdbool.h>
+
+#ifdef LLAMA_SHARED
+# if defined(_WIN32) && !defined(__MINGW32__)
+# ifdef LLAMA_BUILD
+# define LLAMA_API __declspec(dllexport)
+# else
+# define LLAMA_API __declspec(dllimport)
+# endif
+# else
+# define LLAMA_API __attribute__ ((visibility ("default")))
+# endif
+#else
+# define LLAMA_API
+#endif
+
+#ifdef __GNUC__
+# define DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
+#elif defined(_MSC_VER)
+# define DEPRECATED(func, hint) __declspec(deprecated(hint)) func
+#else
+# define DEPRECATED(func, hint) func
+#endif
+
+#define LLAMA_DEFAULT_SEED 0xFFFFFFFF
+
+#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'
+#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'
+#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'
+
+#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN
+#define LLAMA_SESSION_VERSION 8
+
+#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ
+#define LLAMA_STATE_SEQ_VERSION 2
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ //
+ // C interface
+ //
+ // TODO: show sample usage
+ //
+
+ struct llama_model;
+ struct llama_context;
+
+ typedef int32_t llama_pos;
+ typedef int32_t llama_token;
+ typedef int32_t llama_seq_id;
+
+ enum llama_vocab_type {
+ LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab
+ LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback
+ LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE
+ LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece
+ LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram
+ LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization
+ };
+
+ // pre-tokenization types
+ enum llama_vocab_pre_type {
+ LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0,
+ LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1,
+ LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2,
+ LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3,
+ LLAMA_VOCAB_PRE_TYPE_FALCON = 4,
+ LLAMA_VOCAB_PRE_TYPE_MPT = 5,
+ LLAMA_VOCAB_PRE_TYPE_STARCODER = 6,
+ LLAMA_VOCAB_PRE_TYPE_GPT2 = 7,
+ LLAMA_VOCAB_PRE_TYPE_REFACT = 8,
+ LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9,
+ LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10,
+ LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11,
+ LLAMA_VOCAB_PRE_TYPE_OLMO = 12,
+ LLAMA_VOCAB_PRE_TYPE_DBRX = 13,
+ LLAMA_VOCAB_PRE_TYPE_SMAUG = 14,
+ LLAMA_VOCAB_PRE_TYPE_PORO = 15,
+ LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16,
+ LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17,
+ LLAMA_VOCAB_PRE_TYPE_VIKING = 18,
+ LLAMA_VOCAB_PRE_TYPE_JAIS = 19,
+ LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20,
+ LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21,
+ LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22,
+ LLAMA_VOCAB_PRE_TYPE_BLOOM = 23,
+ LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24,
+ LLAMA_VOCAB_PRE_TYPE_EXAONE = 25,
+ };
+
+ enum llama_rope_type {
+ LLAMA_ROPE_TYPE_NONE = -1,
+ LLAMA_ROPE_TYPE_NORM = 0,
+ LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,
+ };
+
+ enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file
+ LLAMA_TOKEN_TYPE_UNDEFINED = 0,
+ LLAMA_TOKEN_TYPE_NORMAL = 1,
+ LLAMA_TOKEN_TYPE_UNKNOWN = 2,
+ LLAMA_TOKEN_TYPE_CONTROL = 3,
+ LLAMA_TOKEN_TYPE_USER_DEFINED = 4,
+ LLAMA_TOKEN_TYPE_UNUSED = 5,
+ LLAMA_TOKEN_TYPE_BYTE = 6,
+ };
+
+ enum llama_token_attr {
+ LLAMA_TOKEN_ATTR_UNDEFINED = 0,
+ LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0,
+ LLAMA_TOKEN_ATTR_UNUSED = 1 << 1,
+ LLAMA_TOKEN_ATTR_NORMAL = 1 << 2,
+ LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL?
+ LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,
+ LLAMA_TOKEN_ATTR_BYTE = 1 << 5,
+ LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6,
+ LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7,
+ LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8,
+ LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9,
+ };
+
+ // model file types
+ enum llama_ftype {
+ LLAMA_FTYPE_ALL_F32 = 0,
+ LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
+ // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
+ // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed
+ // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed
+ LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // except 1d tensors
+
+ LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
+ };
+
+ enum llama_rope_scaling_type {
+ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,
+ LLAMA_ROPE_SCALING_TYPE_NONE = 0,
+ LLAMA_ROPE_SCALING_TYPE_LINEAR = 1,
+ LLAMA_ROPE_SCALING_TYPE_YARN = 2,
+ LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN,
+ };
+
+ enum llama_pooling_type {
+ LLAMA_POOLING_TYPE_UNSPECIFIED = -1,
+ LLAMA_POOLING_TYPE_NONE = 0,
+ LLAMA_POOLING_TYPE_MEAN = 1,
+ LLAMA_POOLING_TYPE_CLS = 2,
+ LLAMA_POOLING_TYPE_LAST = 3,
+ };
+
+ enum llama_attention_type {
+ LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1,
+ LLAMA_ATTENTION_TYPE_CAUSAL = 0,
+ LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1,
+ };
+
+ enum llama_split_mode {
+ LLAMA_SPLIT_MODE_NONE = 0, // single GPU
+ LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs
+ LLAMA_SPLIT_MODE_ROW = 2, // split rows across GPUs
+ };
+
+ typedef struct llama_token_data {
+ llama_token id; // token id
+ float logit; // log-odds of the token
+ float p; // probability of the token
+ } llama_token_data;
+
+ typedef struct llama_token_data_array {
+ llama_token_data * data;
+ size_t size;
+ bool sorted;
+ } llama_token_data_array;
+
+ typedef bool (*llama_progress_callback)(float progress, void * user_data);
+
+ // Input data for llama_decode
+ // A llama_batch object can contain input about one or many sequences
+ // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens
+ //
+ // - token : the token ids of the input (used when embd is NULL)
+ // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
+ // - pos : the positions of the respective token in the sequence
+ // - seq_id : the sequence to which the respective token belongs
+ // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output
+ //
+ typedef struct llama_batch {
+ int32_t n_tokens;
+
+ llama_token * token;
+ float * embd;
+ llama_pos * pos;
+ int32_t * n_seq_id;
+ llama_seq_id ** seq_id;
+ int8_t * logits; // TODO: rename this to "output"
+
+ // NOTE: helpers for smooth API transition - can be deprecated in the future
+ // for future-proof code, use the above fields instead and ignore everything below
+ //
+ // pos[i] = all_pos_0 + i*all_pos_1
+ //
+ llama_pos all_pos_0; // used if pos == NULL
+ llama_pos all_pos_1; // used if pos == NULL
+ llama_seq_id all_seq_id; // used if seq_id == NULL
+ } llama_batch;
+
+ enum llama_model_kv_override_type {
+ LLAMA_KV_OVERRIDE_TYPE_INT,
+ LLAMA_KV_OVERRIDE_TYPE_FLOAT,
+ LLAMA_KV_OVERRIDE_TYPE_BOOL,
+ LLAMA_KV_OVERRIDE_TYPE_STR,
+ };
+
+ struct llama_model_kv_override {
+ enum llama_model_kv_override_type tag;
+
+ char key[128];
+
+ union {
+ int64_t val_i64;
+ double val_f64;
+ bool val_bool;
+ char val_str[128];
+ };
+ };
+
+ struct llama_model_params {
+ int32_t n_gpu_layers; // number of layers to store in VRAM
+ enum llama_split_mode split_mode; // how to split the model across multiple GPUs
+
+ // main_gpu interpretation depends on split_mode:
+ // LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model
+ // LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results
+ // LLAMA_SPLIT_MODE_LAYER: ignored
+ int32_t main_gpu;
+
+ // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
+ const float * tensor_split;
+
+ // comma separated list of RPC servers to use for offloading
+ const char * rpc_servers;
+
+ // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.
+ // If the provided progress_callback returns true, model loading continues.
+ // If it returns false, model loading is immediately aborted.
+ llama_progress_callback progress_callback;
+
+ // context pointer passed to the progress callback
+ void * progress_callback_user_data;
+
+ // override key-value pairs of the model meta data
+ const struct llama_model_kv_override * kv_overrides;
+
+ // Keep the booleans together to avoid misalignment during copy-by-value.
+ bool vocab_only; // only load the vocabulary, no weights
+ bool use_mmap; // use mmap if possible
+ bool use_mlock; // force system to keep model in RAM
+ bool check_tensors; // validate model tensor data
+ };
+
+ // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations
+ // https://github.com/ggerganov/llama.cpp/pull/7544
+ struct llama_context_params {
+ uint32_t seed; // RNG seed, -1 for random
+ uint32_t n_ctx; // text context, 0 = from model
+ uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode
+ uint32_t n_ubatch; // physical maximum batch size
+ uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models)
+ int32_t n_threads; // number of threads to use for generation
+ int32_t n_threads_batch; // number of threads to use for batch processing
+
+ enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`
+ enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id
+ enum llama_attention_type attention_type; // attention type to use for embeddings
+
+ // ref: https://github.com/ggerganov/llama.cpp/pull/2054
+ float rope_freq_base; // RoPE base frequency, 0 = from model
+ float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model
+ float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model
+ float yarn_attn_factor; // YaRN magnitude scaling factor
+ float yarn_beta_fast; // YaRN low correction dim
+ float yarn_beta_slow; // YaRN high correction dim
+ uint32_t yarn_orig_ctx; // YaRN original context size
+ float defrag_thold; // defragment the KV cache if holes/size > thold, < 0 disabled (default)
+
+ ggml_backend_sched_eval_callback cb_eval;
+ void * cb_eval_user_data;
+
+ enum ggml_type type_k; // data type for K cache [EXPERIMENTAL]
+ enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]
+
+ // Keep the booleans together to avoid misalignment during copy-by-value.
+ bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)
+ bool embeddings; // if true, extract embeddings (together with logits)
+ bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU
+ bool flash_attn; // whether to use flash attention [EXPERIMENTAL]
+
+ // Abort callback
+ // if it returns true, execution of llama_decode() will be aborted
+ // currently works only with CPU execution
+ ggml_abort_callback abort_callback;
+ void * abort_callback_data;
+ };
+
+ // model quantization parameters
+ typedef struct llama_model_quantize_params {
+ int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
+ enum llama_ftype ftype; // quantize to this llama_ftype
+ enum ggml_type output_tensor_type; // output tensor type
+ enum ggml_type token_embedding_type; // token embeddings tensor type
+ bool allow_requantize; // allow quantizing non-f32/f16 tensors
+ bool quantize_output_tensor; // quantize output.weight
+ bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
+ bool pure; // quantize all tensors to the default type
+ bool keep_split; // quantize to the same number of shards
+ void * imatrix; // pointer to importance matrix data
+ void * kv_overrides; // pointer to vector containing overrides
+ } llama_model_quantize_params;
+
+ // grammar types
+ struct llama_grammar;
+
+ // grammar element type
+ enum llama_gretype {
+ // end of rule definition
+ LLAMA_GRETYPE_END = 0,
+
+ // start of alternate definition for rule
+ LLAMA_GRETYPE_ALT = 1,
+
+ // non-terminal element: reference to rule
+ LLAMA_GRETYPE_RULE_REF = 2,
+
+ // terminal element: character (code point)
+ LLAMA_GRETYPE_CHAR = 3,
+
+ // inverse char(s) ([^a], [^a-b] [^abc])
+ LLAMA_GRETYPE_CHAR_NOT = 4,
+
+ // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to
+ // be an inclusive range ([a-z])
+ LLAMA_GRETYPE_CHAR_RNG_UPPER = 5,
+
+ // modifies a preceding LLAMA_GRETYPE_CHAR or
+ // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA])
+ LLAMA_GRETYPE_CHAR_ALT = 6,
+
+ // any character (.)
+ LLAMA_GRETYPE_CHAR_ANY = 7,
+ };
+
+ typedef struct llama_grammar_element {
+ enum llama_gretype type;
+ uint32_t value; // Unicode code point or rule ID
+ } llama_grammar_element;
+
+ // performance timing information
+ struct llama_timings {
+ double t_start_ms;
+ double t_end_ms;
+ double t_load_ms;
+ double t_sample_ms;
+ double t_p_eval_ms;
+ double t_eval_ms;
+
+ int32_t n_sample;
+ int32_t n_p_eval;
+ int32_t n_eval;
+ };
+
+ // used in chat template
+ typedef struct llama_chat_message {
+ const char * role;
+ const char * content;
+ } llama_chat_message;
+
+ // lora adapter
+ struct llama_lora_adapter;
+
+ // Helpers for getting default parameters
+ LLAMA_API struct llama_model_params llama_model_default_params(void);
+ LLAMA_API struct llama_context_params llama_context_default_params(void);
+ LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);
+
+ // Initialize the llama + ggml backend
+ // If numa is true, use NUMA optimizations
+ // Call once at the start of the program
+ LLAMA_API void llama_backend_init(void);
+
+ //optional:
+ LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa);
+
+ // Optional: an auto threadpool gets created in ggml if not passed explicitly
+ LLAMA_API void llama_attach_threadpool(
+ struct llama_context * ctx,
+ ggml_threadpool_t threadpool,
+ ggml_threadpool_t threadpool_batch);
+ LLAMA_API void llama_detach_threadpool(struct llama_context * ctx);
+
+ // Call once at the end of the program - currently only used for MPI
+ LLAMA_API void llama_backend_free(void);
+
+ LLAMA_API struct llama_model * llama_load_model_from_file(
+ const char * path_model,
+ struct llama_model_params params);
+
+ LLAMA_API void llama_free_model(struct llama_model * model);
+
+ LLAMA_API struct llama_context * llama_new_context_with_model(
+ struct llama_model * model,
+ struct llama_context_params params);
+
+ // Frees all allocated memory
+ LLAMA_API void llama_free(struct llama_context * ctx);
+
+ LLAMA_API int64_t llama_time_us(void);
+
+ LLAMA_API size_t llama_max_devices(void);
+
+ LLAMA_API bool llama_supports_mmap (void);
+ LLAMA_API bool llama_supports_mlock (void);
+ LLAMA_API bool llama_supports_gpu_offload(void);
+
+ LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx);
+
+ LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx);
+ LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);
+ LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx);
+ LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx);
+
+ LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx);
+
+ LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model);
+ LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model);
+
+ LLAMA_API int32_t llama_n_vocab (const struct llama_model * model);
+ LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model);
+ LLAMA_API int32_t llama_n_embd (const struct llama_model * model);
+ LLAMA_API int32_t llama_n_layer (const struct llama_model * model);
+
+ // Get the model's RoPE frequency scaling factor
+ LLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model);
+
+ // Functions to access the model's GGUF metadata scalar values
+ // - The functions return the length of the string on success, or -1 on failure
+ // - The output string is always null-terminated and cleared on failure
+ // - GGUF array values are not supported by these functions
+
+ // Get metadata value as a string by key name
+ LLAMA_API int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);
+
+ // Get the number of metadata key/value pairs
+ LLAMA_API int32_t llama_model_meta_count(const struct llama_model * model);
+
+ // Get metadata key name by index
+ LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
+
+ // Get metadata value as a string by index
+ LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
+
+ // Get a string describing the model type
+ LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);
+
+ // Returns the total size of all the tensors in the model in bytes
+ LLAMA_API uint64_t llama_model_size(const struct llama_model * model);
+
+ // Returns the total number of parameters in the model
+ LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model);
+
+ // Get a llama model tensor
+ LLAMA_API struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const char * name);
+
+ // Returns true if the model contains an encoder that requires llama_encode() call
+ LLAMA_API bool llama_model_has_encoder(const struct llama_model * model);
+
+ // Returns true if the model contains a decoder that requires llama_decode() call
+ LLAMA_API bool llama_model_has_decoder(const struct llama_model * model);
+
+ // For encoder-decoder models, this function returns id of the token that must be provided
+ // to the decoder to start generating output sequence. For other models, it returns -1.
+ LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model);
+
+ // Returns true if the model is recurrent (like Mamba, RWKV, etc.)
+ LLAMA_API bool llama_model_is_recurrent(const struct llama_model * model);
+
+ // Returns 0 on success
+ LLAMA_API uint32_t llama_model_quantize(
+ const char * fname_inp,
+ const char * fname_out,
+ const llama_model_quantize_params * params);
+
+ // Load a LoRA adapter from file
+ // The loaded adapter will be associated to the given model, and will be free when the model is deleted
+ LLAMA_API struct llama_lora_adapter * llama_lora_adapter_init(
+ struct llama_model * model,
+ const char * path_lora);
+
+ // Add a loaded LoRA adapter to given context
+ // This will not modify model's weight
+ LLAMA_API int32_t llama_lora_adapter_set(
+ struct llama_context * ctx,
+ struct llama_lora_adapter * adapter,
+ float scale);
+
+ // Remove a specific LoRA adapter from given context
+ // Return -1 if the adapter is not present in the context
+ LLAMA_API int32_t llama_lora_adapter_remove(
+ struct llama_context * ctx,
+ struct llama_lora_adapter * adapter);
+
+ // Remove all LoRA adapters from given context
+ LLAMA_API void llama_lora_adapter_clear(
+ struct llama_context * ctx);
+
+ // Manually free a LoRA adapter
+ // Note: loaded adapters will be free when the associated model is deleted
+ LLAMA_API void llama_lora_adapter_free(struct llama_lora_adapter * adapter);
+
+ // Apply a loaded control vector to a llama_context, or if data is NULL, clear
+ // the currently loaded vector.
+ // n_embd should be the size of a single layer's control, and data should point
+ // to an n_embd x n_layers buffer starting from layer 1.
+ // il_start and il_end are the layer range the vector should apply to (both inclusive)
+ // See llama_control_vector_load in common to load a control vector.
+ LLAMA_API int32_t llama_control_vector_apply(
+ struct llama_context * lctx,
+ const float * data,
+ size_t len,
+ int32_t n_embd,
+ int32_t il_start,
+ int32_t il_end);
+
+ //
+ // KV cache
+ //
+
+ // Information associated with an individual cell in the KV cache view.
+ struct llama_kv_cache_view_cell {
+ // The position for this cell. Takes KV cache shifts into account.
+ // May be negative if the cell is not populated.
+ llama_pos pos;
+ };
+
+ // An updateable view of the KV cache.
+ struct llama_kv_cache_view {
+ // Number of KV cache cells. This will be the same as the context size.
+ int32_t n_cells;
+
+ // Maximum number of sequences that can exist in a cell. It's not an error
+ // if there are more sequences in a cell than this value, however they will
+ // not be visible in the view cells_sequences.
+ int32_t n_seq_max;
+
+ // Number of tokens in the cache. For example, if there are two populated
+ // cells, the first with 1 sequence id in it and the second with 2 sequence
+ // ids then you'll have 3 tokens.
+ int32_t token_count;
+
+ // Number of populated cache cells.
+ int32_t used_cells;
+
+ // Maximum contiguous empty slots in the cache.
+ int32_t max_contiguous;
+
+ // Index to the start of the max_contiguous slot range. Can be negative
+ // when cache is full.
+ int32_t max_contiguous_idx;
+
+ // Information for an individual cell.
+ struct llama_kv_cache_view_cell * cells;
+
+ // The sequences for each cell. There will be n_seq_max items per cell.
+ llama_seq_id * cells_sequences;
+ };
+
+ // Create an empty KV cache view. (use only for debugging purposes)
+ LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_seq_max);
+
+ // Free a KV cache view. (use only for debugging purposes)
+ LLAMA_API void llama_kv_cache_view_free(struct llama_kv_cache_view * view);
+
+ // Update the KV cache view structure with the current state of the KV cache. (use only for debugging purposes)
+ LLAMA_API void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kv_cache_view * view);
+
+ // Returns the number of tokens in the KV cache (slow, use only for debug)
+ // If a KV cell has multiple sequences assigned to it, it will be counted multiple times
+ LLAMA_API int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx);
+
+ // Returns the number of used KV cells (i.e. have at least one sequence assigned to them)
+ LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx);
+
+ // Clear the KV cache - both cell info is erased and KV data is zeroed
+ LLAMA_API void llama_kv_cache_clear(
+ struct llama_context * ctx);
+
+ // Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
+ // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails
+ // seq_id < 0 : match any sequence
+ // p0 < 0 : [0, p1]
+ // p1 < 0 : [p0, inf)
+ LLAMA_API bool llama_kv_cache_seq_rm(
+ struct llama_context * ctx,
+ llama_seq_id seq_id,
+ llama_pos p0,
+ llama_pos p1);
+
+ // Copy all tokens that belong to the specified sequence to another sequence
+ // Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence
+ // p0 < 0 : [0, p1]
+ // p1 < 0 : [p0, inf)
+ LLAMA_API void llama_kv_cache_seq_cp(
+ struct llama_context * ctx,
+ llama_seq_id seq_id_src,
+ llama_seq_id seq_id_dst,
+ llama_pos p0,
+ llama_pos p1);
+
+ // Removes all tokens that do not belong to the specified sequence
+ LLAMA_API void llama_kv_cache_seq_keep(
+ struct llama_context * ctx,
+ llama_seq_id seq_id);
+
+ // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
+ // If the KV cache is RoPEd, the KV data is updated accordingly:
+ // - lazily on next llama_decode()
+ // - explicitly with llama_kv_cache_update()
+ // p0 < 0 : [0, p1]
+ // p1 < 0 : [p0, inf)
+ LLAMA_API void llama_kv_cache_seq_add(
+ struct llama_context * ctx,
+ llama_seq_id seq_id,
+ llama_pos p0,
+ llama_pos p1,
+ llama_pos delta);
+
+ // Integer division of the positions by factor of `d > 1`
+ // If the KV cache is RoPEd, the KV data is updated accordingly:
+ // - lazily on next llama_decode()
+ // - explicitly with llama_kv_cache_update()
+ // p0 < 0 : [0, p1]
+ // p1 < 0 : [p0, inf)
+ LLAMA_API void llama_kv_cache_seq_div(
+ struct llama_context * ctx,
+ llama_seq_id seq_id,
+ llama_pos p0,
+ llama_pos p1,
+ int d);
+
+ // Returns the largest position present in the KV cache for the specified sequence
+ LLAMA_API llama_pos llama_kv_cache_seq_pos_max(
+ struct llama_context * ctx,
+ llama_seq_id seq_id);
+
+ // Defragment the KV cache
+ // This will be applied:
+ // - lazily on next llama_decode()
+ // - explicitly with llama_kv_cache_update()
+ LLAMA_API void llama_kv_cache_defrag(struct llama_context * ctx);
+
+ // Apply the KV cache updates (such as K-shifts, defragmentation, etc.)
+ LLAMA_API void llama_kv_cache_update(struct llama_context * ctx);
+
+ //
+ // State / sessions
+ //
+
+ // Returns the *actual* size in bytes of the state
+ // (rng, logits, embedding and kv_cache)
+ // Only use when saving the state, not when restoring it, otherwise the size may be too small.
+ LLAMA_API size_t llama_state_get_size(struct llama_context * ctx);
+ LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx),
+ "use llama_state_get_size instead");
+
+ // Copies the state to the specified destination address.
+ // Destination needs to have allocated enough memory.
+ // Returns the number of bytes copied
+ LLAMA_API size_t llama_state_get_data(
+ struct llama_context * ctx,
+ uint8_t * dst,
+ size_t size);
+ LLAMA_API DEPRECATED(size_t llama_copy_state_data(
+ struct llama_context * ctx,
+ uint8_t * dst),
+ "use llama_state_get_data instead");
+
+ // Set the state reading from the specified address
+ // Returns the number of bytes read
+ LLAMA_API size_t llama_state_set_data(
+ struct llama_context * ctx,
+ const uint8_t * src,
+ size_t size);
+ LLAMA_API DEPRECATED(size_t llama_set_state_data(
+ struct llama_context * ctx,
+ const uint8_t * src),
+ "use llama_state_set_data instead");
+
+ // Save/load session file
+ LLAMA_API bool llama_state_load_file(
+ struct llama_context * ctx,
+ const char * path_session,
+ llama_token * tokens_out,
+ size_t n_token_capacity,
+ size_t * n_token_count_out);
+ LLAMA_API DEPRECATED(bool llama_load_session_file(
+ struct llama_context * ctx,
+ const char * path_session,
+ llama_token * tokens_out,
+ size_t n_token_capacity,
+ size_t * n_token_count_out),
+ "use llama_state_load_file instead");
+
+ LLAMA_API bool llama_state_save_file(
+ struct llama_context * ctx,
+ const char * path_session,
+ const llama_token * tokens,
+ size_t n_token_count);
+ LLAMA_API DEPRECATED(bool llama_save_session_file(
+ struct llama_context * ctx,
+ const char * path_session,
+ const llama_token * tokens,
+ size_t n_token_count),
+ "use llama_state_save_file instead");
+
+ // Get the exact size needed to copy the KV cache of a single sequence
+ LLAMA_API size_t llama_state_seq_get_size(
+ struct llama_context * ctx,
+ llama_seq_id seq_id);
+
+ // Copy the KV cache of a single sequence into the specified buffer
+ LLAMA_API size_t llama_state_seq_get_data(
+ struct llama_context * ctx,
+ uint8_t * dst,
+ size_t size,
+ llama_seq_id seq_id);
+
+ // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence
+ // Returns:
+ // - Positive: Ok
+ // - Zero: Failed to load
+ LLAMA_API size_t llama_state_seq_set_data(
+ struct llama_context * ctx,
+ const uint8_t * src,
+ size_t size,
+ llama_seq_id dest_seq_id);
+
+ LLAMA_API size_t llama_state_seq_save_file(
+ struct llama_context * ctx,
+ const char * filepath,
+ llama_seq_id seq_id,
+ const llama_token * tokens,
+ size_t n_token_count);
+
+ LLAMA_API size_t llama_state_seq_load_file(
+ struct llama_context * ctx,
+ const char * filepath,
+ llama_seq_id dest_seq_id,
+ llama_token * tokens_out,
+ size_t n_token_capacity,
+ size_t * n_token_count_out);
+
+ //
+ // Decoding
+ //
+
+ // Return batch for single sequence of tokens starting at pos_0
+ //
+ // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
+ //
+ LLAMA_API struct llama_batch llama_batch_get_one(
+ llama_token * tokens,
+ int32_t n_tokens,
+ llama_pos pos_0,
+ llama_seq_id seq_id);
+
+ // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
+ // Each token can be assigned up to n_seq_max sequence ids
+ // The batch has to be freed with llama_batch_free()
+ // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
+ // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
+ // The rest of the llama_batch members are allocated with size n_tokens
+ // All members are left uninitialized
+ LLAMA_API struct llama_batch llama_batch_init(
+ int32_t n_tokens,
+ int32_t embd,
+ int32_t n_seq_max);
+
+ // Frees a batch of tokens allocated with llama_batch_init()
+ LLAMA_API void llama_batch_free(struct llama_batch batch);
+
+ // Processes a batch of tokens with the ecoder part of the encoder-decoder model.
+ // Stores the encoder output internally for later use by the decoder cross-attention layers.
+ // 0 - success
+ // < 0 - error
+ LLAMA_API int32_t llama_encode(
+ struct llama_context * ctx,
+ struct llama_batch batch);
+
+ // Positive return values does not mean a fatal error, but rather a warning.
+ // 0 - success
+ // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
+ // < 0 - error
+ LLAMA_API int32_t llama_decode(
+ struct llama_context * ctx,
+ struct llama_batch batch);
+
+ // Set the number of threads used for decoding
+ // n_threads is the number of threads used for generation (single token)
+ // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
+ LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch);
+
+ // Get the number of threads used for generation of a single token.
+ LLAMA_API int32_t llama_n_threads(struct llama_context * ctx);
+
+ // Get the number of threads used for prompt and batch processing (multiple token).
+ LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx);
+
+ // Set whether the model is in embeddings mode or not
+ // If true, embeddings will be returned but logits will not
+ LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings);
+
+ // Set whether to use causal attention or not
+ // If set to true, the model will only attend to the past tokens
+ LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);
+
+ // Set abort callback
+ LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data);
+
+ // Wait until all computations are finished
+ // This is automatically done when using one of the functions below to obtain the computation results
+ // and is not necessary to call it explicitly in most cases
+ LLAMA_API void llama_synchronize(struct llama_context * ctx);
+
+ // Token logits obtained from the last call to llama_decode()
+ // The logits for which llama_batch.logits[i] != 0 are stored contiguously
+ // in the order they have appeared in the batch.
+ // Rows: number of tokens for which llama_batch.logits[i] != 0
+ // Cols: n_vocab
+ LLAMA_API float * llama_get_logits(struct llama_context * ctx);
+
+ // Logits for the ith token. For positive indices, Equivalent to:
+ // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab
+ // Negative indicies can be used to access logits in reverse order, -1 is the last logit.
+ // returns NULL for invalid ids.
+ LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);
+
+ // Get all output token embeddings.
+ // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model,
+ // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously
+ // in the order they have appeared in the batch.
+ // shape: [n_outputs*n_embd]
+ // Otherwise, returns NULL.
+ LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
+
+ // Get the embeddings for the ith token. For positive indices, Equivalent to:
+ // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd
+ // Negative indicies can be used to access embeddings in reverse order, -1 is the last embedding.
+ // shape: [n_embd] (1-dimensional)
+ // returns NULL for invalid ids.
+ LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);
+
+ // Get the embeddings for a sequence id
+ // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE
+ // shape: [n_embd] (1-dimensional)
+ LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id);
+
+ //
+ // Vocab
+ //
+
+ LLAMA_API const char * llama_token_get_text(const struct llama_model * model, llama_token token);
+
+ LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token);
+
+ LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token);
+
+ // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)
+ LLAMA_API bool llama_token_is_eog(const struct llama_model * model, llama_token token);
+
+ // Identify if Token Id is a control token or a render-able token
+ LLAMA_API bool llama_token_is_control(const struct llama_model * model, llama_token token);
+
+ // Special tokens
+ LLAMA_API llama_token llama_token_bos(const struct llama_model * model); // beginning-of-sentence
+ LLAMA_API llama_token llama_token_eos(const struct llama_model * model); // end-of-sentence
+ LLAMA_API llama_token llama_token_cls(const struct llama_model * model); // classification
+ LLAMA_API llama_token llama_token_sep(const struct llama_model * model); // sentence separator
+ LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line
+ LLAMA_API llama_token llama_token_pad(const struct llama_model * model); // padding
+
+ LLAMA_API bool llama_add_bos_token(const struct llama_model * model);
+ LLAMA_API bool llama_add_eos_token(const struct llama_model * model);
+
+ // Codellama infill tokens
+ LLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefix
+ LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle
+ LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix
+ LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle
+
+ //
+ // Tokenization
+ //
+
+ /// @details Convert the provided text into tokens.
+ /// @param tokens The tokens pointer must be large enough to hold the resulting tokens.
+ /// @return Returns the number of tokens on success, no more than n_tokens_max
+ /// @return Returns a negative number on failure - the number of tokens that would have been returned
+ /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so.
+ /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated
+ /// as plaintext. Does not insert a leading space.
+ LLAMA_API int32_t llama_tokenize(
+ const struct llama_model * model,
+ const char * text,
+ int32_t text_len,
+ llama_token * tokens,
+ int32_t n_tokens_max,
+ bool add_special,
+ bool parse_special);
+
+ // Token Id -> Piece.
+ // Uses the vocabulary in the provided context.
+ // Does not write null terminator to the buffer.
+ // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix')
+ // @param special If true, special tokens are rendered in the output.
+ LLAMA_API int32_t llama_token_to_piece(
+ const struct llama_model * model,
+ llama_token token,
+ char * buf,
+ int32_t length,
+ int32_t lstrip,
+ bool special);
+
+ /// @details Convert the provided tokens into text (inverse of llama_tokenize()).
+ /// @param text The char pointer must be large enough to hold the resulting text.
+ /// @return Returns the number of chars/bytes on success, no more than text_len_max.
+ /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned.
+ /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so.
+ /// @param unparse_special If true, special tokens are rendered in the output.
+ LLAMA_API int32_t llama_detokenize(
+ const struct llama_model * model,
+ const llama_token * tokens,
+ int32_t n_tokens,
+ char * text,
+ int32_t text_len_max,
+ bool remove_special,
+ bool unparse_special);
+
+ //
+ // Chat templates
+ //
+
+ /// Apply chat template. Inspired by hf apply_chat_template() on python.
+ /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model"
+ /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template
+ /// @param tmpl A Jinja template to use for this chat. If this is nullptr, the model’s default chat template will be used instead.
+ /// @param chat Pointer to a list of multiple llama_chat_message
+ /// @param n_msg Number of llama_chat_message in this chat
+ /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message.
+ /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages)
+ /// @param length The size of the allocated buffer
+ /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template.
+ LLAMA_API int32_t llama_chat_apply_template(
+ const struct llama_model * model,
+ const char * tmpl,
+ const struct llama_chat_message * chat,
+ size_t n_msg,
+ bool add_ass,
+ char * buf,
+ int32_t length);
+
+ //
+ // Grammar
+ //
+
+ /// Initialize a llama_grammar.
+ ///
+ /// @param rules The rule elements of the grammar to initialize.
+ /// @param n_rules The number of rules.
+ /// @param start_rule_index The index of the root rule (the starting point of the grammar).
+ /// @return The initialized llama_grammar or nullptr if initialization failed.
+ LLAMA_API struct llama_grammar * llama_grammar_init(
+ const llama_grammar_element ** rules,
+ size_t n_rules,
+ size_t start_rule_index);
+
+ LLAMA_API void llama_grammar_free(struct llama_grammar * grammar);
+
+ LLAMA_API struct llama_grammar * llama_grammar_copy(const struct llama_grammar * grammar);
+
+ /// @details Apply constraints from grammar
+ LLAMA_API void llama_grammar_sample(
+ const struct llama_grammar * grammar,
+ const struct llama_context * ctx,
+ llama_token_data_array * candidates);
+ LLAMA_API DEPRECATED(void llama_sample_grammar(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ const struct llama_grammar * grammar),
+ "use llama_grammar_sample instead");
+
+ /// @details Accepts the sampled token into the grammar
+ LLAMA_API void llama_grammar_accept_token(
+ struct llama_grammar * grammar,
+ struct llama_context * ctx,
+ llama_token token);
+
+ //
+ // Sampling functions
+ //
+
+ // Sets the current rng seed.
+ LLAMA_API void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed);
+
+ /// @details Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix.
+ /// @details Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details.
+ LLAMA_API void llama_sample_repetition_penalties(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ const llama_token * last_tokens,
+ size_t penalty_last_n,
+ float penalty_repeat,
+ float penalty_freq,
+ float penalty_present);
+
+ /// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806
+ /// @param logits Logits extracted from the original generation context.
+ /// @param logits_guidance Logits extracted from a separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context.
+ /// @param scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance.
+ LLAMA_API void llama_sample_apply_guidance(
+ struct llama_context * ctx,
+ float * logits,
+ float * logits_guidance,
+ float scale);
+
+ /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.
+ LLAMA_API void llama_sample_softmax(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates);
+
+ /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
+ LLAMA_API void llama_sample_top_k(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ int32_t k,
+ size_t min_keep);
+
+ /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
+ LLAMA_API void llama_sample_top_p(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ float p,
+ size_t min_keep);
+
+ /// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841
+ LLAMA_API void llama_sample_min_p(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ float p,
+ size_t min_keep);
+
+ /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.
+ LLAMA_API void llama_sample_tail_free(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ float z,
+ size_t min_keep);
+
+ /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
+ LLAMA_API void llama_sample_typical(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ float p,
+ size_t min_keep);
+
+ /// @details Dynamic temperature implementation described in the paper https://arxiv.org/abs/2309.02772.
+ LLAMA_API void llama_sample_entropy(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates_p,
+ float min_temp,
+ float max_temp,
+ float exponent_val);
+
+ LLAMA_API void llama_sample_temp(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ float temp);
+
+ /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
+ /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
+ /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
+ /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
+ /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.
+ /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
+ LLAMA_API llama_token llama_sample_token_mirostat(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ float tau,
+ float eta,
+ int32_t m,
+ float * mu);
+
+ /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
+ /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
+ /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
+ /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
+ /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
+ LLAMA_API llama_token llama_sample_token_mirostat_v2(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ float tau,
+ float eta,
+ float * mu);
+
+ /// @details Selects the token with the highest probability.
+ /// Does not compute the token probabilities. Use llama_sample_softmax() instead.
+ LLAMA_API llama_token llama_sample_token_greedy(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates);
+
+ /// @details Randomly selects a token from the candidates based on their probabilities using the RNG of ctx.
+ LLAMA_API llama_token llama_sample_token(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates);
+
+ //
+ // Model split
+ //
+
+ /// @details Build a split GGUF final path for this chunk.
+ /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf"
+ // Returns the split_path length.
+ LLAMA_API int llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int split_no, int split_count);
+
+ /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match.
+ /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0"
+ // Returns the split_prefix length.
+ LLAMA_API int llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int split_no, int split_count);
+
+ // Performance information
+ LLAMA_API struct llama_timings llama_get_timings(struct llama_context * ctx);
+
+ LLAMA_API void llama_print_timings(struct llama_context * ctx);
+ LLAMA_API void llama_reset_timings(struct llama_context * ctx);
+
+ // Print system information
+ LLAMA_API const char * llama_print_system_info(void);
+
+ // Set callback for all future logging events.
+ // If this is not called, or NULL is supplied, everything is output on stderr.
+ LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data);
+
+ LLAMA_API void llama_dump_timing_info_yaml(FILE * stream, const struct llama_context * ctx);
+
+#ifdef __cplusplus
+}
+#endif
+
+// Internal API to be implemented by llama.cpp and used by tests/benchmarks only
+#ifdef LLAMA_API_INTERNAL
+
+#include <random>
+#include <string>
+#include <vector>
+
+struct ggml_tensor;
+
+const std::vector<std::pair<std::string, struct ggml_tensor *>> & llama_internal_get_tensor_map(
+ struct llama_context * ctx
+);
+
+struct llama_partial_utf8 {
+ uint32_t value; // bit value so far (unshifted)
+ int n_remain; // num bytes remaining; -1 indicates invalid sequence
+};
+
+struct llama_grammar_candidate {
+ size_t index;
+ const uint32_t * code_points;
+ llama_partial_utf8 partial_utf8;
+};
+
+using llama_grammar_rule = std::vector< llama_grammar_element>;
+using llama_grammar_stack = std::vector<const llama_grammar_element *>;
+
+using llama_grammar_rules = std::vector<llama_grammar_rule>;
+using llama_grammar_stacks = std::vector<llama_grammar_stack>;
+using llama_grammar_candidates = std::vector<llama_grammar_candidate>;
+
+const llama_grammar_rules & llama_grammar_get_rules (const struct llama_grammar * grammar);
+ llama_grammar_stacks & llama_grammar_get_stacks( struct llama_grammar * grammar);
+
+void llama_grammar_accept(
+ const llama_grammar_rules & rules,
+ const llama_grammar_stacks & stacks,
+ const uint32_t chr,
+ llama_grammar_stacks & new_stacks);
+
+std::vector<llama_grammar_candidate> llama_grammar_reject_candidates_for_stack(
+ const llama_grammar_rules & rules,
+ const llama_grammar_stack & stack,
+ const llama_grammar_candidates & candidates);
+
+std::pair<std::vector<uint32_t>, llama_partial_utf8> decode_utf8(
+ const std::string & src,
+ llama_partial_utf8 partial_start);
+
+// Randomly selects a token from the candidates based on their probabilities using given std::mt19937.
+// This is a temporary workaround in order to fix race conditions when sampling with multiple sequences.
+llama_token llama_sample_token_with_rng(struct llama_context * ctx, llama_token_data_array * candidates, std::mt19937 & rng);
+
+#endif // LLAMA_API_INTERNAL
+
+#endif // LLAMA_H
diff --git a/llama/llama_darwin.c b/llama/llama_darwin.c
new file mode 100644
index 00000000..3d41a75d
--- /dev/null
+++ b/llama/llama_darwin.c
@@ -0,0 +1,2 @@
+const char *ggml_metallib_start;
+const char *ggml_metallib_end;
diff --git a/llama/llama_darwin.go b/llama/llama_darwin.go
new file mode 100644
index 00000000..b5d5b379
--- /dev/null
+++ b/llama/llama_darwin.go
@@ -0,0 +1,24 @@
+package llama
+
+// extern const char *ggml_metallib_start;
+// extern const char *ggml_metallib_end;
+import "C"
+
+import (
+ _ "embed"
+ "strings"
+ "unsafe"
+)
+
+//go:embed ggml-common.h
+var ggmlCommon string
+
+//go:embed ggml-metal.metal
+var ggmlMetal string
+
+func init() {
+ metal := strings.ReplaceAll(ggmlMetal, `#include "ggml-common.h"`, ggmlCommon)
+ cMetal := C.CString(metal)
+ C.ggml_metallib_start = cMetal
+ C.ggml_metallib_end = (*C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(cMetal)) + uintptr(len(metal))))
+}
diff --git a/llama/llama_test.go b/llama/llama_test.go
new file mode 100644
index 00000000..5f835d68
--- /dev/null
+++ b/llama/llama_test.go
@@ -0,0 +1 @@
+package llama
diff --git a/llama/llamafile/sgemm.h b/llama/llamafile/sgemm.h
new file mode 100644
index 00000000..caf6dd55
--- /dev/null
+++ b/llama/llamafile/sgemm.h
@@ -0,0 +1,14 @@
+#pragma once
+#include <stdint.h>
+#include <stdbool.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+bool llamafile_sgemm(int64_t, int64_t, int64_t, const void *, int64_t,
+ const void *, int64_t, void *, int64_t, int, int,
+ int, int, int);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama/llava.cpp b/llama/llava.cpp
new file mode 100644
index 00000000..128e46b9
--- /dev/null
+++ b/llama/llava.cpp
@@ -0,0 +1,513 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "clip.h"
+#include "common.h"
+#include "llama.h"
+#include "llava.h"
+#include "base64.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+#include <numeric>
+
+// RGB uint8 image
+struct clip_image_u8 {
+ int nx;
+ int ny;
+
+ std::vector<uint8_t> buf;
+};
+
+// RGB float32 image (NHWC)
+// Memory layout: RGBRGBRGB...
+struct clip_image_f32 {
+ int nx;
+ int ny;
+
+ std::vector<float> buf;
+};
+
+struct clip_image_grid_shape {
+ int first;
+ int second;
+};
+
+/**
+ * Selects the best resolution from a list of possible resolutions based on the original size.
+ *
+ * @param original_size The original size of the image in the format (width, height).
+ * @param possible_resolutions A list of possible resolutions in the format [(width1, height1), (width2, height2), ...].
+ * @return The best fit resolution in the format (width, height).
+ */
+static std::pair<int, int> select_best_resolution(const std::pair<int, int>& original_size, const std::vector<std::pair<int, int>>& possible_resolutions) {
+ int original_width = original_size.first;
+ int original_height = original_size.second;
+
+ std::pair<int, int> best_fit;
+ int max_effective_resolution = 0;
+ int min_wasted_resolution = std::numeric_limits<int>::max();
+
+ for (const auto& resolution : possible_resolutions) {
+ int width = resolution.first;
+ int height = resolution.second;
+ float scale = std::min(static_cast<float>(width) / original_width, static_cast<float>(height) / original_height);
+ int downscaled_width = static_cast<int>(original_width * scale);
+ int downscaled_height = static_cast<int>(original_height * scale);
+ int effective_resolution = std::min(downscaled_width * downscaled_height, original_width * original_height);
+ int wasted_resolution = (width * height) - effective_resolution;
+ // LOG_TEE("resolution: %d %d, scale: %f, downscaled: %d %d, effective: %d, wasted: %d\n", width, height, scale, downscaled_width, downscaled_height, effective_resolution, wasted_resolution);
+ if (effective_resolution > max_effective_resolution || (effective_resolution == max_effective_resolution && wasted_resolution < min_wasted_resolution)) {
+ max_effective_resolution = effective_resolution;
+ min_wasted_resolution = wasted_resolution;
+ best_fit = resolution;
+ }
+ }
+
+ return best_fit;
+}
+
+/**
+ * @brief Get the anyres image grid shape object
+ *
+ * @param image_size
+ * @param grid_pinpoints
+ * @param image_patch_size
+ * @return <int, int>
+ */
+static struct clip_image_grid_shape get_anyres_image_grid_shape(const std::pair<int, int> & image_size, const std::vector<std::pair<int, int>> & grid_pinpoints, int image_patch_size) {
+ /**
+ Conversion from gguf flat array to vector:
+ std::vector<std::pair<int, int>> possible_resolutions;
+ for (int i = 0; i < 32 && params.image_grid_pinpoints[i] != 0; i+=2) {
+ possible_resolutions.push_back({params.image_grid_pinpoints[i], params.image_grid_pinpoints[i+1]});
+ }
+ */
+ auto best_resolution = select_best_resolution(image_size, grid_pinpoints);
+ return {best_resolution.first / image_patch_size, best_resolution.second / image_patch_size};
+}
+
+// Take the image segments in a grid configuration and return the embeddings and the number of embeddings into preallocated memory (image_embd_out)
+static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector<float *> & image_embd_v, struct clip_image_grid_shape grid_shape, float * image_embd_out, int * n_img_pos_out) {
+ struct {
+ struct ggml_context * ctx;
+ } model;
+
+ const int32_t image_size = clip_image_size(ctx_clip);
+ const int32_t patch_size = clip_patch_size(ctx_clip);
+
+ int32_t num_patches_per_side = image_size / patch_size; // 336 / 14 = 24 - used for embedding-patching boxes (24*24 = 576 patches)
+
+ int num_patches_width = grid_shape.first; // grid 1-4
+ int num_patches_height = grid_shape.second; // grid 1-4
+
+ const size_t num_images = num_patches_width * num_patches_height + 1;
+
+ // TODO: size calculation is not calculated - it's only tens of MB
+ size_t ctx_size = 0;
+
+ {
+ ctx_size += clip_embd_nbytes(ctx_clip) * num_images * 8; // image_features
+ ctx_size += 1024*1024 * ggml_type_size(GGML_TYPE_F32);
+ }
+
+ struct ggml_init_params params {
+ /*.mem_size =*/ ctx_size,
+ /*.mem_buffer =*/ NULL,
+ /*.no_alloc =*/ false, // NOTE: this should be false when using the legacy API
+ };
+
+ // Python reference code for full unpad:
+ /*
+ base_image_feature = image_feature[0]
+ image_feature = image_feature[1:]
+ image_feature = image_feature.permute(4, 0, 2, 1, 3).contiguous()
+ image_feature = image_feature.flatten(1, 2).flatten(2, 3)
+ image_feature = unpad_image(image_feature, image_sizes[image_idx])
+ image_feature = torch.cat((
+ image_feature,
+ self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1)
+ ), dim=-1)
+ image_feature = image_feature.flatten(1, 2).transpose(0, 1)
+ image_feature = torch.cat((base_image_feature, image_feature), dim=0)
+ */
+ // We now have two options: unpad or no unpad. Unpad removes tokens for faster llm eval.
+ // In terms of result quality it appears to make no difference, so we'll start with the easier approach given 5D tensors are not supported in ggml yet.
+ // Without unpad we have to split the sub-image embeddings into patches of 24 features each and permute them.
+ // Once all images are processed to prepended the base_image_features without any changes.
+
+ // Pytorch reference simplified, modified for ggml compatibility - confirmed identical output in python (for a 2x2 grid image (676x676 scaling))
+ /*
+ image_feature = image_feature.view(2, 2, 24, 24, 4096)
+ image_feature = image_feature.permute(0, 2, 1, 3, 4).contiguous()
+ image_feature = image_feature.view(2, 24, 2, 24, 4096)
+ image_feature = image_feature.flatten(0, 3)
+
+ // Reshape to 4D tensor by merging the last two dimensions
+ image_feature = image_feature.view(2, 2, 24, 24*4096)
+ image_feature = image_feature.permute(0, 2, 1, 3).contiguous()
+ image_feature = image_feature.view(-1, 4096)
+ */
+
+ model.ctx = ggml_init(params);
+
+ struct ggml_tensor * image_features = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F32, clip_n_mmproj_embd(ctx_clip), clip_n_patches(ctx_clip), num_images - 1); // example: 4096 x 576 x 4
+ // ggml_tensor_printf(image_features,"image_features",__LINE__,false,false);
+ // fill it with the image embeddings, ignoring the base
+ for (size_t i = 1; i < num_images; i++) {
+ size_t offset = (i-1) * clip_embd_nbytes(ctx_clip);
+ memcpy((uint8_t *)(image_features->data) + offset, image_embd_v[i], clip_embd_nbytes(ctx_clip));
+ }
+
+ struct ggml_cgraph * gf = ggml_new_graph(model.ctx);
+ size_t size_ele = ggml_type_size(GGML_TYPE_F32);
+
+ struct ggml_tensor *image_features_patchview = ggml_view_4d(model.ctx, image_features,
+ num_patches_per_side * clip_n_mmproj_embd(ctx_clip),
+ num_patches_per_side,
+ num_patches_width,
+ num_patches_height,
+ size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip),
+ size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip) * num_patches_per_side,
+ size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip) * num_patches_per_side * num_patches_width, 0);
+ // ggml_tensor_printf(image_features_patchview,"image_features_patchview",__LINE__,false,false);
+ struct ggml_tensor *permuted_cont = ggml_cont(model.ctx, ggml_permute(model.ctx, image_features_patchview, 0, 2, 1, 3));
+ /**
+ At the end of each row we have to add the row_end embeddings, which are the same as the newline embeddings
+ image_feature = torch.cat((
+ image_feature,
+ self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1).to(image_feature.device)
+ ), dim=-1)
+ *
+ */
+
+ // ggml_tensor_printf(permuted_cont,"permuted_cont",__LINE__,false,false);
+ struct ggml_tensor *flatten = ggml_view_2d(model.ctx, permuted_cont, clip_n_mmproj_embd(ctx_clip), num_patches_height * num_patches_width * num_patches_per_side * num_patches_per_side, size_ele * clip_n_mmproj_embd(ctx_clip), 0);
+ // ggml_tensor_printf(flatten,"flatten",__LINE__,false,false);
+ ggml_build_forward_expand(gf, flatten);
+ ggml_graph_compute_with_ctx(model.ctx, gf, 1);
+ struct ggml_tensor* result = gf->nodes[gf->n_nodes - 1];
+
+ memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as global context
+ // append without newline tokens (default behavior in llava_arch when not using unpad ):
+ memcpy(image_embd_out + clip_n_patches(ctx_clip) * clip_n_mmproj_embd(ctx_clip), (float*)result->data, clip_embd_nbytes(ctx_clip) * (num_images-1)); // grid patches
+ *n_img_pos_out = static_cast<int>(result->ne[1]+clip_n_patches(ctx_clip));
+
+ // Debug: Test single segments
+ // Current findings: sending base image, sending a segment embedding all works similar to python
+ // However, permuted embeddings do not work yet (stride issue?)
+ // memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as context
+ // memcpy(image_embd_out, (float*)prepared_cont->data, clip_embd_nbytes(ctx_clip)); // main image as context
+ // *n_img_pos_out=576;
+
+ ggml_free(model.ctx);
+ return true;
+}
+
+static clip_image_f32 * only_v2_5_reshape_by_patch(clip_image_f32 * image, int patch_size) {
+ int width = image->nx;
+ int height = image->ny;
+ int num_patches = (height / patch_size) * (width / patch_size);
+ clip_image_f32 * patch = clip_image_f32_init();
+ patch->nx = patch_size * num_patches;
+ patch->ny = patch_size;
+ patch->buf.resize(3 * patch->nx * patch->ny);
+
+ int patch_index = 0;
+
+ for (int i = 0; i < height; i += patch_size) {
+ for (int j = 0; j < width; j += patch_size) {
+ for (int pi = 0; pi < patch_size; ++pi) {
+ for (int pj = 0; pj < patch_size; ++pj) {
+ int input_index = ((i + pi) * width + (j + pj)) * 3;
+ int output_index = (pi * patch_size * num_patches + patch_index * patch_size + pj) * 3;
+ patch->buf[output_index] = image->buf[input_index];
+ patch->buf[output_index+1] = image->buf[input_index+1];
+ patch->buf[output_index+2] = image->buf[input_index+2];
+ }
+ }
+ patch_index++;
+ }
+ }
+ return patch;
+}
+
+static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float * image_embd, int * n_img_pos) {
+ // std::vector<clip_image_f32*> img_res_v; // format VectN x H x W x RGB (N x 336 x 336 x 3), so interleaved RGB - different to the python implementation which is N x 3 x 336 x 336
+ clip_image_f32_batch img_res_v;
+ img_res_v.size = 0;
+ img_res_v.data = nullptr;
+ if (!clip_image_preprocess(ctx_clip, img, &img_res_v)) {
+ LOG_TEE("%s: unable to preprocess image\n", __func__);
+ delete[] img_res_v.data;
+ return false;
+ }
+
+ const int64_t t_img_enc_start_us = ggml_time_us();
+
+ const char * mm_patch_merge_type = clip_patch_merge_type(ctx_clip);
+
+ if (clip_is_minicpmv(ctx_clip)) {
+ std::vector<float *> image_embd_v;
+ image_embd_v.resize(img_res_v.size);
+ struct clip_image_size * load_image_size = clip_image_size_init();
+ for (size_t i = 0; i < img_res_v.size; i++) {
+ const int64_t t_img_enc_step_start_us = ggml_time_us();
+ image_embd_v[i] = (float *)malloc(clip_embd_nbytes(ctx_clip));
+ int patch_size=14;
+ load_image_size->width = img_res_v.data[i].nx;
+ load_image_size->height = img_res_v.data[i].ny;
+ clip_add_load_image_size(ctx_clip, load_image_size);
+ bool encoded = false;
+ int has_minicpmv_projector = clip_is_minicpmv(ctx_clip);
+ if (has_minicpmv_projector == 2) {
+ encoded = clip_image_encode(ctx_clip, n_threads, only_v2_5_reshape_by_patch(&img_res_v.data[i], patch_size), image_embd_v[i]);
+ }
+ else if (has_minicpmv_projector == 3) {
+ encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[i], image_embd_v[i]);
+ }
+ if (!encoded) {
+ LOG_TEE("Unable to encode image - spatial_unpad - subimage %d of %d\n", (int) i+1, (int) img_res_v.size);
+ return false;
+ }
+ const int64_t t_img_enc_steop_batch_us = ggml_time_us();
+ LOG_TEE("%s: step %d of %d encoded in %8.2f ms\n", __func__, (int)i+1, (int)img_res_v.size, (t_img_enc_steop_batch_us - t_img_enc_step_start_us) / 1000.0);
+ }
+ const int64_t t_img_enc_batch_us = ggml_time_us();
+ LOG_TEE("%s: all %d segments encoded in %8.2f ms\n", __func__, (int)img_res_v.size, (t_img_enc_batch_us - t_img_enc_start_us) / 1000.0);
+
+ int n_img_pos_out = 0;
+ for (size_t i = 0; i < image_embd_v.size(); i++) {
+ std::memcpy(image_embd + n_img_pos_out * clip_n_mmproj_embd(ctx_clip), image_embd_v[i], clip_embd_nbytes(ctx_clip));
+ n_img_pos_out += clip_n_patches(ctx_clip);
+ }
+ *n_img_pos = n_img_pos_out;
+ for (size_t i = 0; i < image_embd_v.size(); i++) {
+ free(image_embd_v[i]);
+ }
+ image_embd_v.clear();
+ load_image_size->width = img->nx;
+ load_image_size->height = img->ny;
+ clip_add_load_image_size(ctx_clip, load_image_size);
+ LOG_TEE("%s: load_image_size %d %d\n", __func__, load_image_size->width, load_image_size->height);
+ }
+ else if (strcmp(mm_patch_merge_type, "spatial_unpad") != 0) {
+ // flat / default llava-1.5 type embedding
+ *n_img_pos = clip_n_patches(ctx_clip);
+ bool encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[0], image_embd); // image_embd shape is 576 x 4096
+ delete[] img_res_v.data;
+ if (!encoded) {
+ LOG_TEE("Unable to encode image\n");
+
+ return false;
+ }
+ }
+ else {
+ // spatial_unpad llava-1.6 type embedding
+ // TODO: CLIP needs batching support - in HF the llm projection is separate after encoding, which might be a solution to quickly get batching working
+ std::vector<float *> image_embd_v;
+ image_embd_v.resize(img_res_v.size);
+ for (size_t i = 0; i < img_res_v.size; i++) {
+ image_embd_v[i] = (float *)malloc(clip_embd_nbytes(ctx_clip)); // 576 patches * 4096 embeddings * 4 bytes = 9437184
+ const bool encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[i], image_embd_v[i]); // image data is in 3x336x336 format and will be converted to 336x336x3 inside
+ if (!encoded) {
+ LOG_TEE("Unable to encode image - spatial_unpad - subimage %d of %d\n", (int) i+1, (int) img_res_v.size);
+ return false;
+ }
+ }
+ const int64_t t_img_enc_batch_us = ggml_time_us();
+ LOG_TEE("%s: %d segments encoded in %8.2f ms\n", __func__, (int)img_res_v.size, (t_img_enc_batch_us - t_img_enc_start_us) / 1000.0);
+
+ const int32_t * image_grid = clip_image_grid(ctx_clip);
+
+ std::vector<std::pair<int, int>> grid_pinpoints;
+ for (int i = 0; i < 32 && image_grid[i] != 0; i += 2) {
+ grid_pinpoints.push_back({image_grid[i], image_grid[i+1]});
+ }
+
+ // free all img_res_v - not needed anymore
+ delete[] img_res_v.data;
+ img_res_v.size = 0;
+ img_res_v.data = nullptr;
+
+ const int32_t image_size = clip_image_size(ctx_clip);
+
+ struct clip_image_grid_shape grid_shape = get_anyres_image_grid_shape({img->nx,img->ny}, grid_pinpoints, image_size);
+
+ int n_img_pos_out;
+ clip_llava_handle_patches(ctx_clip, image_embd_v, grid_shape, image_embd, &n_img_pos_out);
+ *n_img_pos = n_img_pos_out;
+
+ for (size_t i = 0; i < image_embd_v.size(); i++) {
+ free(image_embd_v[i]);
+ }
+ image_embd_v.clear();
+
+ // debug image/segment/normalization content:
+ // clip_image_u8 * tmp = clip_image_u8_init();
+ // clip_image_convert_f32_to_u8(*image_feature, *tmp);
+ // clip_image_save_to_bmp(*tmp, "image_feature.bmp");
+ }
+
+ LOG_TEE("%s: image embedding created: %d tokens\n", __func__, *n_img_pos);
+
+ const int64_t t_img_enc_end_us = ggml_time_us();
+ float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0;
+
+ LOG_TEE("\n%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);
+
+ return true;
+}
+
+bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip) {
+ // make sure that the correct mmproj was used, i.e., compare apples to apples
+ int n_llama_embd = llama_n_embd(llama_get_model(ctx_llama));
+ auto n_image_embd = clip_n_mmproj_embd(ctx_clip);
+ if (n_image_embd != n_llama_embd) {
+ LOG_TEE("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_image_embd, n_llama_embd);
+ return false;
+ }
+ return true;
+}
+
+bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out) {
+ int num_max_patches = 6;
+ if (clip_is_minicpmv(ctx_clip)) {
+ num_max_patches = 10;
+ }
+ float * image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)*num_max_patches); // TODO: base on gridsize/llava model
+ if (!image_embd) {
+ LOG_TEE("Unable to allocate memory for image embeddings\n");
+ return false;
+ }
+
+ int n_img_pos;
+ if (!encode_image_with_clip(ctx_clip, n_threads, img, image_embd, &n_img_pos)) {
+ LOG_TEE("%s: cannot encode image, aborting\n", __func__);
+ free(image_embd);
+ return false;
+ }
+ *image_embd_out = image_embd;
+ *n_img_pos_out = n_img_pos;
+
+ return true;
+}
+
+bool llava_eval_image_embed(llama_context * ctx_llama, const struct llava_image_embed * image_embed, int n_batch, int * n_past) {
+ int n_embd = llama_n_embd(llama_get_model(ctx_llama));
+
+ for (int i = 0; i < image_embed->n_image_pos; i += n_batch) {
+ int n_eval = image_embed->n_image_pos - i;
+ if (n_eval > n_batch) {
+ n_eval = n_batch;
+ }
+ llama_batch batch = {int32_t(n_eval), nullptr, (image_embed->embed+i*n_embd), nullptr, nullptr, nullptr, nullptr, *n_past, 1, 0, };
+ if (llama_decode(ctx_llama, batch)) {
+ LOG_TEE("%s : failed to eval\n", __func__);
+ return false;
+ }
+ *n_past += n_eval;
+ }
+ return true;
+}
+
+struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length) {
+ clip_image_u8 * img = clip_image_u8_init();
+ if (!clip_image_load_from_bytes(image_bytes, image_bytes_length, img)) {
+ clip_image_u8_free(img);
+ LOG_TEE("%s: can't load image from bytes, is it a valid image?", __func__);
+ return NULL;
+ }
+
+ float* image_embed = NULL;
+ int n_image_pos = 0;
+ bool image_embed_result = llava_image_embed_make_with_clip_img(ctx_clip, n_threads, img, &image_embed, &n_image_pos);
+ if (!image_embed_result) {
+ clip_image_u8_free(img);
+ LOG_TEE("%s: coulnd't embed the image\n", __func__);
+ return NULL;
+ }
+
+ clip_image_u8_free(img);
+ auto result = (llava_image_embed*)malloc(sizeof(llava_image_embed));
+ result->embed = image_embed;
+ result->n_image_pos = n_image_pos;
+ return result;
+}
+
+static bool load_file_to_bytes(const char* path, unsigned char** bytesOut, long *sizeOut) {
+ auto file = fopen(path, "rb");
+ if (file == NULL) {
+ LOG_TEE("%s: can't read file %s\n", __func__, path);
+ return false;
+ }
+
+ fseek(file, 0, SEEK_END);
+ auto fileSize = ftell(file);
+ fseek(file, 0, SEEK_SET);
+
+ auto buffer = (unsigned char *)malloc(fileSize); // Allocate memory to hold the file data
+ if (buffer == NULL) {
+ LOG_TEE("%s: failed to alloc %ld bytes for file %s\n", __func__, fileSize, path);
+ perror("Memory allocation error");
+ fclose(file);
+ return false;
+ }
+ errno = 0;
+ size_t ret = fread(buffer, 1, fileSize, file); // Read the file into the buffer
+ if (ferror(file)) {
+ die_fmt("read error: %s", strerror(errno));
+ }
+ if (ret != (size_t) fileSize) {
+ die("unexpectedly reached end of file");
+ }
+ fclose(file); // Close the file
+
+ *bytesOut = buffer;
+ *sizeOut = fileSize;
+ return true;
+}
+
+struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path) {
+ unsigned char* image_bytes;
+ long image_bytes_length;
+ auto loaded = load_file_to_bytes(image_path, &image_bytes, &image_bytes_length);
+ if (!loaded) {
+ LOG_TEE("%s: failed to load %s\n", __func__, image_path);
+ return NULL;
+ }
+
+ llava_image_embed *embed = llava_image_embed_make_with_bytes(ctx_clip, n_threads, image_bytes, image_bytes_length);
+ free(image_bytes);
+
+ return embed;
+}
+
+void llava_image_embed_free(struct llava_image_embed * embed) {
+ free(embed->embed);
+ free(embed);
+}
diff --git a/llama/llava.h b/llama/llava.h
new file mode 100644
index 00000000..859baf37
--- /dev/null
+++ b/llama/llava.h
@@ -0,0 +1,75 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef LLAVA_H
+#define LLAVA_H
+
+#include "ggml.h"
+
+#ifdef LLAMA_SHARED
+# if defined(_WIN32) && !defined(__MINGW32__)
+# ifdef LLAMA_BUILD
+# define LLAVA_API __declspec(dllexport)
+# else
+# define LLAVA_API __declspec(dllimport)
+# endif
+# else
+# define LLAVA_API __attribute__ ((visibility ("default")))
+# endif
+#else
+# define LLAVA_API
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct clip_ctx;
+struct llava_image_embed {
+ float * embed;
+ int n_image_pos;
+};
+
+/** sanity check for clip <-> llava embed size match */
+LLAVA_API bool llava_validate_embed_size(const struct llama_context * ctx_llama, const struct clip_ctx * ctx_clip);
+
+LLAVA_API bool llava_image_embed_make_with_clip_img(struct clip_ctx * ctx_clip, int n_threads, const struct clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out);
+
+/** build an image embed from image file bytes */
+LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length);
+/** build an image embed from a path to an image filename */
+LLAVA_API struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path);
+/** free an embedding made with llava_image_embed_make_* */
+LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed);
+
+/** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */
+LLAVA_API bool llava_eval_image_embed(struct llama_context * ctx_llama, const struct llava_image_embed * embed, int n_batch, int * n_past);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/llama/log.h b/llama/log.h
new file mode 100644
index 00000000..cf552678
--- /dev/null
+++ b/llama/log.h
@@ -0,0 +1,750 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include <chrono>
+#include <cstring>
+#include <sstream>
+#include <iostream>
+#include <thread>
+#include <vector>
+#include <algorithm>
+#include <cinttypes>
+
+// --------------------------------
+//
+// Basic usage:
+//
+// --------
+//
+// The LOG() and LOG_TEE() macros are ready to go by default
+// they do not require any initialization.
+//
+// LOGLN() and LOG_TEELN() are variants which automatically
+// include \n character at the end of the log string.
+//
+// LOG() behaves exactly like printf, by default writing to a logfile.
+// LOG_TEE() additionally, prints to the screen too ( mimics Unix tee command ).
+//
+// Default logfile is named
+// "llama.<threadID>.log"
+// Default LOG_TEE() secondary output target is
+// stderr
+//
+// Logs can be dynamically disabled or enabled using functions:
+// log_disable()
+// and
+// log_enable()
+//
+// A log target can be changed with:
+// log_set_target( string )
+// creating and opening, or re-opening a file by string filename
+// or
+// log_set_target( FILE* )
+// allowing to point at stderr, stdout, or any valid FILE* file handler.
+//
+// --------
+//
+// End of Basic usage.
+//
+// --------------------------------
+
+// Specifies a log target.
+// default uses log_handler() with "llama.log" log file
+// this can be changed, by defining LOG_TARGET
+// like so:
+//
+// #define LOG_TARGET (a valid FILE*)
+// #include "log.h"
+//
+// or it can be simply redirected to stdout or stderr
+// like so:
+//
+// #define LOG_TARGET stderr
+// #include "log.h"
+//
+// The log target can also be redirected to a different function
+// like so:
+//
+// #define LOG_TARGET log_handler_different()
+// #include "log.h"
+//
+// FILE* log_handler_different()
+// {
+// return stderr;
+// }
+//
+// or:
+//
+// #define LOG_TARGET log_handler_another_one("somelog.log")
+// #include "log.h"
+//
+// FILE* log_handler_another_one(char*filename)
+// {
+// static FILE* logfile = nullptr;
+// (...)
+// if( !logfile )
+// {
+// fopen(...)
+// }
+// (...)
+// return logfile
+// }
+//
+#ifndef LOG_TARGET
+ #define LOG_TARGET log_handler()
+#endif
+
+#ifndef LOG_TEE_TARGET
+ #define LOG_TEE_TARGET stderr
+#endif
+
+// Utility for synchronizing log configuration state
+// since std::optional was introduced only in c++17
+enum LogTriState
+{
+ LogTriStateSame,
+ LogTriStateFalse,
+ LogTriStateTrue
+};
+
+// Utility to obtain "pid" like unique process id and use it when creating log files.
+inline std::string log_get_pid()
+{
+ static std::string pid;
+ if (pid.empty())
+ {
+ // std::this_thread::get_id() is the most portable way of obtaining a "process id"
+ // it's not the same as "pid" but is unique enough to solve multiple instances
+ // trying to write to the same log.
+ std::stringstream ss;
+ ss << std::this_thread::get_id();
+ pid = ss.str();
+ }
+
+ return pid;
+}
+
+// Utility function for generating log file names with unique id based on thread id.
+// invocation with log_filename_generator( "llama", "log" ) creates a string "llama.<number>.log"
+// where the number is a runtime id of the current thread.
+
+#define log_filename_generator(log_file_basename, log_file_extension) log_filename_generator_impl(LogTriStateSame, log_file_basename, log_file_extension)
+
+// INTERNAL, DO NOT USE
+inline std::string log_filename_generator_impl(LogTriState multilog, const std::string & log_file_basename, const std::string & log_file_extension)
+{
+ static bool _multilog = false;
+
+ if (multilog != LogTriStateSame)
+ {
+ _multilog = multilog == LogTriStateTrue;
+ }
+
+ std::stringstream buf;
+
+ buf << log_file_basename;
+ if (_multilog)
+ {
+ buf << ".";
+ buf << log_get_pid();
+ }
+ buf << ".";
+ buf << log_file_extension;
+
+ return buf.str();
+}
+
+#ifndef LOG_DEFAULT_FILE_NAME
+ #define LOG_DEFAULT_FILE_NAME log_filename_generator("llama", "log")
+#endif
+
+// Utility for turning #define values into string literals
+// so we can have a define for stderr and
+// we can print "stderr" instead of literal stderr, etc.
+#define LOG_STRINGIZE1(s) #s
+#define LOG_STRINGIZE(s) LOG_STRINGIZE1(s)
+
+#define LOG_TEE_TARGET_STRING LOG_STRINGIZE(LOG_TEE_TARGET)
+
+// Allows disabling timestamps.
+// in order to disable, define LOG_NO_TIMESTAMPS
+// like so:
+//
+// #define LOG_NO_TIMESTAMPS
+// #include "log.h"
+//
+#ifndef LOG_NO_TIMESTAMPS
+ #ifndef _MSC_VER
+ #define LOG_TIMESTAMP_FMT "[%" PRIu64 "] "
+ #define LOG_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count()
+ #else
+ #define LOG_TIMESTAMP_FMT "[%" PRIu64 "] "
+ #define LOG_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count()
+ #endif
+#else
+ #define LOG_TIMESTAMP_FMT "%s"
+ #define LOG_TIMESTAMP_VAL ,""
+#endif
+
+#ifdef LOG_TEE_TIMESTAMPS
+ #ifndef _MSC_VER
+ #define LOG_TEE_TIMESTAMP_FMT "[%" PRIu64 "] "
+ #define LOG_TEE_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count()
+ #else
+ #define LOG_TEE_TIMESTAMP_FMT "[%" PRIu64 "] "
+ #define LOG_TEE_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count()
+ #endif
+#else
+ #define LOG_TEE_TIMESTAMP_FMT "%s"
+ #define LOG_TEE_TIMESTAMP_VAL ,""
+#endif
+
+// Allows disabling file/line/function prefix
+// in order to disable, define LOG_NO_FILE_LINE_FUNCTION
+// like so:
+//
+// #define LOG_NO_FILE_LINE_FUNCTION
+// #include "log.h"
+//
+#ifndef LOG_NO_FILE_LINE_FUNCTION
+ #ifndef _MSC_VER
+ #define LOG_FLF_FMT "[%24s:%5d][%24s] "
+ #define LOG_FLF_VAL , __FILE__, __LINE__, __FUNCTION__
+ #else
+ #define LOG_FLF_FMT "[%24s:%5ld][%24s] "
+ #define LOG_FLF_VAL , __FILE__, (long)__LINE__, __FUNCTION__
+ #endif
+#else
+ #define LOG_FLF_FMT "%s"
+ #define LOG_FLF_VAL ,""
+#endif
+
+#ifdef LOG_TEE_FILE_LINE_FUNCTION
+ #ifndef _MSC_VER
+ #define LOG_TEE_FLF_FMT "[%24s:%5d][%24s] "
+ #define LOG_TEE_FLF_VAL , __FILE__, __LINE__, __FUNCTION__
+ #else
+ #define LOG_TEE_FLF_FMT "[%24s:%5ld][%24s] "
+ #define LOG_TEE_FLF_VAL , __FILE__, (long)__LINE__, __FUNCTION__
+ #endif
+#else
+ #define LOG_TEE_FLF_FMT "%s"
+ #define LOG_TEE_FLF_VAL ,""
+#endif
+
+// INTERNAL, DO NOT USE
+// USE LOG() INSTEAD
+//
+#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER) || defined(__clang__)
+ #define LOG_IMPL(str, ...) \
+ do { \
+ if (LOG_TARGET != nullptr) \
+ { \
+ fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL, __VA_ARGS__); \
+ fflush(LOG_TARGET); \
+ } \
+ } while (0)
+#else
+ #define LOG_IMPL(str, ...) \
+ do { \
+ if (LOG_TARGET != nullptr) \
+ { \
+ fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL "", ##__VA_ARGS__); \
+ fflush(LOG_TARGET); \
+ } \
+ } while (0)
+#endif
+
+// INTERNAL, DO NOT USE
+// USE LOG_TEE() INSTEAD
+//
+#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER) || defined(__clang__)
+ #define LOG_TEE_IMPL(str, ...) \
+ do { \
+ if (LOG_TARGET != nullptr) \
+ { \
+ fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL, __VA_ARGS__); \
+ fflush(LOG_TARGET); \
+ } \
+ if (LOG_TARGET != nullptr && LOG_TARGET != stdout && LOG_TARGET != stderr && LOG_TEE_TARGET != nullptr) \
+ { \
+ fprintf(LOG_TEE_TARGET, LOG_TEE_TIMESTAMP_FMT LOG_TEE_FLF_FMT str "%s" LOG_TEE_TIMESTAMP_VAL LOG_TEE_FLF_VAL, __VA_ARGS__); \
+ fflush(LOG_TEE_TARGET); \
+ } \
+ } while (0)
+#else
+ #define LOG_TEE_IMPL(str, ...) \
+ do { \
+ if (LOG_TARGET != nullptr) \
+ { \
+ fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL "", ##__VA_ARGS__); \
+ fflush(LOG_TARGET); \
+ } \
+ if (LOG_TARGET != nullptr && LOG_TARGET != stdout && LOG_TARGET != stderr && LOG_TEE_TARGET != nullptr) \
+ { \
+ fprintf(LOG_TEE_TARGET, LOG_TEE_TIMESTAMP_FMT LOG_TEE_FLF_FMT str "%s" LOG_TEE_TIMESTAMP_VAL LOG_TEE_FLF_VAL "", ##__VA_ARGS__); \
+ fflush(LOG_TEE_TARGET); \
+ } \
+ } while (0)
+#endif
+
+// The '\0' as a last argument, is a trick to bypass the silly
+// "warning: ISO C++11 requires at least one argument for the "..." in a variadic macro"
+// so we can have a single macro which can be called just like printf.
+
+// Main LOG macro.
+// behaves like printf, and supports arguments the exact same way.
+//
+#if !defined(_MSC_VER) || defined(__clang__)
+ #define LOG(...) LOG_IMPL(__VA_ARGS__, "")
+#else
+ #define LOG(str, ...) LOG_IMPL("%s" str, "", ##__VA_ARGS__, "")
+#endif
+
+// Main TEE macro.
+// does the same as LOG
+// and
+// simultaneously writes stderr.
+//
+// Secondary target can be changed just like LOG_TARGET
+// by defining LOG_TEE_TARGET
+//
+#if !defined(_MSC_VER) || defined(__clang__)
+ #define LOG_TEE(...) LOG_TEE_IMPL(__VA_ARGS__, "")
+#else
+ #define LOG_TEE(str, ...) LOG_TEE_IMPL("%s" str, "", ##__VA_ARGS__, "")
+#endif
+
+// LOG macro variants with auto endline.
+#if !defined(_MSC_VER) || defined(__clang__)
+ #define LOGLN(...) LOG_IMPL(__VA_ARGS__, "\n")
+ #define LOG_TEELN(...) LOG_TEE_IMPL(__VA_ARGS__, "\n")
+#else
+ #define LOGLN(str, ...) LOG_IMPL("%s" str, "", ##__VA_ARGS__, "\n")
+ #define LOG_TEELN(str, ...) LOG_TEE_IMPL("%s" str, "", ##__VA_ARGS__, "\n")
+#endif
+
+// INTERNAL, DO NOT USE
+inline FILE *log_handler1_impl(bool change = false, LogTriState append = LogTriStateSame, LogTriState disable = LogTriStateSame, const std::string & filename = LOG_DEFAULT_FILE_NAME, FILE *target = nullptr)
+{
+ static bool _initialized = false;
+ static bool _append = false;
+ static bool _disabled = filename.empty() && target == nullptr;
+ static std::string log_current_filename{filename};
+ static FILE *log_current_target{target};
+ static FILE *logfile = nullptr;
+
+ if (change)
+ {
+ if (append != LogTriStateSame)
+ {
+ _append = append == LogTriStateTrue;
+ return logfile;
+ }
+
+ if (disable == LogTriStateTrue)
+ {
+ // Disable primary target
+ _disabled = true;
+ }
+ // If previously disabled, only enable, and keep previous target
+ else if (disable == LogTriStateFalse)
+ {
+ _disabled = false;
+ }
+ // Otherwise, process the arguments
+ else if (log_current_filename != filename || log_current_target != target)
+ {
+ _initialized = false;
+ }
+ }
+
+ if (_disabled)
+ {
+ // Log is disabled
+ return nullptr;
+ }
+
+ if (_initialized)
+ {
+ // with fallback in case something went wrong
+ return logfile ? logfile : stderr;
+ }
+
+ // do the (re)initialization
+ if (target != nullptr)
+ {
+ if (logfile != nullptr && logfile != stdout && logfile != stderr)
+ {
+ fclose(logfile);
+ }
+
+ log_current_filename = LOG_DEFAULT_FILE_NAME;
+ log_current_target = target;
+
+ logfile = target;
+ }
+ else
+ {
+ if (log_current_filename != filename)
+ {
+ if (logfile != nullptr && logfile != stdout && logfile != stderr)
+ {
+ fclose(logfile);
+ }
+ }
+
+ logfile = fopen(filename.c_str(), _append ? "a" : "w");
+ }
+
+ if (!logfile)
+ {
+ // Verify whether the file was opened, otherwise fallback to stderr
+ logfile = stderr;
+
+ fprintf(stderr, "Failed to open logfile '%s' with error '%s'\n", filename.c_str(), std::strerror(errno));
+ fflush(stderr);
+
+ // At this point we let the init flag be to true below, and let the target fallback to stderr
+ // otherwise we would repeatedly fopen() which was already unsuccessful
+ }
+
+ _initialized = true;
+
+ return logfile ? logfile : stderr;
+}
+
+// INTERNAL, DO NOT USE
+inline FILE *log_handler2_impl(bool change = false, LogTriState append = LogTriStateSame, LogTriState disable = LogTriStateSame, FILE *target = nullptr, const std::string & filename = LOG_DEFAULT_FILE_NAME)
+{
+ return log_handler1_impl(change, append, disable, filename, target);
+}
+
+// Disables logs entirely at runtime.
+// Makes LOG() and LOG_TEE() produce no output,
+// until enabled back.
+#define log_disable() log_disable_impl()
+
+// INTERNAL, DO NOT USE
+inline FILE *log_disable_impl()
+{
+ return log_handler1_impl(true, LogTriStateSame, LogTriStateTrue);
+}
+
+// Enables logs at runtime.
+#define log_enable() log_enable_impl()
+
+// INTERNAL, DO NOT USE
+inline FILE *log_enable_impl()
+{
+ return log_handler1_impl(true, LogTriStateSame, LogTriStateFalse);
+}
+
+// Sets target fir logs, either by a file name or FILE* pointer (stdout, stderr, or any valid FILE*)
+#define log_set_target(target) log_set_target_impl(target)
+
+// INTERNAL, DO NOT USE
+inline FILE *log_set_target_impl(const std::string & filename) { return log_handler1_impl(true, LogTriStateSame, LogTriStateSame, filename); }
+inline FILE *log_set_target_impl(FILE *target) { return log_handler2_impl(true, LogTriStateSame, LogTriStateSame, target); }
+
+// INTERNAL, DO NOT USE
+inline FILE *log_handler() { return log_handler1_impl(); }
+
+// Enable or disable creating separate log files for each run.
+// can ONLY be invoked BEFORE first log use.
+#define log_multilog(enable) log_filename_generator_impl((enable) ? LogTriStateTrue : LogTriStateFalse, "", "")
+// Enable or disable append mode for log file.
+// can ONLY be invoked BEFORE first log use.
+#define log_append(enable) log_append_impl(enable)
+// INTERNAL, DO NOT USE
+inline FILE *log_append_impl(bool enable)
+{
+ return log_handler1_impl(true, enable ? LogTriStateTrue : LogTriStateFalse, LogTriStateSame);
+}
+
+inline void log_test()
+{
+ log_disable();
+ LOG("01 Hello World to nobody, because logs are disabled!\n");
+ log_enable();
+ LOG("02 Hello World to default output, which is \"%s\" ( Yaaay, arguments! )!\n", LOG_STRINGIZE(LOG_TARGET));
+ LOG_TEE("03 Hello World to **both** default output and " LOG_TEE_TARGET_STRING "!\n");
+ log_set_target(stderr);
+ LOG("04 Hello World to stderr!\n");
+ LOG_TEE("05 Hello World TEE with double printing to stderr prevented!\n");
+ log_set_target(LOG_DEFAULT_FILE_NAME);
+ LOG("06 Hello World to default log file!\n");
+ log_set_target(stdout);
+ LOG("07 Hello World to stdout!\n");
+ log_set_target(LOG_DEFAULT_FILE_NAME);
+ LOG("08 Hello World to default log file again!\n");
+ log_disable();
+ LOG("09 Hello World _1_ into the void!\n");
+ log_enable();
+ LOG("10 Hello World back from the void ( you should not see _1_ in the log or the output )!\n");
+ log_disable();
+ log_set_target("llama.anotherlog.log");
+ LOG("11 Hello World _2_ to nobody, new target was selected but logs are still disabled!\n");
+ log_enable();
+ LOG("12 Hello World this time in a new file ( you should not see _2_ in the log or the output )?\n");
+ log_set_target("llama.yetanotherlog.log");
+ LOG("13 Hello World this time in yet new file?\n");
+ log_set_target(log_filename_generator("llama_autonamed", "log"));
+ LOG("14 Hello World in log with generated filename!\n");
+#ifdef _MSC_VER
+ LOG_TEE("15 Hello msvc TEE without arguments\n");
+ LOG_TEE("16 Hello msvc TEE with (%d)(%s) arguments\n", 1, "test");
+ LOG_TEELN("17 Hello msvc TEELN without arguments\n");
+ LOG_TEELN("18 Hello msvc TEELN with (%d)(%s) arguments\n", 1, "test");
+ LOG("19 Hello msvc LOG without arguments\n");
+ LOG("20 Hello msvc LOG with (%d)(%s) arguments\n", 1, "test");
+ LOGLN("21 Hello msvc LOGLN without arguments\n");
+ LOGLN("22 Hello msvc LOGLN with (%d)(%s) arguments\n", 1, "test");
+#endif
+}
+
+inline bool log_param_single_parse(const std::string & param)
+{
+ if ( param == "--log-test")
+ {
+ log_test();
+ return true;
+ }
+
+ if ( param == "--log-disable")
+ {
+ log_disable();
+ return true;
+ }
+
+ if ( param == "--log-enable")
+ {
+ log_enable();
+ return true;
+ }
+
+ if (param == "--log-new")
+ {
+ log_multilog(true);
+ return true;
+ }
+
+ if (param == "--log-append")
+ {
+ log_append(true);
+ return true;
+ }
+
+ return false;
+}
+
+inline bool log_param_pair_parse(bool check_but_dont_parse, const std::string & param, const std::string & next = std::string())
+{
+ if ( param == "--log-file")
+ {
+ if (!check_but_dont_parse)
+ {
+ log_set_target(log_filename_generator(next.empty() ? "unnamed" : next, "log"));
+ }
+
+ return true;
+ }
+
+ return false;
+}
+
+inline void log_print_usage()
+{
+ printf("log options:\n");
+ /* format
+ printf(" -h, --help show this help message and exit\n");*/
+ /* spacing
+ printf("__-param----------------Description\n");*/
+ printf(" --log-test Run simple logging test\n");
+ printf(" --log-disable Disable trace logs\n");
+ printf(" --log-enable Enable trace logs\n");
+ printf(" --log-file Specify a log filename (without extension)\n");
+ printf(" --log-new Create a separate new log file on start. "
+ "Each log file will have unique name: \"<name>.<ID>.log\"\n");
+ printf(" --log-append Don't truncate the old log file.\n");
+ printf("\n");
+}
+
+#define log_dump_cmdline(argc, argv) log_dump_cmdline_impl(argc, argv)
+
+// INTERNAL, DO NOT USE
+inline void log_dump_cmdline_impl(int argc, char **argv)
+{
+ std::stringstream buf;
+ for (int i = 0; i < argc; ++i)
+ {
+ if (std::string(argv[i]).find(' ') != std::string::npos)
+ {
+ buf << " \"" << argv[i] <<"\"";
+ }
+ else
+ {
+ buf << " " << argv[i];
+ }
+ }
+ LOGLN("Cmd:%s", buf.str().c_str());
+}
+
+#define log_tostr(var) log_var_to_string_impl(var).c_str()
+
+inline std::string log_var_to_string_impl(bool var)
+{
+ return var ? "true" : "false";
+}
+
+inline std::string log_var_to_string_impl(std::string var)
+{
+ return var;
+}
+
+inline std::string log_var_to_string_impl(const std::vector<int> & var)
+{
+ std::stringstream buf;
+ buf << "[ ";
+ bool first = true;
+ for (auto e : var)
+ {
+ if (first)
+ {
+ first = false;
+ }
+ else
+ {
+ buf << ", ";
+ }
+ buf << std::to_string(e);
+ }
+ buf << " ]";
+
+ return buf.str();
+}
+
+template <typename C, typename T>
+inline std::string LOG_TOKENS_TOSTR_PRETTY(const C & ctx, const T & tokens)
+{
+ std::stringstream buf;
+ buf << "[ ";
+
+ bool first = true;
+ for (const auto & token : tokens)
+ {
+ if (!first) {
+ buf << ", ";
+ } else {
+ first = false;
+ }
+
+ auto detokenized = llama_token_to_piece(ctx, token);
+
+ detokenized.erase(
+ std::remove_if(
+ detokenized.begin(),
+ detokenized.end(),
+ [](const unsigned char c) { return !std::isprint(c); }),
+ detokenized.end());
+
+ buf
+ << "'" << detokenized << "'"
+ << ":" << std::to_string(token);
+ }
+ buf << " ]";
+
+ return buf.str();
+}
+
+template <typename C, typename B>
+inline std::string LOG_BATCH_TOSTR_PRETTY(const C & ctx, const B & batch)
+{
+ std::stringstream buf;
+ buf << "[ ";
+
+ bool first = true;
+ for (int i = 0; i < batch.n_tokens; ++i)
+ {
+ if (!first) {
+ buf << ", ";
+ } else {
+ first = false;
+ }
+
+ auto detokenized = llama_token_to_piece(ctx, batch.token[i]);
+
+ detokenized.erase(
+ std::remove_if(
+ detokenized.begin(),
+ detokenized.end(),
+ [](const unsigned char c) { return !std::isprint(c); }),
+ detokenized.end());
+
+ buf
+ << "\n" << std::to_string(i)
+ << ":token '" << detokenized << "'"
+ << ":pos " << std::to_string(batch.pos[i])
+ << ":n_seq_id " << std::to_string(batch.n_seq_id[i])
+ << ":seq_id " << std::to_string(batch.seq_id[i][0])
+ << ":logits " << std::to_string(batch.logits[i]);
+ }
+ buf << " ]";
+
+ return buf.str();
+}
+
+#ifdef LOG_DISABLE_LOGS
+
+#undef LOG
+#define LOG(...) // dummy stub
+#undef LOGLN
+#define LOGLN(...) // dummy stub
+
+#undef LOG_TEE
+#define LOG_TEE(...) fprintf(stderr, __VA_ARGS__) // convert to normal fprintf
+
+#undef LOG_TEELN
+#define LOG_TEELN(...) fprintf(stderr, __VA_ARGS__) // convert to normal fprintf
+
+#undef LOG_DISABLE
+#define LOG_DISABLE() // dummy stub
+
+#undef LOG_ENABLE
+#define LOG_ENABLE() // dummy stub
+
+#undef LOG_ENABLE
+#define LOG_ENABLE() // dummy stub
+
+#undef LOG_SET_TARGET
+#define LOG_SET_TARGET(...) // dummy stub
+
+#undef LOG_DUMP_CMDLINE
+#define LOG_DUMP_CMDLINE(...) // dummy stub
+
+#endif // LOG_DISABLE_LOGS
diff --git a/llama/make/Makefile.cuda_v11 b/llama/make/Makefile.cuda_v11
new file mode 100644
index 00000000..528e0efe
--- /dev/null
+++ b/llama/make/Makefile.cuda_v11
@@ -0,0 +1,12 @@
+# Build rules for CUDA v11 runner
+
+include make/common-defs.make
+
+
+GPU_RUNNER_VARIANT := _v11
+GPU_PATH_ROOT_WIN=$(shell ls -d $(dir $(shell cygpath -m -s "$(CUDA_PATH)\.."))/v11.? 2>/dev/null)
+GPU_PATH_ROOT_LINUX=$(shell ls -d $(CUDA_PATH)-11 2>/dev/null)
+CUDA_ARCHITECTURES?=50;52;53;60;61;62;70;72;75;80;86
+
+include make/cuda.make
+include make/gpu.make
\ No newline at end of file
diff --git a/llama/make/Makefile.cuda_v12 b/llama/make/Makefile.cuda_v12
new file mode 100644
index 00000000..2418ef00
--- /dev/null
+++ b/llama/make/Makefile.cuda_v12
@@ -0,0 +1,12 @@
+# Build rules for CUDA v12 runner
+
+include make/common-defs.make
+
+
+GPU_RUNNER_VARIANT := _v12
+GPU_PATH_ROOT_WIN=$(shell ls -d $(dir $(shell cygpath -m -s "$(CUDA_PATH)\.."))/v12.? 2>/dev/null)
+GPU_PATH_ROOT_LINUX=$(shell ls -d $(CUDA_PATH)-12 2>/dev/null)
+CUDA_ARCHITECTURES?=60;61;62;70;72;75;80;86;87;89;90;90a
+
+include make/cuda.make
+include make/gpu.make
\ No newline at end of file
diff --git a/llama/make/Makefile.default b/llama/make/Makefile.default
new file mode 100644
index 00000000..d07c2468
--- /dev/null
+++ b/llama/make/Makefile.default
@@ -0,0 +1,49 @@
+# Build the default runner(s) for the platform which do not rely on 3rd party GPU libraries
+# On Mac arm64, this builds the metal runner
+# On other platforms this builds the CPU runner(s)
+
+include make/common-defs.make
+
+CPU_GOFLAGS="-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$(VERSION)\" \"-X=github.com/ollama/ollama/llama.CpuFeatures=$(subst $(space),$(comma),$(TARGET_CPU_FLAGS))\" $(TARGET_LDFLAGS)"
+DEFAULT_RUNNER := $(if $(and $(filter darwin,$(OS)),$(filter arm64,$(ARCH))),metal,cpu)
+RUNNERS := $(DEFAULT_RUNNER)
+ifeq ($(ARCH),amd64)
+ RUNNERS += cpu_avx cpu_avx2
+endif
+
+DIST_RUNNERS = $(addprefix $(RUNNERS_DIST_DIR)/,$(addsuffix /ollama_llama_server$(EXE_EXT),$(RUNNERS)))
+ifneq ($(OS),windows)
+PAYLOAD_RUNNERS = $(addprefix $(RUNNERS_PAYLOAD_DIR)/,$(addsuffix /ollama_llama_server$(EXE_EXT).gz,$(RUNNERS)))
+endif
+BUILD_RUNNERS = $(addprefix $(RUNNERS_BUILD_DIR)/,$(addsuffix /ollama_llama_server$(EXE_EXT),$(RUNNERS)))
+
+all: $(BUILD_RUNNERS) $(DIST_RUNNERS) $(PAYLOAD_RUNNERS)
+
+$(RUNNERS_BUILD_DIR)/$(DEFAULT_RUNNER)/ollama_llama_server$(EXE_EXT): TARGET_CPU_FLAGS=
+$(RUNNERS_BUILD_DIR)/$(DEFAULT_RUNNER)/ollama_llama_server$(EXE_EXT): *.go ./runner/*.go $(COMMON_SRCS) $(COMMON_HDRS)
+ @-mkdir -p $(dir $@)
+ GOARCH=$(ARCH) go build $(CPU_GOFLAGS) -o $@ ./runner
+
+$(RUNNERS_BUILD_DIR)/cpu_avx/ollama_llama_server$(EXE_EXT): TARGET_CPU_FLAGS="avx"
+$(RUNNERS_BUILD_DIR)/cpu_avx/ollama_llama_server$(EXE_EXT): *.go ./runner/*.go $(COMMON_SRCS) $(COMMON_HDRS)
+ @-mkdir -p $(dir $@)
+ GOARCH=$(ARCH) go build $(CPU_GOFLAGS) -tags $(subst $(space),$(comma),$(TARGET_CPU_FLAGS)) -o $@ ./runner
+
+$(RUNNERS_BUILD_DIR)/cpu_avx2/ollama_llama_server$(EXE_EXT): TARGET_CPU_FLAGS="avx avx2"
+$(RUNNERS_BUILD_DIR)/cpu_avx2/ollama_llama_server$(EXE_EXT): *.go ./runner/*.go $(COMMON_SRCS) $(COMMON_HDRS)
+ @-mkdir -p $(dir $@)
+ GOARCH=$(ARCH) go build $(CPU_GOFLAGS) -tags $(subst $(space),$(comma),$(TARGET_CPU_FLAGS)) -o $@ ./runner
+
+$(RUNNERS_DIST_DIR)/%: $(RUNNERS_BUILD_DIR)/%
+ @-mkdir -p $(dir $@)
+ cp $< $@
+
+$(RUNNERS_PAYLOAD_DIR)/%/ollama_llama_server$(EXE_EXT).gz: $(RUNNERS_BUILD_DIR)/%/ollama_llama_server$(EXE_EXT)
+ @-mkdir -p $(dir $@)
+ ${GZIP} --best -c $< > $@
+
+clean:
+ rm -f $(BUILD_RUNNERS) $(DIST_RUNNERS) $(PAYLOAD_RUNNERS)
+
+.PHONY: clean all
+
diff --git a/llama/make/Makefile.rocm b/llama/make/Makefile.rocm
new file mode 100644
index 00000000..865714b8
--- /dev/null
+++ b/llama/make/Makefile.rocm
@@ -0,0 +1,101 @@
+# Build rules for ROCm runner
+#
+# Note: at present we only support a single ROCm version (whichever is default on the build system)
+# unlike CUDA where we'll build both a v11 and v12 variant.
+
+include make/common-defs.make
+
+HIP_ARCHS_COMMON := gfx900 gfx940 gfx941 gfx942 gfx1010 gfx1012 gfx1030 gfx1100 gfx1101 gfx1102
+HIP_ARCHS_LINUX := gfx906:xnack- gfx908:xnack- gfx90a:xnack+ gfx90a:xnack-
+
+ifeq ($(OS),windows)
+ GPU_LIB_DIR_WIN := $(shell cygpath -m -s "$(HIP_PATH)\bin")
+ # If HIP_PATH has spaces, hipcc trips over them when subprocessing
+ HIP_PATH := $(shell cygpath -m -s "$(HIP_PATH)\")
+ CGO_EXTRA_LDFLAGS_WIN := -L$(shell cygpath -m -s "$(HIP_PATH)\lib")
+ export HIP_PATH
+ GPU_COMPILER_WIN := $(HIP_PATH)bin/hipcc.bin.exe
+ GPU_COMPILER:=$(GPU_COMPILER_WIN)
+else ifeq ($(OS),linux)
+ HIP_PATH?=/opt/rocm
+ GPU_LIB_DIR_LINUX := $(HIP_PATH)/lib
+ GPU_COMPILER_LINUX := $(shell X=$$(which hipcc 2>/dev/null) && echo $$X)
+ GPU_COMPILER:=$(GPU_COMPILER_LINUX)
+ ROCM_TRANSITIVE_LIBS = $(shell ldd $(ROCM_LIBS) | grep "=>" | cut -f2 -d= | cut -f2 -d' ' | grep -e rocm -e amdgpu -e libtinfo -e libnuma -e libelf | sort -u )
+endif
+
+# TODO future multi-variant support for ROCm
+# ROCM_VERSION = $(subst $(space),.,$(wordlist 1,2,$(subst .,$(space),$(word 3,$(subst -,$(space),$(filter HIP version: %,$(shell $(GPU_COMPILER) --version)))))))
+# ifneq (,$(ROCM_VERSION))
+# GPU_RUNNER_VARIANT = _v$(ROCM_VERSION)
+# endif
+
+GPU_RUNNER_GO_TAGS := rocm
+GPU_RUNNER_NAME := rocm$(GPU_RUNNER_VARIANT)
+GPU_RUNNER_DRIVER_LIB_LINK := -lamdhip64
+GPU_RUNNER_LIBS_SHORT := hipblas rocblas
+GPU_PATH_ROOT_WIN=$(dir $(GPU_LIB_DIR_WIN))
+GPU_PATH_ROOT_LINUX=$(dir $(GPU_LIB_DIR_LINUX))
+GPU_COMPILER_CFLAGS_WIN = $(CFLAGS)
+GPU_COMPILER_CFLAGS_LINUX = $(CFLAGS) -fPIC -D_GNU_SOURCE
+GPU_COMPILER_CXXFLAGS_WIN = $(CXXFLAGS)
+GPU_COMPILER_CXXFLAGS_LINUX = $(CXXFLAGS) -fPIC -D_GNU_SOURCE
+
+ROCM_LIBS = $(wildcard $(addsuffix .$(SHARED_EXT),$(addprefix $(GPU_LIB_DIR)/$(SHARED_PREFIX),$(GPU_RUNNER_LIBS_SHORT))))
+ROCM_DIST_DEPS_DIR = $(abspath $(SRC_DIR)/../dist/$(OS)-$(ARCH)-rocm)/lib/ollama
+ROCM_DIST_DEPS_LIBS = $(addprefix $(ROCM_DIST_DEPS_DIR)/,$(notdir $(ROCM_LIBS)) $(notdir $(ROCM_TRANSITIVE_LIBS)))
+ROCBLAS_DIST_DEP_MANIFEST = $(ROCM_DIST_DEPS_DIR)/rocblas/library/TensileManifest.txt
+
+ifeq ($(OS),linux)
+ GPU_COMPILER_FPIC := -fPIC -Wno-unused-function -std=gnu++11
+ GPU_RUNNER_ARCH_FLAGS := $(foreach arch, $(HIP_ARCHS_COMMON) $(HIP_ARCHS_LINUX), --offload-arch=$(arch))
+else ifeq ($(OS),windows)
+ GPU_COMPILER_FPIC := -Xclang --dependent-lib=msvcrt
+ GPU_RUNNER_ARCH_FLAGS := $(foreach arch, $(HIP_ARCHS_COMMON), --offload-arch=$(arch))
+endif
+
+GPU_COMPILER_CUFLAGS = \
+ $(GPU_COMPILER_FPIC) \
+ $(addprefix -m,$(GPU_RUNNER_CPU_FLAGS)) \
+ -parallel-jobs=2 \
+ -c \
+ -O3 \
+ -DGGML_USE_CUDA \
+ -DGGML_BUILD=1 \
+ -DGGML_SHARED=1 \
+ -DGGML_CUDA_DMMV_X=32 \
+ -DGGML_CUDA_MMV_Y=1 \
+ -DGGML_SCHED_MAX_COPIES=4 \
+ -DGGML_USE_HIPBLAS \
+ -DGGML_USE_LLAMAFILE \
+ -DHIP_FAST_MATH \
+ -D__HIP_PLATFORM_AMD__=1 \
+ -D__HIP_ROCclr__=1 \
+ -DNDEBUG \
+ -DK_QUANTS_PER_ITERATION=2 \
+ -D_CRT_SECURE_NO_WARNINGS \
+ -D_GNU_SOURCE \
+ -D_XOPEN_SOURCE=600 \
+ -mllvm=-amdgpu-early-inline-all=true \
+ -mllvm=-amdgpu-function-calls=false \
+ -Wno-expansion-to-defined \
+ -Wno-invalid-noreturn \
+ -Wno-ignored-attributes \
+ -Wno-pass-failed \
+ -Wno-deprecated-declarations \
+ -Wno-unused-result \
+ -I. \
+ $(foreach arch, $(HIP_ARCHS_COMMON), --offload-arch=$(arch))
+
+include make/gpu.make
+
+# Adjust the rules from gpu.make to handle the ROCm dependencies properly
+$(RUNNERS_DIST_DIR)/$(GPU_RUNNER_NAME)/ollama_llama_server$(EXE_EXT): $(ROCBLAS_DIST_DEP_MANIFEST) $(ROCM_DIST_DEPS_LIBS)
+$(ROCBLAS_DIST_DEP_MANIFEST):
+ @-mkdir -p $(dir $@)
+ @echo "Copying rocblas library..."
+ cd $(GPU_LIB_DIR)/rocblas/library/ && tar cf - . | (cd $(dir $@) && tar xf - )
+ @echo "rocblas library copy complete"
+$(ROCM_DIST_DEPS_LIBS):
+ @-mkdir -p $(dir $@)
+ $(CP) $(dir $(filter %$(notdir $@),$(ROCM_LIBS) $(ROCM_TRANSITIVE_LIBS)))/$(notdir $@)* $(dir $@)
diff --git a/llama/make/common-defs.make b/llama/make/common-defs.make
new file mode 100644
index 00000000..98d51090
--- /dev/null
+++ b/llama/make/common-defs.make
@@ -0,0 +1,74 @@
+# Common definitions for the various Makefiles
+# No rules are defined here so this is safe to include at the beginning of other makefiles
+
+OS := $(shell uname -s)
+ARCH ?= $(subst aarch64,arm64,$(subst x86_64,amd64,$(shell uname -m)))
+ifneq (,$(findstring MINGW,$(OS))$(findstring MSYS,$(OS)))
+ OS := windows
+ ARCH := $(shell systeminfo 2>/dev/null | grep "System Type" | grep ARM64 > /dev/null && echo "arm64" || echo "amd64" )
+else ifeq ($(OS),Linux)
+ OS := linux
+else ifeq ($(OS),Darwin)
+ OS := darwin
+endif
+comma:= ,
+empty:=
+space:= $(empty) $(empty)
+uc = $(subst a,A,$(subst b,B,$(subst c,C,$(subst d,D,$(subst e,E,$(subst f,F,$(subst g,G,$(subst h,H,$(subst i,I,$(subst j,J,$(subst k,K,$(subst l,L,$(subst m,M,$(subst n,N,$(subst o,O,$(subst p,P,$(subst q,Q,$(subst r,R,$(subst s,S,$(subst t,T,$(subst u,U,$(subst v,V,$(subst w,W,$(subst x,X,$(subst y,Y,$(subst z,Z,$1))))))))))))))))))))))))))
+
+export CGO_CFLAGS_ALLOW = -mfma|-mf16c
+export CGO_CXXFLAGS_ALLOW = -mfma|-mf16c
+export HIP_PLATFORM = amd
+export CGO_ENABLED=1
+
+SRC_DIR := $(dir $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST))))))
+BUILD_DIR = $(SRC_DIR)build/$(OS)-$(ARCH)
+DIST_BASE = $(abspath $(SRC_DIR)/../dist/$(OS)-$(ARCH))
+DIST_LIB_DIR = $(DIST_BASE)/lib/ollama
+RUNNERS_DIST_DIR = $(DIST_LIB_DIR)/runners
+RUNNERS_PAYLOAD_DIR = $(abspath $(SRC_DIR)/../build/$(OS)/$(ARCH))
+RUNNERS_BUILD_DIR = $(BUILD_DIR)/runners
+DEFAULT_RUNNER := $(if $(and $(filter darwin,$(OS)),$(filter arm64,$(ARCH))),metal,cpu)
+GZIP:=$(shell command -v pigz 2>/dev/null || echo "gzip")
+ifneq ($(OS),windows)
+ CCACHE:=$(shell command -v ccache 2>/dev/null || echo "")
+endif
+VERSION?=$(shell git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")
+
+# Conditionally enable ccache for cgo builds too
+ifneq ($(CCACHE),)
+ CC=$(CCACHE) gcc
+ CXX=$(CCACHE) g++
+ export CC
+ export CXX
+endif
+
+
+# Override in environment space separated to tune GPU runner CPU vector flags
+ifeq ($(ARCH),amd64)
+# TODO may need a bit more work - setting 'GPU_RUNNER_CPU_FLAGS="avx avx2 avx512f avx512bw"' doesn't yield
+# a system_info showing 'AVX512 = 1' so there may be additional macros that are needed in GGML
+ GPU_RUNNER_CPU_FLAGS ?= avx
+endif
+
+ifeq ($(OS),windows)
+ CP := cp
+ SRC_DIR := $(shell cygpath -m -s "$(SRC_DIR)")
+ OBJ_EXT := obj
+ SHARED_EXT := dll
+ EXE_EXT := .exe
+ SHARED_PREFIX :=
+ CPU_FLAG_PREFIX := /arch:
+else ifeq ($(OS),linux)
+ CP := cp -af
+ OBJ_EXT := o
+ SHARED_EXT := so
+ SHARED_PREFIX := lib
+ CPU_FLAG_PREFIX := -m
+else
+ OBJ_EXT := o
+ SHARED_EXT := so
+ CPU_FLAG_PREFIX := -m
+ CP := cp -af
+endif
+
diff --git a/llama/make/cuda.make b/llama/make/cuda.make
new file mode 100644
index 00000000..d50940bf
--- /dev/null
+++ b/llama/make/cuda.make
@@ -0,0 +1,47 @@
+# Common definitions for all cuda versions
+
+ifndef GPU_RUNNER_VARIANT
+dummy:
+ $(error This makefile is not meant to build directly, but instead included in other Makefiles that set required variables)
+endif
+
+
+GPU_RUNNER_NAME := cuda$(GPU_RUNNER_VARIANT)
+GPU_RUNNER_GO_TAGS := cuda cuda$(GPU_RUNNER_VARIANT)
+GPU_RUNNER_DRIVER_LIB_LINK := -lcuda
+GPU_RUNNER_LIBS_SHORT := cublas cudart cublasLt
+GPU_LIB_DIR_WIN = $(GPU_PATH_ROOT_WIN)/bin
+GPU_LIB_DIR_LINUX = $(GPU_PATH_ROOT_LINUX)/lib64
+CGO_EXTRA_LDFLAGS_WIN = -L"$(GPU_PATH_ROOT_WIN)/lib/x64"
+GPU_COMPILER_WIN = $(GPU_PATH_ROOT_WIN)/bin/nvcc
+GPU_COMPILER_LINUX = $(GPU_PATH_ROOT_LINUX)/bin/nvcc
+GPU_COMPILER_CFLAGS_WIN = $(CFLAGS) -D_WIN32_WINNT=0x602
+GPU_COMPILER_CFLAGS_LINUX = $(CFLAGS) -Xcompiler -fPIC -D_GNU_SOURCE
+GPU_COMPILER_CXXFLAGS_WIN = $(CXXFLAGS) -D_WIN32_WINNT=0x602
+GPU_COMPILER_CXXFLAGS_LINUX = $(CXXFLAGS) -Xcompiler -fPIC -D_GNU_SOURCE
+ifeq ($(OS),linux)
+ CUDA_PATH?=/usr/local/cuda
+ GPU_COMPILER_FPIC = -fPIC -Wno-unused-function -std=c++11
+endif
+GPU_RUNNER_ARCH_FLAGS := $(foreach arch,$(subst ;,$(space),$(CUDA_ARCHITECTURES)),--generate-code=arch=compute_$(arch)$(comma)code=[compute_$(arch)$(comma)sm_$(arch)]) \
+ -DGGML_CUDA_USE_GRAPHS=1
+GPU_COMPILER_CUFLAGS = \
+ $(GPU_COMPILER_FPIC) \
+ -Xcompiler "$(addprefix $(CPU_FLAG_PREFIX),$(_OS_GPU_RUNNER_CPU_FLAGS))" \
+ -t2 \
+ -DGGML_CUDA_DMMV_X=32 \
+ -DGGML_CUDA_MMV_Y=1 \
+ -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 \
+ -DGGML_USE_CUDA=1 \
+ -DGGML_SHARED=1 \
+ -DGGML_BUILD=1 \
+ -DGGML_USE_LLAMAFILE \
+ -DK_QUANTS_PER_ITERATION=2 \
+ -DNDEBUG \
+ -D_GNU_SOURCE \
+ -D_XOPEN_SOURCE=600 \
+ -Wno-deprecated-gpu-targets \
+ --forward-unknown-to-host-compiler \
+ -use_fast_math \
+ -I. \
+ -O3
diff --git a/llama/make/gpu.make b/llama/make/gpu.make
new file mode 100644
index 00000000..db941739
--- /dev/null
+++ b/llama/make/gpu.make
@@ -0,0 +1,126 @@
+# Generalized GPU runner build
+
+ifndef GPU_RUNNER_NAME
+dummy:
+ $(error This makefile is not meant to build directly, but instead included in other Makefiles that set required variables)
+endif
+
+ifeq ($(OS),windows)
+ GPU_COMPILER:=$(GPU_COMPILER_WIN)
+ GPU_LIB_DIR:=$(GPU_LIB_DIR_WIN)
+ CGO_EXTRA_LDFLAGS:=$(CGO_EXTRA_LDFLAGS_WIN)
+ GPU_COMPILER_CFLAGS = $(GPU_COMPILER_CFLAGS_WIN)
+ GPU_COMPILER_CXXFLAGS = $(GPU_COMPILER_CXXFLAGS_WIN)
+else ifeq ($(OS),linux)
+ GPU_COMPILER:=$(GPU_COMPILER_LINUX)
+ GPU_LIB_DIR:=$(GPU_LIB_DIR_LINUX)
+ CGO_EXTRA_LDFLAGS:=$(CGO_EXTRA_LDFLAGS_LINUX)
+ GPU_COMPILER_CFLAGS = $(GPU_COMPILER_CFLAGS_LINUX)
+ GPU_COMPILER_CXXFLAGS = $(GPU_COMPILER_CXXFLAGS_LINUX)
+endif
+
+GPU_GOFLAGS="-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$(VERSION)\" \"-X=github.com/ollama/ollama/llama.CpuFeatures=$(subst $(space),$(comma),$(GPU_RUNNER_CPU_FLAGS))\" $(TARGET_LDFLAGS)"
+
+# TODO Unify how we handle dependencies in the dist/packaging and install flow
+# today, cuda is bundled, but rocm is split out. Should split them each out by runner
+DIST_GPU_RUNNER_DEPS_DIR = $(DIST_LIB_DIR)
+
+ifeq ($(OS),windows)
+ _OS_GPU_RUNNER_CPU_FLAGS=$(call uc,$(GPU_RUNNER_CPU_FLAGS))
+else ifeq ($(OS),linux)
+ _OS_GPU_RUNNER_CPU_FLAGS=$(GPU_RUNNER_CPU_FLAGS)
+endif
+
+GPU_RUNNER_LIBS = $(wildcard $(addsuffix .$(SHARED_EXT).*,$(addprefix $(GPU_LIB_DIR)/$(SHARED_PREFIX),$(GPU_RUNNER_LIBS_SHORT))))
+DIST_GPU_RUNNER_LIB_DEPS = $(addprefix $(DIST_GPU_RUNNER_DEPS_DIR)/,$(notdir $(GPU_RUNNER_LIBS)))
+
+COMMON_SRCS := \
+ $(wildcard *.c) \
+ $(wildcard *.cpp)
+COMMON_HDRS := \
+ $(wildcard *.h) \
+ $(wildcard *.hpp)
+
+GPU_RUNNER_SRCS := \
+ ggml-cuda.cu \
+ $(filter-out $(wildcard ggml-cuda/fattn*.cu),$(wildcard ggml-cuda/*.cu)) \
+ $(wildcard ggml-cuda/template-instances/mmq*.cu) \
+ ggml.c ggml-backend.c ggml-alloc.c ggml-quants.c sgemm.cpp ggml-aarch64.c
+GPU_RUNNER_HDRS := \
+ $(wildcard ggml-cuda/*.cuh)
+
+
+# Conditional flags and components to speed up developer builds
+ifneq ($(OLLAMA_FAST_BUILD),)
+ GPU_COMPILER_CUFLAGS += \
+ -DGGML_DISABLE_FLASH_ATTN
+else
+ GPU_RUNNER_SRCS += \
+ $(wildcard ggml-cuda/fattn*.cu) \
+ $(wildcard ggml-cuda/template-instances/fattn-wmma*.cu) \
+ $(wildcard ggml-cuda/template-instances/fattn-vec*q4_0-q4_0.cu) \
+ $(wildcard ggml-cuda/template-instances/fattn-vec*q8_0-q8_0.cu) \
+ $(wildcard ggml-cuda/template-instances/fattn-vec*f16-f16.cu)
+endif
+
+GPU_RUNNER_OBJS := $(GPU_RUNNER_SRCS:.cu=.$(GPU_RUNNER_NAME).$(OBJ_EXT))
+GPU_RUNNER_OBJS := $(GPU_RUNNER_OBJS:.c=.$(GPU_RUNNER_NAME).$(OBJ_EXT))
+GPU_RUNNER_OBJS := $(addprefix $(BUILD_DIR)/,$(GPU_RUNNER_OBJS:.cpp=.$(GPU_RUNNER_NAME).$(OBJ_EXT)))
+
+DIST_RUNNERS = $(addprefix $(RUNNERS_DIST_DIR)/,$(addsuffix /ollama_llama_server$(EXE_EXT),$(GPU_RUNNER_NAME)))
+ifneq ($(OS),windows)
+PAYLOAD_RUNNERS = $(addprefix $(RUNNERS_PAYLOAD_DIR)/,$(addsuffix /ollama_llama_server$(EXE_EXT).gz,$(GPU_RUNNER_NAME)))
+endif
+BUILD_RUNNERS = $(addprefix $(RUNNERS_BUILD_DIR)/,$(addsuffix /ollama_llama_server$(EXE_EXT),$(GPU_RUNNER_NAME)))
+
+
+$(GPU_RUNNER_NAME): $(BUILD_RUNNERS) $(DIST_RUNNERS) $(PAYLOAD_RUNNERS)
+
+# Build targets
+$(BUILD_DIR)/%.$(GPU_RUNNER_NAME).$(OBJ_EXT): %.cu
+ @-mkdir -p $(dir $@)
+ $(CCACHE) $(GPU_COMPILER) -c $(GPU_COMPILER_CUFLAGS) $(GPU_RUNNER_ARCH_FLAGS) -o $@ $<
+$(BUILD_DIR)/%.$(GPU_RUNNER_NAME).$(OBJ_EXT): %.c
+ @-mkdir -p $(dir $@)
+ $(CCACHE) $(GPU_COMPILER) -c $(GPU_COMPILER_CFLAGS) -o $@ $<
+$(BUILD_DIR)/%.$(GPU_RUNNER_NAME).$(OBJ_EXT): %.cpp
+ @-mkdir -p $(dir $@)
+ $(CCACHE) $(GPU_COMPILER) -c $(GPU_COMPILER_CXXFLAGS) -o $@ $<
+$(RUNNERS_BUILD_DIR)/$(GPU_RUNNER_NAME)/ollama_llama_server$(EXE_EXT): TARGET_CGO_LDFLAGS = -L"$(RUNNERS_BUILD_DIR)/$(GPU_RUNNER_NAME)/" $(CGO_EXTRA_LDFLAGS)
+$(RUNNERS_BUILD_DIR)/$(GPU_RUNNER_NAME)/ollama_llama_server$(EXE_EXT): $(RUNNERS_BUILD_DIR)/$(GPU_RUNNER_NAME)/$(SHARED_PREFIX)ggml_$(GPU_RUNNER_NAME).$(SHARED_EXT) *.go ./runner/*.go $(COMMON_SRCS) $(COMMON_HDRS)
+ @-mkdir -p $(dir $@)
+ GOARCH=$(ARCH) CGO_LDFLAGS="$(TARGET_CGO_LDFLAGS)" go build $(GPU_GOFLAGS) -tags $(subst $(space),$(comma),$(GPU_RUNNER_CPU_FLAGS) $(GPU_RUNNER_GO_TAGS)) -o $@ ./runner
+$(RUNNERS_BUILD_DIR)/$(GPU_RUNNER_NAME)/$(SHARED_PREFIX)ggml_$(GPU_RUNNER_NAME).$(SHARED_EXT): $(GPU_RUNNER_OBJS) $(DIST_GPU_RUNNER_LIB_DEPS) $(COMMON_HDRS) $(GPU_RUNNER_HDRS)
+ @-mkdir -p $(dir $@)
+ $(CCACHE) $(GPU_COMPILER) --shared $(GPU_RUNNER_DRIVER_LIB_LINK) -L${DIST_GPU_RUNNER_DEPS_DIR} $(foreach lib, $(GPU_RUNNER_LIBS_SHORT), -l$(lib)) $(GPU_RUNNER_OBJS) -o $@
+
+# Distribution targets
+$(RUNNERS_DIST_DIR)/%: $(RUNNERS_BUILD_DIR)/%
+ @-mkdir -p $(dir $@)
+ cp $< $@
+$(RUNNERS_DIST_DIR)/$(GPU_RUNNER_NAME)/ollama_llama_server$(EXE_EXT): $(DIST_LIB_DIR)/$(SHARED_PREFIX)ggml_$(GPU_RUNNER_NAME).$(SHARED_EXT)
+$(DIST_LIB_DIR)/$(SHARED_PREFIX)ggml_$(GPU_RUNNER_NAME).$(SHARED_EXT): $(RUNNERS_BUILD_DIR)/$(GPU_RUNNER_NAME)/$(SHARED_PREFIX)ggml_$(GPU_RUNNER_NAME).$(SHARED_EXT)
+ @-mkdir -p $(dir $@)
+ cp $< $@
+$(DIST_GPU_RUNNER_LIB_DEPS):
+ @-mkdir -p $(dir $@)
+ $(CP) $(GPU_LIB_DIR)/$(notdir $@)* $(dir $@)
+
+# Payload targets
+$(RUNNERS_PAYLOAD_DIR)/%/ollama_llama_server.gz: $(RUNNERS_BUILD_DIR)/%/ollama_llama_server
+ @-mkdir -p $(dir $@)
+ ${GZIP} --best -c $< > $@
+$(RUNNERS_PAYLOAD_DIR)/$(GPU_RUNNER_NAME)/%.gz: $(RUNNERS_BUILD_DIR)/$(GPU_RUNNER_NAME)/%
+ @-mkdir -p $(dir $@)
+ ${GZIP} --best -c $< > $@
+
+clean:
+ rm -f $(GPU_RUNNER_OBJS) $(BUILD_RUNNERS) $(DIST_RUNNERS) $(PAYLOAD_RUNNERS)
+
+.PHONY: clean $(GPU_RUNNER_NAME)
+
+
+# Handy debugging for make variables
+print-%:
+ @echo '$*=$($*)'
+
diff --git a/llama/patches/01-cuda.diff b/llama/patches/01-cuda.diff
new file mode 100644
index 00000000..0ae13db1
--- /dev/null
+++ b/llama/patches/01-cuda.diff
@@ -0,0 +1,54 @@
+diff --git a/ggml/src/ggml-backend.c b/ggml/src/ggml-backend.c
+index 9e35ce98..179be840 100644
+--- a/ggml/src/ggml-backend.c
++++ b/ggml/src/ggml-backend.c
+@@ -87,7 +87,12 @@ void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {
+ if (buffer->iface.free_buffer != NULL) {
+ buffer->iface.free_buffer(buffer);
+ }
++
++// TODO: this needs to be freed in cuda and hipblas backends because
++// the cuda backend implementation compiled with msvc
++#if !defined(GGML_USE_CUDA) && !defined(GGML_USE_HIPBLAS)
+ free(buffer);
++#endif
+ }
+
+ size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {
+diff --git a/ggml/src/ggml-cuda.cu b/ggml/src/ggml-cuda.cu
+index 04b6e528..43b12bdf 100644
+--- a/ggml/src/ggml-cuda.cu
++++ b/ggml/src/ggml-cuda.cu
+@@ -392,6 +392,10 @@ GGML_CALL static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer)
+ GGML_CALL static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+ delete ctx;
++
++ // TODO: this needs to be freed in cuda and hipblas backends because
++ // the cuda backend implementation compiled with msvc
++ free(buffer);
+ }
+
+ GGML_CALL static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) {
+@@ -3028,8 +3032,6 @@ GGML_CALL static ggml_backend_t ggml_backend_reg_cuda_init(const char * params,
+ GGML_UNUSED(params);
+ }
+
+-extern "C" GGML_CALL int ggml_backend_cuda_reg_devices();
+-
+ GGML_CALL int ggml_backend_cuda_reg_devices() {
+ int device_count = ggml_backend_cuda_get_device_count();
+ //int device_count = 1; // DEBUG: some tools require delaying CUDA initialization
+diff --git a/ggml/include/ggml-cuda.h b/ggml/include/ggml-cuda.h
+index 5eb4af40..50b91009 100644
+--- a/ggml/include/ggml-cuda.h
++++ b/ggml/include/ggml-cuda.h
+@@ -31,6 +31,8 @@ GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_typ
+ // pinned host buffer for use with the CPU backend for faster copies between CPU and GPU
+ GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);
+
++GGML_API GGML_CALL int ggml_backend_cuda_reg_devices();
++
+ GGML_API GGML_CALL int ggml_backend_cuda_get_device_count(void);
+ GGML_API GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);
+ GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);
diff --git a/llama/patches/02-pretokenizer.diff b/llama/patches/02-pretokenizer.diff
new file mode 100644
index 00000000..351bcaef
--- /dev/null
+++ b/llama/patches/02-pretokenizer.diff
@@ -0,0 +1,32 @@
+diff --git a/src/llama.cpp b/src/llama.cpp
+index 88355971..dd7d41ed 100644
+--- a/src/llama.cpp
++++ b/src/llama.cpp
+@@ -6083,16 +6083,7 @@ static void llm_load_vocab(
+ if (vocab.type == LLAMA_VOCAB_TYPE_BPE) {
+ vocab.tokenizer_add_space_prefix = false;
+ vocab.tokenizer_clean_spaces = true;
+- if (tokenizer_pre.empty()) {
+- LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__);
+- LLAMA_LOG_WARN("%s: \n", __func__);
+- LLAMA_LOG_WARN("%s: ************************************ \n", __func__);
+- LLAMA_LOG_WARN("%s: GENERATION QUALITY WILL BE DEGRADED! \n", __func__);
+- LLAMA_LOG_WARN("%s: CONSIDER REGENERATING THE MODEL \n", __func__);
+- LLAMA_LOG_WARN("%s: ************************************ \n", __func__);
+- LLAMA_LOG_WARN("%s: \n", __func__);
+- vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+- } else if (tokenizer_pre == "default") {
++ if (tokenizer_pre == "default") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ } else if (
+ tokenizer_pre == "llama3" ||
+@@ -6188,7 +6179,8 @@ static void llm_load_vocab(
+ tokenizer_pre == "exaone") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_EXAONE;
+ } else {
+- throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
++ LLAMA_LOG_WARN("%s: missing or unrecognized pre-tokenizer type, using: 'default'\n", __func__);
++ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ }
+ } else if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
diff --git a/llama/patches/03-metal.diff b/llama/patches/03-metal.diff
new file mode 100644
index 00000000..e63732e7
--- /dev/null
+++ b/llama/patches/03-metal.diff
@@ -0,0 +1,45 @@
+diff --git a/ggml/src/ggml-metal.m b/ggml/src/ggml-metal.m
+index 0207b787..b5e9884b 100644
+--- a/ggml/src/ggml-metal.m
++++ b/ggml/src/ggml-metal.m
+@@ -1396,27 +1396,23 @@ static enum ggml_status ggml_metal_graph_compute(
+ // to the matrix-vector kernel
+ int ne11_mm_min = 1;
+
+-#if 0
+ // the numbers below are measured on M2 Ultra for 7B and 13B models
+ // these numbers do not translate to other devices or model sizes
+ // TODO: need to find a better approach
+- if ([ctx->device.name isEqualToString:@"Apple M2 Ultra"]) {
+- switch (src0t) {
+- case GGML_TYPE_F16: ne11_mm_min = 2; break;
+- case GGML_TYPE_Q8_0: ne11_mm_min = 7; break;
+- case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;
+- case GGML_TYPE_Q3_K: ne11_mm_min = 7; break;
+- case GGML_TYPE_Q4_0:
+- case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;
+- case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;
+- case GGML_TYPE_Q5_0: // not tested yet
+- case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet
+- case GGML_TYPE_Q5_K: ne11_mm_min = 7; break;
+- case GGML_TYPE_Q6_K: ne11_mm_min = 7; break;
+- default: ne11_mm_min = 1; break;
+- }
++ switch (src0t) {
++ case GGML_TYPE_F16: ne11_mm_min = 2; break;
++ case GGML_TYPE_Q8_0: ne11_mm_min = 7; break;
++ case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;
++ case GGML_TYPE_Q3_K: ne11_mm_min = 7; break;
++ case GGML_TYPE_Q4_0:
++ case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;
++ case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;
++ case GGML_TYPE_Q5_0: // not tested yet
++ case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet
++ case GGML_TYPE_Q5_K: ne11_mm_min = 7; break;
++ case GGML_TYPE_Q6_K: ne11_mm_min = 7; break;
++ default: ne11_mm_min = 1; break;
+ }
+-#endif
+
+ // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
+ // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
diff --git a/llama/patches/04-ggml-metal.diff b/llama/patches/04-ggml-metal.diff
new file mode 100644
index 00000000..b3b7f14c
--- /dev/null
+++ b/llama/patches/04-ggml-metal.diff
@@ -0,0 +1,15 @@
+diff --git a/ggml/src/ggml-metal.m b/ggml/src/ggml-metal.m
+index b56c3604..400d43f4 100644
+--- a/ggml/src/ggml-metal.m
++++ b/ggml/src/ggml-metal.m
+@@ -377,8 +377,8 @@ static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){
+ #if GGML_METAL_EMBED_LIBRARY
+ GGML_METAL_LOG_INFO("%s: using embedded metal library\n", __func__);
+
+- extern const char ggml_metallib_start[];
+- extern const char ggml_metallib_end[];
++ extern const char *ggml_metallib_start;
++ extern const char *ggml_metallib_end;
+
+ NSString * src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding];
+ #else
diff --git a/llama/patches/05-embeddings.diff b/llama/patches/05-embeddings.diff
new file mode 100644
index 00000000..f3c071cb
--- /dev/null
+++ b/llama/patches/05-embeddings.diff
@@ -0,0 +1,43 @@
+diff --git a/src/llama.cpp b/src/llama.cpp
+index 88355971..d7db689b 100644
+--- a/src/llama.cpp
++++ b/src/llama.cpp
+@@ -15906,7 +15906,7 @@ static size_t llama_output_reserve(llama_context & lctx, size_t n_outputs) {
+ const auto n_embd = hparams.n_embd;
+
+ // TODO: use a per-batch flag for logits presence instead
+- const bool has_logits = !cparams.embeddings;
++ const bool has_logits = cparams.causal_attn;
+ const bool has_embd = cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE);
+
+ const size_t logits_size = has_logits ? n_vocab*n_outputs_max : 0;
+@@ -16175,20 +16175,23 @@ static int llama_decode_internal(
+ // no output
+ res = nullptr;
+ embd = nullptr;
+- } else if (cparams.embeddings) {
+- res = nullptr; // do not extract logits for embedding case
+- embd = nullptr;
++ }
++
++ if (cparams.embeddings) {
+ for (int i = gf->n_nodes - 1; i >= 0; --i) {
+- if (strcmp(gf->nodes[i]->name, "result_embd_pooled") == 0) {
+- embd = gf->nodes[i];
++ embd = gf->nodes[i];
++ if (strcmp(embd->name, "result_embd_pooled") == 0) {
+ break;
+ }
+ }
+- GGML_ASSERT(embd != nullptr && "missing embeddings tensor");
+ } else {
+ embd = nullptr; // do not extract embeddings when not needed
+ GGML_ASSERT(strcmp(res->name, "result_output") == 0 && "missing result_output tensor");
+ }
++
++ if (!cparams.causal_attn) {
++ res = nullptr; // do not extract logits when not needed
++ }
+ // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);
+
+ ggml_backend_sched_alloc_graph(lctx.sched, gf);
diff --git a/llama/patches/07-clip-unicode.diff b/llama/patches/07-clip-unicode.diff
new file mode 100644
index 00000000..f5152ba3
--- /dev/null
+++ b/llama/patches/07-clip-unicode.diff
@@ -0,0 +1,66 @@
+diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp
+index dcc65f02..1a990306 100644
+--- a/examples/llava/clip.cpp
++++ b/examples/llava/clip.cpp
+@@ -66,6 +66,19 @@
+ #include <cinttypes>
+ #include <limits>
+
++#if defined(_WIN32)
++#define WIN32_LEAN_AND_MEAN
++#ifndef NOMINMAX
++ #define NOMINMAX
++#endif
++#include <windows.h>
++#if __GLIBCXX__
++#include <cstdio>
++#include <ext/stdio_filebuf.h>
++#include <fcntl.h>
++#endif
++#endif
++
+ //#define CLIP_DEBUG_FUNCTIONS
+
+ // RGB uint8 image
+@@ -1248,7 +1261,29 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
+ return nullptr;
+ }
+
++#ifdef _WIN32
++ int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
++ if (!wlen) {
++ return NULL;
++ }
++ wchar_t * wbuf = (wchar_t *) malloc(wlen * sizeof(wchar_t));
++ wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, wbuf, wlen);
++ if (!wlen) {
++ free(wbuf);
++ return NULL;
++ }
++#if __GLIBCXX__
++ int fd = _wopen(wbuf, _O_RDONLY | _O_BINARY);
++ __gnu_cxx::stdio_filebuf<char> buffer(fd, std::ios_base::in);
++ std::istream fin(&buffer);
++#else // MSVC
++ // unused in our current build
++ auto fin = std::ifstream(wbuf, std::ios::binary);
++#endif
++ free(wbuf);
++#else
+ auto fin = std::ifstream(fname, std::ios::binary);
++#endif
+ if (!fin) {
+ LOG_TEE("cannot open model file for loading tensors\n");
+ clip_free(new_clip);
+@@ -1288,7 +1323,11 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
+ ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes);
+ }
+ }
++#if defined(_WIN32) && defined(__GLIBCXX__)
++ close(fd);
++#else
+ fin.close();
++#endif
+ }
+
+ // vision model
diff --git a/llama/patches/08-solar-pro.diff b/llama/patches/08-solar-pro.diff
new file mode 100644
index 00000000..616c8791
--- /dev/null
+++ b/llama/patches/08-solar-pro.diff
@@ -0,0 +1,386 @@
+diff --git a/src/llama.cpp b/src/llama.cpp
+index f79bd782..b7771f53 100644
+--- a/src/llama.cpp
++++ b/src/llama.cpp
+@@ -213,6 +213,7 @@ enum llm_arch {
+ LLM_ARCH_NEMOTRON,
+ LLM_ARCH_EXAONE,
+ LLM_ARCH_RWKV6,
++ LLM_ARCH_SOLAR,
+ LLM_ARCH_UNKNOWN,
+ };
+
+@@ -261,6 +262,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
+ { LLM_ARCH_NEMOTRON, "nemotron" },
+ { LLM_ARCH_EXAONE, "exaone" },
+ { LLM_ARCH_RWKV6, "rwkv6" },
++ { LLM_ARCH_SOLAR, "solar" },
+ { LLM_ARCH_UNKNOWN, "(unknown)" },
+ };
+
+@@ -314,6 +316,7 @@ enum llm_kv {
+ LLM_KV_ATTENTION_KV_LORA_RANK,
+ LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT,
+ LLM_KV_ATTENTION_SLIDING_WINDOW,
++ LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION,
+
+ LLM_KV_ROPE_DIMENSION_COUNT,
+ LLM_KV_ROPE_FREQ_BASE,
+@@ -405,19 +408,20 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
+ { LLM_KV_TIME_MIX_EXTRA_DIM, "%s.time_mix_extra_dim" },
+ { LLM_KV_TIME_DECAY_EXTRA_DIM, "%s.time_decay_extra_dim" },
+
+- { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },
+- { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },
+- { LLM_KV_ATTENTION_MAX_ALIBI_BIAS, "%s.attention.max_alibi_bias" },
+- { LLM_KV_ATTENTION_CLAMP_KQV, "%s.attention.clamp_kqv" },
+- { LLM_KV_ATTENTION_KEY_LENGTH, "%s.attention.key_length" },
+- { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" },
+- { LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" },
+- { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" },
+- { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" },
+- { LLM_KV_ATTENTION_Q_LORA_RANK, "%s.attention.q_lora_rank" },
+- { LLM_KV_ATTENTION_KV_LORA_RANK, "%s.attention.kv_lora_rank" },
+- { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" },
+- { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" },
++ { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },
++ { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },
++ { LLM_KV_ATTENTION_MAX_ALIBI_BIAS, "%s.attention.max_alibi_bias" },
++ { LLM_KV_ATTENTION_CLAMP_KQV, "%s.attention.clamp_kqv" },
++ { LLM_KV_ATTENTION_KEY_LENGTH, "%s.attention.key_length" },
++ { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" },
++ { LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" },
++ { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" },
++ { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" },
++ { LLM_KV_ATTENTION_Q_LORA_RANK, "%s.attention.q_lora_rank" },
++ { LLM_KV_ATTENTION_KV_LORA_RANK, "%s.attention.kv_lora_rank" },
++ { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" },
++ { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" },
++ { LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION, "%s.attention.block_skip_connection.%d" },
+
+ { LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" },
+ { LLM_KV_ROPE_FREQ_BASE, "%s.rope.freq_base" },
+@@ -589,6 +593,7 @@ enum llm_tensor {
+ LLM_TENSOR_ENC_FFN_DOWN,
+ LLM_TENSOR_ENC_FFN_UP,
+ LLM_TENSOR_ENC_OUTPUT_NORM,
++ LLM_TENSOR_BSKCN_TV,
+ };
+
+ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES = {
+@@ -1408,6 +1413,24 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
+ { LLM_TENSOR_CHANNEL_MIX_RECEPTANCE, "blk.%d.channel_mix_receptance" },
+ },
+ },
++ {
++ LLM_ARCH_SOLAR,
++ {
++ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
++ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
++ { LLM_TENSOR_OUTPUT, "output" },
++ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
++ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
++ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
++ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
++ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
++ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
++ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
++ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
++ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
++ { LLM_TENSOR_BSKCN_TV, "bskcn_tv" },
++ },
++ },
+ {
+ LLM_ARCH_UNKNOWN,
+ {
+@@ -2237,6 +2260,7 @@ enum e_model {
+ MODEL_15B,
+ MODEL_16B,
+ MODEL_20B,
++ MODEL_22B,
+ MODEL_30B,
+ MODEL_34B,
+ MODEL_35B,
+@@ -2284,6 +2308,8 @@ struct llama_hparams {
+ std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_kv_arr;
+ std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_arr;
+
++ std::array<std::array<uint32_t, LLAMA_MAX_LAYERS>, 4> n_bskcn_arr;
++
+ uint32_t n_layer_dense_lead = 0;
+ uint32_t n_lora_q = 0;
+ uint32_t n_lora_kv = 0;
+@@ -2349,6 +2375,7 @@ struct llama_hparams {
+ if (this->n_head_arr != other.n_head_arr) return true;
+ if (this->n_head_kv_arr != other.n_head_kv_arr) return true;
+ if (this->n_ff_arr != other.n_ff_arr) return true;
++ if (this->n_bskcn_arr != other.n_bskcn_arr) return true;
+
+ if (this->n_rel_attn_bkts != other.n_rel_attn_bkts) return true;
+ if (this->n_layer_dense_lead != other.n_layer_dense_lead) return true;
+@@ -2455,6 +2482,14 @@ struct llama_hparams {
+ return ssm_d_state * ssm_d_inner;
+ }
+ }
++
++ bool n_bskcn(uint32_t n, uint32_t il = 0) const {
++ if (il < n_layer) {
++ return n_bskcn_arr[n][il] > 0;
++ }
++
++ GGML_ABORT("fatal error");
++ }
+ };
+
+ static_assert(std::is_trivially_copyable<llama_hparams>::value, "llama_hparams must be trivially copyable");
+@@ -2635,6 +2670,8 @@ struct llama_layer {
+ struct ggml_tensor * ffn_gate_scale;
+ struct ggml_tensor * ffn_up_scale;
+ struct ggml_tensor * ffn_down_scale;
++
++ struct ggml_tensor * bskcn_tv;
+ };
+
+ // very similar to llama_batch,
+@@ -5937,6 +5974,21 @@ static void llm_load_hparams(
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
++ case LLM_ARCH_SOLAR:
++ {
++ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
++
++ for (int i = 0; i < hparams.n_bskcn_arr.max_size(); ++i) {
++ auto & bskcn = hparams.n_bskcn_arr.at(i);
++ bskcn.fill(0);
++ ml.get_key_or_arr(::format(LLM_KV_NAMES.at(LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION), LLM_ARCH_NAMES.at(ml.llm_kv.arch), i), bskcn, hparams.n_layer, false);
++ }
++
++ switch (hparams.n_layer) {
++ case 64: model.type = e_model::MODEL_22B; break;
++ default: model.type = e_model::MODEL_UNKNOWN;
++ }
++ }
+ default: (void)0;
+ }
+
+@@ -8420,6 +8472,38 @@ static bool llm_load_tensors(
+ }
+
+ } break;
++ case LLM_ARCH_SOLAR:
++ {
++ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
++
++ // output
++ {
++ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
++ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
++ }
++
++ for (int i = 0; i < n_layer; ++i) {
++ ggml_context * ctx_layer = ctx_for_layer(i);
++ ggml_context * ctx_split = ctx_for_layer_split(i);
++
++ auto & layer = model.layers[i];
++
++ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
++
++ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head});
++ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
++ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
++ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});
++
++ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
++
++ layer.bskcn_tv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_BSKCN_TV, "weight"), {2}, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));
++
++ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
++ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
++ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
++ }
++ } break;
+ default:
+ throw std::runtime_error("unknown architecture");
+ }
+@@ -15173,6 +15257,158 @@ struct llm_build_context {
+
+ return gf;
+ }
++
++ ggml_cgraph * build_solar() {
++ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
++
++ // mutable variable, needed during the last layer of the computation to skip unused tokens
++ int32_t n_tokens = this->n_tokens;
++
++ const int64_t n_embd_head = hparams.n_embd_head_v;
++ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
++ GGML_ASSERT(n_embd_head == hparams.n_rot);
++
++ struct ggml_tensor * cur;
++ struct ggml_tensor * inpL;
++
++ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
++
++ // inp_pos - contains the positions
++ struct ggml_tensor * inp_pos = build_inp_pos();
++
++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
++ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
++
++ struct ggml_tensor * bskcn_1;
++ struct ggml_tensor * bskcn_2;
++
++ for (int il = 0; il < n_layer; ++il) {
++ struct ggml_tensor * inpSA = inpL;
++
++ if (hparams.n_bskcn(0, il)) {
++ bskcn_1 = inpSA;
++ }
++
++ if (hparams.n_bskcn(1, il)) {
++ bskcn_2 = inpSA;
++ }
++
++ if (hparams.n_bskcn(2, il)) {
++ inpSA = ggml_add(
++ ctx0,
++ ggml_mul(ctx0, bskcn_1, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, 0)),
++ ggml_mul(ctx0, inpSA, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, ggml_element_size(model.layers[il].bskcn_tv))));
++ }
++
++ if (hparams.n_bskcn(3, il)) {
++ inpSA = ggml_add(
++ ctx0,
++ ggml_mul(ctx0, bskcn_2, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, 0)),
++ ggml_mul(ctx0, inpSA, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, ggml_element_size(model.layers[il].bskcn_tv))));
++ }
++
++ // norm
++ cur = llm_build_norm(ctx0, inpL, hparams,
++ model.layers[il].attn_norm, NULL,
++ LLM_NORM_RMS, cb, il);
++ cb(cur, "attn_norm", il);
++
++ // self-attention
++ {
++ // rope freq factors for llama3; may return nullptr for llama2 and other models
++ struct ggml_tensor * rope_factors = build_rope_factors(il);
++
++ // compute Q and K and RoPE them
++ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
++ cb(Qcur, "Qcur", il);
++ if (model.layers[il].bq) {
++ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
++ cb(Qcur, "Qcur", il);
++ }
++
++ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
++ cb(Kcur, "Kcur", il);
++ if (model.layers[il].bk) {
++ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
++ cb(Kcur, "Kcur", il);
++ }
++
++ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
++ cb(Vcur, "Vcur", il);
++ if (model.layers[il].bv) {
++ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
++ cb(Vcur, "Vcur", il);
++ }
++
++ Qcur = ggml_rope_ext(
++ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors,
++ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
++ ext_factor, attn_factor, beta_fast, beta_slow
++ );
++ cb(Qcur, "Qcur", il);
++
++ Kcur = ggml_rope_ext(
++ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors,
++ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
++ ext_factor, attn_factor, beta_fast, beta_slow
++ );
++ cb(Kcur, "Kcur", il);
++
++ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
++ model.layers[il].wo, model.layers[il].bo,
++ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
++ }
++
++ if (il == n_layer - 1) {
++ // skip computing output for unused tokens
++ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
++ n_tokens = n_outputs;
++ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
++ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
++ }
++
++ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
++ cb(ffn_inp, "ffn_inp", il);
++
++ // feed-forward network
++ cur = llm_build_norm(ctx0, ffn_inp, hparams,
++ model.layers[il].ffn_norm, NULL,
++ LLM_NORM_RMS, cb, il);
++ cb(cur, "ffn_norm", il);
++
++ cur = llm_build_ffn(ctx0, lctx, cur,
++ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
++ model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,
++ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
++ NULL,
++ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
++ cb(cur, "ffn_out", il);
++
++ cur = ggml_add(ctx0, cur, ffn_inp);
++ cb(cur, "ffn_out", il);
++
++ cur = lctx.cvec.apply_to(ctx0, cur, il);
++ cb(cur, "l_out", il);
++
++ // input for next layer
++ inpL = cur;
++ }
++
++ cur = inpL;
++
++ cur = llm_build_norm(ctx0, cur, hparams,
++ model.output_norm, NULL,
++ LLM_NORM_RMS, cb, -1);
++ cb(cur, "result_norm", -1);
++
++ // lm_head
++ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
++ cb(cur, "result_output", -1);
++
++ ggml_build_forward_expand(gf, cur);
++
++ return gf;
++ }
+ };
+
+ static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector<uint32_t> & ids) {
+@@ -15423,6 +15659,10 @@ static struct ggml_cgraph * llama_build_graph(
+ {
+ result = llm.build_rwkv6();
+ } break;
++ case LLM_ARCH_SOLAR:
++ {
++ result = llm.build_solar();
++ } break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+@@ -18503,6 +18743,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {
+ case LLM_ARCH_ARCTIC:
+ case LLM_ARCH_DEEPSEEK2:
+ case LLM_ARCH_CHATGLM:
++ case LLM_ARCH_SOLAR:
+ return LLAMA_ROPE_TYPE_NORM;
+
+ // the pairs of head values are offset by n_rot/2
+--
+2.46.0
+
diff --git a/llama/patches/10-conditional-fattn.diff b/llama/patches/10-conditional-fattn.diff
new file mode 100644
index 00000000..0ec0236c
--- /dev/null
+++ b/llama/patches/10-conditional-fattn.diff
@@ -0,0 +1,16 @@
+diff --git a/ggml/src/ggml-cuda.cu b/ggml/src/ggml-cuda.cu
+index 8a844b02..61d61542 100644
+--- a/ggml/src/ggml-cuda.cu
++++ b/ggml/src/ggml-cuda.cu
+@@ -2310,9 +2310,11 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
+ case GGML_OP_ARGSORT:
+ ggml_cuda_op_argsort(ctx, dst);
+ break;
++#if !defined(GGML_DISABLE_FLASH_ATTN)
+ case GGML_OP_FLASH_ATTN_EXT:
+ ggml_cuda_flash_attn_ext(ctx, dst);
+ break;
++#endif
+ case GGML_OP_CROSS_ENTROPY_LOSS:
+ ggml_cuda_cross_entropy_loss(ctx, dst);
+ break;
diff --git a/llama/patches/11-blas.diff b/llama/patches/11-blas.diff
new file mode 100644
index 00000000..c354cbc8
--- /dev/null
+++ b/llama/patches/11-blas.diff
@@ -0,0 +1,16 @@
+diff --git a/ggml/src/ggml-blas.cpp b/ggml/src/ggml-blas.cpp
+index 71373173..1309c451 100644
+--- a/ggml/src/ggml-blas.cpp
++++ b/ggml/src/ggml-blas.cpp
+@@ -1,3 +1,5 @@
++#ifdef GGML_USE_BLAS
++
+ #include "ggml-blas.h"
+ #include "ggml-backend-impl.h"
+
+@@ -365,3 +367,5 @@ void ggml_backend_blas_set_n_threads(ggml_backend_t backend_blas, int n_threads)
+ ggml_backend_blas_context * ctx = (ggml_backend_blas_context *)backend_blas->context;
+ ctx->n_threads = n_threads;
+ }
++
++#endif
diff --git a/llama/runner/README.md b/llama/runner/README.md
new file mode 100644
index 00000000..98f21e09
--- /dev/null
+++ b/llama/runner/README.md
@@ -0,0 +1,21 @@
+# `runner`
+
+> Note: this is a work in progress
+
+A minimial runner for loading a model and running inference via a http web server.
+
+```
+./runner -model <model binary>
+```
+
+### Completion
+
+```
+curl -X POST -H "Content-Type: application/json" -d '{"prompt": "hi"}' http://localhost:8080/completion
+```
+
+### Embeddings
+
+```
+curl -X POST -H "Content-Type: application/json" -d '{"prompt": "turn me into an embedding"}' http://localhost:8080/embeddings
+```
diff --git a/llama/runner/cache.go b/llama/runner/cache.go
new file mode 100644
index 00000000..ef8f6cfb
--- /dev/null
+++ b/llama/runner/cache.go
@@ -0,0 +1,265 @@
+package main
+
+import (
+ "errors"
+ "hash/maphash"
+ "log/slog"
+ "reflect"
+ "time"
+
+ "github.com/ollama/ollama/llama"
+)
+
+type InputCache struct {
+ // context window size (per slot)
+ numCtx int
+
+ // individual KV caches
+ slots []InputCacheSlot
+
+ // optimize cache eviction for multiple users
+ multiUserCache bool
+
+ // cache of images to embeddings
+ images []imageCache
+ imageHash maphash.Hash
+
+ lc *llama.Context
+}
+
+func NewInputCache(lc *llama.Context, kvSize int, numSlots int, multiUserCache bool) *InputCache {
+ slots := make([]InputCacheSlot, numSlots)
+
+ for i := range slots {
+ slots[i] = InputCacheSlot{
+ Id: i,
+ Inputs: make([]input, 0),
+ }
+ }
+
+ return &InputCache{
+ numCtx: kvSize / numSlots,
+ slots: slots,
+ multiUserCache: multiUserCache,
+ images: make([]imageCache, numSlots),
+ lc: lc,
+ }
+}
+
+// Locking: Operations on InputCacheSlot (including finding one
+// through LoadCacheSlot) require a lock to be be held that serializes
+// these operations with each other and llama.Decode
+
+type InputCacheSlot struct {
+ // Index in the KV cache
+ Id int
+
+ // Inputs that are stored in the KV cache
+ Inputs []input
+
+ // is this cache actively being processed as part of a sequence?
+ InUse bool
+
+ // last time this cache was used (as of start of processing)
+ lastUsed time.Time
+}
+
+func (c *InputCache) LoadCacheSlot(prompt []input, cachePrompt bool) (*InputCacheSlot, []input, int, error) {
+ var slot *InputCacheSlot
+ var numPast int
+ var err error
+
+ // In single-user scenarios, the longest cache slot works fine for getting good input
+ // cache hit rates and it reuses the same VRAM over and over again, which is good for
+ // GPU performance in situations where we miss the input cache.
+ // For multiple users, the "best" cache slot produces better input cache hit rates
+ // at the cost of worse performance when we miss the input cache (because it causes
+ // GPU L2 cache misses due to spreading out accesses across VRAM).
+ if !c.multiUserCache {
+ slot, numPast, err = c.findLongestCacheSlot(prompt)
+ } else {
+ slot, numPast, err = c.findBestCacheSlot(prompt)
+ }
+ if err != nil {
+ return nil, nil, 0, err
+ }
+
+ if !cachePrompt {
+ numPast = 0
+ }
+
+ slot.InUse = true
+ slot.lastUsed = time.Now()
+
+ if numPast == len(prompt) {
+ // Leave one input to sample so we can get a response
+ numPast--
+ }
+
+ if !c.lc.KvCacheSeqRm(slot.Id, numPast, -1) {
+ // Some models don't support partial erasure
+ c.lc.KvCacheSeqRm(slot.Id, 0, -1)
+ numPast = 0
+ }
+
+ slog.Debug("loading cache slot", "id", slot.Id, "cache", len(slot.Inputs), "prompt", len(prompt),
+ "used", numPast, "remaining", len(prompt)-numPast)
+
+ prompt = prompt[numPast:]
+ slot.Inputs = slot.Inputs[:numPast]
+
+ return slot, prompt, numPast, nil
+}
+
+func (c *InputCache) findLongestCacheSlot(prompt []input) (*InputCacheSlot, int, error) {
+ longest := -1
+ var longestSlot *InputCacheSlot
+
+ for i, s := range c.slots {
+ if s.InUse {
+ continue
+ }
+
+ count := countCommonPrefix(s.Inputs, prompt)
+ if count > longest {
+ longest = count
+ longestSlot = &c.slots[i]
+ }
+ }
+
+ if longestSlot == nil {
+ return nil, 0, errors.New("no available cache slots")
+ }
+
+ return longestSlot, longest, nil
+}
+
+func (c *InputCache) findBestCacheSlot(prompt []input) (*InputCacheSlot, int, error) {
+ oldest := time.Now()
+ var oldestSlot *InputCacheSlot
+
+ longest := -1
+ var longestSlot *InputCacheSlot
+
+ for i, s := range c.slots {
+ count := countCommonPrefix(s.Inputs, prompt)
+ if count > longest {
+ longest = count
+ longestSlot = &c.slots[i]
+ }
+
+ if s.lastUsed.Compare(oldest) < 0 && !s.InUse {
+ oldest = s.lastUsed
+ oldestSlot = &c.slots[i]
+ }
+ }
+
+ if longest == len(longestSlot.Inputs) && !longestSlot.InUse {
+ return longestSlot, longest, nil
+ }
+
+ if oldestSlot.InUse {
+ return nil, 0, errors.New("no available cache slots")
+ }
+
+ if len(oldestSlot.Inputs) != 0 {
+ slog.Debug("evicting cache slot", "id", oldestSlot.Id, "inputs", len(oldestSlot.Inputs),
+ "used", oldestSlot.lastUsed)
+ }
+
+ if longest > 0 && longestSlot != oldestSlot {
+ slog.Debug("forking cache slot", "src", longestSlot.Id, "dst", oldestSlot.Id, "inputs", longest, "total",
+ len(longestSlot.Inputs))
+ oldestSlot.Inputs = make([]input, longest)
+ copy(oldestSlot.Inputs, longestSlot.Inputs[:longest])
+ // This is only nil for unit tests
+ if c.lc != nil {
+ c.lc.KvCacheSeqRm(oldestSlot.Id, 0, -1)
+ c.lc.KvCacheSeqCp(longestSlot.Id, oldestSlot.Id, 0, longest)
+ }
+ }
+
+ return oldestSlot, longest, nil
+}
+
+func countCommonPrefix(a []input, b []input) int {
+ var count int
+
+ for i := range a {
+ if i >= len(b) {
+ break
+ }
+
+ if !reflect.DeepEqual(a[i], b[i]) {
+ break
+ }
+
+ count++
+ }
+
+ return count
+}
+
+func (c *InputCache) ShiftCacheSlot(slot *InputCacheSlot, numKeep int, numDiscard int, numPast int) {
+ // TODO (jessegross): KV cache removal can fail for certain types of models
+ // server.cpp doesn't handle this, though we can be more graceful
+ c.lc.KvCacheSeqRm(slot.Id, numKeep, numKeep+numDiscard)
+ c.lc.KvCacheSeqAdd(slot.Id, numKeep+numDiscard, numPast, -numDiscard)
+
+ for i := numKeep + numDiscard; i < len(slot.Inputs); i++ {
+ slot.Inputs[i-numDiscard] = slot.Inputs[i]
+ }
+ slot.Inputs = slot.Inputs[:len(slot.Inputs)-numDiscard]
+}
+
+// Locking: Lookup and store operations on imageCache require a lock
+// to be held that serializes these with each other. Hash does not
+// require a lock nor they need to be serialized with InputCacheSlot.
+
+type imageCache struct {
+ key uint64
+ val [][]float32
+ lastUsed time.Time
+}
+
+func (c *InputCache) HashImage(image []byte) uint64 {
+ c.imageHash.Reset()
+ _, _ = c.imageHash.Write(image)
+ return c.imageHash.Sum64()
+}
+
+var ErrImageNotFound = errors.New("image not found in cache")
+
+func (c *InputCache) FindImage(hash uint64) ([][]float32, error) {
+ for i := range c.images {
+ if c.images[i].key == hash {
+ slog.Debug("loading image embeddings from cache", "entry", i)
+ c.images[i].lastUsed = time.Now()
+ return c.images[i].val, nil
+ }
+ }
+
+ return nil, ErrImageNotFound
+}
+
+func (c *InputCache) AddImage(hash uint64, embed [][]float32) {
+ best := time.Now()
+ var bestImage int
+
+ for i := range c.images {
+ if c.images[i].key == hash {
+ bestImage = i
+ break
+ }
+
+ if c.images[i].lastUsed.Compare(best) < 0 {
+ best = c.images[i].lastUsed
+ bestImage = i
+ }
+ }
+
+ slog.Debug("storing image embeddings in cache", "entry", bestImage, "used", c.images[bestImage].lastUsed)
+ c.images[bestImage].key = hash
+ c.images[bestImage].val = embed
+ c.images[bestImage].lastUsed = time.Now()
+}
diff --git a/llama/runner/cache_test.go b/llama/runner/cache_test.go
new file mode 100644
index 00000000..cc13b5f2
--- /dev/null
+++ b/llama/runner/cache_test.go
@@ -0,0 +1,304 @@
+package main
+
+import (
+ "reflect"
+ "testing"
+ "time"
+)
+
+func TestCountCommon(t *testing.T) {
+ tests := []struct {
+ name string
+ t1 []input
+ t2 []input
+ expected int
+ }{
+ {
+ name: "Equal",
+ t1: []input{{token: 1}, {token: 2}, {token: 3}},
+ t2: []input{{token: 1}, {token: 2}, {token: 3}},
+ expected: 3,
+ },
+ {
+ name: "Prefix",
+ t1: []input{{token: 1}},
+ t2: []input{{token: 1}, {token: 2}, {token: 3}},
+ expected: 1,
+ },
+ {
+ name: "Embeddings Prefix",
+ t1: []input{{embed: []float32{0.1, 0.2, 0.3}}},
+ t2: []input{{embed: []float32{0.1, 0.2, 0.3}}, {embed: []float32{0.4, 0.5, 0.6}}, {embed: []float32{0.7}}},
+ expected: 1,
+ },
+ {
+ name: "Embeddings Prefix Partial",
+ t1: []input{{embed: []float32{0.1, 0.2, 0.3}}},
+ t2: []input{{embed: []float32{0.1, 0.2}}, {embed: []float32{0.4, 0.5, 0.6}}, {embed: []float32{0.7}}},
+ expected: 0,
+ },
+ {
+ name: "Mixed",
+ t1: []input{{token: 1}, {embed: []float32{0.2, 0.3, 0.4}}},
+ t2: []input{{token: 1}, {embed: []float32{0.2, 0.3, 0.4}}, {token: 5}},
+ expected: 2,
+ },
+ {
+ name: "Empty",
+ t1: []input{},
+ t2: []input{{token: 1}, {token: 2}, {token: 3}},
+ expected: 0,
+ },
+ {
+ name: "Both Empty",
+ t1: []input{},
+ t2: []input{},
+ expected: 0,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := countCommonPrefix(tt.t1, tt.t2)
+ if result != tt.expected {
+ t.Errorf("countCommonPrefix(%v, %v): have %v; want %v", tt.t1, tt.t2, result, tt.expected)
+ }
+ })
+ }
+}
+
+func TestFindCacheSlot(t *testing.T) {
+ type expected struct {
+ result int
+ len int
+ }
+
+ tests := []struct {
+ name string
+ cache InputCache
+ prompt []input
+ longest expected
+ best expected
+ }{
+ {
+ name: "Empty",
+ cache: InputCache{slots: []InputCacheSlot{
+ {
+ Id: 0,
+ Inputs: []input{},
+ InUse: false,
+ lastUsed: time.Time{},
+ },
+ {
+ Id: 1,
+ Inputs: []input{},
+ InUse: false,
+ lastUsed: time.Time{},
+ },
+ }},
+ prompt: []input{{token: 1}},
+ longest: expected{result: 0, len: 0},
+ best: expected{result: 0, len: 0},
+ },
+ {
+ name: "Extend",
+ cache: InputCache{slots: []InputCacheSlot{
+ {
+ Id: 0,
+ Inputs: []input{{token: 1}},
+ InUse: false,
+ lastUsed: time.Now().Add(-time.Second),
+ },
+ {
+ Id: 1,
+ Inputs: []input{{token: 1}, {token: 2}},
+ InUse: false,
+ lastUsed: time.Now().Add(-2 * time.Second),
+ },
+ }},
+ prompt: []input{{token: 1}, {token: 2}},
+ longest: expected{result: 1, len: 2},
+ best: expected{result: 1, len: 2},
+ },
+ {
+ name: "New",
+ cache: InputCache{slots: []InputCacheSlot{
+ {
+ Id: 0,
+ Inputs: []input{{token: 1}, {token: 2}},
+ InUse: false,
+ lastUsed: time.Now().Add(-time.Second),
+ },
+ {
+ Id: 1,
+ Inputs: []input{},
+ InUse: false,
+ lastUsed: time.Time{},
+ },
+ }},
+ prompt: []input{{token: 2}},
+ longest: expected{result: 0, len: 0},
+ best: expected{result: 1, len: 0},
+ },
+ {
+ name: "Fork",
+ cache: InputCache{
+ slots: []InputCacheSlot{
+ {
+ Id: 0,
+ Inputs: []input{{token: 1}, {token: 2}},
+ InUse: false,
+ lastUsed: time.Now().Add(-time.Second),
+ },
+ {
+ Id: 1,
+ Inputs: []input{},
+ InUse: false,
+ lastUsed: time.Time{},
+ },
+ },
+ },
+ prompt: []input{{token: 1}},
+ longest: expected{result: 0, len: 1},
+ best: expected{result: 1, len: 1},
+ },
+ {
+ name: "Evict",
+ cache: InputCache{slots: []InputCacheSlot{
+ {
+ Id: 0,
+ Inputs: []input{{token: 1}},
+ InUse: false,
+ lastUsed: time.Now().Add(-time.Second),
+ },
+ {
+ Id: 1,
+ Inputs: []input{{token: 1}, {token: 2}},
+ InUse: false,
+ lastUsed: time.Now().Add(-2 * time.Second),
+ },
+ }},
+ prompt: []input{{token: 2}, {token: 3}},
+ longest: expected{result: 0, len: 0},
+ best: expected{result: 1, len: 0},
+ },
+ {
+ name: "In use",
+ cache: InputCache{slots: []InputCacheSlot{
+ {
+ Id: 0,
+ Inputs: []input{{token: 1}, {token: 2}},
+ InUse: true,
+ lastUsed: time.Now().Add(-time.Second),
+ },
+ {
+ Id: 1,
+ Inputs: []input{{token: 1}},
+ InUse: false,
+ lastUsed: time.Now().Add(-2 * time.Second),
+ },
+ }},
+ prompt: []input{{token: 1}, {token: 2}},
+ longest: expected{result: 1, len: 1},
+ best: expected{result: 1, len: 2},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run("Longest-"+tt.name, func(t *testing.T) {
+ result, resultLen, err := tt.cache.findLongestCacheSlot(tt.prompt)
+ if err != nil {
+ t.Errorf("findLongestCacheSlot: err %v", err)
+ } else if result.Id != tt.longest.result || resultLen != tt.longest.len {
+ t.Errorf("findLongestCacheSlot: slot have %v, want %v len have %v, want %v",
+ result.Id, tt.longest.result, resultLen, tt.longest.len)
+ }
+ })
+ }
+
+ for _, tt := range tests {
+ t.Run("Best-"+tt.name, func(t *testing.T) {
+ result, resultLen, err := tt.cache.findBestCacheSlot(tt.prompt)
+ if err != nil {
+ t.Errorf("findBestCacheSlot: err %v", err)
+ } else if result.Id != tt.best.result || resultLen != tt.best.len {
+ t.Errorf("findBestCacheSlot: slot have %v, want %v len have %v, want %v",
+ result.Id, tt.best.result, resultLen, tt.best.len)
+ }
+ })
+ }
+}
+
+func TestImageCache(t *testing.T) {
+ cache := NewInputCache(nil, 2048, 4, false)
+
+ valA := [][]float32{{0.1, 0.2}, {0.3}}
+ valB := [][]float32{{0.4}, {0.5}, {0.6}}
+ valC := [][]float32{{0.7}}
+ valD := [][]float32{{0.8}}
+ valE := [][]float32{{0.9}}
+
+ // Empty cache
+ result, err := cache.FindImage(0x5adb61d31933a946)
+ if err != ErrImageNotFound {
+ t.Errorf("found result in empty cache: result %v, err %v", result, err)
+ }
+
+ // Insert A
+ cache.AddImage(0x5adb61d31933a946, valA)
+
+ result, err = cache.FindImage(0x5adb61d31933a946)
+ if !reflect.DeepEqual(result, valA) {
+ t.Errorf("failed to find expected value: result %v, err %v", result, err)
+ }
+
+ // Insert B
+ cache.AddImage(0x011551369a34a901, valB)
+
+ result, err = cache.FindImage(0x5adb61d31933a946)
+ if !reflect.DeepEqual(result, valA) {
+ t.Errorf("failed to find expected value: result %v, err %v", result, err)
+ }
+ result, err = cache.FindImage(0x011551369a34a901)
+ if !reflect.DeepEqual(result, valB) {
+ t.Errorf("failed to find expected value: result %v, err %v", result, err)
+ }
+
+ // Replace B with C
+ cache.AddImage(0x011551369a34a901, valC)
+
+ result, err = cache.FindImage(0x5adb61d31933a946)
+ if !reflect.DeepEqual(result, valA) {
+ t.Errorf("failed to find expected value: result %v, err %v", result, err)
+ }
+ result, err = cache.FindImage(0x011551369a34a901)
+ if !reflect.DeepEqual(result, valC) {
+ t.Errorf("failed to find expected value: result %v, err %v", result, err)
+ }
+
+ // Evict A
+ cache.AddImage(0x756b218a517e7353, valB)
+ cache.AddImage(0x75e5e8d35d7e3967, valD)
+ cache.AddImage(0xd96f7f268ca0646e, valE)
+
+ result, err = cache.FindImage(0x5adb61d31933a946)
+ if reflect.DeepEqual(result, valA) {
+ t.Errorf("failed to find expected value: result %v, err %v", result, err)
+ }
+ result, err = cache.FindImage(0x756b218a517e7353)
+ if !reflect.DeepEqual(result, valB) {
+ t.Errorf("failed to find expected value: result %v, err %v", result, err)
+ }
+ result, err = cache.FindImage(0x011551369a34a901)
+ if !reflect.DeepEqual(result, valC) {
+ t.Errorf("failed to find expected value: result %v, err %v", result, err)
+ }
+ result, err = cache.FindImage(0x75e5e8d35d7e3967)
+ if !reflect.DeepEqual(result, valD) {
+ t.Errorf("failed to find expected value: result %v, err %v", result, err)
+ }
+ result, err = cache.FindImage(0xd96f7f268ca0646e)
+ if !reflect.DeepEqual(result, valE) {
+ t.Errorf("failed to find expected value: result %v, err %v", result, err)
+ }
+}
diff --git a/llama/runner/requirements.go b/llama/runner/requirements.go
new file mode 100644
index 00000000..71b3b9aa
--- /dev/null
+++ b/llama/runner/requirements.go
@@ -0,0 +1,19 @@
+package main
+
+import (
+ "encoding/json"
+ "os"
+
+ "github.com/ollama/ollama/llama"
+ "github.com/ollama/ollama/version"
+)
+
+func printRequirements(fp *os.File) {
+ attrs := map[string]string{
+ "system_info": llama.PrintSystemInfo(),
+ "version": version.Version,
+ "cpu_features": llama.CpuFeatures,
+ }
+ enc := json.NewEncoder(fp)
+ _ = enc.Encode(attrs)
+}
diff --git a/llama/runner/runner.go b/llama/runner/runner.go
new file mode 100644
index 00000000..bf799d37
--- /dev/null
+++ b/llama/runner/runner.go
@@ -0,0 +1,891 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "log"
+ "log/slog"
+ "net"
+ "net/http"
+ "os"
+ "path/filepath"
+ "regexp"
+ "runtime"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llama"
+)
+
+// input is an element of the prompt to process, either
+// a token or an image embedding (generated from a vision projector)
+type input struct {
+ token int
+
+ // embed is an image embedding
+ embed []float32
+}
+
+type Sequence struct {
+ // number of inputs evaluated
+ numPast int
+
+ // batch index
+ iBatch int
+
+ // number of tokens predicted so far
+ numPredicted int
+
+ // prompt inputs left to evaluate
+ inputs []input
+
+ // tokens that have been generated but not returned yet (e.g. for stop sequences)
+ pendingResponses []string
+
+ // input cache being used by this sequence
+ cache *InputCacheSlot
+
+ // channel to send responses over
+ responses chan string
+
+ // channel to stop decoding (such as if the remote connection is closed)
+ quit chan bool
+
+ // number of tokens to predict
+ numPredict int
+
+ samplingCtx *llama.SamplingContext
+
+ // channel to send back the embedding if embedding only
+ embedding chan []float32
+
+ // stop sequences
+ stop []string
+
+ // number of inputs to keep at the beginning when shifting context window
+ numKeep int
+
+ // true if an embedding are to be returned instead of text generation
+ embeddingOnly bool
+
+ doneReason string
+
+ // Metrics
+ startProcessingTime time.Time
+ startGenerationTime time.Time
+ numDecoded int
+ numPromptInputs int
+}
+
+type NewSequenceParams struct {
+ numPredict int
+ stop []string
+ numKeep int
+ samplingParams *llama.SamplingParams
+ embedding bool
+}
+
+func (s *Server) NewSequence(prompt string, images []ImageData, params NewSequenceParams) (*Sequence, error) {
+ s.ready.Wait()
+
+ startTime := time.Now()
+
+ inputs, err := s.inputs(prompt, images)
+ if err != nil {
+ return nil, fmt.Errorf("failed to process inputs: %w", err)
+ } else if len(inputs) == 0 {
+ return nil, errors.New("no input provided")
+ }
+
+ if params.numKeep < 0 {
+ params.numKeep = len(inputs)
+ }
+
+ if !params.embedding {
+ // Subtracting 4 ensures that at least 1 input can be discarded during shift
+ params.numKeep = min(params.numKeep, s.cache.numCtx-4)
+ params.numKeep += s.bosToken
+ } else {
+ // Embeddings are 1 shot - just truncate to the context window, without ever shifting
+ params.numKeep = min(params.numKeep, s.cache.numCtx)
+ }
+
+ // truncate to fit in context window
+ if len(inputs) > s.cache.numCtx {
+ slog.Warn("truncating input prompt", "limit", s.cache.numCtx, "prompt", len(inputs), "numKeep", params.numKeep)
+ newInputs := inputs[:params.numKeep]
+ newInputs = append(newInputs, inputs[len(inputs)-s.cache.numCtx+params.numKeep:]...)
+ inputs = newInputs
+ }
+
+ var sc *llama.SamplingContext
+ if params.samplingParams != nil {
+ sc = llama.NewSamplingContext(*params.samplingParams)
+ for _, input := range inputs {
+ if input.embed == nil {
+ sc.Accept(s.lc, input.token, false)
+ }
+ }
+ }
+
+ return &Sequence{
+ inputs: inputs,
+ numPromptInputs: len(inputs),
+ startProcessingTime: startTime,
+ numPredict: params.numPredict,
+ pendingResponses: make([]string, 0),
+ responses: make(chan string, 100),
+ quit: make(chan bool, 1),
+ embedding: make(chan []float32, 1),
+ samplingCtx: sc,
+ embeddingOnly: params.embedding,
+ stop: params.stop,
+ numKeep: params.numKeep,
+ }, nil
+}
+
+// inputs processes the prompt and images into a list of inputs
+// by splitting the prompt on [img-<n>] tags, tokenizing text and
+// generating image embeddings for each image
+func (s *Server) inputs(prompt string, images []ImageData) ([]input, error) {
+ var inputs []input
+
+ re := regexp.MustCompile(`\[img-(\d+)\]`)
+ parts := re.Split(prompt, -1)
+ matches := re.FindAllStringSubmatch(prompt, -1)
+
+ for i, part := range parts {
+ // text - tokenize
+ if strings.TrimSpace(part) != "" {
+ tokens, err := s.lc.Model().Tokenize(part, i == 0, true)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, t := range tokens {
+ inputs = append(inputs, input{token: t})
+ }
+ }
+
+ // image - generate image embedding
+ if i < len(matches) {
+ n, _ := strconv.Atoi(matches[i][1])
+
+ imageIndex := -1
+ for j := range images {
+ if images[j].ID == n {
+ imageIndex = j
+ break
+ }
+ }
+
+ if imageIndex < 0 {
+ return nil, fmt.Errorf("invalid image index: %d", n)
+ }
+
+ hash := s.cache.HashImage(images[imageIndex].Data)
+
+ // Vision models cannot be accessed concurrently
+ s.clip.mu.Lock()
+ embed, err := s.cache.FindImage(hash)
+ if err != nil {
+ embed = llama.NewLlavaImageEmbed(s.lc, s.clip.cc, images[imageIndex].Data)
+ s.cache.AddImage(hash, embed)
+ }
+ s.clip.mu.Unlock()
+
+ for _, e := range embed {
+ inputs = append(inputs, input{embed: e})
+ }
+ }
+ }
+
+ return inputs, nil
+}
+
+type clip struct {
+ cc *llama.ClipContext
+ mu sync.Mutex
+}
+
+type Server struct {
+ model *llama.Model
+ lc *llama.Context
+
+ // required for image embeddings
+ clip clip
+
+ batchSize int
+
+ // parallel is the number of parallel requests to handle
+ parallel int
+
+ // seqs is the list of parallel sequences being evaluated
+ // TODO (jmorganca): this can probably be moved into run()
+ seqs []*Sequence
+
+ // KV cache
+ cache *InputCache
+
+ // does this model require a beginning of sequence token?
+ bosToken int
+
+ // next sequence for prompt processing to avoid starvation
+ nextSeq int
+
+ // is the server ready to process requests?
+ ready sync.WaitGroup
+
+ mu sync.Mutex
+
+ cond *sync.Cond
+
+ progress float32
+
+ status ServerStatus
+}
+
+func (s *Server) allNil() bool {
+ for _, item := range s.seqs {
+ if item != nil {
+ return false
+ }
+ }
+ return true
+}
+
+func (s *Server) shiftContext(seq *Sequence) {
+ numLeft := seq.numPast - seq.numKeep
+ numDiscard := numLeft / 2
+
+ slog.Debug("context limit hit - shifting", "limit", s.cache.numCtx, "numPast", seq.numPast,
+ "numKeep", seq.numKeep, "numLeft", numLeft, "numDiscard", numDiscard)
+
+ s.cache.ShiftCacheSlot(seq.cache, seq.numKeep, numDiscard, seq.numPast)
+
+ seq.numPast -= numDiscard
+}
+
+func flushPending(seq *Sequence) bool {
+ for _, p := range seq.pendingResponses {
+ select {
+ case seq.responses <- p:
+ case <-seq.quit:
+ seq.pendingResponses = []string{}
+ return false
+ }
+ }
+
+ seq.pendingResponses = []string{}
+ return true
+}
+
+func (s *Server) removeSequence(seqIndex int, reason string) {
+ seq := s.seqs[seqIndex]
+
+ flushPending(seq)
+ seq.doneReason = reason
+ close(seq.responses)
+ close(seq.embedding)
+ seq.cache.InUse = false
+ s.seqs[seqIndex] = nil
+}
+
+func (s *Server) run(ctx context.Context) {
+ s.ready.Wait()
+
+ // logically these batches are used only within the context of processBatch
+ // but it is better for performance to allocate them once here
+ tokenBatch := llama.NewBatch(s.batchSize*len(s.seqs), 0, len(s.seqs))
+ defer tokenBatch.Free()
+
+ embedBatch := llama.NewBatch(s.batchSize*len(s.seqs), s.lc.Model().NEmbd(), len(s.seqs))
+ defer embedBatch.Free()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ s.processBatch(tokenBatch, embedBatch)
+ tokenBatch.Clear()
+ embedBatch.Clear()
+ }
+ }
+}
+
+// TODO (jmorganca): processBatch should be simplified, removing:
+// * sampling
+// * stop token checking
+// * metrics
+// these should instead be handled by the handlers
+// it should only be responsible for accepting tokens or embeddings and
+// processing batches as fast as possible
+func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch) {
+ s.mu.Lock()
+ for s.allNil() {
+ s.cond.Wait() // Wait until an item is added
+ }
+ defer s.mu.Unlock()
+
+ var batch *llama.Batch
+
+ seqIdx := s.nextSeq - 1
+ for range s.seqs {
+ seqIdx = (seqIdx + 1) % len(s.seqs)
+ seq := s.seqs[seqIdx]
+
+ if seq == nil {
+ continue
+ }
+
+ // if past the num predict limit
+ if seq.numPredict > 0 && seq.numPredicted > seq.numPredict {
+ s.removeSequence(seqIdx, "limit")
+ continue
+ }
+
+ if seq.numPast+len(seq.inputs) > s.cache.numCtx {
+ s.shiftContext(seq)
+ }
+
+ var numInputsProcessed int
+ for i, input := range seq.inputs {
+ embedding := input.embed != nil
+
+ // If we don't currently have a batch, use one of the correct type and
+ // fill it up as much as possible across all sequences. If we encounter an
+ // input of the opppsite type, stop for that sequence but then pick up from
+ // there for the next batch, ensuring that we alternate types
+ if batch == nil {
+ if !embedding {
+ batch = tokenBatch
+ } else {
+ batch = embedBatch
+ }
+ } else if embedding != batch.IsEmbedding() {
+ s.nextSeq = seqIdx
+ break
+ }
+
+ // todo: make this n_batch
+ if i >= s.batchSize {
+ break
+ }
+
+ batch.Add(input.token, input.embed, seq.numPast, []int{seq.cache.Id}, numInputsProcessed+1 == len(seq.inputs))
+ seq.numPast++
+ numInputsProcessed++
+ }
+
+ if numInputsProcessed > 0 {
+ seq.cache.Inputs = append(seq.cache.Inputs, seq.inputs[:numInputsProcessed]...)
+ seq.inputs = seq.inputs[numInputsProcessed:]
+ seq.iBatch = batch.NumTokens() - 1
+ }
+ }
+
+ if batch == nil || batch.NumTokens() == 0 {
+ return
+ }
+
+ err := s.lc.Decode(batch)
+ if err != nil {
+ slog.Error("failed to decode batch", "error", err)
+ return
+ }
+
+ for i, seq := range s.seqs {
+ if seq == nil {
+ continue
+ }
+
+ // don't sample prompt processing
+ if len(seq.inputs) != 0 {
+ continue
+ }
+
+ seq.numDecoded += 1
+ if seq.numDecoded == 1 {
+ seq.startGenerationTime = time.Now()
+ }
+
+ // if done processing the prompt, generate an embedding and return
+ if seq.embeddingOnly {
+ embed := s.lc.GetEmbeddingsSeq(i)
+ if embed == nil {
+ embed = s.lc.GetEmbeddingsIth(seq.iBatch)
+ }
+
+ seq.embedding <- embed
+ s.removeSequence(i, "")
+ continue
+ }
+
+ // sample a token
+ token := seq.samplingCtx.Sample(s.lc, nil, seq.iBatch)
+ seq.samplingCtx.Accept(s.lc, token, true)
+ piece := s.model.TokenToPiece(token)
+
+ seq.numPredicted++
+
+ // if it's an end of sequence token, break
+ if s.model.TokenIsEog(token) {
+ // TODO (jmorganca): we should send this back
+ // as it's important for the /api/generate context
+ // seq.responses <- piece
+
+ s.removeSequence(i, "stop")
+ continue
+ }
+
+ seq.inputs = []input{{token: token}}
+
+ seq.pendingResponses = append(seq.pendingResponses, piece)
+ sequence := strings.Join(seq.pendingResponses, "")
+
+ if ok, stop := findStop(sequence, seq.stop); ok {
+ slog.Debug("hit stop token", "pending", seq.pendingResponses, "stop", stop)
+
+ var tokenTruncated bool
+ origLen := len(seq.pendingResponses)
+ seq.pendingResponses, tokenTruncated = truncateStop(seq.pendingResponses, stop)
+ newLen := len(seq.pendingResponses)
+
+ // Update the cache based on the tokens that will be returned:
+ // - We have 1 token more than is currently in the cache because
+ // the last one generated wasn't submitted to Decode
+ // - Remove any stop sequences that we stripped out
+ // - If truncateStop removed a portion of a token, drop that
+ // - As defense-in-depth, if truncatedToken didn't find a stop token
+ // remove the extra one that we added to the cache len
+ tokenLen := len(seq.cache.Inputs) + 1
+ tokenLen -= origLen - newLen
+ if tokenTruncated || origLen == newLen {
+ tokenLen--
+ }
+ seq.cache.Inputs = seq.cache.Inputs[:tokenLen]
+
+ s.removeSequence(i, "stop")
+ continue
+ }
+
+ if containsStopSuffix(sequence, seq.stop) {
+ continue
+ }
+
+ if incompleteUnicode(sequence) {
+ continue
+ }
+
+ if !flushPending(seq) {
+ s.removeSequence(i, "connection")
+ }
+ }
+}
+
+// TODO (jmorganca): use structs from the api package to avoid duplication
+// this way the api acts as a proxy instead of using a different api for the
+// runner
+type Options struct {
+ api.Runner
+
+ NumKeep int `json:"n_keep"`
+ Seed int `json:"seed"`
+ NumPredict int `json:"n_predict"`
+ TopK int `json:"top_k"`
+ TopP float32 `json:"top_p"`
+ MinP float32 `json:"min_p"`
+ TFSZ float32 `json:"tfs_z"`
+ TypicalP float32 `json:"typical_p"`
+ RepeatLastN int `json:"repeat_last_n"`
+ Temperature float32 `json:"temperature"`
+ RepeatPenalty float32 `json:"repeat_penalty"`
+ PresencePenalty float32 `json:"presence_penalty"`
+ FrequencyPenalty float32 `json:"frequency_penalty"`
+ Mirostat int `json:"mirostat"`
+ MirostatTau float32 `json:"mirostat_tau"`
+ MirostatEta float32 `json:"mirostat_eta"`
+ PenalizeNewline bool `json:"penalize_nl"`
+ Stop []string `json:"stop"`
+}
+
+type ImageData struct {
+ Data []byte `json:"data"`
+ ID int `json:"id"`
+}
+
+type CompletionRequest struct {
+ Prompt string `json:"prompt"`
+ Images []ImageData `json:"image_data"`
+ Grammar string `json:"grammar"`
+ CachePrompt bool `json:"cache_prompt"`
+
+ Options
+}
+
+type Timings struct {
+ PredictedN int `json:"predicted_n"`
+ PredictedMS float64 `json:"predicted_ms"`
+ PromptN int `json:"prompt_n"`
+ PromptMS float64 `json:"prompt_ms"`
+}
+
+type CompletionResponse struct {
+ Content string `json:"content"`
+ Stop bool `json:"stop"`
+
+ Model string `json:"model,omitempty"`
+ Prompt string `json:"prompt,omitempty"`
+ StoppedLimit bool `json:"stopped_limit,omitempty"`
+ PredictedN int `json:"predicted_n,omitempty"`
+ PredictedMS float64 `json:"predicted_ms,omitempty"`
+ PromptN int `json:"prompt_n,omitempty"`
+ PromptMS float64 `json:"prompt_ms,omitempty"`
+
+ Timings Timings `json:"timings"`
+}
+
+func (s *Server) completion(w http.ResponseWriter, r *http.Request) {
+ var req CompletionRequest
+ req.Options = Options(api.DefaultOptions())
+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+ http.Error(w, "Bad request", http.StatusBadRequest)
+ return
+ }
+
+ // Set the headers to indicate streaming
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("Transfer-Encoding", "chunked")
+
+ flusher, ok := w.(http.Flusher)
+ if !ok {
+ http.Error(w, "Streaming not supported", http.StatusInternalServerError)
+ return
+ }
+
+ var samplingParams llama.SamplingParams
+ samplingParams.TopK = req.TopK
+ samplingParams.TopP = req.TopP
+ samplingParams.MinP = req.MinP
+ samplingParams.TfsZ = req.TFSZ
+ samplingParams.TypicalP = req.TypicalP
+ samplingParams.Temp = req.Temperature
+ samplingParams.RepeatLastN = req.RepeatLastN
+ samplingParams.PenaltyRepeat = req.RepeatPenalty
+ samplingParams.PenaltyFreq = req.FrequencyPenalty
+ samplingParams.PenaltyPresent = req.PresencePenalty
+ samplingParams.Mirostat = req.Mirostat
+ samplingParams.MirostatTau = req.MirostatTau
+ samplingParams.MirostatEta = req.MirostatEta
+ samplingParams.PenalizeNl = req.PenalizeNewline
+ samplingParams.Seed = uint32(req.Seed)
+ samplingParams.Grammar = req.Grammar
+
+ seq, err := s.NewSequence(req.Prompt, req.Images, NewSequenceParams{
+ numPredict: req.NumPredict,
+ stop: req.Stop,
+ numKeep: req.NumKeep,
+ samplingParams: &samplingParams,
+ embedding: false,
+ })
+ if err != nil {
+ http.Error(w, fmt.Sprintf("Failed to create new sequence: %v", err), http.StatusInternalServerError)
+ return
+ }
+
+ // TODO (jmorganca): add to sequence queue instead of
+ // failing if a slot isn't available
+ s.mu.Lock()
+ for i, sq := range s.seqs {
+ if sq == nil {
+ seq.cache, seq.inputs, seq.numPast, err = s.cache.LoadCacheSlot(seq.inputs, req.CachePrompt)
+ if err != nil {
+ s.mu.Unlock()
+ http.Error(w, fmt.Sprintf("Failed to load cache: %v", err), http.StatusInternalServerError)
+ return
+ }
+ s.seqs[i] = seq
+ s.cond.Signal()
+ break
+ }
+ }
+ s.mu.Unlock()
+
+ for {
+ select {
+ case <-r.Context().Done():
+ close(seq.quit)
+ return
+ case content, ok := <-seq.responses:
+ if ok {
+ if err := json.NewEncoder(w).Encode(&CompletionResponse{
+ Content: content,
+ }); err != nil {
+ http.Error(w, fmt.Sprintf("failed to encode response: %v", err), http.StatusInternalServerError)
+ close(seq.quit)
+ return
+ }
+
+ flusher.Flush()
+ } else {
+ // Send the final response
+ if err := json.NewEncoder(w).Encode(&CompletionResponse{
+ Stop: true,
+ StoppedLimit: seq.doneReason == "limit",
+ Timings: Timings{
+ PromptN: seq.numPromptInputs,
+ PromptMS: float64(seq.startGenerationTime.Sub(seq.startProcessingTime).Milliseconds()),
+ PredictedN: seq.numDecoded,
+ PredictedMS: float64(time.Since(seq.startGenerationTime).Milliseconds()),
+ },
+ }); err != nil {
+ http.Error(w, fmt.Sprintf("failed to encode final response: %v", err), http.StatusInternalServerError)
+ }
+
+ return
+ }
+ }
+ }
+}
+
+type EmbeddingRequest struct {
+ Content string `json:"content"`
+ CachePrompt bool `json:"cache_prompt"`
+}
+
+type EmbeddingResponse struct {
+ Embedding []float32 `json:"embedding"`
+}
+
+func (s *Server) embeddings(w http.ResponseWriter, r *http.Request) {
+ var req EmbeddingRequest
+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+ http.Error(w, fmt.Sprintf("bad request: %s", err), http.StatusBadRequest)
+ return
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+
+ slog.Debug("embedding request", "content", req.Content)
+
+ seq, err := s.NewSequence(req.Content, nil, NewSequenceParams{embedding: true})
+ if err != nil {
+ http.Error(w, fmt.Sprintf("Failed to create new sequence: %v", err), http.StatusInternalServerError)
+ return
+ }
+
+ // TODO (jessegross): Wait for a free slot instead of failing and blocking forever
+ s.mu.Lock()
+ for i, sq := range s.seqs {
+ if sq == nil {
+ seq.cache, seq.inputs, seq.numPast, err = s.cache.LoadCacheSlot(seq.inputs, req.CachePrompt)
+ if err != nil {
+ s.mu.Unlock()
+ http.Error(w, fmt.Sprintf("Failed to load cache: %v", err), http.StatusInternalServerError)
+ return
+ }
+ s.seqs[i] = seq
+ s.cond.Signal()
+ break
+ }
+ }
+ s.mu.Unlock()
+
+ embedding := <-seq.embedding
+
+ if err := json.NewEncoder(w).Encode(&EmbeddingResponse{
+ Embedding: embedding,
+ }); err != nil {
+ http.Error(w, fmt.Sprintf("failed to encode response: %v", err), http.StatusInternalServerError)
+ }
+}
+
+type HealthResponse struct {
+ Status string `json:"status"`
+ Progress float32 `json:"progress"`
+}
+
+type ServerStatus int
+
+const (
+ ServerStatusReady ServerStatus = iota
+ ServerStatusLoadingModel
+ ServerStatusError
+)
+
+func (s ServerStatus) ToString() string {
+ switch s {
+ case ServerStatusReady:
+ return "ok"
+ case ServerStatusLoadingModel:
+ return "loading model"
+ default:
+ return "server error"
+ }
+}
+
+func (s *Server) health(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(&HealthResponse{
+ Status: s.status.ToString(),
+ Progress: s.progress,
+ }); err != nil {
+ http.Error(w, fmt.Sprintf("failed to encode response: %v", err), http.StatusInternalServerError)
+ }
+}
+
+func (s *Server) loadModel(
+ params llama.ModelParams,
+ mpath string,
+ lpath string,
+ ppath string,
+ kvSize int,
+ flashAttention bool,
+ threads int,
+ multiUserCache bool,
+) {
+ llama.BackendInit()
+
+ s.model = llama.LoadModelFromFile(mpath, params)
+
+ ctxParams := llama.NewContextParams(kvSize, s.batchSize*s.parallel, s.parallel, threads, flashAttention)
+ s.lc = llama.NewContextWithModel(s.model, ctxParams)
+
+ if lpath != "" {
+ err := s.model.ApplyLoraFromFile(s.lc, lpath, 1.0, threads)
+ if err != nil {
+ panic(err)
+ }
+ }
+
+ if s.model.AddBOSToken() {
+ s.bosToken = 1
+ }
+
+ if ppath != "" {
+ s.clip.cc = llama.NewClipContext(ppath)
+ }
+
+ s.cache = NewInputCache(s.lc, kvSize, s.parallel, multiUserCache)
+
+ s.status = ServerStatusReady
+ s.ready.Done()
+}
+
+func main() {
+ mpath := flag.String("model", "", "Path to model binary file")
+ ppath := flag.String("mmproj", "", "Path to projector binary file")
+ parallel := flag.Int("parallel", 1, "Number of sequences to handle simultaneously")
+ batchSize := flag.Int("batch-size", 512, "Batch size")
+ nGpuLayers := flag.Int("n-gpu-layers", 0, "Number of layers to offload to GPU")
+ mainGpu := flag.Int("main-gpu", 0, "Main GPU")
+ flashAttention := flag.Bool("flash-attn", false, "Enable flash attention")
+ kvSize := flag.Int("ctx-size", 2048, "Context (or KV cache) size")
+ lpath := flag.String("lora", "", "Path to lora layer file")
+ port := flag.Int("port", 8080, "Port to expose the server on")
+ threads := flag.Int("threads", runtime.NumCPU(), "Number of threads to use during generation")
+ verbose := flag.Bool("verbose", false, "verbose output (default: disabled)")
+ noMmap := flag.Bool("no-mmap", false, "do not memory-map model (slower load but may reduce pageouts if not using mlock)")
+ mlock := flag.Bool("mlock", false, "force system to keep model in RAM rather than swapping or compressing")
+ tensorSplit := flag.String("tensor-split", "", "fraction of the model to offload to each GPU, comma-separated list of proportions")
+ multiUserCache := flag.Bool("multiuser-cache", false, "optimize input cache algorithm for multiple users")
+ // Expose requirements as a JSON output to stdout
+ requirements := flag.Bool("requirements", false, "print json requirement information")
+
+ // These are either ignored by llama.cpp or have no significance to us
+ _ = flag.Bool("embedding", false, "enable embedding vector output (default: disabled)")
+ _ = flag.Bool("log-disable", false, "disables logging to a file")
+ _ = flag.Bool("memory-f32", false, "use f32 instead of f16 for memory key+value (default: disabled) not recommended: doubles context memory required and no measurable increase in quality")
+
+ flag.Parse()
+ if *requirements {
+ printRequirements(os.Stdout)
+ return
+ }
+ level := slog.LevelInfo
+ if *verbose {
+ level = slog.LevelDebug
+ }
+ handler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+ Level: level,
+ AddSource: true,
+ ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr {
+ if attr.Key == slog.SourceKey {
+ source := attr.Value.Any().(*slog.Source)
+ source.File = filepath.Base(source.File)
+ }
+ return attr
+ },
+ })
+ slog.SetDefault(slog.New(handler))
+ slog.Info("starting go runner")
+ slog.Debug("system info", "cpu", llama.PrintSystemInfo(), "threads", *threads)
+
+ server := &Server{
+ batchSize: *batchSize,
+ parallel: *parallel,
+ seqs: make([]*Sequence, *parallel),
+ status: ServerStatusLoadingModel,
+ }
+
+ var tensorSplitFloats []float32
+ if *tensorSplit != "" {
+ stringFloats := regexp.MustCompile(",").Split(*tensorSplit, -1)
+
+ tensorSplitFloats = make([]float32, 0, len(stringFloats))
+ for _, s := range stringFloats {
+ f, _ := strconv.ParseFloat(s, 32)
+ tensorSplitFloats = append(tensorSplitFloats, float32(f))
+ }
+ }
+
+ params := llama.ModelParams{
+ NumGpuLayers: *nGpuLayers,
+ MainGpu: *mainGpu,
+ UseMmap: !*noMmap && *lpath == "",
+ UseMlock: *mlock,
+ TensorSplit: tensorSplitFloats,
+ Progress: func(progress float32) {
+ server.progress = progress
+ },
+ }
+
+ server.ready.Add(1)
+ go server.loadModel(params, *mpath, *lpath, *ppath, *kvSize, *flashAttention, *threads, *multiUserCache)
+
+ server.cond = sync.NewCond(&server.mu)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ go server.run(ctx)
+
+ addr := "127.0.0.1:" + strconv.Itoa(*port)
+ listener, err := net.Listen("tcp", addr)
+ if err != nil {
+ fmt.Println("Listen error:", err)
+ return
+ }
+ defer listener.Close()
+
+ mux := http.NewServeMux()
+ mux.HandleFunc("/embedding", server.embeddings)
+ mux.HandleFunc("/completion", server.completion)
+ mux.HandleFunc("/health", server.health)
+
+ httpServer := http.Server{
+ Handler: mux,
+ }
+
+ log.Println("Server listening on", addr)
+ if err := httpServer.Serve(listener); err != nil {
+ log.Fatal("server error:", err)
+ }
+
+ cancel()
+}
diff --git a/llama/runner/stop.go b/llama/runner/stop.go
new file mode 100644
index 00000000..c05f5e3d
--- /dev/null
+++ b/llama/runner/stop.go
@@ -0,0 +1,96 @@
+package main
+
+import (
+ "strings"
+)
+
+func findStop(sequence string, stops []string) (bool, string) {
+ for _, stop := range stops {
+ if strings.Contains(sequence, stop) {
+ return true, stop
+ }
+ }
+
+ return false, ""
+}
+
+func containsStopSuffix(sequence string, stops []string) bool {
+ for _, stop := range stops {
+ for i := 1; i <= len(stop); i++ {
+ if strings.HasSuffix(sequence, stop[:i]) {
+ return true
+ }
+ }
+ }
+
+ return false
+}
+
+// truncateStop removes the provided stop string from pieces,
+// returning the partial pieces with stop removed, including truncating
+// the last piece if required (and signalling if this was the case)
+func truncateStop(pieces []string, stop string) ([]string, bool) {
+ joined := strings.Join(pieces, "")
+
+ index := strings.Index(joined, stop)
+ if index == -1 {
+ return pieces, false
+ }
+
+ joined = joined[:index]
+
+ // Split truncated string back into pieces of original lengths
+ lengths := make([]int, len(pieces))
+ for i, piece := range pieces {
+ lengths[i] = len(piece)
+ }
+
+ var result []string
+ tokenTruncated := false
+ start := 0
+ for _, length := range lengths {
+ if start >= len(joined) {
+ break
+ }
+
+ end := start + length
+ if end > len(joined) {
+ end = len(joined)
+ tokenTruncated = true
+ }
+ result = append(result, joined[start:end])
+ start = end
+ }
+
+ return result, tokenTruncated
+}
+
+func incompleteUnicode(token string) bool {
+ incomplete := false
+
+ // check if there is incomplete UTF-8 character at the end
+ for i := 1; i < 5 && i <= len(token); i++ {
+ c := token[len(token)-i]
+
+ if (c & 0xc0) == 0x80 {
+ // continuation byte: 10xxxxxx
+ continue
+ }
+
+ if (c & 0xe0) == 0xc0 {
+ // 2-byte character: 110xxxxx ...
+ incomplete = i < 2
+ } else if (c & 0xf0) == 0xe0 {
+ // 3-byte character: 1110xxxx ...
+ incomplete = i < 3
+ } else if (c & 0xf8) == 0xf0 {
+ // 4-byte character: 11110xxx ...
+ incomplete = i < 4
+ }
+
+ // else 1-byte character or invalid byte
+ break
+ }
+
+ return incomplete
+}
diff --git a/llama/runner/stop_test.go b/llama/runner/stop_test.go
new file mode 100644
index 00000000..51b35fde
--- /dev/null
+++ b/llama/runner/stop_test.go
@@ -0,0 +1,129 @@
+package main
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestTruncateStop(t *testing.T) {
+ tests := []struct {
+ name string
+ pieces []string
+ stop string
+ expected []string
+ expectedTrunc bool
+ }{
+ {
+ name: "Single word",
+ pieces: []string{"hello", "world"},
+ stop: "world",
+ expected: []string{"hello"},
+ expectedTrunc: false,
+ },
+ {
+ name: "Partial",
+ pieces: []string{"hello", "wor"},
+ stop: "or",
+ expected: []string{"hello", "w"},
+ expectedTrunc: true,
+ },
+ {
+ name: "Suffix",
+ pieces: []string{"Hello", " there", "!"},
+ stop: "!",
+ expected: []string{"Hello", " there"},
+ expectedTrunc: false,
+ },
+ {
+ name: "Suffix partial",
+ pieces: []string{"Hello", " the", "re!"},
+ stop: "there!",
+ expected: []string{"Hello", " "},
+ expectedTrunc: true,
+ },
+ {
+ name: "Middle",
+ pieces: []string{"hello", " wor"},
+ stop: "llo w",
+ expected: []string{"he"},
+ expectedTrunc: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result, resultTrunc := truncateStop(tt.pieces, tt.stop)
+ if !reflect.DeepEqual(result, tt.expected) || resultTrunc != tt.expectedTrunc {
+ t.Errorf("truncateStop(%v, %s): have %v (%v); want %v (%v)", tt.pieces, tt.stop, result, resultTrunc, tt.expected, tt.expectedTrunc)
+ }
+ })
+ }
+}
+
+func TestIncompleteUnicode(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ expected bool
+ }{
+ {
+ name: "Basic",
+ input: "hi",
+ expected: false,
+ },
+ {
+ name: "Two byte",
+ input: "hi" + string([]byte{0xc2, 0xa3}),
+ expected: false,
+ },
+ {
+ name: "Two byte - missing last",
+ input: "hi" + string([]byte{0xc2}),
+ expected: true,
+ },
+ {
+ name: "Three byte",
+ input: "hi" + string([]byte{0xe0, 0xA0, 0x80}),
+ expected: false,
+ },
+ {
+ name: "Three byte - missing last",
+ input: "hi" + string([]byte{0xe0, 0xA0}),
+ expected: true,
+ },
+ {
+ name: "Three byte - missing last 2",
+ input: "hi" + string([]byte{0xe0}),
+ expected: true,
+ },
+ {
+ name: "Four byte",
+ input: "hi" + string([]byte{0xf0, 0x92, 0x8a, 0xb7}),
+ expected: false,
+ },
+ {
+ name: "Four byte - missing last",
+ input: "hi" + string([]byte{0xf0, 0x92, 0x8a}),
+ expected: true,
+ },
+ {
+ name: "Four byte - missing last 2",
+ input: "hi" + string([]byte{0xf0, 0x92}),
+ expected: true,
+ },
+ {
+ name: "Four byte - missing last 3",
+ input: "hi" + string([]byte{0xf0}),
+ expected: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := incompleteUnicode(tt.input)
+ if result != tt.expected {
+ t.Errorf("incompleteUnicode(%s): have %v; want %v", tt.input, result, tt.expected)
+ }
+ })
+ }
+}
diff --git a/llama/sampling.cpp b/llama/sampling.cpp
new file mode 100644
index 00000000..604e99c9
--- /dev/null
+++ b/llama/sampling.cpp
@@ -0,0 +1,486 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define LLAMA_API_INTERNAL
+#include "sampling.h"
+#include <random>
+
+struct llama_sampling_context * llama_sampling_init(const struct llama_sampling_params & params) {
+ struct llama_sampling_context * result = new llama_sampling_context();
+
+ result->params = params;
+ result->grammar = nullptr;
+
+ // if there is a grammar, parse it
+ if (!params.grammar.empty()) {
+ result->parsed_grammar = grammar_parser::parse(params.grammar.c_str());
+
+ // will be empty (default) if there are parse errors
+ if (result->parsed_grammar.rules.empty()) {
+ fprintf(stderr, "%s: failed to parse grammar\n", __func__);
+ delete result;
+ return nullptr;
+ }
+
+ // Ensure that there is a "root" node.
+ if (result->parsed_grammar.symbol_ids.find("root") == result->parsed_grammar.symbol_ids.end()) {
+ fprintf(stderr, "%s: grammar does not contain a 'root' symbol\n", __func__);
+ delete result;
+ return nullptr;
+ }
+
+ std::vector<const llama_grammar_element *> grammar_rules(result->parsed_grammar.c_rules());
+
+ struct llama_grammar * grammar = llama_grammar_init(
+ grammar_rules.data(),
+ grammar_rules.size(), result->parsed_grammar.symbol_ids.at("root"));
+ if (grammar == nullptr) {
+ throw std::runtime_error("Failed to initialize llama_grammar");
+ }
+ result->grammar = grammar;
+ }
+
+ result->prev.resize(params.n_prev);
+
+ result->n_valid = 0;
+
+ llama_sampling_set_rng_seed(result, params.seed);
+
+ return result;
+}
+
+void llama_sampling_free(struct llama_sampling_context * ctx) {
+ if (ctx->grammar != NULL) {
+ llama_grammar_free(ctx->grammar);
+ }
+
+ delete ctx;
+}
+
+void llama_sampling_reset(llama_sampling_context * ctx) {
+ if (ctx->grammar != NULL) {
+ llama_grammar_free(ctx->grammar);
+ ctx->grammar = NULL;
+ }
+
+ if (!ctx->parsed_grammar.rules.empty()) {
+ std::vector<const llama_grammar_element *> grammar_rules(ctx->parsed_grammar.c_rules());
+
+ struct llama_grammar * grammar = llama_grammar_init(
+ grammar_rules.data(),
+ grammar_rules.size(), ctx->parsed_grammar.symbol_ids.at("root"));
+ if (grammar == nullptr) {
+ throw std::runtime_error("Failed to initialize llama_grammar");
+ }
+ ctx->grammar = grammar;
+ }
+
+ std::fill(ctx->prev.begin(), ctx->prev.end(), 0);
+ ctx->cur.clear();
+ ctx->n_valid = 0;
+}
+
+void llama_sampling_set_rng_seed(struct llama_sampling_context * ctx, uint32_t seed) {
+ if (seed == LLAMA_DEFAULT_SEED) {
+ seed = std::random_device{}();
+ }
+ ctx->rng.seed(seed);
+}
+
+void llama_sampling_cp(llama_sampling_context * src, llama_sampling_context * dst) {
+ if (dst->grammar) {
+ llama_grammar_free(dst->grammar);
+ dst->grammar = nullptr;
+ }
+
+ if (src->grammar) {
+ dst->grammar = llama_grammar_copy(src->grammar);
+ }
+
+ dst->prev = src->prev;
+}
+
+llama_token llama_sampling_last(llama_sampling_context * ctx) {
+ return ctx->prev.back();
+}
+
+std::string llama_sampling_prev_str(llama_sampling_context * ctx_sampling, llama_context * ctx_main, int n) {
+ const int size = ctx_sampling->prev.size();
+
+ n = std::min(n, size);
+
+ std::string result;
+
+ for (int i = size - n; i < size; i++) {
+ result += llama_token_to_piece(ctx_main, ctx_sampling->prev[i]);
+ }
+
+ return result;
+}
+
+std::string llama_sampling_print(const llama_sampling_params & params) {
+ char result[1024];
+
+ snprintf(result, sizeof(result),
+ "\trepeat_last_n = %d, repeat_penalty = %.3f, frequency_penalty = %.3f, presence_penalty = %.3f\n"
+ "\ttop_k = %d, tfs_z = %.3f, top_p = %.3f, min_p = %.3f, typical_p = %.3f, temp = %.3f\n"
+ "\tmirostat = %d, mirostat_lr = %.3f, mirostat_ent = %.3f",
+ params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present,
+ params.top_k, params.tfs_z, params.top_p, params.min_p, params.typical_p, params.temp,
+ params.mirostat, params.mirostat_eta, params.mirostat_tau);
+
+ return std::string(result);
+}
+
+std::string llama_sampling_order_print(const llama_sampling_params & params) {
+ std::string result = "CFG -> Penalties ";
+ if (params.mirostat == 0) {
+ for (auto sampler_type : params.samplers_sequence) {
+ const auto sampler_type_name = llama_sampling_type_to_str(sampler_type);
+ if (!sampler_type_name.empty()) {
+ result += "-> " + sampler_type_name + " ";
+ }
+ }
+ } else {
+ result += "-> mirostat ";
+ }
+
+ return result;
+}
+
+std::string llama_sampling_type_to_str(llama_sampler_type sampler_type) {
+ switch (sampler_type) {
+ case llama_sampler_type::TOP_K: return "top_k";
+ case llama_sampler_type::TFS_Z: return "tfs_z";
+ case llama_sampler_type::TYPICAL_P: return "typical_p";
+ case llama_sampler_type::TOP_P: return "top_p";
+ case llama_sampler_type::MIN_P: return "min_p";
+ case llama_sampler_type::TEMPERATURE: return "temperature";
+ default : return "";
+ }
+}
+
+std::vector<llama_sampler_type> llama_sampling_types_from_names(const std::vector<std::string> & names, bool allow_alt_names) {
+ std::unordered_map<std::string, llama_sampler_type> sampler_canonical_name_map {
+ {"top_k", llama_sampler_type::TOP_K},
+ {"top_p", llama_sampler_type::TOP_P},
+ {"typical_p", llama_sampler_type::TYPICAL_P},
+ {"min_p", llama_sampler_type::MIN_P},
+ {"tfs_z", llama_sampler_type::TFS_Z},
+ {"temperature", llama_sampler_type::TEMPERATURE}
+ };
+
+ // since samplers names are written multiple ways
+ // make it ready for both system names and input names
+ std::unordered_map<std::string, llama_sampler_type> sampler_alt_name_map {
+ {"top-k", llama_sampler_type::TOP_K},
+ {"top-p", llama_sampler_type::TOP_P},
+ {"nucleus", llama_sampler_type::TOP_P},
+ {"typical-p", llama_sampler_type::TYPICAL_P},
+ {"typical", llama_sampler_type::TYPICAL_P},
+ {"min-p", llama_sampler_type::MIN_P},
+ {"tfs-z", llama_sampler_type::TFS_Z},
+ {"tfs", llama_sampler_type::TFS_Z},
+ {"temp", llama_sampler_type::TEMPERATURE}
+ };
+
+ std::vector<llama_sampler_type> sampler_types;
+ sampler_types.reserve(names.size());
+ for (const auto & name : names)
+ {
+ auto sampler_item = sampler_canonical_name_map.find(name);
+ if (sampler_item != sampler_canonical_name_map.end())
+ {
+ sampler_types.push_back(sampler_item->second);
+ }
+ else
+ {
+ if (allow_alt_names)
+ {
+ sampler_item = sampler_alt_name_map.find(name);
+ if (sampler_item != sampler_alt_name_map.end())
+ {
+ sampler_types.push_back(sampler_item->second);
+ }
+ }
+ }
+ }
+ return sampler_types;
+}
+
+std::vector<llama_sampler_type> llama_sampling_types_from_chars(const std::string & names_string) {
+ std::unordered_map<char, llama_sampler_type> sampler_name_map {
+ {'k', llama_sampler_type::TOP_K},
+ {'p', llama_sampler_type::TOP_P},
+ {'y', llama_sampler_type::TYPICAL_P},
+ {'m', llama_sampler_type::MIN_P},
+ {'f', llama_sampler_type::TFS_Z},
+ {'t', llama_sampler_type::TEMPERATURE}
+ };
+
+ std::vector<llama_sampler_type> sampler_types;
+ sampler_types.reserve(names_string.size());
+ for (const auto & c : names_string) {
+ const auto sampler_item = sampler_name_map.find(c);
+ if (sampler_item != sampler_name_map.end()) {
+ sampler_types.push_back(sampler_item->second);
+ }
+ }
+ return sampler_types;
+}
+
+// no reasons to expose this function in header
+static void sampler_queue(
+ struct llama_context * ctx_main,
+ const llama_sampling_params & params,
+ llama_token_data_array & cur_p,
+ size_t min_keep) {
+ const float temp = params.temp;
+ const float dynatemp_range = params.dynatemp_range;
+ const float dynatemp_exponent = params.dynatemp_exponent;
+ const int32_t top_k = params.top_k;
+ const float top_p = params.top_p;
+ const float min_p = params.min_p;
+ const float tfs_z = params.tfs_z;
+ const float typical_p = params.typical_p;
+ const std::vector<llama_sampler_type> & samplers_sequence = params.samplers_sequence;
+
+ for (auto sampler_type : samplers_sequence) {
+ switch (sampler_type) {
+ case llama_sampler_type::TOP_K : llama_sample_top_k (ctx_main, &cur_p, top_k, min_keep); break;
+ case llama_sampler_type::TFS_Z : llama_sample_tail_free(ctx_main, &cur_p, tfs_z, min_keep); break;
+ case llama_sampler_type::TYPICAL_P: llama_sample_typical (ctx_main, &cur_p, typical_p, min_keep); break;
+ case llama_sampler_type::TOP_P : llama_sample_top_p (ctx_main, &cur_p, top_p, min_keep); break;
+ case llama_sampler_type::MIN_P : llama_sample_min_p (ctx_main, &cur_p, min_p, min_keep); break;
+ case llama_sampler_type::TEMPERATURE:
+ if (dynatemp_range > 0) {
+ float dynatemp_min = std::max(0.0f, temp - dynatemp_range);
+ float dynatemp_max = std::max(0.0f, temp + dynatemp_range);
+ llama_sample_entropy(ctx_main, &cur_p, dynatemp_min, dynatemp_max, dynatemp_exponent);
+ } else {
+ llama_sample_temp(ctx_main, &cur_p, temp);
+ }
+ break;
+ default : break;
+ }
+ }
+}
+
+static llama_token llama_sampling_sample_impl(
+ struct llama_sampling_context * ctx_sampling,
+ struct llama_context * ctx_main,
+ struct llama_context * ctx_cfg,
+ const int idx,
+ bool is_resampling) {
+ const llama_sampling_params & params = ctx_sampling->params;
+
+ const float temp = params.temp;
+ const int mirostat = params.mirostat;
+ const float mirostat_tau = params.mirostat_tau;
+ const float mirostat_eta = params.mirostat_eta;
+
+ std::vector<float> original_logits;
+ auto cur_p = llama_sampling_prepare(ctx_sampling, ctx_main, ctx_cfg, idx, /* apply_grammar= */ is_resampling, &original_logits);
+ if (ctx_sampling->grammar != NULL && !is_resampling) {
+ GGML_ASSERT(!original_logits.empty());
+ }
+ llama_token id = 0;
+
+ if (temp < 0.0) {
+ // greedy sampling, with probs
+ llama_sample_softmax(ctx_main, &cur_p);
+ id = cur_p.data[0].id;
+ } else if (temp == 0.0) {
+ // greedy sampling, no probs
+ id = llama_sample_token_greedy(ctx_main, &cur_p);
+ } else {
+ if (mirostat == 1) {
+ const int mirostat_m = 100;
+ llama_sample_temp(ctx_main, &cur_p, temp);
+ id = llama_sample_token_mirostat(ctx_main, &cur_p, mirostat_tau, mirostat_eta, mirostat_m, &ctx_sampling->mirostat_mu);
+ } else if (mirostat == 2) {
+ llama_sample_temp(ctx_main, &cur_p, temp);
+ id = llama_sample_token_mirostat_v2(ctx_main, &cur_p, mirostat_tau, mirostat_eta, &ctx_sampling->mirostat_mu);
+ } else {
+ // temperature sampling
+ size_t min_keep = std::max(1, params.min_keep);
+
+ sampler_queue(ctx_main, params, cur_p, min_keep);
+
+ id = llama_sample_token_with_rng(ctx_main, &cur_p, ctx_sampling->rng);
+
+ //{
+ // const int n_top = 10;
+ // LOG("top %d candidates:\n", n_top);
+
+ // for (int i = 0; i < n_top; i++) {
+ // const llama_token id = cur_p.data[i].id;
+ // (void)id; // To avoid a warning that id is unused when logging is disabled.
+ // LOG(" - %5d: '%12s' (%.3f)\n", id, llama_token_to_piece(ctx_main, id).c_str(), cur_p.data[i].p);
+ // }
+ //}
+
+ //LOG("sampled token: %5d: '%s'\n", id, llama_token_to_piece(ctx_main, id).c_str());
+ }
+ }
+
+ if (ctx_sampling->grammar != NULL && !is_resampling) {
+ // Get a pointer to the logits
+ float * logits = llama_get_logits_ith(ctx_main, idx);
+
+ // Create an array with a single token data element for the sampled id
+ llama_token_data single_token_data = {id, logits[id], 0.0f};
+ llama_token_data_array single_token_data_array = { &single_token_data, 1, false };
+
+ // Apply grammar constraints to the single token
+ llama_grammar_sample(ctx_sampling->grammar, ctx_main, &single_token_data_array);
+
+ // Check if the token is valid according to the grammar by seeing if its logit has been set to -INFINITY
+ bool is_valid = single_token_data_array.data[0].logit != -INFINITY;
+
+ // If the token is not valid according to the grammar, perform resampling
+ if (!is_valid) {
+ LOG("Resampling because token %d: '%s' does not meet grammar rules\n", id, llama_token_to_piece(ctx_main, id).c_str());
+
+ // Restore logits from the copy
+ std::copy(original_logits.begin(), original_logits.end(), logits);
+
+ return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, /* is_resampling= */ true);
+ }
+ }
+
+ ctx_sampling->n_valid = temp == 0.0f ? 0 : cur_p.size;
+
+ return id;
+}
+
+static llama_token_data_array llama_sampling_prepare_impl(
+ struct llama_sampling_context * ctx_sampling,
+ struct llama_context * ctx_main,
+ struct llama_context * ctx_cfg,
+ const int idx,
+ bool apply_grammar,
+ std::vector<float> * original_logits) {
+ const llama_sampling_params & params = ctx_sampling->params;
+
+ const int n_vocab = llama_n_vocab(llama_get_model(ctx_main));
+
+ const int32_t penalty_last_n = params.penalty_last_n < 0 ? params.n_prev : params.penalty_last_n;
+ const float penalty_repeat = params.penalty_repeat;
+ const float penalty_freq = params.penalty_freq;
+ const float penalty_present = params.penalty_present;
+
+ const bool penalize_nl = params.penalize_nl;
+
+ auto & prev = ctx_sampling->prev;
+ auto & cur = ctx_sampling->cur;
+
+ // Get a pointer to the logits
+ float * logits = llama_get_logits_ith(ctx_main, idx);
+
+ if (ctx_sampling->grammar != NULL && !apply_grammar) {
+ GGML_ASSERT(original_logits != NULL);
+ // Only make a copy of the original logits if we are not applying grammar checks, not sure if I actually have to do this.
+ *original_logits = {logits, logits + n_vocab};
+ }
+
+ // apply params.logit_bias map
+ for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) {
+ logits[it->first] += it->second;
+ }
+
+ if (ctx_cfg) {
+ float * logits_guidance = llama_get_logits_ith(ctx_cfg, idx);
+ llama_sample_apply_guidance(ctx_main, logits, logits_guidance, params.cfg_scale);
+ }
+
+ cur.resize(n_vocab);
+
+ for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
+ cur[token_id] = llama_token_data{token_id, logits[token_id], 0.0f};
+ }
+
+ llama_token_data_array cur_p = { cur.data(), cur.size(), false };
+
+ // apply penalties
+ const auto& penalty_tokens = params.use_penalty_prompt_tokens ? params.penalty_prompt_tokens : prev;
+ const int penalty_tokens_used_size = std::min((int)penalty_tokens.size(), penalty_last_n);
+ if (penalty_tokens_used_size) {
+ const float nl_logit = logits[llama_token_nl(llama_get_model(ctx_main))];
+
+ llama_sample_repetition_penalties(ctx_main, &cur_p,
+ penalty_tokens.data() + penalty_tokens.size() - penalty_tokens_used_size,
+ penalty_tokens_used_size, penalty_repeat, penalty_freq, penalty_present);
+
+ if (!penalize_nl) {
+ for (size_t idx = 0; idx < cur_p.size; idx++) {
+ if (cur_p.data[idx].id == llama_token_nl(llama_get_model(ctx_main))) {
+ cur_p.data[idx].logit = nl_logit;
+ break;
+ }
+ }
+ }
+ }
+
+ // apply grammar checks before sampling logic
+ if (apply_grammar && ctx_sampling->grammar != NULL) {
+ llama_grammar_sample(ctx_sampling->grammar, ctx_main, &cur_p);
+ }
+
+ return cur_p;
+}
+
+llama_token llama_sampling_sample(
+ struct llama_sampling_context * ctx_sampling,
+ struct llama_context * ctx_main,
+ struct llama_context * ctx_cfg,
+ const int idx) {
+ // Call the implementation function with is_resampling set to false by default
+ return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, /* is_resampling= */ false);
+}
+
+llama_token_data_array llama_sampling_prepare(
+ struct llama_sampling_context * ctx_sampling,
+ struct llama_context * ctx_main,
+ struct llama_context * ctx_cfg,
+ const int idx,
+ bool apply_grammar,
+ std::vector<float> * original_logits) {
+ return llama_sampling_prepare_impl(ctx_sampling,ctx_main, ctx_cfg, idx, apply_grammar, original_logits);
+}
+
+void llama_sampling_accept(
+ struct llama_sampling_context * ctx_sampling,
+ struct llama_context * ctx_main,
+ llama_token id,
+ bool apply_grammar) {
+ ctx_sampling->prev.erase(ctx_sampling->prev.begin());
+ ctx_sampling->prev.push_back(id);
+
+ if (ctx_sampling->grammar != NULL && apply_grammar) {
+ llama_grammar_accept_token(ctx_sampling->grammar, ctx_main, id);
+ }
+}
diff --git a/llama/sampling.h b/llama/sampling.h
new file mode 100644
index 00000000..b5cb7982
--- /dev/null
+++ b/llama/sampling.h
@@ -0,0 +1,186 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "llama.h"
+
+#include "grammar-parser.h"
+
+#include <random>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+// sampler types
+enum class llama_sampler_type : char {
+ TOP_K = 'k',
+ TOP_P = 'p',
+ MIN_P = 'm',
+ TFS_Z = 'f',
+ TYPICAL_P = 'y',
+ TEMPERATURE = 't'
+};
+
+// sampling parameters
+typedef struct llama_sampling_params {
+ int32_t n_prev = 64; // number of previous tokens to remember
+ int32_t n_probs = 0; // if greater than 0, output the probabilities of top n_probs tokens.
+ int32_t min_keep = 0; // 0 = disabled, otherwise samplers should return at least min_keep tokens
+ int32_t top_k = 40; // <= 0 to use vocab size
+ float top_p = 0.95f; // 1.0 = disabled
+ float min_p = 0.05f; // 0.0 = disabled
+ float tfs_z = 1.00f; // 1.0 = disabled
+ float typical_p = 1.00f; // 1.0 = disabled
+ float temp = 0.80f; // <= 0.0 to sample greedily, 0.0 to not output probabilities
+ float dynatemp_range = 0.00f; // 0.0 = disabled
+ float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature sampler
+ int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size)
+ float penalty_repeat = 1.00f; // 1.0 = disabled
+ float penalty_freq = 0.00f; // 0.0 = disabled
+ float penalty_present = 0.00f; // 0.0 = disabled
+ int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0
+ float mirostat_tau = 5.00f; // target entropy
+ float mirostat_eta = 0.10f; // learning rate
+ bool penalize_nl = false; // consider newlines as a repeatable token
+ uint32_t seed = LLAMA_DEFAULT_SEED; // the seed used to initialize llama_sampling_context
+
+ std::vector<llama_sampler_type> samplers_sequence = {
+ llama_sampler_type::TOP_K,
+ llama_sampler_type::TFS_Z,
+ llama_sampler_type::TYPICAL_P,
+ llama_sampler_type::TOP_P,
+ llama_sampler_type::MIN_P,
+ llama_sampler_type::TEMPERATURE
+ };
+
+ std::string grammar; // optional BNF-like grammar to constrain sampling
+
+ // Classifier-Free Guidance
+ // https://arxiv.org/abs/2306.17806
+ std::string cfg_negative_prompt; // string to help guidance
+ float cfg_scale = 1.f; // how strong is guidance
+
+ std::unordered_map<llama_token, float> logit_bias; // logit bias for specific tokens
+
+ std::vector<llama_token> penalty_prompt_tokens;
+ bool use_penalty_prompt_tokens = false;
+} llama_sampling_params;
+
+// general sampler context
+// TODO: move to llama.h
+struct llama_sampling_context {
+ // parameters that will be used for sampling
+ llama_sampling_params params;
+
+ // mirostat sampler state
+ float mirostat_mu;
+
+ llama_grammar * grammar;
+
+ // internal
+ grammar_parser::parse_state parsed_grammar;
+
+ // TODO: replace with ring-buffer
+ std::vector<llama_token> prev;
+ std::vector<llama_token_data> cur;
+ size_t n_valid; // Number of correct top tokens with correct probabilities.
+
+ std::mt19937 rng;
+};
+
+#include "common.h"
+
+// Create a new sampling context instance.
+struct llama_sampling_context * llama_sampling_init(const struct llama_sampling_params & params);
+
+void llama_sampling_free(struct llama_sampling_context * ctx);
+
+// Reset the sampler context
+// - clear prev tokens
+// - reset grammar
+void llama_sampling_reset(llama_sampling_context * ctx);
+
+// Set the sampler seed
+void llama_sampling_set_rng_seed(struct llama_sampling_context * ctx, uint32_t seed);
+
+// Copy the sampler context
+void llama_sampling_cp(llama_sampling_context * src, llama_sampling_context * dst);
+
+// Get the last sampled token
+llama_token llama_sampling_last(llama_sampling_context * ctx);
+
+// Get a string representation of the last sampled tokens
+std::string llama_sampling_prev_str(llama_sampling_context * ctx_sampling, llama_context * ctx_main, int n);
+
+// Print sampling parameters into a string
+std::string llama_sampling_print(const llama_sampling_params & params);
+
+// Print sampling order into a string
+std::string llama_sampling_order_print(const llama_sampling_params & params);
+
+std::string llama_sampling_type_to_str(llama_sampler_type sampler_type);
+
+std::vector<llama_sampler_type> llama_sampling_types_from_names(const std::vector<std::string> & names, bool allow_alt_names);
+std::vector<llama_sampler_type> llama_sampling_types_from_chars(const std::string & names_string);
+
+// this is a common sampling function used across the examples for convenience
+// it can serve as a starting point for implementing your own sampling function
+// Note: When using multiple sequences, it is the caller's responsibility to call
+// llama_sampling_reset when a sequence ends
+//
+// required:
+// - ctx_main: context to use for sampling
+// - ctx_sampling: sampling-specific context
+//
+// optional:
+// - ctx_cfg: context to use for classifier-free guidance
+// - idx: sample from llama_get_logits_ith(ctx, idx)
+//
+// returns:
+// - token: sampled token
+// - candidates: vector of candidate tokens
+//
+llama_token llama_sampling_sample(
+ struct llama_sampling_context * ctx_sampling,
+ struct llama_context * ctx_main,
+ struct llama_context * ctx_cfg,
+ int idx = -1);
+
+// Prepares and adjusts the set of token candidates for sampling based on penalties, biases, and sampling parameters.
+llama_token_data_array llama_sampling_prepare(
+ struct llama_sampling_context * ctx_sampling,
+ struct llama_context * ctx_main,
+ struct llama_context * ctx_cfg,
+ int idx = 0,
+ bool apply_grammar = true,
+ std::vector<float> * original_logits = nullptr);
+
+void llama_sampling_accept(
+ struct llama_sampling_context * ctx_sampling,
+ struct llama_context * ctx_main,
+ llama_token id,
+ bool apply_grammar);
diff --git a/llama/sampling_ext.cpp b/llama/sampling_ext.cpp
new file mode 100644
index 00000000..da92cedf
--- /dev/null
+++ b/llama/sampling_ext.cpp
@@ -0,0 +1,53 @@
+// TODO: this is a temporary wrapper to allow calling C++ code from CGo
+#include "sampling.h"
+#include "sampling_ext.h"
+
+struct llama_sampling_context *llama_sampling_cinit(struct llama_sampling_cparams *params)
+{
+ llama_sampling_params sparams;
+ sparams.top_k = params->top_k;
+ sparams.top_p = params->top_p;
+ sparams.min_p = params->min_p;
+ sparams.tfs_z = params->tfs_z;
+ sparams.typical_p = params->typical_p;
+ sparams.temp = params->temp;
+ sparams.penalty_last_n = params->penalty_last_n;
+ sparams.penalty_repeat = params->penalty_repeat;
+ sparams.penalty_freq = params->penalty_freq;
+ sparams.penalty_present = params->penalty_present;
+ sparams.mirostat = params->mirostat;
+ sparams.mirostat_tau = params->mirostat_tau;
+ sparams.mirostat_eta = params->mirostat_eta;
+ sparams.penalize_nl = params->penalize_nl;
+ sparams.seed = params->seed;
+ sparams.grammar = params->grammar;
+ return llama_sampling_init(sparams);
+}
+
+void llama_sampling_cfree(struct llama_sampling_context *ctx)
+{
+ llama_sampling_free(ctx);
+}
+
+void llama_sampling_creset(struct llama_sampling_context *ctx)
+{
+ llama_sampling_reset(ctx);
+}
+
+llama_token llama_sampling_csample(
+ struct llama_sampling_context *ctx_sampling,
+ struct llama_context *ctx_main,
+ struct llama_context *ctx_cfg,
+ int idx)
+{
+ return llama_sampling_sample(ctx_sampling, ctx_main, ctx_cfg, idx);
+}
+
+void llama_sampling_caccept(
+ struct llama_sampling_context *ctx_sampling,
+ struct llama_context *ctx_main,
+ llama_token id,
+ bool apply_grammar)
+{
+ llama_sampling_accept(ctx_sampling, ctx_main, id, apply_grammar);
+}
diff --git a/llama/sampling_ext.h b/llama/sampling_ext.h
new file mode 100644
index 00000000..588ed5c1
--- /dev/null
+++ b/llama/sampling_ext.h
@@ -0,0 +1,52 @@
+// TODO: this is a temporary wrapper to allow calling C++ code from CGo
+#ifndef LLAMA_SAMPLING_EXT_H
+#define LLAMA_SAMPLING_EXT_H
+
+#include "llama.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+ struct llama_sampling_cparams
+ {
+ int32_t top_k;
+ float top_p;
+ float min_p;
+ float tfs_z;
+ float typical_p;
+ float temp;
+ int32_t penalty_last_n;
+ float penalty_repeat;
+ float penalty_freq;
+ float penalty_present;
+ int32_t mirostat;
+ float mirostat_tau;
+ float mirostat_eta;
+ bool penalize_nl;
+ uint32_t seed;
+ char *grammar;
+ };
+
+ struct llama_sampling_context *llama_sampling_cinit(struct llama_sampling_cparams *params);
+ void llama_sampling_cfree(struct llama_sampling_context *ctx);
+ void llama_sampling_creset(struct llama_sampling_context *ctx);
+
+ llama_token llama_sampling_csample(
+ struct llama_sampling_context *ctx_sampling,
+ struct llama_context *ctx_main,
+ struct llama_context *ctx_cfg,
+ int idx);
+
+ void llama_sampling_caccept(
+ struct llama_sampling_context *ctx_sampling,
+ struct llama_context *ctx_main,
+ llama_token id,
+ bool apply_grammar);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // LLAMA_SAMPLING_EXT_H
diff --git a/llama/sgemm.cpp b/llama/sgemm.cpp
new file mode 100644
index 00000000..f0988ba7
--- /dev/null
+++ b/llama/sgemm.cpp
@@ -0,0 +1,1176 @@
+// Copyright 2024 Mozilla Foundation
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//
+// _ _ ___ _ _ ___
+// | |_(_)_ _ _ _| _ ) | /_\ / __|
+// | _| | ' \ || | _ \ |__ / _ \\__ \.
+// \__|_|_||_\_, |___/____/_/ \_\___/
+// |__/
+//
+// BASIC LINEAR ALGEBRA SUBPROGRAMS
+//
+//
+// This file implements multithreaded CPU matrix multiplication for the
+// common contiguous use case C = Aᵀ * B. These kernels are designed to
+// have excellent performance[1] for matrices that fit in the CPU cache
+// without imposing any overhead such as cache filling or malloc calls.
+//
+// This implementation does not guarantee any upper bound with rounding
+// errors, which grow along with k. Our goal's to maximally exploit the
+// hardware for performance, and then use whatever resources remain for
+// improving numerical accuracy.
+//
+// [1] J. Tunney, ‘LLaMA Now Goes Faster on CPUs’, Mar. 2024. [Online].
+// Available: https://justine.lol/matmul/. [Accessed: 29-Mar-2024].
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wpedantic"
+#pragma GCC diagnostic ignored "-Wignored-attributes"
+#endif
+
+#include "sgemm.h"
+#include "ggml-impl.h"
+#include "ggml-quants.h"
+
+#ifdef _MSC_VER
+#define NOINLINE __declspec(noinline)
+#else
+#define NOINLINE __attribute__((__noinline__))
+#endif
+
+#if defined(__ARM_NEON) || defined(__AVX512F__)
+#define VECTOR_REGISTERS 32
+#else
+#define VECTOR_REGISTERS 16
+#endif
+
+#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
+
+namespace {
+
+inline float unhalf(ggml_fp16_t d) {
+ return GGML_FP16_TO_FP32(d);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// VECTORIZED ARITHMETIC OPERATIONS
+
+#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+inline __m128 add(__m128 x, __m128 y) { return _mm_add_ps(x, y); }
+inline __m128 sub(__m128 x, __m128 y) { return _mm_sub_ps(x, y); }
+inline __m128 mul(__m128 x, __m128 y) { return _mm_mul_ps(x, y); }
+#endif // __SSE__
+
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+inline __m256 add(__m256 x, __m256 y) { return _mm256_add_ps(x, y); }
+inline __m256 sub(__m256 x, __m256 y) { return _mm256_sub_ps(x, y); }
+inline __m256 mul(__m256 x, __m256 y) { return _mm256_mul_ps(x, y); }
+#endif // __AVX__
+
+#if defined(__AVX512F__)
+inline __m512 add(__m512 x, __m512 y) { return _mm512_add_ps(x, y); }
+inline __m512 sub(__m512 x, __m512 y) { return _mm512_sub_ps(x, y); }
+inline __m512 mul(__m512 x, __m512 y) { return _mm512_mul_ps(x, y); }
+#endif // __AVX512F__
+
+#if defined(__ARM_NEON)
+inline float32x4_t add(float32x4_t x, float32x4_t y) { return vaddq_f32(x, y); }
+inline float32x4_t sub(float32x4_t x, float32x4_t y) { return vsubq_f32(x, y); }
+inline float32x4_t mul(float32x4_t x, float32x4_t y) { return vmulq_f32(x, y); }
+#endif // __ARM_NEON
+
+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+inline float16x8_t add(float16x8_t x, float16x8_t y) { return vaddq_f16(x, y); }
+inline float16x8_t sub(float16x8_t x, float16x8_t y) { return vsubq_f16(x, y); }
+inline float16x8_t mul(float16x8_t x, float16x8_t y) { return vmulq_f16(x, y); }
+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// VECTORIZED FUSED MULTIPLY ADD
+
+/**
+ * Computes a * b + c.
+ */
+template <typename T, typename U>
+inline U madd(T a, T b, U c) {
+ return add(mul(a, b), c);
+}
+
+#if defined(__FMA__)
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+template <>
+inline __m256 madd(__m256 a, __m256 b, __m256 c) {
+ return _mm256_fmadd_ps(a, b, c);
+}
+#endif
+#if defined(__AVX512F__)
+template <>
+inline __m512 madd(__m512 a, __m512 b, __m512 c) {
+ return _mm512_fmadd_ps(a, b, c);
+}
+#endif
+#endif
+
+#if defined(__ARM_FEATURE_FMA)
+template <>
+inline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) {
+ return vfmaq_f32(c, b, a);
+}
+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
+template <>
+inline float16x8_t madd(float16x8_t a, float16x8_t b, float16x8_t c) {
+ return vfmaq_f16(c, b, a);
+}
+#endif
+#endif
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// VECTORIZED HORIZONTAL SUM
+
+#if defined(__ARM_NEON)
+inline float hsum(float32x4_t x) {
+ return vaddvq_f32(x);
+}
+#endif // __ARM_NEON
+
+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
+inline float hsum(float16x8_t x) {
+ return vaddvq_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(x)),
+ vcvt_f32_f16(vget_high_f16(x))));
+}
+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+
+#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+inline float hsum(__m128 x) {
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+ x = _mm_add_ps(x, _mm_movehl_ps(x, x));
+ x = _mm_add_ss(x, _mm_movehdup_ps(x));
+#else
+ __m128 t;
+ t = _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1));
+ x = _mm_add_ps(x, t);
+ t = _mm_movehl_ps(t, x);
+ x = _mm_add_ss(x, t);
+#endif
+ return _mm_cvtss_f32(x);
+}
+#endif
+
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+inline float hsum(__m256 x) {
+ return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1),
+ _mm256_castps256_ps128(x)));
+}
+#endif // __AVX__
+
+#if defined(__AVX512F__)
+inline float hsum(__m512 x) {
+ return _mm512_reduce_add_ps(x);
+}
+#endif // __AVX512F__
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// VECTORIZED MEMORY LOADING
+
+template <typename T, typename U> T load(const U *);
+
+#if defined(__ARM_NEON)
+template <> inline float32x4_t load(const float *p) {
+ return vld1q_f32(p);
+}
+#if !defined(_MSC_VER)
+template <> inline float16x8_t load(const ggml_fp16_t *p) {
+ return vld1q_f16((const float16_t *)p);
+}
+template <> inline float32x4_t load(const ggml_fp16_t *p) {
+ return vcvt_f32_f16(vld1_f16((const float16_t *)p));
+}
+#endif // _MSC_VER
+#endif // __ARM_NEON
+
+#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+template <> inline __m128 load(const float *p) {
+ return _mm_loadu_ps(p);
+}
+#endif // __SSE__
+
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+template <> inline __m256 load(const float *p) {
+ return _mm256_loadu_ps(p);
+}
+#endif // __AVX__
+
+#if defined(__F16C__)
+template <> inline __m256 load(const ggml_fp16_t *p) {
+ return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));
+}
+#endif // __F16C__
+
+#if defined(__AVX512F__)
+template <> inline __m512 load(const float *p) {
+ return _mm512_loadu_ps(p);
+}
+template <> inline __m512 load(const ggml_fp16_t *p) {
+ return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));
+}
+#endif // __AVX512F__
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// FLOATING POINT MATRIX MULTIPLICATION
+
+template <int KN, typename D, typename V, typename TA, typename TB, typename TC>
+class tinyBLAS {
+ public:
+ tinyBLAS(int64_t k,
+ const TA *A, int64_t lda,
+ const TB *B, int64_t ldb,
+ TC *C, int64_t ldc,
+ int ith, int nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int64_t m, int64_t n) {
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ NOINLINE void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {
+ int64_t mc, nc, mp, np;
+ switch ((MIN(m - m0, 5) << 4) | MIN(n - n0, 5)) {
+#if VECTOR_REGISTERS == 32
+ case 0x55:
+ mc = 5;
+ nc = 5;
+ gemm<5, 5>(m0, m, n0, n);
+ break;
+ case 0x45:
+ mc = 4;
+ nc = 5;
+ gemm<4, 5>(m0, m, n0, n);
+ break;
+ case 0x54:
+ mc = 5;
+ nc = 4;
+ gemm<5, 4>(m0, m, n0, n);
+ break;
+ case 0x44:
+ mc = 4;
+ nc = 4;
+ gemm<4, 4>(m0, m, n0, n);
+ break;
+ case 0x53:
+ mc = 5;
+ nc = 3;
+ gemm<5, 3>(m0, m, n0, n);
+ break;
+ case 0x35:
+ mc = 3;
+ nc = 5;
+ gemm<3, 5>(m0, m, n0, n);
+ break;
+ case 0x43:
+ mc = 4;
+ nc = 3;
+ gemm<4, 3>(m0, m, n0, n);
+ break;
+#else
+ case 0x55:
+ case 0x54:
+ case 0x53:
+ case 0x45:
+ case 0x44:
+ case 0x43:
+ mc = 4;
+ nc = 3;
+ gemm<4, 3>(m0, m, n0, n);
+ break;
+ case 0x35:
+#endif
+ case 0x34:
+ mc = 3;
+ nc = 4;
+ gemm<3, 4>(m0, m, n0, n);
+ break;
+ case 0x52:
+ mc = 5;
+ nc = 2;
+ gemm<5, 2>(m0, m, n0, n);
+ break;
+ case 0x33:
+ mc = 3;
+ nc = 3;
+ gemm<3, 3>(m0, m, n0, n);
+ break;
+ case 0x25:
+ mc = 2;
+ nc = 5;
+ gemm<2, 5>(m0, m, n0, n);
+ break;
+ case 0x42:
+ mc = 4;
+ nc = 2;
+ gemm<4, 2>(m0, m, n0, n);
+ break;
+ case 0x24:
+ mc = 2;
+ nc = 4;
+ gemm<2, 4>(m0, m, n0, n);
+ break;
+ case 0x32:
+ mc = 3;
+ nc = 2;
+ gemm<3, 2>(m0, m, n0, n);
+ break;
+ case 0x23:
+ mc = 2;
+ nc = 3;
+ gemm<2, 3>(m0, m, n0, n);
+ break;
+ case 0x51:
+ mc = 5;
+ nc = 1;
+ gemm<5, 1>(m0, m, n0, n);
+ break;
+ case 0x41:
+ mc = 4;
+ nc = 1;
+ gemm<4, 1>(m0, m, n0, n);
+ break;
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2>(m0, m, n0, n);
+ break;
+ case 0x15:
+ mc = 1;
+ nc = 5;
+ gemm<1, 5>(m0, m, n0, n);
+ break;
+ case 0x14:
+ mc = 1;
+ nc = 4;
+ gemm<1, 4>(m0, m, n0, n);
+ break;
+ case 0x31:
+ mc = 3;
+ nc = 1;
+ gemm<3, 1>(m0, m, n0, n);
+ break;
+ case 0x13:
+ mc = 1;
+ nc = 3;
+ gemm<1, 3>(m0, m, n0, n);
+ break;
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1>(m0, m, n0, n);
+ break;
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, m, np, n);
+ }
+
+ template <int RM, int RN>
+ NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {
+ int64_t ytiles = (m - m0) / RM;
+ int64_t xtiles = (n - n0) / RN;
+ int64_t tiles = xtiles * ytiles;
+ int64_t duty = (tiles + nth - 1) / nth;
+ int64_t start = duty * ith;
+ int64_t end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (int64_t job = start; job < end; ++job) {
+ int64_t ii = m0 + job / xtiles * RM;
+ int64_t jj = n0 + job % xtiles * RN;
+ D Cv[RN][RM] = {};
+ for (int64_t l = 0; l < k; l += KN)
+ for (int64_t j = 0; j < RN; ++j)
+ for (int64_t i = 0; i < RM; ++i)
+ Cv[j][i] = madd(load<V>(A + lda * (ii + i) + l),
+ load<V>(B + ldb * (jj + j) + l),
+ Cv[j][i]);
+ for (int64_t j = 0; j < RN; ++j)
+ for (int64_t i = 0; i < RM; ++i)
+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);
+ }
+ }
+
+ const TA *const A;
+ const TB *const B;
+ TC *const C;
+ const int64_t k;
+ const int64_t lda;
+ const int64_t ldb;
+ const int64_t ldc;
+ const int ith;
+ const int nth;
+};
+
+//////////////////////////////////////////////////////////////////////////////////////////
+// QUANT ZERO MATRIX MULTIPLICATION
+
+#if defined(__ARM_FEATURE_DOTPROD)
+template <typename TA>
+class tinyBLAS_Q0_ARM {
+ public:
+ tinyBLAS_Q0_ARM(int64_t k,
+ const TA *A, int64_t lda,
+ const block_q8_0 *B, int64_t ldb,
+ float *C, int64_t ldc,
+ int ith, int nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int64_t m, int64_t n) {
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ NOINLINE void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {
+ int64_t mc, nc, mp, np;
+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3ll)) {
+ case 0x33:
+ mc = 3;
+ nc = 3;
+ gemm<3, 3>(m0, m, n0, n);
+ break;
+ case 0x32:
+ mc = 3;
+ nc = 2;
+ gemm<3, 2>(m0, m, n0, n);
+ break;
+ case 0x23:
+ mc = 2;
+ nc = 3;
+ gemm<2, 3>(m0, m, n0, n);
+ break;
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2>(m0, m, n0, n);
+ break;
+ case 0x31:
+ mc = 3;
+ nc = 1;
+ gemm<3, 1>(m0, m, n0, n);
+ break;
+ case 0x13:
+ mc = 1;
+ nc = 3;
+ gemm<1, 3>(m0, m, n0, n);
+ break;
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1>(m0, m, n0, n);
+ break;
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, m, np, n);
+ }
+
+ template <int RM, int RN>
+ NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {
+ int64_t ytiles = (m - m0) / RM;
+ int64_t xtiles = (n - n0) / RN;
+ int64_t tiles = xtiles * ytiles;
+ int64_t duty = (tiles + nth - 1) / nth;
+ int64_t start = duty * ith;
+ int64_t end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (int64_t job = start; job < end; ++job) {
+ int64_t ii = m0 + job / xtiles * RM;
+ int64_t jj = n0 + job % xtiles * RN;
+ float32x4_t Cv[RN][RM] = {};
+ for (int64_t l = 0; l < k; ++l)
+ for (int64_t j = 0; j < RN; ++j)
+ for (int64_t i = 0; i < RM; ++i)
+ Cv[j][i] = vmlaq_n_f32(Cv[j][i],
+ vcvtq_f32_s32(vdotq_s32(
+ vdotq_s32(vdupq_n_s32(0),
+ load_lo(A + lda * (ii + i) + l),
+ load_lo(B + ldb * (jj + j) + l)),
+ load_hi(A + lda * (ii + i) + l),
+ load_hi(B + ldb * (jj + j) + l))),
+ unhalf(A[lda * (ii + i) + l].d) *
+ unhalf(B[ldb * (jj + j) + l].d));
+ for (int64_t j = 0; j < RN; ++j)
+ for (int64_t i = 0; i < RM; ++i)
+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);
+ }
+ }
+
+ inline int8x16_t load_lo(const block_q8_0 *b) {
+ return vld1q_s8(b->qs);
+ }
+
+ inline int8x16_t load_hi(const block_q8_0 *b) {
+ return vld1q_s8(b->qs + 16);
+ }
+
+ inline int8x16_t load_lo(const block_q4_0 *b) {
+ return vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vld1q_u8(b->qs),
+ vdupq_n_u8(0x0f))),
+ vdupq_n_s8(0x8));
+ }
+
+ inline int8x16_t load_hi(const block_q4_0 *b) {
+ return vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(vld1q_u8(b->qs), 4)),
+ vdupq_n_s8(0x8));
+ }
+
+ const TA *const A;
+ const block_q8_0 *const B;
+ float *const C;
+ const int64_t k;
+ const int64_t lda;
+ const int64_t ldb;
+ const int64_t ldc;
+ const int ith;
+ const int nth;
+};
+#endif // __ARM_FEATURE_DOTPROD
+
+#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)
+template <typename TA, typename TB, typename TC>
+class tinyBLAS_Q0_AVX {
+ public:
+ tinyBLAS_Q0_AVX(int64_t k,
+ const TA *A, int64_t lda,
+ const TB *B, int64_t ldb,
+ TC *C, int64_t ldc,
+ int ith, int nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int64_t m, int64_t n) {
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {
+ int64_t mc, nc, mp, np;
+ switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 4)) {
+#if VECTOR_REGISTERS == 32
+ case 0x44:
+ mc = 4;
+ nc = 4;
+#if defined(__AVX2__) && defined(__F16C__)
+ gemm4xN<4>(m0, m, n0, n);
+#else
+ gemm<4, 4>(m0, m, n0, n);
+#endif
+ break;
+ case 0x43:
+ mc = 4;
+ nc = 3;
+#if defined(__AVX2__) && defined(__F16C__)
+ gemm4xN<3>(m0, m, n0, n);
+#else
+ gemm<4, 3>(m0, m, n0, n);
+#endif
+ break;
+ case 0x34:
+ mc = 3;
+ nc = 4;
+#if defined(__AVX2__) && defined(__F16C__)
+ gemmMx4<3>(m0, m, n0, n);
+#else
+ gemm<3, 4>(m0, m, n0, n);
+#endif
+ break;
+ case 0x33:
+ mc = 3;
+ nc = 3;
+ gemm<3, 3>(m0, m, n0, n);
+ break;
+ case 0x42:
+ mc = 4;
+ nc = 2;
+#if defined(__AVX2__) && defined(__F16C__)
+ gemm4xN<2>(m0, m, n0, n);
+#else
+ gemm<4, 2>(m0, m, n0, n);
+#endif
+ break;
+ case 0x24:
+ mc = 2;
+ nc = 4;
+#if defined(__AVX2__) && defined(__F16C__)
+ gemmMx4<2>(m0, m, n0, n);
+#else
+ gemm<2, 4>(m0, m, n0, n);
+#endif
+ break;
+#else
+ case 0x44:
+ case 0x43:
+ case 0x42:
+ mc = 4;
+ nc = 2;
+#if defined(__AVX2__) && defined(__F16C__)
+ gemm4xN<2>(m0, m, n0, n);
+#else
+ gemm<4, 2>(m0, m, n0, n);
+#endif
+ break;
+ case 0x34:
+ case 0x24:
+ mc = 2;
+ nc = 4;
+#if defined(__AVX2__) && defined(__F16C__)
+ gemmMx4<2>(m0, m, n0, n);
+#else
+ gemm<2, 4>(m0, m, n0, n);
+#endif
+ break;
+ case 0x33:
+#endif
+ case 0x32:
+ mc = 3;
+ nc = 2;
+ gemm<3, 2>(m0, m, n0, n);
+ break;
+ case 0x23:
+ mc = 2;
+ nc = 3;
+ gemm<2, 3>(m0, m, n0, n);
+ break;
+ case 0x41:
+ mc = 4;
+ nc = 1;
+#if defined(__AVX2__) && defined(__F16C__)
+ gemm4xN<1>(m0, m, n0, n);
+#else
+ gemm<4, 1>(m0, m, n0, n);
+#endif
+ break;
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2>(m0, m, n0, n);
+ break;
+ case 0x14:
+ mc = 1;
+ nc = 4;
+#if defined(__AVX2__) && defined(__F16C__)
+ gemmMx4<1>(m0, m, n0, n);
+#else
+ gemm<1, 4>(m0, m, n0, n);
+#endif
+ break;
+ case 0x31:
+ mc = 3;
+ nc = 1;
+ gemm<3, 1>(m0, m, n0, n);
+ break;
+ case 0x13:
+ mc = 1;
+ nc = 3;
+ gemm<1, 3>(m0, m, n0, n);
+ break;
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1>(m0, m, n0, n);
+ break;
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, m, np, n);
+ }
+
+#if defined(__AVX2__) && defined(__F16C__)
+// Templated functions for gemm of dimensions 4xN
+ template <int RN>
+ NOINLINE void gemm4xN(int64_t m0, int64_t m, int64_t n0, int64_t n) {
+ int64_t ytiles = (m - m0) / 4;
+ int64_t xtiles = (n - n0) / RN;
+ int64_t tiles = xtiles * ytiles;
+ int64_t duty = (tiles + nth - 1) / nth;
+ int64_t start = duty * ith;
+ int64_t end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (int64_t job = start; job < end; ++job) {
+ int64_t ii = m0 + job / xtiles * 4;
+ int64_t jj = n0 + job % xtiles * RN;
+ __m256 Cv[RN][4] = {};
+ for (int64_t l = 0; l < k; ++l) {
+ uint64_t a_delta = ((uint64_t)A[lda * (ii + 3) + l].d << 48) | ((uint64_t)A[lda * (ii + 2) + l].d << 32) | ((uint64_t)A[lda * (ii + 1) + l].d << 16) | (A[lda * (ii + 0) + l].d);
+ // Convert delta values for four blocks to float values
+ __m128 da = _mm_cvtph_ps(_mm_set_epi64x(0, a_delta));
+ __m256i avec0 = load(A + lda * (ii + 0) + l);
+ __m256i avec1 = load(A + lda * (ii + 1) + l);
+ __m256i avec2 = load(A + lda * (ii + 2) + l);
+ __m256i avec3 = load(A + lda * (ii + 3) + l);
+ for (int64_t j = 0; j < RN; ++j) {
+ __m128 db = _mm_set1_ps(unhalf(B[ldb * (jj + j) + l].d));
+ // Computation of product of delta values for four blocks and replicate it across 256 bit lane
+ __m256 dvec = _mm256_castps128_ps256(_mm_mul_ps(da, db));
+ dvec = _mm256_permute2f128_ps(dvec ,dvec, 0);
+ // Computation of dot product and multiplication with appropriate delta value products
+ Cv[j][0] = madd(_mm256_shuffle_ps(dvec, dvec, 0),
+ updot(_mm256_sign_epi8(avec0, avec0),
+ _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec0)),
+ Cv[j][0]);
+ Cv[j][1] = madd(_mm256_shuffle_ps(dvec, dvec, 85),
+ updot(_mm256_sign_epi8(avec1, avec1),
+ _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec1)),
+ Cv[j][1]);
+ Cv[j][2] = madd(_mm256_shuffle_ps(dvec, dvec, 170),
+ updot(_mm256_sign_epi8(avec2, avec2),
+ _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec2)),
+ Cv[j][2]);
+ Cv[j][3] = madd(_mm256_shuffle_ps(dvec, dvec, 255),
+ updot(_mm256_sign_epi8(avec3, avec3),
+ _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec3)),
+ Cv[j][3]);
+ }
+ }
+
+ for (int64_t j = 0; j < RN; ++j)
+ for (int64_t i = 0; i < 4; ++i)
+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);
+ }
+ }
+
+ // Templated functions for gemm of dimensions Mx4
+ template <int RM>
+ NOINLINE void gemmMx4(int64_t m0, int64_t m, int64_t n0, int64_t n) {
+ int64_t ytiles = (m - m0) / RM;
+ int64_t xtiles = (n - n0) / 4;
+ int64_t tiles = xtiles * ytiles;
+ int64_t duty = (tiles + nth - 1) / nth;
+ int64_t start = duty * ith;
+ int64_t end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (int64_t job = start; job < end; ++job) {
+ int64_t ii = m0 + job / xtiles * RM;
+ int64_t jj = n0 + job % xtiles * 4;
+ __m256 Cv[4][RM] = {};
+ for (int64_t l = 0; l < k; ++l) {
+ uint64_t b_delta = ((uint64_t)B[ldb * (jj + 3) + l].d << 48) | ((uint64_t)B[ldb * (jj + 2) + l].d << 32) | ((uint64_t)B[ldb * (jj + 1) + l].d << 16) | (B[ldb * (jj + 0) + l].d);
+ // Convert delta values for four blocks to float values
+ __m128 db = _mm_cvtph_ps(_mm_set_epi64x(0, b_delta));
+ __m256i bvec0 = load(B + ldb * (jj + 0) + l);
+ __m256i bvec1 = load(B + ldb * (jj + 1) + l);
+ __m256i bvec2 = load(B + ldb * (jj + 2) + l);
+ __m256i bvec3 = load(B + ldb * (jj + 3) + l);
+ for (int64_t i = 0; i < RM; ++i) {
+ __m128 da = _mm_set1_ps(unhalf((A[lda * (ii + i) + l].d)));
+ // Computation of product of delta values for four blocks and replicate it across 256 bit lane
+ __m256 dvec = _mm256_castps128_ps256(_mm_mul_ps(da, db));
+ dvec = _mm256_permute2f128_ps(dvec ,dvec, 0);
+ // Computation of dot product and multiplication with appropriate delta value products
+ Cv[0][i] = madd(_mm256_shuffle_ps(dvec, dvec, 0),
+ updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),
+ load(A + lda * (ii + i) + l)),
+ _mm256_sign_epi8(bvec0, load(A + lda * (ii + i) + l))),
+ Cv[0][i]);
+ Cv[1][i] = madd(_mm256_shuffle_ps(dvec, dvec, 85),
+ updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),
+ load(A + lda * (ii + i) + l)),
+ _mm256_sign_epi8(bvec1, load(A + lda * (ii + i) + l))),
+ Cv[1][i]);
+ Cv[2][i] = madd(_mm256_shuffle_ps(dvec, dvec, 170),
+ updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),
+ load(A + lda * (ii + i) + l)),
+ _mm256_sign_epi8(bvec2, load(A + lda * (ii + i) + l))),
+ Cv[2][i]);
+ Cv[3][i] = madd(_mm256_shuffle_ps(dvec, dvec, 255),
+ updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),
+ load(A + lda * (ii + i) + l)),
+ _mm256_sign_epi8(bvec3, load(A + lda * (ii + i) + l))),
+ Cv[3][i]);
+ }
+ }
+ for (int64_t j = 0; j < 4; ++j)
+ for (int64_t i = 0; i < RM; ++i)
+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);
+ }
+ }
+#endif
+
+ template <int RM, int RN>
+ NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {
+ int64_t ytiles = (m - m0) / RM;
+ int64_t xtiles = (n - n0) / RN;
+ int64_t tiles = xtiles * ytiles;
+ int64_t duty = (tiles + nth - 1) / nth;
+ int64_t start = duty * ith;
+ int64_t end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (int64_t job = start; job < end; ++job) {
+ int64_t ii = m0 + job / xtiles * RM;
+ int64_t jj = n0 + job % xtiles * RN;
+ __m256 Cv[RN][RM] = {};
+ for (int64_t l = 0; l < k; ++l)
+ for (int64_t j = 0; j < RN; ++j)
+ for (int64_t i = 0; i < RM; ++i) {
+#if defined(__AVX2__)
+ __m256 udTmp = updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),
+ load(A + lda * (ii + i) + l)),
+ _mm256_sign_epi8(load(B + ldb * (jj + j) + l),
+ load(A + lda * (ii + i) + l)));
+#else
+ __m128i ali0 = load0(A + lda * (ii + i) + l);
+ __m128i ali1 = load1(A + lda * (ii + i) + l);
+ __m128i blj0 = load0(B + ldb * (jj + j) + l);
+ __m128i blj1 = load1(B + ldb * (jj + j) + l);
+
+ __m128i sepAA0 = _mm_sign_epi8(ali0, ali0);
+ __m128i sepAA1 = _mm_sign_epi8(ali1, ali1);
+ __m128i sepBA0 = _mm_sign_epi8(blj0, ali0);
+ __m128i sepBA1 = _mm_sign_epi8(blj1, ali1);
+
+ // updot
+ const __m128i oneFill = _mm_set1_epi16(1);
+ __m128i mad0 = _mm_maddubs_epi16(sepAA0, sepBA0);
+ __m128i mad1 = _mm_maddubs_epi16(sepAA1, sepBA1);
+ __m256 udTmp = _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_madd_epi16(oneFill, mad1), _mm_madd_epi16(oneFill, mad0)));
+#endif
+ Cv[j][i] = madd(_mm256_set1_ps(unhalf(A[lda * (ii + i) + l].d) *
+ unhalf(B[ldb * (jj + j) + l].d)),
+ udTmp,
+ Cv[j][i]);
+ }
+ for (int64_t j = 0; j < RN; ++j)
+ for (int64_t i = 0; i < RM; ++i)
+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);
+ }
+ }
+
+ inline __m256i load(const block_q8_0 *b) {
+ return _mm256_loadu_si256((const __m256i *)b->qs);
+ }
+
+ inline __m128i load0(const block_q8_0 *b) {
+ return _mm_loadu_si128((const __m128i *)b->qs);
+ }
+
+ inline __m128i load1(const block_q8_0 *b) {
+ return _mm_loadu_si128(((const __m128i *)b->qs) + 1);
+ }
+
+ inline __m256i load(const block_q4_0 *b) {
+ return _mm256_sub_epi8(denibble(b->qs), _mm256_set1_epi8(8));
+ }
+
+ inline __m128i load0(const block_q4_0 *b) {
+ const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));
+ return _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), x), _mm_set1_epi8(8));
+ }
+
+ inline __m128i load1(const block_q4_0 *b) {
+ const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));
+ return _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(x, 4)), _mm_set1_epi8(8));
+ }
+
+ inline __m256 updot(__m256i u, __m256i s) {
+ __m256i res;
+#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))
+ res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);
+#else
+ res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));
+#endif
+ return _mm256_cvtepi32_ps(res);
+ }
+
+ static inline __m256i denibble(const uint8_t *p) {
+ __m128i x = _mm_loadu_si128((const __m128i *)p);
+ return _mm256_and_si256(_mm256_set1_epi8(15),
+ _mm256_insertf128_si256(_mm256_castsi128_si256(x),
+ _mm_srli_epi16(x, 4), 1));
+ }
+
+ const TA *const A;
+ const TB *const B;
+ TC *const C;
+ const int64_t k;
+ const int64_t lda;
+ const int64_t ldb;
+ const int64_t ldc;
+ const int ith;
+ const int nth;
+};
+#endif // __AVX__
+
+} // namespace
+
+/**
+ * Performs optimized matrix multiplication on CPU.
+ *
+ * This subroutine may compute C = Aᵀ * B with column major ordering.
+ * Despite its name, this isn't a generalized implementation. Work is
+ * only performed when a handwritten kernel is written and available.
+ * Otherwise the caller should fall back to a general matmul routine.
+ *
+ * For example, for single-threaded single-precision GEMM you can say
+ *
+ * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc,
+ * 0, 1,
+ * GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32);
+ *
+ * @param m is rows in `A` and `C`
+ * @param n is cols in `B` and `C`
+ * @param k is cols in `A` and rows in `B`
+ * @param A is first input matrix (always transposed)
+ * @param lda is row stride of `A`
+ * @param B is second input matrix (never transposed)
+ * @param ldb is row stride of `B`
+ * @param C is input/output array of output matrices
+ * @param ldc is row stride of `C`
+ * @param ith is thread id (must be less than `nth`)
+ * @param nth is number of threads (must be greater than zero)
+ * @param Atype is GGML data type of `A`
+ * @param Btype is GGML data type of `B`
+ * @param Ctype is GGML data type of `C`
+ * @return true if this function was able to service the matmul request
+ */
+bool llamafile_sgemm(int64_t m, int64_t n, int64_t k, const void *A, int64_t lda, const void *B, int64_t ldb, void *C,
+ int64_t ldc, int ith, int nth, int Atype, int Btype, int Ctype) {
+
+ assert(m >= 0);
+ assert(n >= 0);
+ assert(k >= 0);
+ assert(lda >= k);
+ assert(ldb >= k);
+ assert(ldc >= m);
+ assert(nth > 0);
+ assert(ith < nth);
+
+ if (Ctype != GGML_TYPE_F32)
+ return false;
+
+ switch (Atype) {
+
+ case GGML_TYPE_F32: {
+ if (Btype != GGML_TYPE_F32)
+ return false;
+#if defined(__AVX512F__)
+ if (k % 16)
+ return false;
+ tinyBLAS<16, __m512, __m512, float, float, float> tb{
+ k, (const float *)A, lda,
+ (const float *)B, ldb,
+ (float *)C, ldc,
+ ith, nth};
+ tb.matmul(m, n);
+ return true;
+#elif defined(__AVX__) || defined(__AVX2__)
+ if (k % 8)
+ return false;
+ tinyBLAS<8, __m256, __m256, float, float, float> tb{
+ k, (const float *)A, lda,
+ (const float *)B, ldb,
+ (float *)C, ldc,
+ ith, nth};
+ tb.matmul(m, n);
+ return true;
+#elif defined(__ARM_NEON)
+ if (n < 4)
+ return false;
+ if (k % 4)
+ return false;
+ tinyBLAS<4, float32x4_t, float32x4_t, float, float, float> tb{
+ k, (const float *)A, lda,
+ (const float *)B, ldb,
+ (float *)C, ldc,
+ ith, nth};
+ tb.matmul(m, n);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ case GGML_TYPE_F16: {
+#if defined(__AVX512F__)
+ if (k % 16)
+ return false;
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ tinyBLAS<16, __m512, __m512, ggml_fp16_t, float, float> tb{
+ k, (const ggml_fp16_t *)A, lda,
+ (const float *)B, ldb,
+ (float *)C, ldc,
+ ith, nth};
+ tb.matmul(m, n);
+ return true;
+#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
+ if (k % 8)
+ return false;
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ tinyBLAS<8, __m256, __m256, ggml_fp16_t, float, float> tb{
+ k, (const ggml_fp16_t *)A, lda,
+ (const float *)B, ldb,
+ (float *)C, ldc,
+ ith, nth};
+ tb.matmul(m, n);
+ return true;
+#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
+ if (n < 8)
+ return false;
+ if (k % 8)
+ return false;
+ if (Btype != GGML_TYPE_F16)
+ return false;
+ tinyBLAS<8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float> tb{
+ k, (const ggml_fp16_t *)A, lda,
+ (const ggml_fp16_t *)B, ldb,
+ (float *)C, ldc,
+ ith, nth};
+ tb.matmul(m, n);
+ return true;
+#elif defined(__ARM_NEON) && !defined(_MSC_VER)
+ if (k % 4)
+ return false;
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ tinyBLAS<4, float32x4_t, float32x4_t, ggml_fp16_t, float, float> tb{
+ k, (const ggml_fp16_t *)A, lda,
+ (const float *)B, ldb,
+ (float *)C, ldc,
+ ith, nth};
+ tb.matmul(m, n);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ case GGML_TYPE_Q8_0: {
+ if (Btype != GGML_TYPE_Q8_0)
+ return false;
+#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)
+ tinyBLAS_Q0_AVX<block_q8_0, block_q8_0, float> tb{
+ k, (const block_q8_0 *)A, lda,
+ (const block_q8_0 *)B, ldb,
+ (float *)C, ldc,
+ ith, nth};
+ tb.matmul(m, n);
+ return true;
+#elif defined(__ARM_FEATURE_DOTPROD)
+ tinyBLAS_Q0_ARM<block_q8_0> tb{
+ k, (const block_q8_0 *)A, lda,
+ (const block_q8_0 *)B, ldb,
+ (float *)C, ldc,
+ ith, nth};
+ tb.matmul(m, n);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ case GGML_TYPE_Q4_0: {
+ if (Btype != GGML_TYPE_Q8_0)
+ return false;
+#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)
+ tinyBLAS_Q0_AVX<block_q4_0, block_q8_0, float> tb{
+ k, (const block_q4_0 *)A, lda,
+ (const block_q8_0 *)B, ldb,
+ (float *)C, ldc,
+ ith, nth};
+ tb.matmul(m, n);
+ return true;
+#elif defined(__ARM_FEATURE_DOTPROD)
+ tinyBLAS_Q0_ARM<block_q4_0> tb{
+ k, (const block_q4_0 *)A, lda,
+ (const block_q8_0 *)B, ldb,
+ (float *)C, ldc,
+ ith, nth};
+ tb.matmul(m, n);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ default:
+ return false;
+ }
+
+ (void)m;
+ (void)n;
+ (void)k;
+ (void)A;
+ (void)lda;
+ (void)B;
+ (void)ldb;
+ (void)C;
+ (void)ldc;
+ (void)ith;
+ (void)nth;
+ (void)Atype;
+ (void)Btype;
+ (void)Ctype;
+}
diff --git a/llama/sgemm.h b/llama/sgemm.h
new file mode 100644
index 00000000..caf6dd55
--- /dev/null
+++ b/llama/sgemm.h
@@ -0,0 +1,14 @@
+#pragma once
+#include <stdint.h>
+#include <stdbool.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+bool llamafile_sgemm(int64_t, int64_t, int64_t, const void *, int64_t,
+ const void *, int64_t, void *, int64_t, int, int,
+ int, int, int);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama/stb_image.h b/llama/stb_image.h
new file mode 100644
index 00000000..9eedabed
--- /dev/null
+++ b/llama/stb_image.h
@@ -0,0 +1,7988 @@
+/* stb_image - v2.30 - public domain image loader - http://nothings.org/stb
+ no warranty implied; use at your own risk
+
+ Do this:
+ #define STB_IMAGE_IMPLEMENTATION
+ before you include this file in *one* C or C++ file to create the implementation.
+
+ // i.e. it should look like this:
+ #include ...
+ #include ...
+ #include ...
+ #define STB_IMAGE_IMPLEMENTATION
+ #include "stb_image.h"
+
+ You can #define STBI_ASSERT(x) before the #include to avoid using assert.h.
+ And #define STBI_MALLOC, STBI_REALLOC, and STBI_FREE to avoid using malloc,realloc,free
+
+
+ QUICK NOTES:
+ Primarily of interest to game developers and other people who can
+ avoid problematic images and only need the trivial interface
+
+ JPEG baseline & progressive (12 bpc/arithmetic not supported, same as stock IJG lib)
+ PNG 1/2/4/8/16-bit-per-channel
+
+ TGA (not sure what subset, if a subset)
+ BMP non-1bpp, non-RLE
+ PSD (composited view only, no extra channels, 8/16 bit-per-channel)
+
+ GIF (*comp always reports as 4-channel)
+ HDR (radiance rgbE format)
+ PIC (Softimage PIC)
+ PNM (PPM and PGM binary only)
+
+ Animated GIF still needs a proper API, but here's one way to do it:
+ http://gist.github.com/urraka/685d9a6340b26b830d49
+
+ - decode from memory or through FILE (define STBI_NO_STDIO to remove code)
+ - decode from arbitrary I/O callbacks
+ - SIMD acceleration on x86/x64 (SSE2) and ARM (NEON)
+
+ Full documentation under "DOCUMENTATION" below.
+
+
+LICENSE
+
+ See end of file for license information.
+
+RECENT REVISION HISTORY:
+
+ 2.30 (2024-05-31) avoid erroneous gcc warning
+ 2.29 (2023-05-xx) optimizations
+ 2.28 (2023-01-29) many error fixes, security errors, just tons of stuff
+ 2.27 (2021-07-11) document stbi_info better, 16-bit PNM support, bug fixes
+ 2.26 (2020-07-13) many minor fixes
+ 2.25 (2020-02-02) fix warnings
+ 2.24 (2020-02-02) fix warnings; thread-local failure_reason and flip_vertically
+ 2.23 (2019-08-11) fix clang static analysis warning
+ 2.22 (2019-03-04) gif fixes, fix warnings
+ 2.21 (2019-02-25) fix typo in comment
+ 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs
+ 2.19 (2018-02-11) fix warning
+ 2.18 (2018-01-30) fix warnings
+ 2.17 (2018-01-29) bugfix, 1-bit BMP, 16-bitness query, fix warnings
+ 2.16 (2017-07-23) all functions have 16-bit variants; optimizations; bugfixes
+ 2.15 (2017-03-18) fix png-1,2,4; all Imagenet JPGs; no runtime SSE detection on GCC
+ 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs
+ 2.13 (2016-12-04) experimental 16-bit API, only for PNG so far; fixes
+ 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes
+ 2.11 (2016-04-02) 16-bit PNGS; enable SSE2 in non-gcc x64
+ RGB-format JPEG; remove white matting in PSD;
+ allocate large structures on the stack;
+ correct channel count for PNG & BMP
+ 2.10 (2016-01-22) avoid warning introduced in 2.09
+ 2.09 (2016-01-16) 16-bit TGA; comments in PNM files; STBI_REALLOC_SIZED
+
+ See end of file for full revision history.
+
+
+ ============================ Contributors =========================
+
+ Image formats Extensions, features
+ Sean Barrett (jpeg, png, bmp) Jetro Lauha (stbi_info)
+ Nicolas Schulz (hdr, psd) Martin "SpartanJ" Golini (stbi_info)
+ Jonathan Dummer (tga) James "moose2000" Brown (iPhone PNG)
+ Jean-Marc Lienher (gif) Ben "Disch" Wenger (io callbacks)
+ Tom Seddon (pic) Omar Cornut (1/2/4-bit PNG)
+ Thatcher Ulrich (psd) Nicolas Guillemot (vertical flip)
+ Ken Miller (pgm, ppm) Richard Mitton (16-bit PSD)
+ github:urraka (animated gif) Junggon Kim (PNM comments)
+ Christopher Forseth (animated gif) Daniel Gibson (16-bit TGA)
+ socks-the-fox (16-bit PNG)
+ Jeremy Sawicki (handle all ImageNet JPGs)
+ Optimizations & bugfixes Mikhail Morozov (1-bit BMP)
+ Fabian "ryg" Giesen Anael Seghezzi (is-16-bit query)
+ Arseny Kapoulkine Simon Breuss (16-bit PNM)
+ John-Mark Allen
+ Carmelo J Fdez-Aguera
+
+ Bug & warning fixes
+ Marc LeBlanc David Woo Guillaume George Martins Mozeiko
+ Christpher Lloyd Jerry Jansson Joseph Thomson Blazej Dariusz Roszkowski
+ Phil Jordan Dave Moore Roy Eltham
+ Hayaki Saito Nathan Reed Won Chun
+ Luke Graham Johan Duparc Nick Verigakis the Horde3D community
+ Thomas Ruf Ronny Chevalier github:rlyeh
+ Janez Zemva John Bartholomew Michal Cichon github:romigrou
+ Jonathan Blow Ken Hamada Tero Hanninen github:svdijk
+ Eugene Golushkov Laurent Gomila Cort Stratton github:snagar
+ Aruelien Pocheville Sergio Gonzalez Thibault Reuille github:Zelex
+ Cass Everitt Ryamond Barbiero github:grim210
+ Paul Du Bois Engin Manap Aldo Culquicondor github:sammyhw
+ Philipp Wiesemann Dale Weiler Oriol Ferrer Mesia github:phprus
+ Josh Tobin Neil Bickford Matthew Gregan github:poppolopoppo
+ Julian Raschke Gregory Mullen Christian Floisand github:darealshinji
+ Baldur Karlsson Kevin Schmidt JR Smith github:Michaelangel007
+ Brad Weinberger Matvey Cherevko github:mosra
+ Luca Sas Alexander Veselov Zack Middleton [reserved]
+ Ryan C. Gordon [reserved] [reserved]
+ DO NOT ADD YOUR NAME HERE
+
+ Jacko Dirks
+
+ To add your name to the credits, pick a random blank space in the middle and fill it.
+ 80% of merge conflicts on stb PRs are due to people adding their name at the end
+ of the credits.
+*/
+
+#ifndef STBI_INCLUDE_STB_IMAGE_H
+#define STBI_INCLUDE_STB_IMAGE_H
+
+// DOCUMENTATION
+//
+// Limitations:
+// - no 12-bit-per-channel JPEG
+// - no JPEGs with arithmetic coding
+// - GIF always returns *comp=4
+//
+// Basic usage (see HDR discussion below for HDR usage):
+// int x,y,n;
+// unsigned char *data = stbi_load(filename, &x, &y, &n, 0);
+// // ... process data if not NULL ...
+// // ... x = width, y = height, n = # 8-bit components per pixel ...
+// // ... replace '0' with '1'..'4' to force that many components per pixel
+// // ... but 'n' will always be the number that it would have been if you said 0
+// stbi_image_free(data);
+//
+// Standard parameters:
+// int *x -- outputs image width in pixels
+// int *y -- outputs image height in pixels
+// int *channels_in_file -- outputs # of image components in image file
+// int desired_channels -- if non-zero, # of image components requested in result
+//
+// The return value from an image loader is an 'unsigned char *' which points
+// to the pixel data, or NULL on an allocation failure or if the image is
+// corrupt or invalid. The pixel data consists of *y scanlines of *x pixels,
+// with each pixel consisting of N interleaved 8-bit components; the first
+// pixel pointed to is top-left-most in the image. There is no padding between
+// image scanlines or between pixels, regardless of format. The number of
+// components N is 'desired_channels' if desired_channels is non-zero, or
+// *channels_in_file otherwise. If desired_channels is non-zero,
+// *channels_in_file has the number of components that _would_ have been
+// output otherwise. E.g. if you set desired_channels to 4, you will always
+// get RGBA output, but you can check *channels_in_file to see if it's trivially
+// opaque because e.g. there were only 3 channels in the source image.
+//
+// An output image with N components has the following components interleaved
+// in this order in each pixel:
+//
+// N=#comp components
+// 1 grey
+// 2 grey, alpha
+// 3 red, green, blue
+// 4 red, green, blue, alpha
+//
+// If image loading fails for any reason, the return value will be NULL,
+// and *x, *y, *channels_in_file will be unchanged. The function
+// stbi_failure_reason() can be queried for an extremely brief, end-user
+// unfriendly explanation of why the load failed. Define STBI_NO_FAILURE_STRINGS
+// to avoid compiling these strings at all, and STBI_FAILURE_USERMSG to get slightly
+// more user-friendly ones.
+//
+// Paletted PNG, BMP, GIF, and PIC images are automatically depalettized.
+//
+// To query the width, height and component count of an image without having to
+// decode the full file, you can use the stbi_info family of functions:
+//
+// int x,y,n,ok;
+// ok = stbi_info(filename, &x, &y, &n);
+// // returns ok=1 and sets x, y, n if image is a supported format,
+// // 0 otherwise.
+//
+// Note that stb_image pervasively uses ints in its public API for sizes,
+// including sizes of memory buffers. This is now part of the API and thus
+// hard to change without causing breakage. As a result, the various image
+// loaders all have certain limits on image size; these differ somewhat
+// by format but generally boil down to either just under 2GB or just under
+// 1GB. When the decoded image would be larger than this, stb_image decoding
+// will fail.
+//
+// Additionally, stb_image will reject image files that have any of their
+// dimensions set to a larger value than the configurable STBI_MAX_DIMENSIONS,
+// which defaults to 2**24 = 16777216 pixels. Due to the above memory limit,
+// the only way to have an image with such dimensions load correctly
+// is for it to have a rather extreme aspect ratio. Either way, the
+// assumption here is that such larger images are likely to be malformed
+// or malicious. If you do need to load an image with individual dimensions
+// larger than that, and it still fits in the overall size limit, you can
+// #define STBI_MAX_DIMENSIONS on your own to be something larger.
+//
+// ===========================================================================
+//
+// UNICODE:
+//
+// If compiling for Windows and you wish to use Unicode filenames, compile
+// with
+// #define STBI_WINDOWS_UTF8
+// and pass utf8-encoded filenames. Call stbi_convert_wchar_to_utf8 to convert
+// Windows wchar_t filenames to utf8.
+//
+// ===========================================================================
+//
+// Philosophy
+//
+// stb libraries are designed with the following priorities:
+//
+// 1. easy to use
+// 2. easy to maintain
+// 3. good performance
+//
+// Sometimes I let "good performance" creep up in priority over "easy to maintain",
+// and for best performance I may provide less-easy-to-use APIs that give higher
+// performance, in addition to the easy-to-use ones. Nevertheless, it's important
+// to keep in mind that from the standpoint of you, a client of this library,
+// all you care about is #1 and #3, and stb libraries DO NOT emphasize #3 above all.
+//
+// Some secondary priorities arise directly from the first two, some of which
+// provide more explicit reasons why performance can't be emphasized.
+//
+// - Portable ("ease of use")
+// - Small source code footprint ("easy to maintain")
+// - No dependencies ("ease of use")
+//
+// ===========================================================================
+//
+// I/O callbacks
+//
+// I/O callbacks allow you to read from arbitrary sources, like packaged
+// files or some other source. Data read from callbacks are processed
+// through a small internal buffer (currently 128 bytes) to try to reduce
+// overhead.
+//
+// The three functions you must define are "read" (reads some bytes of data),
+// "skip" (skips some bytes of data), "eof" (reports if the stream is at the end).
+//
+// ===========================================================================
+//
+// SIMD support
+//
+// The JPEG decoder will try to automatically use SIMD kernels on x86 when
+// supported by the compiler. For ARM Neon support, you must explicitly
+// request it.
+//
+// (The old do-it-yourself SIMD API is no longer supported in the current
+// code.)
+//
+// On x86, SSE2 will automatically be used when available based on a run-time
+// test; if not, the generic C versions are used as a fall-back. On ARM targets,
+// the typical path is to have separate builds for NEON and non-NEON devices
+// (at least this is true for iOS and Android). Therefore, the NEON support is
+// toggled by a build flag: define STBI_NEON to get NEON loops.
+//
+// If for some reason you do not want to use any of SIMD code, or if
+// you have issues compiling it, you can disable it entirely by
+// defining STBI_NO_SIMD.
+//
+// ===========================================================================
+//
+// HDR image support (disable by defining STBI_NO_HDR)
+//
+// stb_image supports loading HDR images in general, and currently the Radiance
+// .HDR file format specifically. You can still load any file through the existing
+// interface; if you attempt to load an HDR file, it will be automatically remapped
+// to LDR, assuming gamma 2.2 and an arbitrary scale factor defaulting to 1;
+// both of these constants can be reconfigured through this interface:
+//
+// stbi_hdr_to_ldr_gamma(2.2f);
+// stbi_hdr_to_ldr_scale(1.0f);
+//
+// (note, do not use _inverse_ constants; stbi_image will invert them
+// appropriately).
+//
+// Additionally, there is a new, parallel interface for loading files as
+// (linear) floats to preserve the full dynamic range:
+//
+// float *data = stbi_loadf(filename, &x, &y, &n, 0);
+//
+// If you load LDR images through this interface, those images will
+// be promoted to floating point values, run through the inverse of
+// constants corresponding to the above:
+//
+// stbi_ldr_to_hdr_scale(1.0f);
+// stbi_ldr_to_hdr_gamma(2.2f);
+//
+// Finally, given a filename (or an open file or memory block--see header
+// file for details) containing image data, you can query for the "most
+// appropriate" interface to use (that is, whether the image is HDR or
+// not), using:
+//
+// stbi_is_hdr(char *filename);
+//
+// ===========================================================================
+//
+// iPhone PNG support:
+//
+// We optionally support converting iPhone-formatted PNGs (which store
+// premultiplied BGRA) back to RGB, even though they're internally encoded
+// differently. To enable this conversion, call
+// stbi_convert_iphone_png_to_rgb(1).
+//
+// Call stbi_set_unpremultiply_on_load(1) as well to force a divide per
+// pixel to remove any premultiplied alpha *only* if the image file explicitly
+// says there's premultiplied data (currently only happens in iPhone images,
+// and only if iPhone convert-to-rgb processing is on).
+//
+// ===========================================================================
+//
+// ADDITIONAL CONFIGURATION
+//
+// - You can suppress implementation of any of the decoders to reduce
+// your code footprint by #defining one or more of the following
+// symbols before creating the implementation.
+//
+// STBI_NO_JPEG
+// STBI_NO_PNG
+// STBI_NO_BMP
+// STBI_NO_PSD
+// STBI_NO_TGA
+// STBI_NO_GIF
+// STBI_NO_HDR
+// STBI_NO_PIC
+// STBI_NO_PNM (.ppm and .pgm)
+//
+// - You can request *only* certain decoders and suppress all other ones
+// (this will be more forward-compatible, as addition of new decoders
+// doesn't require you to disable them explicitly):
+//
+// STBI_ONLY_JPEG
+// STBI_ONLY_PNG
+// STBI_ONLY_BMP
+// STBI_ONLY_PSD
+// STBI_ONLY_TGA
+// STBI_ONLY_GIF
+// STBI_ONLY_HDR
+// STBI_ONLY_PIC
+// STBI_ONLY_PNM (.ppm and .pgm)
+//
+// - If you use STBI_NO_PNG (or _ONLY_ without PNG), and you still
+// want the zlib decoder to be available, #define STBI_SUPPORT_ZLIB
+//
+// - If you define STBI_MAX_DIMENSIONS, stb_image will reject images greater
+// than that size (in either width or height) without further processing.
+// This is to let programs in the wild set an upper bound to prevent
+// denial-of-service attacks on untrusted data, as one could generate a
+// valid image of gigantic dimensions and force stb_image to allocate a
+// huge block of memory and spend disproportionate time decoding it. By
+// default this is set to (1 << 24), which is 16777216, but that's still
+// very big.
+
+#ifndef STBI_NO_STDIO
+#include <stdio.h>
+#endif // STBI_NO_STDIO
+
+#define STBI_VERSION 1
+
+enum
+{
+ STBI_default = 0, // only used for desired_channels
+
+ STBI_grey = 1,
+ STBI_grey_alpha = 2,
+ STBI_rgb = 3,
+ STBI_rgb_alpha = 4
+};
+
+#include <stdlib.h>
+typedef unsigned char stbi_uc;
+typedef unsigned short stbi_us;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef STBIDEF
+#ifdef STB_IMAGE_STATIC
+#define STBIDEF static
+#else
+#define STBIDEF extern
+#endif
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// PRIMARY API - works on images of any type
+//
+
+//
+// load image by filename, open file, or memory buffer
+//
+
+typedef struct
+{
+ int (*read) (void *user,char *data,int size); // fill 'data' with 'size' bytes. return number of bytes actually read
+ void (*skip) (void *user,int n); // skip the next 'n' bytes, or 'unget' the last -n bytes if negative
+ int (*eof) (void *user); // returns nonzero if we are at end of file/data
+} stbi_io_callbacks;
+
+////////////////////////////////////
+//
+// 8-bits-per-channel interface
+//
+
+STBIDEF stbi_uc *stbi_load_from_memory (stbi_uc const *buffer, int len , int *x, int *y, int *channels_in_file, int desired_channels);
+STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk , void *user, int *x, int *y, int *channels_in_file, int desired_channels);
+
+#ifndef STBI_NO_STDIO
+STBIDEF stbi_uc *stbi_load (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels);
+STBIDEF stbi_uc *stbi_load_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels);
+// for stbi_load_from_file, file pointer is left pointing immediately after image
+#endif
+
+#ifndef STBI_NO_GIF
+STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp);
+#endif
+
+#ifdef STBI_WINDOWS_UTF8
+STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input);
+#endif
+
+////////////////////////////////////
+//
+// 16-bits-per-channel interface
+//
+
+STBIDEF stbi_us *stbi_load_16_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels);
+STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels);
+
+#ifndef STBI_NO_STDIO
+STBIDEF stbi_us *stbi_load_16 (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels);
+STBIDEF stbi_us *stbi_load_from_file_16(FILE *f, int *x, int *y, int *channels_in_file, int desired_channels);
+#endif
+
+////////////////////////////////////
+//
+// float-per-channel interface
+//
+#ifndef STBI_NO_LINEAR
+ STBIDEF float *stbi_loadf_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels);
+ STBIDEF float *stbi_loadf_from_callbacks (stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels);
+
+ #ifndef STBI_NO_STDIO
+ STBIDEF float *stbi_loadf (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels);
+ STBIDEF float *stbi_loadf_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels);
+ #endif
+#endif
+
+#ifndef STBI_NO_HDR
+ STBIDEF void stbi_hdr_to_ldr_gamma(float gamma);
+ STBIDEF void stbi_hdr_to_ldr_scale(float scale);
+#endif // STBI_NO_HDR
+
+#ifndef STBI_NO_LINEAR
+ STBIDEF void stbi_ldr_to_hdr_gamma(float gamma);
+ STBIDEF void stbi_ldr_to_hdr_scale(float scale);
+#endif // STBI_NO_LINEAR
+
+// stbi_is_hdr is always defined, but always returns false if STBI_NO_HDR
+STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user);
+STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len);
+#ifndef STBI_NO_STDIO
+STBIDEF int stbi_is_hdr (char const *filename);
+STBIDEF int stbi_is_hdr_from_file(FILE *f);
+#endif // STBI_NO_STDIO
+
+
+// get a VERY brief reason for failure
+// on most compilers (and ALL modern mainstream compilers) this is threadsafe
+STBIDEF const char *stbi_failure_reason (void);
+
+// free the loaded image -- this is just free()
+STBIDEF void stbi_image_free (void *retval_from_stbi_load);
+
+// get image dimensions & components without fully decoding
+STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp);
+STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp);
+STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len);
+STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *clbk, void *user);
+
+#ifndef STBI_NO_STDIO
+STBIDEF int stbi_info (char const *filename, int *x, int *y, int *comp);
+STBIDEF int stbi_info_from_file (FILE *f, int *x, int *y, int *comp);
+STBIDEF int stbi_is_16_bit (char const *filename);
+STBIDEF int stbi_is_16_bit_from_file(FILE *f);
+#endif
+
+
+
+// for image formats that explicitly notate that they have premultiplied alpha,
+// we just return the colors as stored in the file. set this flag to force
+// unpremultiplication. results are undefined if the unpremultiply overflow.
+STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply);
+
+// indicate whether we should process iphone images back to canonical format,
+// or just pass them through "as-is"
+STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert);
+
+// flip the image vertically, so the first pixel in the output array is the bottom left
+STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip);
+
+// as above, but only applies to images loaded on the thread that calls the function
+// this function is only available if your compiler supports thread-local variables;
+// calling it will fail to link if your compiler doesn't
+STBIDEF void stbi_set_unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply);
+STBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert);
+STBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip);
+
+// ZLIB client - used by PNG, available for other purposes
+
+STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen);
+STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header);
+STBIDEF char *stbi_zlib_decode_malloc(const char *buffer, int len, int *outlen);
+STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, const char *ibuffer, int ilen);
+
+STBIDEF char *stbi_zlib_decode_noheader_malloc(const char *buffer, int len, int *outlen);
+STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+//
+//
+//// end header file /////////////////////////////////////////////////////
+#endif // STBI_INCLUDE_STB_IMAGE_H
+
+#ifdef STB_IMAGE_IMPLEMENTATION
+
+#if defined(STBI_ONLY_JPEG) || defined(STBI_ONLY_PNG) || defined(STBI_ONLY_BMP) \
+ || defined(STBI_ONLY_TGA) || defined(STBI_ONLY_GIF) || defined(STBI_ONLY_PSD) \
+ || defined(STBI_ONLY_HDR) || defined(STBI_ONLY_PIC) || defined(STBI_ONLY_PNM) \
+ || defined(STBI_ONLY_ZLIB)
+ #ifndef STBI_ONLY_JPEG
+ #define STBI_NO_JPEG
+ #endif
+ #ifndef STBI_ONLY_PNG
+ #define STBI_NO_PNG
+ #endif
+ #ifndef STBI_ONLY_BMP
+ #define STBI_NO_BMP
+ #endif
+ #ifndef STBI_ONLY_PSD
+ #define STBI_NO_PSD
+ #endif
+ #ifndef STBI_ONLY_TGA
+ #define STBI_NO_TGA
+ #endif
+ #ifndef STBI_ONLY_GIF
+ #define STBI_NO_GIF
+ #endif
+ #ifndef STBI_ONLY_HDR
+ #define STBI_NO_HDR
+ #endif
+ #ifndef STBI_ONLY_PIC
+ #define STBI_NO_PIC
+ #endif
+ #ifndef STBI_ONLY_PNM
+ #define STBI_NO_PNM
+ #endif
+#endif
+
+#if defined(STBI_NO_PNG) && !defined(STBI_SUPPORT_ZLIB) && !defined(STBI_NO_ZLIB)
+#define STBI_NO_ZLIB
+#endif
+
+
+#include <stdarg.h>
+#include <stddef.h> // ptrdiff_t on osx
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR)
+#include <math.h> // ldexp, pow
+#endif
+
+#ifndef STBI_NO_STDIO
+#include <stdio.h>
+#endif
+
+#ifndef STBI_ASSERT
+#include <assert.h>
+#define STBI_ASSERT(x) assert(x)
+#endif
+
+#ifdef __cplusplus
+#define STBI_EXTERN extern "C"
+#else
+#define STBI_EXTERN extern
+#endif
+
+
+#ifndef _MSC_VER
+ #ifdef __cplusplus
+ #define stbi_inline inline
+ #else
+ #define stbi_inline
+ #endif
+#else
+ #define stbi_inline __forceinline
+#endif
+
+#ifndef STBI_NO_THREAD_LOCALS
+ #if defined(__cplusplus) && __cplusplus >= 201103L
+ #define STBI_THREAD_LOCAL thread_local
+ #elif defined(__GNUC__) && __GNUC__ < 5
+ #define STBI_THREAD_LOCAL __thread
+ #elif defined(_MSC_VER)
+ #define STBI_THREAD_LOCAL __declspec(thread)
+ #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__)
+ #define STBI_THREAD_LOCAL _Thread_local
+ #endif
+
+ #ifndef STBI_THREAD_LOCAL
+ #if defined(__GNUC__)
+ #define STBI_THREAD_LOCAL __thread
+ #endif
+ #endif
+#endif
+
+#if defined(_MSC_VER) || defined(__SYMBIAN32__)
+typedef unsigned short stbi__uint16;
+typedef signed short stbi__int16;
+typedef unsigned int stbi__uint32;
+typedef signed int stbi__int32;
+#else
+#include <stdint.h>
+typedef uint16_t stbi__uint16;
+typedef int16_t stbi__int16;
+typedef uint32_t stbi__uint32;
+typedef int32_t stbi__int32;
+#endif
+
+// should produce compiler error if size is wrong
+typedef unsigned char validate_uint32[sizeof(stbi__uint32)==4 ? 1 : -1];
+
+#ifdef _MSC_VER
+#define STBI_NOTUSED(v) (void)(v)
+#else
+#define STBI_NOTUSED(v) (void)sizeof(v)
+#endif
+
+#ifdef _MSC_VER
+#define STBI_HAS_LROTL
+#endif
+
+#ifdef STBI_HAS_LROTL
+ #define stbi_lrot(x,y) _lrotl(x,y)
+#else
+ #define stbi_lrot(x,y) (((x) << (y)) | ((x) >> (-(y) & 31)))
+#endif
+
+#if defined(STBI_MALLOC) && defined(STBI_FREE) && (defined(STBI_REALLOC) || defined(STBI_REALLOC_SIZED))
+// ok
+#elif !defined(STBI_MALLOC) && !defined(STBI_FREE) && !defined(STBI_REALLOC) && !defined(STBI_REALLOC_SIZED)
+// ok
+#else
+#error "Must define all or none of STBI_MALLOC, STBI_FREE, and STBI_REALLOC (or STBI_REALLOC_SIZED)."
+#endif
+
+#ifndef STBI_MALLOC
+#define STBI_MALLOC(sz) malloc(sz)
+#define STBI_REALLOC(p,newsz) realloc(p,newsz)
+#define STBI_FREE(p) free(p)
+#endif
+
+#ifndef STBI_REALLOC_SIZED
+#define STBI_REALLOC_SIZED(p,oldsz,newsz) STBI_REALLOC(p,newsz)
+#endif
+
+// x86/x64 detection
+#if defined(__x86_64__) || defined(_M_X64)
+#define STBI__X64_TARGET
+#elif defined(__i386) || defined(_M_IX86)
+#define STBI__X86_TARGET
+#endif
+
+#if defined(__GNUC__) && defined(STBI__X86_TARGET) && !defined(__SSE2__) && !defined(STBI_NO_SIMD)
+// gcc doesn't support sse2 intrinsics unless you compile with -msse2,
+// which in turn means it gets to use SSE2 everywhere. This is unfortunate,
+// but previous attempts to provide the SSE2 functions with runtime
+// detection caused numerous issues. The way architecture extensions are
+// exposed in GCC/Clang is, sadly, not really suited for one-file libs.
+// New behavior: if compiled with -msse2, we use SSE2 without any
+// detection; if not, we don't use it at all.
+#define STBI_NO_SIMD
+#endif
+
+#if defined(__MINGW32__) && defined(STBI__X86_TARGET) && !defined(STBI_MINGW_ENABLE_SSE2) && !defined(STBI_NO_SIMD)
+// Note that __MINGW32__ doesn't actually mean 32-bit, so we have to avoid STBI__X64_TARGET
+//
+// 32-bit MinGW wants ESP to be 16-byte aligned, but this is not in the
+// Windows ABI and VC++ as well as Windows DLLs don't maintain that invariant.
+// As a result, enabling SSE2 on 32-bit MinGW is dangerous when not
+// simultaneously enabling "-mstackrealign".
+//
+// See https://github.com/nothings/stb/issues/81 for more information.
+//
+// So default to no SSE2 on 32-bit MinGW. If you've read this far and added
+// -mstackrealign to your build settings, feel free to #define STBI_MINGW_ENABLE_SSE2.
+#define STBI_NO_SIMD
+#endif
+
+#if !defined(STBI_NO_SIMD) && (defined(STBI__X86_TARGET) || defined(STBI__X64_TARGET))
+#define STBI_SSE2
+#include <emmintrin.h>
+
+#ifdef _MSC_VER
+
+#if _MSC_VER >= 1400 // not VC6
+#include <intrin.h> // __cpuid
+static int stbi__cpuid3(void)
+{
+ int info[4];
+ __cpuid(info,1);
+ return info[3];
+}
+#else
+static int stbi__cpuid3(void)
+{
+ int res;
+ __asm {
+ mov eax,1
+ cpuid
+ mov res,edx
+ }
+ return res;
+}
+#endif
+
+#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name
+
+#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2)
+static int stbi__sse2_available(void)
+{
+ int info3 = stbi__cpuid3();
+ return ((info3 >> 26) & 1) != 0;
+}
+#endif
+
+#else // assume GCC-style if not VC++
+#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16)))
+
+#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2)
+static int stbi__sse2_available(void)
+{
+ // If we're even attempting to compile this on GCC/Clang, that means
+ // -msse2 is on, which means the compiler is allowed to use SSE2
+ // instructions at will, and so are we.
+ return 1;
+}
+#endif
+
+#endif
+#endif
+
+// ARM NEON
+#if defined(STBI_NO_SIMD) && defined(STBI_NEON)
+#undef STBI_NEON
+#endif
+
+#ifdef STBI_NEON
+#include <arm_neon.h>
+#ifdef _MSC_VER
+#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name
+#else
+#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16)))
+#endif
+#endif
+
+#ifndef STBI_SIMD_ALIGN
+#define STBI_SIMD_ALIGN(type, name) type name
+#endif
+
+#ifndef STBI_MAX_DIMENSIONS
+#define STBI_MAX_DIMENSIONS (1 << 24)
+#endif
+
+///////////////////////////////////////////////
+//
+// stbi__context struct and start_xxx functions
+
+// stbi__context structure is our basic context used by all images, so it
+// contains all the IO context, plus some basic image information
+typedef struct
+{
+ stbi__uint32 img_x, img_y;
+ int img_n, img_out_n;
+
+ stbi_io_callbacks io;
+ void *io_user_data;
+
+ int read_from_callbacks;
+ int buflen;
+ stbi_uc buffer_start[128];
+ int callback_already_read;
+
+ stbi_uc *img_buffer, *img_buffer_end;
+ stbi_uc *img_buffer_original, *img_buffer_original_end;
+} stbi__context;
+
+
+static void stbi__refill_buffer(stbi__context *s);
+
+// initialize a memory-decode context
+static void stbi__start_mem(stbi__context *s, stbi_uc const *buffer, int len)
+{
+ s->io.read = NULL;
+ s->read_from_callbacks = 0;
+ s->callback_already_read = 0;
+ s->img_buffer = s->img_buffer_original = (stbi_uc *) buffer;
+ s->img_buffer_end = s->img_buffer_original_end = (stbi_uc *) buffer+len;
+}
+
+// initialize a callback-based context
+static void stbi__start_callbacks(stbi__context *s, stbi_io_callbacks *c, void *user)
+{
+ s->io = *c;
+ s->io_user_data = user;
+ s->buflen = sizeof(s->buffer_start);
+ s->read_from_callbacks = 1;
+ s->callback_already_read = 0;
+ s->img_buffer = s->img_buffer_original = s->buffer_start;
+ stbi__refill_buffer(s);
+ s->img_buffer_original_end = s->img_buffer_end;
+}
+
+#ifndef STBI_NO_STDIO
+
+static int stbi__stdio_read(void *user, char *data, int size)
+{
+ return (int) fread(data,1,size,(FILE*) user);
+}
+
+static void stbi__stdio_skip(void *user, int n)
+{
+ int ch;
+ fseek((FILE*) user, n, SEEK_CUR);
+ ch = fgetc((FILE*) user); /* have to read a byte to reset feof()'s flag */
+ if (ch != EOF) {
+ ungetc(ch, (FILE *) user); /* push byte back onto stream if valid. */
+ }
+}
+
+static int stbi__stdio_eof(void *user)
+{
+ return feof((FILE*) user) || ferror((FILE *) user);
+}
+
+static stbi_io_callbacks stbi__stdio_callbacks =
+{
+ stbi__stdio_read,
+ stbi__stdio_skip,
+ stbi__stdio_eof,
+};
+
+static void stbi__start_file(stbi__context *s, FILE *f)
+{
+ stbi__start_callbacks(s, &stbi__stdio_callbacks, (void *) f);
+}
+
+//static void stop_file(stbi__context *s) { }
+
+#endif // !STBI_NO_STDIO
+
+static void stbi__rewind(stbi__context *s)
+{
+ // conceptually rewind SHOULD rewind to the beginning of the stream,
+ // but we just rewind to the beginning of the initial buffer, because
+ // we only use it after doing 'test', which only ever looks at at most 92 bytes
+ s->img_buffer = s->img_buffer_original;
+ s->img_buffer_end = s->img_buffer_original_end;
+}
+
+enum
+{
+ STBI_ORDER_RGB,
+ STBI_ORDER_BGR
+};
+
+typedef struct
+{
+ int bits_per_channel;
+ int num_channels;
+ int channel_order;
+} stbi__result_info;
+
+#ifndef STBI_NO_JPEG
+static int stbi__jpeg_test(stbi__context *s);
+static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);
+static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp);
+#endif
+
+#ifndef STBI_NO_PNG
+static int stbi__png_test(stbi__context *s);
+static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);
+static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp);
+static int stbi__png_is16(stbi__context *s);
+#endif
+
+#ifndef STBI_NO_BMP
+static int stbi__bmp_test(stbi__context *s);
+static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);
+static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp);
+#endif
+
+#ifndef STBI_NO_TGA
+static int stbi__tga_test(stbi__context *s);
+static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);
+static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp);
+#endif
+
+#ifndef STBI_NO_PSD
+static int stbi__psd_test(stbi__context *s);
+static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc);
+static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp);
+static int stbi__psd_is16(stbi__context *s);
+#endif
+
+#ifndef STBI_NO_HDR
+static int stbi__hdr_test(stbi__context *s);
+static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);
+static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp);
+#endif
+
+#ifndef STBI_NO_PIC
+static int stbi__pic_test(stbi__context *s);
+static void *stbi__pic_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);
+static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp);
+#endif
+
+#ifndef STBI_NO_GIF
+static int stbi__gif_test(stbi__context *s);
+static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);
+static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp);
+static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp);
+#endif
+
+#ifndef STBI_NO_PNM
+static int stbi__pnm_test(stbi__context *s);
+static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);
+static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp);
+static int stbi__pnm_is16(stbi__context *s);
+#endif
+
+static
+#ifdef STBI_THREAD_LOCAL
+STBI_THREAD_LOCAL
+#endif
+const char *stbi__g_failure_reason;
+
+STBIDEF const char *stbi_failure_reason(void)
+{
+ return stbi__g_failure_reason;
+}
+
+#ifndef STBI_NO_FAILURE_STRINGS
+static int stbi__err(const char *str)
+{
+ stbi__g_failure_reason = str;
+ return 0;
+}
+#endif
+
+static void *stbi__malloc(size_t size)
+{
+ return STBI_MALLOC(size);
+}
+
+// stb_image uses ints pervasively, including for offset calculations.
+// therefore the largest decoded image size we can support with the
+// current code, even on 64-bit targets, is INT_MAX. this is not a
+// significant limitation for the intended use case.
+//
+// we do, however, need to make sure our size calculations don't
+// overflow. hence a few helper functions for size calculations that
+// multiply integers together, making sure that they're non-negative
+// and no overflow occurs.
+
+// return 1 if the sum is valid, 0 on overflow.
+// negative terms are considered invalid.
+static int stbi__addsizes_valid(int a, int b)
+{
+ if (b < 0) return 0;
+ // now 0 <= b <= INT_MAX, hence also
+ // 0 <= INT_MAX - b <= INTMAX.
+ // And "a + b <= INT_MAX" (which might overflow) is the
+ // same as a <= INT_MAX - b (no overflow)
+ return a <= INT_MAX - b;
+}
+
+// returns 1 if the product is valid, 0 on overflow.
+// negative factors are considered invalid.
+static int stbi__mul2sizes_valid(int a, int b)
+{
+ if (a < 0 || b < 0) return 0;
+ if (b == 0) return 1; // mul-by-0 is always safe
+ // portable way to check for no overflows in a*b
+ return a <= INT_MAX/b;
+}
+
+#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR)
+// returns 1 if "a*b + add" has no negative terms/factors and doesn't overflow
+static int stbi__mad2sizes_valid(int a, int b, int add)
+{
+ return stbi__mul2sizes_valid(a, b) && stbi__addsizes_valid(a*b, add);
+}
+#endif
+
+// returns 1 if "a*b*c + add" has no negative terms/factors and doesn't overflow
+static int stbi__mad3sizes_valid(int a, int b, int c, int add)
+{
+ return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) &&
+ stbi__addsizes_valid(a*b*c, add);
+}
+
+// returns 1 if "a*b*c*d + add" has no negative terms/factors and doesn't overflow
+#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM)
+static int stbi__mad4sizes_valid(int a, int b, int c, int d, int add)
+{
+ return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) &&
+ stbi__mul2sizes_valid(a*b*c, d) && stbi__addsizes_valid(a*b*c*d, add);
+}
+#endif
+
+#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR)
+// mallocs with size overflow checking
+static void *stbi__malloc_mad2(int a, int b, int add)
+{
+ if (!stbi__mad2sizes_valid(a, b, add)) return NULL;
+ return stbi__malloc(a*b + add);
+}
+#endif
+
+static void *stbi__malloc_mad3(int a, int b, int c, int add)
+{
+ if (!stbi__mad3sizes_valid(a, b, c, add)) return NULL;
+ return stbi__malloc(a*b*c + add);
+}
+
+#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM)
+static void *stbi__malloc_mad4(int a, int b, int c, int d, int add)
+{
+ if (!stbi__mad4sizes_valid(a, b, c, d, add)) return NULL;
+ return stbi__malloc(a*b*c*d + add);
+}
+#endif
+
+// returns 1 if the sum of two signed ints is valid (between -2^31 and 2^31-1 inclusive), 0 on overflow.
+static int stbi__addints_valid(int a, int b)
+{
+ if ((a >= 0) != (b >= 0)) return 1; // a and b have different signs, so no overflow
+ if (a < 0 && b < 0) return a >= INT_MIN - b; // same as a + b >= INT_MIN; INT_MIN - b cannot overflow since b < 0.
+ return a <= INT_MAX - b;
+}
+
+// returns 1 if the product of two ints fits in a signed short, 0 on overflow.
+static int stbi__mul2shorts_valid(int a, int b)
+{
+ if (b == 0 || b == -1) return 1; // multiplication by 0 is always 0; check for -1 so SHRT_MIN/b doesn't overflow
+ if ((a >= 0) == (b >= 0)) return a <= SHRT_MAX/b; // product is positive, so similar to mul2sizes_valid
+ if (b < 0) return a <= SHRT_MIN / b; // same as a * b >= SHRT_MIN
+ return a >= SHRT_MIN / b;
+}
+
+// stbi__err - error
+// stbi__errpf - error returning pointer to float
+// stbi__errpuc - error returning pointer to unsigned char
+
+#ifdef STBI_NO_FAILURE_STRINGS
+ #define stbi__err(x,y) 0
+#elif defined(STBI_FAILURE_USERMSG)
+ #define stbi__err(x,y) stbi__err(y)
+#else
+ #define stbi__err(x,y) stbi__err(x)
+#endif
+
+#define stbi__errpf(x,y) ((float *)(size_t) (stbi__err(x,y)?NULL:NULL))
+#define stbi__errpuc(x,y) ((unsigned char *)(size_t) (stbi__err(x,y)?NULL:NULL))
+
+STBIDEF void stbi_image_free(void *retval_from_stbi_load)
+{
+ STBI_FREE(retval_from_stbi_load);
+}
+
+#ifndef STBI_NO_LINEAR
+static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp);
+#endif
+
+#ifndef STBI_NO_HDR
+static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp);
+#endif
+
+static int stbi__vertically_flip_on_load_global = 0;
+
+STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip)
+{
+ stbi__vertically_flip_on_load_global = flag_true_if_should_flip;
+}
+
+#ifndef STBI_THREAD_LOCAL
+#define stbi__vertically_flip_on_load stbi__vertically_flip_on_load_global
+#else
+static STBI_THREAD_LOCAL int stbi__vertically_flip_on_load_local, stbi__vertically_flip_on_load_set;
+
+STBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip)
+{
+ stbi__vertically_flip_on_load_local = flag_true_if_should_flip;
+ stbi__vertically_flip_on_load_set = 1;
+}
+
+#define stbi__vertically_flip_on_load (stbi__vertically_flip_on_load_set \
+ ? stbi__vertically_flip_on_load_local \
+ : stbi__vertically_flip_on_load_global)
+#endif // STBI_THREAD_LOCAL
+
+static void *stbi__load_main(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc)
+{
+ memset(ri, 0, sizeof(*ri)); // make sure it's initialized if we add new fields
+ ri->bits_per_channel = 8; // default is 8 so most paths don't have to be changed
+ ri->channel_order = STBI_ORDER_RGB; // all current input & output are this, but this is here so we can add BGR order
+ ri->num_channels = 0;
+
+ // test the formats with a very explicit header first (at least a FOURCC
+ // or distinctive magic number first)
+ #ifndef STBI_NO_PNG
+ if (stbi__png_test(s)) return stbi__png_load(s,x,y,comp,req_comp, ri);
+ #endif
+ #ifndef STBI_NO_BMP
+ if (stbi__bmp_test(s)) return stbi__bmp_load(s,x,y,comp,req_comp, ri);
+ #endif
+ #ifndef STBI_NO_GIF
+ if (stbi__gif_test(s)) return stbi__gif_load(s,x,y,comp,req_comp, ri);
+ #endif
+ #ifndef STBI_NO_PSD
+ if (stbi__psd_test(s)) return stbi__psd_load(s,x,y,comp,req_comp, ri, bpc);
+ #else
+ STBI_NOTUSED(bpc);
+ #endif
+ #ifndef STBI_NO_PIC
+ if (stbi__pic_test(s)) return stbi__pic_load(s,x,y,comp,req_comp, ri);
+ #endif
+
+ // then the formats that can end up attempting to load with just 1 or 2
+ // bytes matching expectations; these are prone to false positives, so
+ // try them later
+ #ifndef STBI_NO_JPEG
+ if (stbi__jpeg_test(s)) return stbi__jpeg_load(s,x,y,comp,req_comp, ri);
+ #endif
+ #ifndef STBI_NO_PNM
+ if (stbi__pnm_test(s)) return stbi__pnm_load(s,x,y,comp,req_comp, ri);
+ #endif
+
+ #ifndef STBI_NO_HDR
+ if (stbi__hdr_test(s)) {
+ float *hdr = stbi__hdr_load(s, x,y,comp,req_comp, ri);
+ return stbi__hdr_to_ldr(hdr, *x, *y, req_comp ? req_comp : *comp);
+ }
+ #endif
+
+ #ifndef STBI_NO_TGA
+ // test tga last because it's a crappy test!
+ if (stbi__tga_test(s))
+ return stbi__tga_load(s,x,y,comp,req_comp, ri);
+ #endif
+
+ return stbi__errpuc("unknown image type", "Image not of any known type, or corrupt");
+}
+
+static stbi_uc *stbi__convert_16_to_8(stbi__uint16 *orig, int w, int h, int channels)
+{
+ int i;
+ int img_len = w * h * channels;
+ stbi_uc *reduced;
+
+ reduced = (stbi_uc *) stbi__malloc(img_len);
+ if (reduced == NULL) return stbi__errpuc("outofmem", "Out of memory");
+
+ for (i = 0; i < img_len; ++i)
+ reduced[i] = (stbi_uc)((orig[i] >> 8) & 0xFF); // top half of each byte is sufficient approx of 16->8 bit scaling
+
+ STBI_FREE(orig);
+ return reduced;
+}
+
+static stbi__uint16 *stbi__convert_8_to_16(stbi_uc *orig, int w, int h, int channels)
+{
+ int i;
+ int img_len = w * h * channels;
+ stbi__uint16 *enlarged;
+
+ enlarged = (stbi__uint16 *) stbi__malloc(img_len*2);
+ if (enlarged == NULL) return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory");
+
+ for (i = 0; i < img_len; ++i)
+ enlarged[i] = (stbi__uint16)((orig[i] << 8) + orig[i]); // replicate to high and low byte, maps 0->0, 255->0xffff
+
+ STBI_FREE(orig);
+ return enlarged;
+}
+
+static void stbi__vertical_flip(void *image, int w, int h, int bytes_per_pixel)
+{
+ int row;
+ size_t bytes_per_row = (size_t)w * bytes_per_pixel;
+ stbi_uc temp[2048];
+ stbi_uc *bytes = (stbi_uc *)image;
+
+ for (row = 0; row < (h>>1); row++) {
+ stbi_uc *row0 = bytes + row*bytes_per_row;
+ stbi_uc *row1 = bytes + (h - row - 1)*bytes_per_row;
+ // swap row0 with row1
+ size_t bytes_left = bytes_per_row;
+ while (bytes_left) {
+ size_t bytes_copy = (bytes_left < sizeof(temp)) ? bytes_left : sizeof(temp);
+ memcpy(temp, row0, bytes_copy);
+ memcpy(row0, row1, bytes_copy);
+ memcpy(row1, temp, bytes_copy);
+ row0 += bytes_copy;
+ row1 += bytes_copy;
+ bytes_left -= bytes_copy;
+ }
+ }
+}
+
+#ifndef STBI_NO_GIF
+static void stbi__vertical_flip_slices(void *image, int w, int h, int z, int bytes_per_pixel)
+{
+ int slice;
+ int slice_size = w * h * bytes_per_pixel;
+
+ stbi_uc *bytes = (stbi_uc *)image;
+ for (slice = 0; slice < z; ++slice) {
+ stbi__vertical_flip(bytes, w, h, bytes_per_pixel);
+ bytes += slice_size;
+ }
+}
+#endif
+
+static unsigned char *stbi__load_and_postprocess_8bit(stbi__context *s, int *x, int *y, int *comp, int req_comp)
+{
+ stbi__result_info ri;
+ void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 8);
+
+ if (result == NULL)
+ return NULL;
+
+ // it is the responsibility of the loaders to make sure we get either 8 or 16 bit.
+ STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16);
+
+ if (ri.bits_per_channel != 8) {
+ result = stbi__convert_16_to_8((stbi__uint16 *) result, *x, *y, req_comp == 0 ? *comp : req_comp);
+ ri.bits_per_channel = 8;
+ }
+
+ // @TODO: move stbi__convert_format to here
+
+ if (stbi__vertically_flip_on_load) {
+ int channels = req_comp ? req_comp : *comp;
+ stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi_uc));
+ }
+
+ return (unsigned char *) result;
+}
+
+static stbi__uint16 *stbi__load_and_postprocess_16bit(stbi__context *s, int *x, int *y, int *comp, int req_comp)
+{
+ stbi__result_info ri;
+ void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 16);
+
+ if (result == NULL)
+ return NULL;
+
+ // it is the responsibility of the loaders to make sure we get either 8 or 16 bit.
+ STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16);
+
+ if (ri.bits_per_channel != 16) {
+ result = stbi__convert_8_to_16((stbi_uc *) result, *x, *y, req_comp == 0 ? *comp : req_comp);
+ ri.bits_per_channel = 16;
+ }
+
+ // @TODO: move stbi__convert_format16 to here
+ // @TODO: special case RGB-to-Y (and RGBA-to-YA) for 8-bit-to-16-bit case to keep more precision
+
+ if (stbi__vertically_flip_on_load) {
+ int channels = req_comp ? req_comp : *comp;
+ stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi__uint16));
+ }
+
+ return (stbi__uint16 *) result;
+}
+
+#if !defined(STBI_NO_HDR) && !defined(STBI_NO_LINEAR)
+static void stbi__float_postprocess(float *result, int *x, int *y, int *comp, int req_comp)
+{
+ if (stbi__vertically_flip_on_load && result != NULL) {
+ int channels = req_comp ? req_comp : *comp;
+ stbi__vertical_flip(result, *x, *y, channels * sizeof(float));
+ }
+}
+#endif
+
+#ifndef STBI_NO_STDIO
+
+#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8)
+STBI_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide);
+STBI_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default);
+#endif
+
+#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8)
+STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input)
+{
+ return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, (int) bufferlen, NULL, NULL);
+}
+#endif
+
+static FILE *stbi__fopen(char const *filename, char const *mode)
+{
+ FILE *f;
+#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8)
+ wchar_t wMode[64];
+ wchar_t wFilename[1024];
+ if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename)/sizeof(*wFilename)))
+ return 0;
+
+ if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode)/sizeof(*wMode)))
+ return 0;
+
+#if defined(_MSC_VER) && _MSC_VER >= 1400
+ if (0 != _wfopen_s(&f, wFilename, wMode))
+ f = 0;
+#else
+ f = _wfopen(wFilename, wMode);
+#endif
+
+#elif defined(_MSC_VER) && _MSC_VER >= 1400
+ if (0 != fopen_s(&f, filename, mode))
+ f=0;
+#else
+ f = fopen(filename, mode);
+#endif
+ return f;
+}
+
+
+STBIDEF stbi_uc *stbi_load(char const *filename, int *x, int *y, int *comp, int req_comp)
+{
+ FILE *f = stbi__fopen(filename, "rb");
+ unsigned char *result;
+ if (!f) return stbi__errpuc("can't fopen", "Unable to open file");
+ result = stbi_load_from_file(f,x,y,comp,req_comp);
+ fclose(f);
+ return result;
+}
+
+STBIDEF stbi_uc *stbi_load_from_file(FILE *f, int *x, int *y, int *comp, int req_comp)
+{
+ unsigned char *result;
+ stbi__context s;
+ stbi__start_file(&s,f);
+ result = stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp);
+ if (result) {
+ // need to 'unget' all the characters in the IO buffer
+ fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR);
+ }
+ return result;
+}
+
+STBIDEF stbi__uint16 *stbi_load_from_file_16(FILE *f, int *x, int *y, int *comp, int req_comp)
+{
+ stbi__uint16 *result;
+ stbi__context s;
+ stbi__start_file(&s,f);
+ result = stbi__load_and_postprocess_16bit(&s,x,y,comp,req_comp);
+ if (result) {
+ // need to 'unget' all the characters in the IO buffer
+ fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR);
+ }
+ return result;
+}
+
+STBIDEF stbi_us *stbi_load_16(char const *filename, int *x, int *y, int *comp, int req_comp)
+{
+ FILE *f = stbi__fopen(filename, "rb");
+ stbi__uint16 *result;
+ if (!f) return (stbi_us *) stbi__errpuc("can't fopen", "Unable to open file");
+ result = stbi_load_from_file_16(f,x,y,comp,req_comp);
+ fclose(f);
+ return result;
+}
+
+
+#endif //!STBI_NO_STDIO
+
+STBIDEF stbi_us *stbi_load_16_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels)
+{
+ stbi__context s;
+ stbi__start_mem(&s,buffer,len);
+ return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels);
+}
+
+STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels)
+{
+ stbi__context s;
+ stbi__start_callbacks(&s, (stbi_io_callbacks *)clbk, user);
+ return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels);
+}
+
+STBIDEF stbi_uc *stbi_load_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp)
+{
+ stbi__context s;
+ stbi__start_mem(&s,buffer,len);
+ return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp);
+}
+
+STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp)
+{
+ stbi__context s;
+ stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user);
+ return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp);
+}
+
+#ifndef STBI_NO_GIF
+STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp)
+{
+ unsigned char *result;
+ stbi__context s;
+ stbi__start_mem(&s,buffer,len);
+
+ result = (unsigned char*) stbi__load_gif_main(&s, delays, x, y, z, comp, req_comp);
+ if (stbi__vertically_flip_on_load) {
+ stbi__vertical_flip_slices( result, *x, *y, *z, *comp );
+ }
+
+ return result;
+}
+#endif
+
+#ifndef STBI_NO_LINEAR
+static float *stbi__loadf_main(stbi__context *s, int *x, int *y, int *comp, int req_comp)
+{
+ unsigned char *data;
+ #ifndef STBI_NO_HDR
+ if (stbi__hdr_test(s)) {
+ stbi__result_info ri;
+ float *hdr_data = stbi__hdr_load(s,x,y,comp,req_comp, &ri);
+ if (hdr_data)
+ stbi__float_postprocess(hdr_data,x,y,comp,req_comp);
+ return hdr_data;
+ }
+ #endif
+ data = stbi__load_and_postprocess_8bit(s, x, y, comp, req_comp);
+ if (data)
+ return stbi__ldr_to_hdr(data, *x, *y, req_comp ? req_comp : *comp);
+ return stbi__errpf("unknown image type", "Image not of any known type, or corrupt");
+}
+
+STBIDEF float *stbi_loadf_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp)
+{
+ stbi__context s;
+ stbi__start_mem(&s,buffer,len);
+ return stbi__loadf_main(&s,x,y,comp,req_comp);
+}
+
+STBIDEF float *stbi_loadf_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp)
+{
+ stbi__context s;
+ stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user);
+ return stbi__loadf_main(&s,x,y,comp,req_comp);
+}
+
+#ifndef STBI_NO_STDIO
+STBIDEF float *stbi_loadf(char const *filename, int *x, int *y, int *comp, int req_comp)
+{
+ float *result;
+ FILE *f = stbi__fopen(filename, "rb");
+ if (!f) return stbi__errpf("can't fopen", "Unable to open file");
+ result = stbi_loadf_from_file(f,x,y,comp,req_comp);
+ fclose(f);
+ return result;
+}
+
+STBIDEF float *stbi_loadf_from_file(FILE *f, int *x, int *y, int *comp, int req_comp)
+{
+ stbi__context s;
+ stbi__start_file(&s,f);
+ return stbi__loadf_main(&s,x,y,comp,req_comp);
+}
+#endif // !STBI_NO_STDIO
+
+#endif // !STBI_NO_LINEAR
+
+// these is-hdr-or-not is defined independent of whether STBI_NO_LINEAR is
+// defined, for API simplicity; if STBI_NO_LINEAR is defined, it always
+// reports false!
+
+STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len)
+{
+ #ifndef STBI_NO_HDR
+ stbi__context s;
+ stbi__start_mem(&s,buffer,len);
+ return stbi__hdr_test(&s);
+ #else
+ STBI_NOTUSED(buffer);
+ STBI_NOTUSED(len);
+ return 0;
+ #endif
+}
+
+#ifndef STBI_NO_STDIO
+STBIDEF int stbi_is_hdr (char const *filename)
+{
+ FILE *f = stbi__fopen(filename, "rb");
+ int result=0;
+ if (f) {
+ result = stbi_is_hdr_from_file(f);
+ fclose(f);
+ }
+ return result;
+}
+
+STBIDEF int stbi_is_hdr_from_file(FILE *f)
+{
+ #ifndef STBI_NO_HDR
+ long pos = ftell(f);
+ int res;
+ stbi__context s;
+ stbi__start_file(&s,f);
+ res = stbi__hdr_test(&s);
+ fseek(f, pos, SEEK_SET);
+ return res;
+ #else
+ STBI_NOTUSED(f);
+ return 0;
+ #endif
+}
+#endif // !STBI_NO_STDIO
+
+STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user)
+{
+ #ifndef STBI_NO_HDR
+ stbi__context s;
+ stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user);
+ return stbi__hdr_test(&s);
+ #else
+ STBI_NOTUSED(clbk);
+ STBI_NOTUSED(user);
+ return 0;
+ #endif
+}
+
+#ifndef STBI_NO_LINEAR
+static float stbi__l2h_gamma=2.2f, stbi__l2h_scale=1.0f;
+
+STBIDEF void stbi_ldr_to_hdr_gamma(float gamma) { stbi__l2h_gamma = gamma; }
+STBIDEF void stbi_ldr_to_hdr_scale(float scale) { stbi__l2h_scale = scale; }
+#endif
+
+static float stbi__h2l_gamma_i=1.0f/2.2f, stbi__h2l_scale_i=1.0f;
+
+STBIDEF void stbi_hdr_to_ldr_gamma(float gamma) { stbi__h2l_gamma_i = 1/gamma; }
+STBIDEF void stbi_hdr_to_ldr_scale(float scale) { stbi__h2l_scale_i = 1/scale; }
+
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Common code used by all image loaders
+//
+
+enum
+{
+ STBI__SCAN_load=0,
+ STBI__SCAN_type,
+ STBI__SCAN_header
+};
+
+static void stbi__refill_buffer(stbi__context *s)
+{
+ int n = (s->io.read)(s->io_user_data,(char*)s->buffer_start,s->buflen);
+ s->callback_already_read += (int) (s->img_buffer - s->img_buffer_original);
+ if (n == 0) {
+ // at end of file, treat same as if from memory, but need to handle case
+ // where s->img_buffer isn't pointing to safe memory, e.g. 0-byte file
+ s->read_from_callbacks = 0;
+ s->img_buffer = s->buffer_start;
+ s->img_buffer_end = s->buffer_start+1;
+ *s->img_buffer = 0;
+ } else {
+ s->img_buffer = s->buffer_start;
+ s->img_buffer_end = s->buffer_start + n;
+ }
+}
+
+stbi_inline static stbi_uc stbi__get8(stbi__context *s)
+{
+ if (s->img_buffer < s->img_buffer_end)
+ return *s->img_buffer++;
+ if (s->read_from_callbacks) {
+ stbi__refill_buffer(s);
+ return *s->img_buffer++;
+ }
+ return 0;
+}
+
+#if defined(STBI_NO_JPEG) && defined(STBI_NO_HDR) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM)
+// nothing
+#else
+stbi_inline static int stbi__at_eof(stbi__context *s)
+{
+ if (s->io.read) {
+ if (!(s->io.eof)(s->io_user_data)) return 0;
+ // if feof() is true, check if buffer = end
+ // special case: we've only got the special 0 character at the end
+ if (s->read_from_callbacks == 0) return 1;
+ }
+
+ return s->img_buffer >= s->img_buffer_end;
+}
+#endif
+
+#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC)
+// nothing
+#else
+static void stbi__skip(stbi__context *s, int n)
+{
+ if (n == 0) return; // already there!
+ if (n < 0) {
+ s->img_buffer = s->img_buffer_end;
+ return;
+ }
+ if (s->io.read) {
+ int blen = (int) (s->img_buffer_end - s->img_buffer);
+ if (blen < n) {
+ s->img_buffer = s->img_buffer_end;
+ (s->io.skip)(s->io_user_data, n - blen);
+ return;
+ }
+ }
+ s->img_buffer += n;
+}
+#endif
+
+#if defined(STBI_NO_PNG) && defined(STBI_NO_TGA) && defined(STBI_NO_HDR) && defined(STBI_NO_PNM)
+// nothing
+#else
+static int stbi__getn(stbi__context *s, stbi_uc *buffer, int n)
+{
+ if (s->io.read) {
+ int blen = (int) (s->img_buffer_end - s->img_buffer);
+ if (blen < n) {
+ int res, count;
+
+ memcpy(buffer, s->img_buffer, blen);
+
+ count = (s->io.read)(s->io_user_data, (char*) buffer + blen, n - blen);
+ res = (count == (n-blen));
+ s->img_buffer = s->img_buffer_end;
+ return res;
+ }
+ }
+
+ if (s->img_buffer+n <= s->img_buffer_end) {
+ memcpy(buffer, s->img_buffer, n);
+ s->img_buffer += n;
+ return 1;
+ } else
+ return 0;
+}
+#endif
+
+#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC)
+// nothing
+#else
+static int stbi__get16be(stbi__context *s)
+{
+ int z = stbi__get8(s);
+ return (z << 8) + stbi__get8(s);
+}
+#endif
+
+#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC)
+// nothing
+#else
+static stbi__uint32 stbi__get32be(stbi__context *s)
+{
+ stbi__uint32 z = stbi__get16be(s);
+ return (z << 16) + stbi__get16be(s);
+}
+#endif
+
+#if defined(STBI_NO_BMP) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF)
+// nothing
+#else
+static int stbi__get16le(stbi__context *s)
+{
+ int z = stbi__get8(s);
+ return z + (stbi__get8(s) << 8);
+}
+#endif
+
+#ifndef STBI_NO_BMP
+static stbi__uint32 stbi__get32le(stbi__context *s)
+{
+ stbi__uint32 z = stbi__get16le(s);
+ z += (stbi__uint32)stbi__get16le(s) << 16;
+ return z;
+}
+#endif
+
+#define STBI__BYTECAST(x) ((stbi_uc) ((x) & 255)) // truncate int to byte without warnings
+
+#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM)
+// nothing
+#else
+//////////////////////////////////////////////////////////////////////////////
+//
+// generic converter from built-in img_n to req_comp
+// individual types do this automatically as much as possible (e.g. jpeg
+// does all cases internally since it needs to colorspace convert anyway,
+// and it never has alpha, so very few cases ). png can automatically
+// interleave an alpha=255 channel, but falls back to this for other cases
+//
+// assume data buffer is malloced, so malloc a new one and free that one
+// only failure mode is malloc failing
+
+static stbi_uc stbi__compute_y(int r, int g, int b)
+{
+ return (stbi_uc) (((r*77) + (g*150) + (29*b)) >> 8);
+}
+#endif
+
+#if defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM)
+// nothing
+#else
+static unsigned char *stbi__convert_format(unsigned char *data, int img_n, int req_comp, unsigned int x, unsigned int y)
+{
+ int i,j;
+ unsigned char *good;
+
+ if (req_comp == img_n) return data;
+ STBI_ASSERT(req_comp >= 1 && req_comp <= 4);
+
+ good = (unsigned char *) stbi__malloc_mad3(req_comp, x, y, 0);
+ if (good == NULL) {
+ STBI_FREE(data);
+ return stbi__errpuc("outofmem", "Out of memory");
+ }
+
+ for (j=0; j < (int) y; ++j) {
+ unsigned char *src = data + j * x * img_n ;
+ unsigned char *dest = good + j * x * req_comp;
+
+ #define STBI__COMBO(a,b) ((a)*8+(b))
+ #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b)
+ // convert source image with img_n components to one with req_comp components;
+ // avoid switch per pixel, so use switch per scanline and massive macros
+ switch (STBI__COMBO(img_n, req_comp)) {
+ STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=255; } break;
+ STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break;
+ STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=255; } break;
+ STBI__CASE(2,1) { dest[0]=src[0]; } break;
+ STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break;
+ STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break;
+ STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=255; } break;
+ STBI__CASE(3,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break;
+ STBI__CASE(3,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = 255; } break;
+ STBI__CASE(4,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break;
+ STBI__CASE(4,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = src[3]; } break;
+ STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break;
+ default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return stbi__errpuc("unsupported", "Unsupported format conversion");
+ }
+ #undef STBI__CASE
+ }
+
+ STBI_FREE(data);
+ return good;
+}
+#endif
+
+#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD)
+// nothing
+#else
+static stbi__uint16 stbi__compute_y_16(int r, int g, int b)
+{
+ return (stbi__uint16) (((r*77) + (g*150) + (29*b)) >> 8);
+}
+#endif
+
+#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD)
+// nothing
+#else
+static stbi__uint16 *stbi__convert_format16(stbi__uint16 *data, int img_n, int req_comp, unsigned int x, unsigned int y)
+{
+ int i,j;
+ stbi__uint16 *good;
+
+ if (req_comp == img_n) return data;
+ STBI_ASSERT(req_comp >= 1 && req_comp <= 4);
+
+ good = (stbi__uint16 *) stbi__malloc(req_comp * x * y * 2);
+ if (good == NULL) {
+ STBI_FREE(data);
+ return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory");
+ }
+
+ for (j=0; j < (int) y; ++j) {
+ stbi__uint16 *src = data + j * x * img_n ;
+ stbi__uint16 *dest = good + j * x * req_comp;
+
+ #define STBI__COMBO(a,b) ((a)*8+(b))
+ #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b)
+ // convert source image with img_n components to one with req_comp components;
+ // avoid switch per pixel, so use switch per scanline and massive macros
+ switch (STBI__COMBO(img_n, req_comp)) {
+ STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=0xffff; } break;
+ STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break;
+ STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=0xffff; } break;
+ STBI__CASE(2,1) { dest[0]=src[0]; } break;
+ STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break;
+ STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break;
+ STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=0xffff; } break;
+ STBI__CASE(3,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break;
+ STBI__CASE(3,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = 0xffff; } break;
+ STBI__CASE(4,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break;
+ STBI__CASE(4,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = src[3]; } break;
+ STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break;
+ default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return (stbi__uint16*) stbi__errpuc("unsupported", "Unsupported format conversion");
+ }
+ #undef STBI__CASE
+ }
+
+ STBI_FREE(data);
+ return good;
+}
+#endif
+
+#ifndef STBI_NO_LINEAR
+static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp)
+{
+ int i,k,n;
+ float *output;
+ if (!data) return NULL;
+ output = (float *) stbi__malloc_mad4(x, y, comp, sizeof(float), 0);
+ if (output == NULL) { STBI_FREE(data); return stbi__errpf("outofmem", "Out of memory"); }
+ // compute number of non-alpha components
+ if (comp & 1) n = comp; else n = comp-1;
+ for (i=0; i < x*y; ++i) {
+ for (k=0; k < n; ++k) {
+ output[i*comp + k] = (float) (pow(data[i*comp+k]/255.0f, stbi__l2h_gamma) * stbi__l2h_scale);
+ }
+ }
+ if (n < comp) {
+ for (i=0; i < x*y; ++i) {
+ output[i*comp + n] = data[i*comp + n]/255.0f;
+ }
+ }
+ STBI_FREE(data);
+ return output;
+}
+#endif
+
+#ifndef STBI_NO_HDR
+#define stbi__float2int(x) ((int) (x))
+static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp)
+{
+ int i,k,n;
+ stbi_uc *output;
+ if (!data) return NULL;
+ output = (stbi_uc *) stbi__malloc_mad3(x, y, comp, 0);
+ if (output == NULL) { STBI_FREE(data); return stbi__errpuc("outofmem", "Out of memory"); }
+ // compute number of non-alpha components
+ if (comp & 1) n = comp; else n = comp-1;
+ for (i=0; i < x*y; ++i) {
+ for (k=0; k < n; ++k) {
+ float z = (float) pow(data[i*comp+k]*stbi__h2l_scale_i, stbi__h2l_gamma_i) * 255 + 0.5f;
+ if (z < 0) z = 0;
+ if (z > 255) z = 255;
+ output[i*comp + k] = (stbi_uc) stbi__float2int(z);
+ }
+ if (k < comp) {
+ float z = data[i*comp+k] * 255 + 0.5f;
+ if (z < 0) z = 0;
+ if (z > 255) z = 255;
+ output[i*comp + k] = (stbi_uc) stbi__float2int(z);
+ }
+ }
+ STBI_FREE(data);
+ return output;
+}
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// "baseline" JPEG/JFIF decoder
+//
+// simple implementation
+// - doesn't support delayed output of y-dimension
+// - simple interface (only one output format: 8-bit interleaved RGB)
+// - doesn't try to recover corrupt jpegs
+// - doesn't allow partial loading, loading multiple at once
+// - still fast on x86 (copying globals into locals doesn't help x86)
+// - allocates lots of intermediate memory (full size of all components)
+// - non-interleaved case requires this anyway
+// - allows good upsampling (see next)
+// high-quality
+// - upsampled channels are bilinearly interpolated, even across blocks
+// - quality integer IDCT derived from IJG's 'slow'
+// performance
+// - fast huffman; reasonable integer IDCT
+// - some SIMD kernels for common paths on targets with SSE2/NEON
+// - uses a lot of intermediate memory, could cache poorly
+
+#ifndef STBI_NO_JPEG
+
+// huffman decoding acceleration
+#define FAST_BITS 9 // larger handles more cases; smaller stomps less cache
+
+typedef struct
+{
+ stbi_uc fast[1 << FAST_BITS];
+ // weirdly, repacking this into AoS is a 10% speed loss, instead of a win
+ stbi__uint16 code[256];
+ stbi_uc values[256];
+ stbi_uc size[257];
+ unsigned int maxcode[18];
+ int delta[17]; // old 'firstsymbol' - old 'firstcode'
+} stbi__huffman;
+
+typedef struct
+{
+ stbi__context *s;
+ stbi__huffman huff_dc[4];
+ stbi__huffman huff_ac[4];
+ stbi__uint16 dequant[4][64];
+ stbi__int16 fast_ac[4][1 << FAST_BITS];
+
+// sizes for components, interleaved MCUs
+ int img_h_max, img_v_max;
+ int img_mcu_x, img_mcu_y;
+ int img_mcu_w, img_mcu_h;
+
+// definition of jpeg image component
+ struct
+ {
+ int id;
+ int h,v;
+ int tq;
+ int hd,ha;
+ int dc_pred;
+
+ int x,y,w2,h2;
+ stbi_uc *data;
+ void *raw_data, *raw_coeff;
+ stbi_uc *linebuf;
+ short *coeff; // progressive only
+ int coeff_w, coeff_h; // number of 8x8 coefficient blocks
+ } img_comp[4];
+
+ stbi__uint32 code_buffer; // jpeg entropy-coded buffer
+ int code_bits; // number of valid bits
+ unsigned char marker; // marker seen while filling entropy buffer
+ int nomore; // flag if we saw a marker so must stop
+
+ int progressive;
+ int spec_start;
+ int spec_end;
+ int succ_high;
+ int succ_low;
+ int eob_run;
+ int jfif;
+ int app14_color_transform; // Adobe APP14 tag
+ int rgb;
+
+ int scan_n, order[4];
+ int restart_interval, todo;
+
+// kernels
+ void (*idct_block_kernel)(stbi_uc *out, int out_stride, short data[64]);
+ void (*YCbCr_to_RGB_kernel)(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step);
+ stbi_uc *(*resample_row_hv_2_kernel)(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs);
+} stbi__jpeg;
+
+static int stbi__build_huffman(stbi__huffman *h, int *count)
+{
+ int i,j,k=0;
+ unsigned int code;
+ // build size list for each symbol (from JPEG spec)
+ for (i=0; i < 16; ++i) {
+ for (j=0; j < count[i]; ++j) {
+ h->size[k++] = (stbi_uc) (i+1);
+ if(k >= 257) return stbi__err("bad size list","Corrupt JPEG");
+ }
+ }
+ h->size[k] = 0;
+
+ // compute actual symbols (from jpeg spec)
+ code = 0;
+ k = 0;
+ for(j=1; j <= 16; ++j) {
+ // compute delta to add to code to compute symbol id
+ h->delta[j] = k - code;
+ if (h->size[k] == j) {
+ while (h->size[k] == j)
+ h->code[k++] = (stbi__uint16) (code++);
+ if (code-1 >= (1u << j)) return stbi__err("bad code lengths","Corrupt JPEG");
+ }
+ // compute largest code + 1 for this size, preshifted as needed later
+ h->maxcode[j] = code << (16-j);
+ code <<= 1;
+ }
+ h->maxcode[j] = 0xffffffff;
+
+ // build non-spec acceleration table; 255 is flag for not-accelerated
+ memset(h->fast, 255, 1 << FAST_BITS);
+ for (i=0; i < k; ++i) {
+ int s = h->size[i];
+ if (s <= FAST_BITS) {
+ int c = h->code[i] << (FAST_BITS-s);
+ int m = 1 << (FAST_BITS-s);
+ for (j=0; j < m; ++j) {
+ h->fast[c+j] = (stbi_uc) i;
+ }
+ }
+ }
+ return 1;
+}
+
+// build a table that decodes both magnitude and value of small ACs in
+// one go.
+static void stbi__build_fast_ac(stbi__int16 *fast_ac, stbi__huffman *h)
+{
+ int i;
+ for (i=0; i < (1 << FAST_BITS); ++i) {
+ stbi_uc fast = h->fast[i];
+ fast_ac[i] = 0;
+ if (fast < 255) {
+ int rs = h->values[fast];
+ int run = (rs >> 4) & 15;
+ int magbits = rs & 15;
+ int len = h->size[fast];
+
+ if (magbits && len + magbits <= FAST_BITS) {
+ // magnitude code followed by receive_extend code
+ int k = ((i << len) & ((1 << FAST_BITS) - 1)) >> (FAST_BITS - magbits);
+ int m = 1 << (magbits - 1);
+ if (k < m) k += (~0U << magbits) + 1;
+ // if the result is small enough, we can fit it in fast_ac table
+ if (k >= -128 && k <= 127)
+ fast_ac[i] = (stbi__int16) ((k * 256) + (run * 16) + (len + magbits));
+ }
+ }
+ }
+}
+
+static void stbi__grow_buffer_unsafe(stbi__jpeg *j)
+{
+ do {
+ unsigned int b = j->nomore ? 0 : stbi__get8(j->s);
+ if (b == 0xff) {
+ int c = stbi__get8(j->s);
+ while (c == 0xff) c = stbi__get8(j->s); // consume fill bytes
+ if (c != 0) {
+ j->marker = (unsigned char) c;
+ j->nomore = 1;
+ return;
+ }
+ }
+ j->code_buffer |= b << (24 - j->code_bits);
+ j->code_bits += 8;
+ } while (j->code_bits <= 24);
+}
+
+// (1 << n) - 1
+static const stbi__uint32 stbi__bmask[17]={0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535};
+
+// decode a jpeg huffman value from the bitstream
+stbi_inline static int stbi__jpeg_huff_decode(stbi__jpeg *j, stbi__huffman *h)
+{
+ unsigned int temp;
+ int c,k;
+
+ if (j->code_bits < 16) stbi__grow_buffer_unsafe(j);
+
+ // look at the top FAST_BITS and determine what symbol ID it is,
+ // if the code is <= FAST_BITS
+ c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1);
+ k = h->fast[c];
+ if (k < 255) {
+ int s = h->size[k];
+ if (s > j->code_bits)
+ return -1;
+ j->code_buffer <<= s;
+ j->code_bits -= s;
+ return h->values[k];
+ }
+
+ // naive test is to shift the code_buffer down so k bits are
+ // valid, then test against maxcode. To speed this up, we've
+ // preshifted maxcode left so that it has (16-k) 0s at the
+ // end; in other words, regardless of the number of bits, it
+ // wants to be compared against something shifted to have 16;
+ // that way we don't need to shift inside the loop.
+ temp = j->code_buffer >> 16;
+ for (k=FAST_BITS+1 ; ; ++k)
+ if (temp < h->maxcode[k])
+ break;
+ if (k == 17) {
+ // error! code not found
+ j->code_bits -= 16;
+ return -1;
+ }
+
+ if (k > j->code_bits)
+ return -1;
+
+ // convert the huffman code to the symbol id
+ c = ((j->code_buffer >> (32 - k)) & stbi__bmask[k]) + h->delta[k];
+ if(c < 0 || c >= 256) // symbol id out of bounds!
+ return -1;
+ STBI_ASSERT((((j->code_buffer) >> (32 - h->size[c])) & stbi__bmask[h->size[c]]) == h->code[c]);
+
+ // convert the id to a symbol
+ j->code_bits -= k;
+ j->code_buffer <<= k;
+ return h->values[c];
+}
+
+// bias[n] = (-1<<n) + 1
+static const int stbi__jbias[16] = {0,-1,-3,-7,-15,-31,-63,-127,-255,-511,-1023,-2047,-4095,-8191,-16383,-32767};
+
+// combined JPEG 'receive' and JPEG 'extend', since baseline
+// always extends everything it receives.
+stbi_inline static int stbi__extend_receive(stbi__jpeg *j, int n)
+{
+ unsigned int k;
+ int sgn;
+ if (j->code_bits < n) stbi__grow_buffer_unsafe(j);
+ if (j->code_bits < n) return 0; // ran out of bits from stream, return 0s intead of continuing
+
+ sgn = j->code_buffer >> 31; // sign bit always in MSB; 0 if MSB clear (positive), 1 if MSB set (negative)
+ k = stbi_lrot(j->code_buffer, n);
+ j->code_buffer = k & ~stbi__bmask[n];
+ k &= stbi__bmask[n];
+ j->code_bits -= n;
+ return k + (stbi__jbias[n] & (sgn - 1));
+}
+
+// get some unsigned bits
+stbi_inline static int stbi__jpeg_get_bits(stbi__jpeg *j, int n)
+{
+ unsigned int k;
+ if (j->code_bits < n) stbi__grow_buffer_unsafe(j);
+ if (j->code_bits < n) return 0; // ran out of bits from stream, return 0s intead of continuing
+ k = stbi_lrot(j->code_buffer, n);
+ j->code_buffer = k & ~stbi__bmask[n];
+ k &= stbi__bmask[n];
+ j->code_bits -= n;
+ return k;
+}
+
+stbi_inline static int stbi__jpeg_get_bit(stbi__jpeg *j)
+{
+ unsigned int k;
+ if (j->code_bits < 1) stbi__grow_buffer_unsafe(j);
+ if (j->code_bits < 1) return 0; // ran out of bits from stream, return 0s intead of continuing
+ k = j->code_buffer;
+ j->code_buffer <<= 1;
+ --j->code_bits;
+ return k & 0x80000000;
+}
+
+// given a value that's at position X in the zigzag stream,
+// where does it appear in the 8x8 matrix coded as row-major?
+static const stbi_uc stbi__jpeg_dezigzag[64+15] =
+{
+ 0, 1, 8, 16, 9, 2, 3, 10,
+ 17, 24, 32, 25, 18, 11, 4, 5,
+ 12, 19, 26, 33, 40, 48, 41, 34,
+ 27, 20, 13, 6, 7, 14, 21, 28,
+ 35, 42, 49, 56, 57, 50, 43, 36,
+ 29, 22, 15, 23, 30, 37, 44, 51,
+ 58, 59, 52, 45, 38, 31, 39, 46,
+ 53, 60, 61, 54, 47, 55, 62, 63,
+ // let corrupt input sample past end
+ 63, 63, 63, 63, 63, 63, 63, 63,
+ 63, 63, 63, 63, 63, 63, 63
+};
+
+// decode one 64-entry block--
+static int stbi__jpeg_decode_block(stbi__jpeg *j, short data[64], stbi__huffman *hdc, stbi__huffman *hac, stbi__int16 *fac, int b, stbi__uint16 *dequant)
+{
+ int diff,dc,k;
+ int t;
+
+ if (j->code_bits < 16) stbi__grow_buffer_unsafe(j);
+ t = stbi__jpeg_huff_decode(j, hdc);
+ if (t < 0 || t > 15) return stbi__err("bad huffman code","Corrupt JPEG");
+
+ // 0 all the ac values now so we can do it 32-bits at a time
+ memset(data,0,64*sizeof(data[0]));
+
+ diff = t ? stbi__extend_receive(j, t) : 0;
+ if (!stbi__addints_valid(j->img_comp[b].dc_pred, diff)) return stbi__err("bad delta","Corrupt JPEG");
+ dc = j->img_comp[b].dc_pred + diff;
+ j->img_comp[b].dc_pred = dc;
+ if (!stbi__mul2shorts_valid(dc, dequant[0])) return stbi__err("can't merge dc and ac", "Corrupt JPEG");
+ data[0] = (short) (dc * dequant[0]);
+
+ // decode AC components, see JPEG spec
+ k = 1;
+ do {
+ unsigned int zig;
+ int c,r,s;
+ if (j->code_bits < 16) stbi__grow_buffer_unsafe(j);
+ c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1);
+ r = fac[c];
+ if (r) { // fast-AC path
+ k += (r >> 4) & 15; // run
+ s = r & 15; // combined length
+ if (s > j->code_bits) return stbi__err("bad huffman code", "Combined length longer than code bits available");
+ j->code_buffer <<= s;
+ j->code_bits -= s;
+ // decode into unzigzag'd location
+ zig = stbi__jpeg_dezigzag[k++];
+ data[zig] = (short) ((r >> 8) * dequant[zig]);
+ } else {
+ int rs = stbi__jpeg_huff_decode(j, hac);
+ if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG");
+ s = rs & 15;
+ r = rs >> 4;
+ if (s == 0) {
+ if (rs != 0xf0) break; // end block
+ k += 16;
+ } else {
+ k += r;
+ // decode into unzigzag'd location
+ zig = stbi__jpeg_dezigzag[k++];
+ data[zig] = (short) (stbi__extend_receive(j,s) * dequant[zig]);
+ }
+ }
+ } while (k < 64);
+ return 1;
+}
+
+static int stbi__jpeg_decode_block_prog_dc(stbi__jpeg *j, short data[64], stbi__huffman *hdc, int b)
+{
+ int diff,dc;
+ int t;
+ if (j->spec_end != 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG");
+
+ if (j->code_bits < 16) stbi__grow_buffer_unsafe(j);
+
+ if (j->succ_high == 0) {
+ // first scan for DC coefficient, must be first
+ memset(data,0,64*sizeof(data[0])); // 0 all the ac values now
+ t = stbi__jpeg_huff_decode(j, hdc);
+ if (t < 0 || t > 15) return stbi__err("can't merge dc and ac", "Corrupt JPEG");
+ diff = t ? stbi__extend_receive(j, t) : 0;
+
+ if (!stbi__addints_valid(j->img_comp[b].dc_pred, diff)) return stbi__err("bad delta", "Corrupt JPEG");
+ dc = j->img_comp[b].dc_pred + diff;
+ j->img_comp[b].dc_pred = dc;
+ if (!stbi__mul2shorts_valid(dc, 1 << j->succ_low)) return stbi__err("can't merge dc and ac", "Corrupt JPEG");
+ data[0] = (short) (dc * (1 << j->succ_low));
+ } else {
+ // refinement scan for DC coefficient
+ if (stbi__jpeg_get_bit(j))
+ data[0] += (short) (1 << j->succ_low);
+ }
+ return 1;
+}
+
+// @OPTIMIZE: store non-zigzagged during the decode passes,
+// and only de-zigzag when dequantizing
+static int stbi__jpeg_decode_block_prog_ac(stbi__jpeg *j, short data[64], stbi__huffman *hac, stbi__int16 *fac)
+{
+ int k;
+ if (j->spec_start == 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG");
+
+ if (j->succ_high == 0) {
+ int shift = j->succ_low;
+
+ if (j->eob_run) {
+ --j->eob_run;
+ return 1;
+ }
+
+ k = j->spec_start;
+ do {
+ unsigned int zig;
+ int c,r,s;
+ if (j->code_bits < 16) stbi__grow_buffer_unsafe(j);
+ c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1);
+ r = fac[c];
+ if (r) { // fast-AC path
+ k += (r >> 4) & 15; // run
+ s = r & 15; // combined length
+ if (s > j->code_bits) return stbi__err("bad huffman code", "Combined length longer than code bits available");
+ j->code_buffer <<= s;
+ j->code_bits -= s;
+ zig = stbi__jpeg_dezigzag[k++];
+ data[zig] = (short) ((r >> 8) * (1 << shift));
+ } else {
+ int rs = stbi__jpeg_huff_decode(j, hac);
+ if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG");
+ s = rs & 15;
+ r = rs >> 4;
+ if (s == 0) {
+ if (r < 15) {
+ j->eob_run = (1 << r);
+ if (r)
+ j->eob_run += stbi__jpeg_get_bits(j, r);
+ --j->eob_run;
+ break;
+ }
+ k += 16;
+ } else {
+ k += r;
+ zig = stbi__jpeg_dezigzag[k++];
+ data[zig] = (short) (stbi__extend_receive(j,s) * (1 << shift));
+ }
+ }
+ } while (k <= j->spec_end);
+ } else {
+ // refinement scan for these AC coefficients
+
+ short bit = (short) (1 << j->succ_low);
+
+ if (j->eob_run) {
+ --j->eob_run;
+ for (k = j->spec_start; k <= j->spec_end; ++k) {
+ short *p = &data[stbi__jpeg_dezigzag[k]];
+ if (*p != 0)
+ if (stbi__jpeg_get_bit(j))
+ if ((*p & bit)==0) {
+ if (*p > 0)
+ *p += bit;
+ else
+ *p -= bit;
+ }
+ }
+ } else {
+ k = j->spec_start;
+ do {
+ int r,s;
+ int rs = stbi__jpeg_huff_decode(j, hac); // @OPTIMIZE see if we can use the fast path here, advance-by-r is so slow, eh
+ if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG");
+ s = rs & 15;
+ r = rs >> 4;
+ if (s == 0) {
+ if (r < 15) {
+ j->eob_run = (1 << r) - 1;
+ if (r)
+ j->eob_run += stbi__jpeg_get_bits(j, r);
+ r = 64; // force end of block
+ } else {
+ // r=15 s=0 should write 16 0s, so we just do
+ // a run of 15 0s and then write s (which is 0),
+ // so we don't have to do anything special here
+ }
+ } else {
+ if (s != 1) return stbi__err("bad huffman code", "Corrupt JPEG");
+ // sign bit
+ if (stbi__jpeg_get_bit(j))
+ s = bit;
+ else
+ s = -bit;
+ }
+
+ // advance by r
+ while (k <= j->spec_end) {
+ short *p = &data[stbi__jpeg_dezigzag[k++]];
+ if (*p != 0) {
+ if (stbi__jpeg_get_bit(j))
+ if ((*p & bit)==0) {
+ if (*p > 0)
+ *p += bit;
+ else
+ *p -= bit;
+ }
+ } else {
+ if (r == 0) {
+ *p = (short) s;
+ break;
+ }
+ --r;
+ }
+ }
+ } while (k <= j->spec_end);
+ }
+ }
+ return 1;
+}
+
+// take a -128..127 value and stbi__clamp it and convert to 0..255
+stbi_inline static stbi_uc stbi__clamp(int x)
+{
+ // trick to use a single test to catch both cases
+ if ((unsigned int) x > 255) {
+ if (x < 0) return 0;
+ if (x > 255) return 255;
+ }
+ return (stbi_uc) x;
+}
+
+#define stbi__f2f(x) ((int) (((x) * 4096 + 0.5)))
+#define stbi__fsh(x) ((x) * 4096)
+
+// derived from jidctint -- DCT_ISLOW
+#define STBI__IDCT_1D(s0,s1,s2,s3,s4,s5,s6,s7) \
+ int t0,t1,t2,t3,p1,p2,p3,p4,p5,x0,x1,x2,x3; \
+ p2 = s2; \
+ p3 = s6; \
+ p1 = (p2+p3) * stbi__f2f(0.5411961f); \
+ t2 = p1 + p3*stbi__f2f(-1.847759065f); \
+ t3 = p1 + p2*stbi__f2f( 0.765366865f); \
+ p2 = s0; \
+ p3 = s4; \
+ t0 = stbi__fsh(p2+p3); \
+ t1 = stbi__fsh(p2-p3); \
+ x0 = t0+t3; \
+ x3 = t0-t3; \
+ x1 = t1+t2; \
+ x2 = t1-t2; \
+ t0 = s7; \
+ t1 = s5; \
+ t2 = s3; \
+ t3 = s1; \
+ p3 = t0+t2; \
+ p4 = t1+t3; \
+ p1 = t0+t3; \
+ p2 = t1+t2; \
+ p5 = (p3+p4)*stbi__f2f( 1.175875602f); \
+ t0 = t0*stbi__f2f( 0.298631336f); \
+ t1 = t1*stbi__f2f( 2.053119869f); \
+ t2 = t2*stbi__f2f( 3.072711026f); \
+ t3 = t3*stbi__f2f( 1.501321110f); \
+ p1 = p5 + p1*stbi__f2f(-0.899976223f); \
+ p2 = p5 + p2*stbi__f2f(-2.562915447f); \
+ p3 = p3*stbi__f2f(-1.961570560f); \
+ p4 = p4*stbi__f2f(-0.390180644f); \
+ t3 += p1+p4; \
+ t2 += p2+p3; \
+ t1 += p2+p4; \
+ t0 += p1+p3;
+
+static void stbi__idct_block(stbi_uc *out, int out_stride, short data[64])
+{
+ int i,val[64],*v=val;
+ stbi_uc *o;
+ short *d = data;
+
+ // columns
+ for (i=0; i < 8; ++i,++d, ++v) {
+ // if all zeroes, shortcut -- this avoids dequantizing 0s and IDCTing
+ if (d[ 8]==0 && d[16]==0 && d[24]==0 && d[32]==0
+ && d[40]==0 && d[48]==0 && d[56]==0) {
+ // no shortcut 0 seconds
+ // (1|2|3|4|5|6|7)==0 0 seconds
+ // all separate -0.047 seconds
+ // 1 && 2|3 && 4|5 && 6|7: -0.047 seconds
+ int dcterm = d[0]*4;
+ v[0] = v[8] = v[16] = v[24] = v[32] = v[40] = v[48] = v[56] = dcterm;
+ } else {
+ STBI__IDCT_1D(d[ 0],d[ 8],d[16],d[24],d[32],d[40],d[48],d[56])
+ // constants scaled things up by 1<<12; let's bring them back
+ // down, but keep 2 extra bits of precision
+ x0 += 512; x1 += 512; x2 += 512; x3 += 512;
+ v[ 0] = (x0+t3) >> 10;
+ v[56] = (x0-t3) >> 10;
+ v[ 8] = (x1+t2) >> 10;
+ v[48] = (x1-t2) >> 10;
+ v[16] = (x2+t1) >> 10;
+ v[40] = (x2-t1) >> 10;
+ v[24] = (x3+t0) >> 10;
+ v[32] = (x3-t0) >> 10;
+ }
+ }
+
+ for (i=0, v=val, o=out; i < 8; ++i,v+=8,o+=out_stride) {
+ // no fast case since the first 1D IDCT spread components out
+ STBI__IDCT_1D(v[0],v[1],v[2],v[3],v[4],v[5],v[6],v[7])
+ // constants scaled things up by 1<<12, plus we had 1<<2 from first
+ // loop, plus horizontal and vertical each scale by sqrt(8) so together
+ // we've got an extra 1<<3, so 1<<17 total we need to remove.
+ // so we want to round that, which means adding 0.5 * 1<<17,
+ // aka 65536. Also, we'll end up with -128 to 127 that we want
+ // to encode as 0..255 by adding 128, so we'll add that before the shift
+ x0 += 65536 + (128<<17);
+ x1 += 65536 + (128<<17);
+ x2 += 65536 + (128<<17);
+ x3 += 65536 + (128<<17);
+ // tried computing the shifts into temps, or'ing the temps to see
+ // if any were out of range, but that was slower
+ o[0] = stbi__clamp((x0+t3) >> 17);
+ o[7] = stbi__clamp((x0-t3) >> 17);
+ o[1] = stbi__clamp((x1+t2) >> 17);
+ o[6] = stbi__clamp((x1-t2) >> 17);
+ o[2] = stbi__clamp((x2+t1) >> 17);
+ o[5] = stbi__clamp((x2-t1) >> 17);
+ o[3] = stbi__clamp((x3+t0) >> 17);
+ o[4] = stbi__clamp((x3-t0) >> 17);
+ }
+}
+
+#ifdef STBI_SSE2
+// sse2 integer IDCT. not the fastest possible implementation but it
+// produces bit-identical results to the generic C version so it's
+// fully "transparent".
+static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64])
+{
+ // This is constructed to match our regular (generic) integer IDCT exactly.
+ __m128i row0, row1, row2, row3, row4, row5, row6, row7;
+ __m128i tmp;
+
+ // dot product constant: even elems=x, odd elems=y
+ #define dct_const(x,y) _mm_setr_epi16((x),(y),(x),(y),(x),(y),(x),(y))
+
+ // out(0) = c0[even]*x + c0[odd]*y (c0, x, y 16-bit, out 32-bit)
+ // out(1) = c1[even]*x + c1[odd]*y
+ #define dct_rot(out0,out1, x,y,c0,c1) \
+ __m128i c0##lo = _mm_unpacklo_epi16((x),(y)); \
+ __m128i c0##hi = _mm_unpackhi_epi16((x),(y)); \
+ __m128i out0##_l = _mm_madd_epi16(c0##lo, c0); \
+ __m128i out0##_h = _mm_madd_epi16(c0##hi, c0); \
+ __m128i out1##_l = _mm_madd_epi16(c0##lo, c1); \
+ __m128i out1##_h = _mm_madd_epi16(c0##hi, c1)
+
+ // out = in << 12 (in 16-bit, out 32-bit)
+ #define dct_widen(out, in) \
+ __m128i out##_l = _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), (in)), 4); \
+ __m128i out##_h = _mm_srai_epi32(_mm_unpackhi_epi16(_mm_setzero_si128(), (in)), 4)
+
+ // wide add
+ #define dct_wadd(out, a, b) \
+ __m128i out##_l = _mm_add_epi32(a##_l, b##_l); \
+ __m128i out##_h = _mm_add_epi32(a##_h, b##_h)
+
+ // wide sub
+ #define dct_wsub(out, a, b) \
+ __m128i out##_l = _mm_sub_epi32(a##_l, b##_l); \
+ __m128i out##_h = _mm_sub_epi32(a##_h, b##_h)
+
+ // butterfly a/b, add bias, then shift by "s" and pack
+ #define dct_bfly32o(out0, out1, a,b,bias,s) \
+ { \
+ __m128i abiased_l = _mm_add_epi32(a##_l, bias); \
+ __m128i abiased_h = _mm_add_epi32(a##_h, bias); \
+ dct_wadd(sum, abiased, b); \
+ dct_wsub(dif, abiased, b); \
+ out0 = _mm_packs_epi32(_mm_srai_epi32(sum_l, s), _mm_srai_epi32(sum_h, s)); \
+ out1 = _mm_packs_epi32(_mm_srai_epi32(dif_l, s), _mm_srai_epi32(dif_h, s)); \
+ }
+
+ // 8-bit interleave step (for transposes)
+ #define dct_interleave8(a, b) \
+ tmp = a; \
+ a = _mm_unpacklo_epi8(a, b); \
+ b = _mm_unpackhi_epi8(tmp, b)
+
+ // 16-bit interleave step (for transposes)
+ #define dct_interleave16(a, b) \
+ tmp = a; \
+ a = _mm_unpacklo_epi16(a, b); \
+ b = _mm_unpackhi_epi16(tmp, b)
+
+ #define dct_pass(bias,shift) \
+ { \
+ /* even part */ \
+ dct_rot(t2e,t3e, row2,row6, rot0_0,rot0_1); \
+ __m128i sum04 = _mm_add_epi16(row0, row4); \
+ __m128i dif04 = _mm_sub_epi16(row0, row4); \
+ dct_widen(t0e, sum04); \
+ dct_widen(t1e, dif04); \
+ dct_wadd(x0, t0e, t3e); \
+ dct_wsub(x3, t0e, t3e); \
+ dct_wadd(x1, t1e, t2e); \
+ dct_wsub(x2, t1e, t2e); \
+ /* odd part */ \
+ dct_rot(y0o,y2o, row7,row3, rot2_0,rot2_1); \
+ dct_rot(y1o,y3o, row5,row1, rot3_0,rot3_1); \
+ __m128i sum17 = _mm_add_epi16(row1, row7); \
+ __m128i sum35 = _mm_add_epi16(row3, row5); \
+ dct_rot(y4o,y5o, sum17,sum35, rot1_0,rot1_1); \
+ dct_wadd(x4, y0o, y4o); \
+ dct_wadd(x5, y1o, y5o); \
+ dct_wadd(x6, y2o, y5o); \
+ dct_wadd(x7, y3o, y4o); \
+ dct_bfly32o(row0,row7, x0,x7,bias,shift); \
+ dct_bfly32o(row1,row6, x1,x6,bias,shift); \
+ dct_bfly32o(row2,row5, x2,x5,bias,shift); \
+ dct_bfly32o(row3,row4, x3,x4,bias,shift); \
+ }
+
+ __m128i rot0_0 = dct_const(stbi__f2f(0.5411961f), stbi__f2f(0.5411961f) + stbi__f2f(-1.847759065f));
+ __m128i rot0_1 = dct_const(stbi__f2f(0.5411961f) + stbi__f2f( 0.765366865f), stbi__f2f(0.5411961f));
+ __m128i rot1_0 = dct_const(stbi__f2f(1.175875602f) + stbi__f2f(-0.899976223f), stbi__f2f(1.175875602f));
+ __m128i rot1_1 = dct_const(stbi__f2f(1.175875602f), stbi__f2f(1.175875602f) + stbi__f2f(-2.562915447f));
+ __m128i rot2_0 = dct_const(stbi__f2f(-1.961570560f) + stbi__f2f( 0.298631336f), stbi__f2f(-1.961570560f));
+ __m128i rot2_1 = dct_const(stbi__f2f(-1.961570560f), stbi__f2f(-1.961570560f) + stbi__f2f( 3.072711026f));
+ __m128i rot3_0 = dct_const(stbi__f2f(-0.390180644f) + stbi__f2f( 2.053119869f), stbi__f2f(-0.390180644f));
+ __m128i rot3_1 = dct_const(stbi__f2f(-0.390180644f), stbi__f2f(-0.390180644f) + stbi__f2f( 1.501321110f));
+
+ // rounding biases in column/row passes, see stbi__idct_block for explanation.
+ __m128i bias_0 = _mm_set1_epi32(512);
+ __m128i bias_1 = _mm_set1_epi32(65536 + (128<<17));
+
+ // load
+ row0 = _mm_load_si128((const __m128i *) (data + 0*8));
+ row1 = _mm_load_si128((const __m128i *) (data + 1*8));
+ row2 = _mm_load_si128((const __m128i *) (data + 2*8));
+ row3 = _mm_load_si128((const __m128i *) (data + 3*8));
+ row4 = _mm_load_si128((const __m128i *) (data + 4*8));
+ row5 = _mm_load_si128((const __m128i *) (data + 5*8));
+ row6 = _mm_load_si128((const __m128i *) (data + 6*8));
+ row7 = _mm_load_si128((const __m128i *) (data + 7*8));
+
+ // column pass
+ dct_pass(bias_0, 10);
+
+ {
+ // 16bit 8x8 transpose pass 1
+ dct_interleave16(row0, row4);
+ dct_interleave16(row1, row5);
+ dct_interleave16(row2, row6);
+ dct_interleave16(row3, row7);
+
+ // transpose pass 2
+ dct_interleave16(row0, row2);
+ dct_interleave16(row1, row3);
+ dct_interleave16(row4, row6);
+ dct_interleave16(row5, row7);
+
+ // transpose pass 3
+ dct_interleave16(row0, row1);
+ dct_interleave16(row2, row3);
+ dct_interleave16(row4, row5);
+ dct_interleave16(row6, row7);
+ }
+
+ // row pass
+ dct_pass(bias_1, 17);
+
+ {
+ // pack
+ __m128i p0 = _mm_packus_epi16(row0, row1); // a0a1a2a3...a7b0b1b2b3...b7
+ __m128i p1 = _mm_packus_epi16(row2, row3);
+ __m128i p2 = _mm_packus_epi16(row4, row5);
+ __m128i p3 = _mm_packus_epi16(row6, row7);
+
+ // 8bit 8x8 transpose pass 1
+ dct_interleave8(p0, p2); // a0e0a1e1...
+ dct_interleave8(p1, p3); // c0g0c1g1...
+
+ // transpose pass 2
+ dct_interleave8(p0, p1); // a0c0e0g0...
+ dct_interleave8(p2, p3); // b0d0f0h0...
+
+ // transpose pass 3
+ dct_interleave8(p0, p2); // a0b0c0d0...
+ dct_interleave8(p1, p3); // a4b4c4d4...
+
+ // store
+ _mm_storel_epi64((__m128i *) out, p0); out += out_stride;
+ _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p0, 0x4e)); out += out_stride;
+ _mm_storel_epi64((__m128i *) out, p2); out += out_stride;
+ _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p2, 0x4e)); out += out_stride;
+ _mm_storel_epi64((__m128i *) out, p1); out += out_stride;
+ _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p1, 0x4e)); out += out_stride;
+ _mm_storel_epi64((__m128i *) out, p3); out += out_stride;
+ _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p3, 0x4e));
+ }
+
+#undef dct_const
+#undef dct_rot
+#undef dct_widen
+#undef dct_wadd
+#undef dct_wsub
+#undef dct_bfly32o
+#undef dct_interleave8
+#undef dct_interleave16
+#undef dct_pass
+}
+
+#endif // STBI_SSE2
+
+#ifdef STBI_NEON
+
+// NEON integer IDCT. should produce bit-identical
+// results to the generic C version.
+static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64])
+{
+ int16x8_t row0, row1, row2, row3, row4, row5, row6, row7;
+
+ int16x4_t rot0_0 = vdup_n_s16(stbi__f2f(0.5411961f));
+ int16x4_t rot0_1 = vdup_n_s16(stbi__f2f(-1.847759065f));
+ int16x4_t rot0_2 = vdup_n_s16(stbi__f2f( 0.765366865f));
+ int16x4_t rot1_0 = vdup_n_s16(stbi__f2f( 1.175875602f));
+ int16x4_t rot1_1 = vdup_n_s16(stbi__f2f(-0.899976223f));
+ int16x4_t rot1_2 = vdup_n_s16(stbi__f2f(-2.562915447f));
+ int16x4_t rot2_0 = vdup_n_s16(stbi__f2f(-1.961570560f));
+ int16x4_t rot2_1 = vdup_n_s16(stbi__f2f(-0.390180644f));
+ int16x4_t rot3_0 = vdup_n_s16(stbi__f2f( 0.298631336f));
+ int16x4_t rot3_1 = vdup_n_s16(stbi__f2f( 2.053119869f));
+ int16x4_t rot3_2 = vdup_n_s16(stbi__f2f( 3.072711026f));
+ int16x4_t rot3_3 = vdup_n_s16(stbi__f2f( 1.501321110f));
+
+#define dct_long_mul(out, inq, coeff) \
+ int32x4_t out##_l = vmull_s16(vget_low_s16(inq), coeff); \
+ int32x4_t out##_h = vmull_s16(vget_high_s16(inq), coeff)
+
+#define dct_long_mac(out, acc, inq, coeff) \
+ int32x4_t out##_l = vmlal_s16(acc##_l, vget_low_s16(inq), coeff); \
+ int32x4_t out##_h = vmlal_s16(acc##_h, vget_high_s16(inq), coeff)
+
+#define dct_widen(out, inq) \
+ int32x4_t out##_l = vshll_n_s16(vget_low_s16(inq), 12); \
+ int32x4_t out##_h = vshll_n_s16(vget_high_s16(inq), 12)
+
+// wide add
+#define dct_wadd(out, a, b) \
+ int32x4_t out##_l = vaddq_s32(a##_l, b##_l); \
+ int32x4_t out##_h = vaddq_s32(a##_h, b##_h)
+
+// wide sub
+#define dct_wsub(out, a, b) \
+ int32x4_t out##_l = vsubq_s32(a##_l, b##_l); \
+ int32x4_t out##_h = vsubq_s32(a##_h, b##_h)
+
+// butterfly a/b, then shift using "shiftop" by "s" and pack
+#define dct_bfly32o(out0,out1, a,b,shiftop,s) \
+ { \
+ dct_wadd(sum, a, b); \
+ dct_wsub(dif, a, b); \
+ out0 = vcombine_s16(shiftop(sum_l, s), shiftop(sum_h, s)); \
+ out1 = vcombine_s16(shiftop(dif_l, s), shiftop(dif_h, s)); \
+ }
+
+#define dct_pass(shiftop, shift) \
+ { \
+ /* even part */ \
+ int16x8_t sum26 = vaddq_s16(row2, row6); \
+ dct_long_mul(p1e, sum26, rot0_0); \
+ dct_long_mac(t2e, p1e, row6, rot0_1); \
+ dct_long_mac(t3e, p1e, row2, rot0_2); \
+ int16x8_t sum04 = vaddq_s16(row0, row4); \
+ int16x8_t dif04 = vsubq_s16(row0, row4); \
+ dct_widen(t0e, sum04); \
+ dct_widen(t1e, dif04); \
+ dct_wadd(x0, t0e, t3e); \
+ dct_wsub(x3, t0e, t3e); \
+ dct_wadd(x1, t1e, t2e); \
+ dct_wsub(x2, t1e, t2e); \
+ /* odd part */ \
+ int16x8_t sum15 = vaddq_s16(row1, row5); \
+ int16x8_t sum17 = vaddq_s16(row1, row7); \
+ int16x8_t sum35 = vaddq_s16(row3, row5); \
+ int16x8_t sum37 = vaddq_s16(row3, row7); \
+ int16x8_t sumodd = vaddq_s16(sum17, sum35); \
+ dct_long_mul(p5o, sumodd, rot1_0); \
+ dct_long_mac(p1o, p5o, sum17, rot1_1); \
+ dct_long_mac(p2o, p5o, sum35, rot1_2); \
+ dct_long_mul(p3o, sum37, rot2_0); \
+ dct_long_mul(p4o, sum15, rot2_1); \
+ dct_wadd(sump13o, p1o, p3o); \
+ dct_wadd(sump24o, p2o, p4o); \
+ dct_wadd(sump23o, p2o, p3o); \
+ dct_wadd(sump14o, p1o, p4o); \
+ dct_long_mac(x4, sump13o, row7, rot3_0); \
+ dct_long_mac(x5, sump24o, row5, rot3_1); \
+ dct_long_mac(x6, sump23o, row3, rot3_2); \
+ dct_long_mac(x7, sump14o, row1, rot3_3); \
+ dct_bfly32o(row0,row7, x0,x7,shiftop,shift); \
+ dct_bfly32o(row1,row6, x1,x6,shiftop,shift); \
+ dct_bfly32o(row2,row5, x2,x5,shiftop,shift); \
+ dct_bfly32o(row3,row4, x3,x4,shiftop,shift); \
+ }
+
+ // load
+ row0 = vld1q_s16(data + 0*8);
+ row1 = vld1q_s16(data + 1*8);
+ row2 = vld1q_s16(data + 2*8);
+ row3 = vld1q_s16(data + 3*8);
+ row4 = vld1q_s16(data + 4*8);
+ row5 = vld1q_s16(data + 5*8);
+ row6 = vld1q_s16(data + 6*8);
+ row7 = vld1q_s16(data + 7*8);
+
+ // add DC bias
+ row0 = vaddq_s16(row0, vsetq_lane_s16(1024, vdupq_n_s16(0), 0));
+
+ // column pass
+ dct_pass(vrshrn_n_s32, 10);
+
+ // 16bit 8x8 transpose
+ {
+// these three map to a single VTRN.16, VTRN.32, and VSWP, respectively.
+// whether compilers actually get this is another story, sadly.
+#define dct_trn16(x, y) { int16x8x2_t t = vtrnq_s16(x, y); x = t.val[0]; y = t.val[1]; }
+#define dct_trn32(x, y) { int32x4x2_t t = vtrnq_s32(vreinterpretq_s32_s16(x), vreinterpretq_s32_s16(y)); x = vreinterpretq_s16_s32(t.val[0]); y = vreinterpretq_s16_s32(t.val[1]); }
+#define dct_trn64(x, y) { int16x8_t x0 = x; int16x8_t y0 = y; x = vcombine_s16(vget_low_s16(x0), vget_low_s16(y0)); y = vcombine_s16(vget_high_s16(x0), vget_high_s16(y0)); }
+
+ // pass 1
+ dct_trn16(row0, row1); // a0b0a2b2a4b4a6b6
+ dct_trn16(row2, row3);
+ dct_trn16(row4, row5);
+ dct_trn16(row6, row7);
+
+ // pass 2
+ dct_trn32(row0, row2); // a0b0c0d0a4b4c4d4
+ dct_trn32(row1, row3);
+ dct_trn32(row4, row6);
+ dct_trn32(row5, row7);
+
+ // pass 3
+ dct_trn64(row0, row4); // a0b0c0d0e0f0g0h0
+ dct_trn64(row1, row5);
+ dct_trn64(row2, row6);
+ dct_trn64(row3, row7);
+
+#undef dct_trn16
+#undef dct_trn32
+#undef dct_trn64
+ }
+
+ // row pass
+ // vrshrn_n_s32 only supports shifts up to 16, we need
+ // 17. so do a non-rounding shift of 16 first then follow
+ // up with a rounding shift by 1.
+ dct_pass(vshrn_n_s32, 16);
+
+ {
+ // pack and round
+ uint8x8_t p0 = vqrshrun_n_s16(row0, 1);
+ uint8x8_t p1 = vqrshrun_n_s16(row1, 1);
+ uint8x8_t p2 = vqrshrun_n_s16(row2, 1);
+ uint8x8_t p3 = vqrshrun_n_s16(row3, 1);
+ uint8x8_t p4 = vqrshrun_n_s16(row4, 1);
+ uint8x8_t p5 = vqrshrun_n_s16(row5, 1);
+ uint8x8_t p6 = vqrshrun_n_s16(row6, 1);
+ uint8x8_t p7 = vqrshrun_n_s16(row7, 1);
+
+ // again, these can translate into one instruction, but often don't.
+#define dct_trn8_8(x, y) { uint8x8x2_t t = vtrn_u8(x, y); x = t.val[0]; y = t.val[1]; }
+#define dct_trn8_16(x, y) { uint16x4x2_t t = vtrn_u16(vreinterpret_u16_u8(x), vreinterpret_u16_u8(y)); x = vreinterpret_u8_u16(t.val[0]); y = vreinterpret_u8_u16(t.val[1]); }
+#define dct_trn8_32(x, y) { uint32x2x2_t t = vtrn_u32(vreinterpret_u32_u8(x), vreinterpret_u32_u8(y)); x = vreinterpret_u8_u32(t.val[0]); y = vreinterpret_u8_u32(t.val[1]); }
+
+ // sadly can't use interleaved stores here since we only write
+ // 8 bytes to each scan line!
+
+ // 8x8 8-bit transpose pass 1
+ dct_trn8_8(p0, p1);
+ dct_trn8_8(p2, p3);
+ dct_trn8_8(p4, p5);
+ dct_trn8_8(p6, p7);
+
+ // pass 2
+ dct_trn8_16(p0, p2);
+ dct_trn8_16(p1, p3);
+ dct_trn8_16(p4, p6);
+ dct_trn8_16(p5, p7);
+
+ // pass 3
+ dct_trn8_32(p0, p4);
+ dct_trn8_32(p1, p5);
+ dct_trn8_32(p2, p6);
+ dct_trn8_32(p3, p7);
+
+ // store
+ vst1_u8(out, p0); out += out_stride;
+ vst1_u8(out, p1); out += out_stride;
+ vst1_u8(out, p2); out += out_stride;
+ vst1_u8(out, p3); out += out_stride;
+ vst1_u8(out, p4); out += out_stride;
+ vst1_u8(out, p5); out += out_stride;
+ vst1_u8(out, p6); out += out_stride;
+ vst1_u8(out, p7);
+
+#undef dct_trn8_8
+#undef dct_trn8_16
+#undef dct_trn8_32
+ }
+
+#undef dct_long_mul
+#undef dct_long_mac
+#undef dct_widen
+#undef dct_wadd
+#undef dct_wsub
+#undef dct_bfly32o
+#undef dct_pass
+}
+
+#endif // STBI_NEON
+
+#define STBI__MARKER_none 0xff
+// if there's a pending marker from the entropy stream, return that
+// otherwise, fetch from the stream and get a marker. if there's no
+// marker, return 0xff, which is never a valid marker value
+static stbi_uc stbi__get_marker(stbi__jpeg *j)
+{
+ stbi_uc x;
+ if (j->marker != STBI__MARKER_none) { x = j->marker; j->marker = STBI__MARKER_none; return x; }
+ x = stbi__get8(j->s);
+ if (x != 0xff) return STBI__MARKER_none;
+ while (x == 0xff)
+ x = stbi__get8(j->s); // consume repeated 0xff fill bytes
+ return x;
+}
+
+// in each scan, we'll have scan_n components, and the order
+// of the components is specified by order[]
+#define STBI__RESTART(x) ((x) >= 0xd0 && (x) <= 0xd7)
+
+// after a restart interval, stbi__jpeg_reset the entropy decoder and
+// the dc prediction
+static void stbi__jpeg_reset(stbi__jpeg *j)
+{
+ j->code_bits = 0;
+ j->code_buffer = 0;
+ j->nomore = 0;
+ j->img_comp[0].dc_pred = j->img_comp[1].dc_pred = j->img_comp[2].dc_pred = j->img_comp[3].dc_pred = 0;
+ j->marker = STBI__MARKER_none;
+ j->todo = j->restart_interval ? j->restart_interval : 0x7fffffff;
+ j->eob_run = 0;
+ // no more than 1<<31 MCUs if no restart_interal? that's plenty safe,
+ // since we don't even allow 1<<30 pixels
+}
+
+static int stbi__parse_entropy_coded_data(stbi__jpeg *z)
+{
+ stbi__jpeg_reset(z);
+ if (!z->progressive) {
+ if (z->scan_n == 1) {
+ int i,j;
+ STBI_SIMD_ALIGN(short, data[64]);
+ int n = z->order[0];
+ // non-interleaved data, we just need to process one block at a time,
+ // in trivial scanline order
+ // number of blocks to do just depends on how many actual "pixels" this
+ // component has, independent of interleaved MCU blocking and such
+ int w = (z->img_comp[n].x+7) >> 3;
+ int h = (z->img_comp[n].y+7) >> 3;
+ for (j=0; j < h; ++j) {
+ for (i=0; i < w; ++i) {
+ int ha = z->img_comp[n].ha;
+ if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0;
+ z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data);
+ // every data block is an MCU, so countdown the restart interval
+ if (--z->todo <= 0) {
+ if (z->code_bits < 24) stbi__grow_buffer_unsafe(z);
+ // if it's NOT a restart, then just bail, so we get corrupt data
+ // rather than no data
+ if (!STBI__RESTART(z->marker)) return 1;
+ stbi__jpeg_reset(z);
+ }
+ }
+ }
+ return 1;
+ } else { // interleaved
+ int i,j,k,x,y;
+ STBI_SIMD_ALIGN(short, data[64]);
+ for (j=0; j < z->img_mcu_y; ++j) {
+ for (i=0; i < z->img_mcu_x; ++i) {
+ // scan an interleaved mcu... process scan_n components in order
+ for (k=0; k < z->scan_n; ++k) {
+ int n = z->order[k];
+ // scan out an mcu's worth of this component; that's just determined
+ // by the basic H and V specified for the component
+ for (y=0; y < z->img_comp[n].v; ++y) {
+ for (x=0; x < z->img_comp[n].h; ++x) {
+ int x2 = (i*z->img_comp[n].h + x)*8;
+ int y2 = (j*z->img_comp[n].v + y)*8;
+ int ha = z->img_comp[n].ha;
+ if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0;
+ z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*y2+x2, z->img_comp[n].w2, data);
+ }
+ }
+ }
+ // after all interleaved components, that's an interleaved MCU,
+ // so now count down the restart interval
+ if (--z->todo <= 0) {
+ if (z->code_bits < 24) stbi__grow_buffer_unsafe(z);
+ if (!STBI__RESTART(z->marker)) return 1;
+ stbi__jpeg_reset(z);
+ }
+ }
+ }
+ return 1;
+ }
+ } else {
+ if (z->scan_n == 1) {
+ int i,j;
+ int n = z->order[0];
+ // non-interleaved data, we just need to process one block at a time,
+ // in trivial scanline order
+ // number of blocks to do just depends on how many actual "pixels" this
+ // component has, independent of interleaved MCU blocking and such
+ int w = (z->img_comp[n].x+7) >> 3;
+ int h = (z->img_comp[n].y+7) >> 3;
+ for (j=0; j < h; ++j) {
+ for (i=0; i < w; ++i) {
+ short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w);
+ if (z->spec_start == 0) {
+ if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n))
+ return 0;
+ } else {
+ int ha = z->img_comp[n].ha;
+ if (!stbi__jpeg_decode_block_prog_ac(z, data, &z->huff_ac[ha], z->fast_ac[ha]))
+ return 0;
+ }
+ // every data block is an MCU, so countdown the restart interval
+ if (--z->todo <= 0) {
+ if (z->code_bits < 24) stbi__grow_buffer_unsafe(z);
+ if (!STBI__RESTART(z->marker)) return 1;
+ stbi__jpeg_reset(z);
+ }
+ }
+ }
+ return 1;
+ } else { // interleaved
+ int i,j,k,x,y;
+ for (j=0; j < z->img_mcu_y; ++j) {
+ for (i=0; i < z->img_mcu_x; ++i) {
+ // scan an interleaved mcu... process scan_n components in order
+ for (k=0; k < z->scan_n; ++k) {
+ int n = z->order[k];
+ // scan out an mcu's worth of this component; that's just determined
+ // by the basic H and V specified for the component
+ for (y=0; y < z->img_comp[n].v; ++y) {
+ for (x=0; x < z->img_comp[n].h; ++x) {
+ int x2 = (i*z->img_comp[n].h + x);
+ int y2 = (j*z->img_comp[n].v + y);
+ short *data = z->img_comp[n].coeff + 64 * (x2 + y2 * z->img_comp[n].coeff_w);
+ if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n))
+ return 0;
+ }
+ }
+ }
+ // after all interleaved components, that's an interleaved MCU,
+ // so now count down the restart interval
+ if (--z->todo <= 0) {
+ if (z->code_bits < 24) stbi__grow_buffer_unsafe(z);
+ if (!STBI__RESTART(z->marker)) return 1;
+ stbi__jpeg_reset(z);
+ }
+ }
+ }
+ return 1;
+ }
+ }
+}
+
+static void stbi__jpeg_dequantize(short *data, stbi__uint16 *dequant)
+{
+ int i;
+ for (i=0; i < 64; ++i)
+ data[i] *= dequant[i];
+}
+
+static void stbi__jpeg_finish(stbi__jpeg *z)
+{
+ if (z->progressive) {
+ // dequantize and idct the data
+ int i,j,n;
+ for (n=0; n < z->s->img_n; ++n) {
+ int w = (z->img_comp[n].x+7) >> 3;
+ int h = (z->img_comp[n].y+7) >> 3;
+ for (j=0; j < h; ++j) {
+ for (i=0; i < w; ++i) {
+ short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w);
+ stbi__jpeg_dequantize(data, z->dequant[z->img_comp[n].tq]);
+ z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data);
+ }
+ }
+ }
+ }
+}
+
+static int stbi__process_marker(stbi__jpeg *z, int m)
+{
+ int L;
+ switch (m) {
+ case STBI__MARKER_none: // no marker found
+ return stbi__err("expected marker","Corrupt JPEG");
+
+ case 0xDD: // DRI - specify restart interval
+ if (stbi__get16be(z->s) != 4) return stbi__err("bad DRI len","Corrupt JPEG");
+ z->restart_interval = stbi__get16be(z->s);
+ return 1;
+
+ case 0xDB: // DQT - define quantization table
+ L = stbi__get16be(z->s)-2;
+ while (L > 0) {
+ int q = stbi__get8(z->s);
+ int p = q >> 4, sixteen = (p != 0);
+ int t = q & 15,i;
+ if (p != 0 && p != 1) return stbi__err("bad DQT type","Corrupt JPEG");
+ if (t > 3) return stbi__err("bad DQT table","Corrupt JPEG");
+
+ for (i=0; i < 64; ++i)
+ z->dequant[t][stbi__jpeg_dezigzag[i]] = (stbi__uint16)(sixteen ? stbi__get16be(z->s) : stbi__get8(z->s));
+ L -= (sixteen ? 129 : 65);
+ }
+ return L==0;
+
+ case 0xC4: // DHT - define huffman table
+ L = stbi__get16be(z->s)-2;
+ while (L > 0) {
+ stbi_uc *v;
+ int sizes[16],i,n=0;
+ int q = stbi__get8(z->s);
+ int tc = q >> 4;
+ int th = q & 15;
+ if (tc > 1 || th > 3) return stbi__err("bad DHT header","Corrupt JPEG");
+ for (i=0; i < 16; ++i) {
+ sizes[i] = stbi__get8(z->s);
+ n += sizes[i];
+ }
+ if(n > 256) return stbi__err("bad DHT header","Corrupt JPEG"); // Loop over i < n would write past end of values!
+ L -= 17;
+ if (tc == 0) {
+ if (!stbi__build_huffman(z->huff_dc+th, sizes)) return 0;
+ v = z->huff_dc[th].values;
+ } else {
+ if (!stbi__build_huffman(z->huff_ac+th, sizes)) return 0;
+ v = z->huff_ac[th].values;
+ }
+ for (i=0; i < n; ++i)
+ v[i] = stbi__get8(z->s);
+ if (tc != 0)
+ stbi__build_fast_ac(z->fast_ac[th], z->huff_ac + th);
+ L -= n;
+ }
+ return L==0;
+ }
+
+ // check for comment block or APP blocks
+ if ((m >= 0xE0 && m <= 0xEF) || m == 0xFE) {
+ L = stbi__get16be(z->s);
+ if (L < 2) {
+ if (m == 0xFE)
+ return stbi__err("bad COM len","Corrupt JPEG");
+ else
+ return stbi__err("bad APP len","Corrupt JPEG");
+ }
+ L -= 2;
+
+ if (m == 0xE0 && L >= 5) { // JFIF APP0 segment
+ static const unsigned char tag[5] = {'J','F','I','F','\0'};
+ int ok = 1;
+ int i;
+ for (i=0; i < 5; ++i)
+ if (stbi__get8(z->s) != tag[i])
+ ok = 0;
+ L -= 5;
+ if (ok)
+ z->jfif = 1;
+ } else if (m == 0xEE && L >= 12) { // Adobe APP14 segment
+ static const unsigned char tag[6] = {'A','d','o','b','e','\0'};
+ int ok = 1;
+ int i;
+ for (i=0; i < 6; ++i)
+ if (stbi__get8(z->s) != tag[i])
+ ok = 0;
+ L -= 6;
+ if (ok) {
+ stbi__get8(z->s); // version
+ stbi__get16be(z->s); // flags0
+ stbi__get16be(z->s); // flags1
+ z->app14_color_transform = stbi__get8(z->s); // color transform
+ L -= 6;
+ }
+ }
+
+ stbi__skip(z->s, L);
+ return 1;
+ }
+
+ return stbi__err("unknown marker","Corrupt JPEG");
+}
+
+// after we see SOS
+static int stbi__process_scan_header(stbi__jpeg *z)
+{
+ int i;
+ int Ls = stbi__get16be(z->s);
+ z->scan_n = stbi__get8(z->s);
+ if (z->scan_n < 1 || z->scan_n > 4 || z->scan_n > (int) z->s->img_n) return stbi__err("bad SOS component count","Corrupt JPEG");
+ if (Ls != 6+2*z->scan_n) return stbi__err("bad SOS len","Corrupt JPEG");
+ for (i=0; i < z->scan_n; ++i) {
+ int id = stbi__get8(z->s), which;
+ int q = stbi__get8(z->s);
+ for (which = 0; which < z->s->img_n; ++which)
+ if (z->img_comp[which].id == id)
+ break;
+ if (which == z->s->img_n) return 0; // no match
+ z->img_comp[which].hd = q >> 4; if (z->img_comp[which].hd > 3) return stbi__err("bad DC huff","Corrupt JPEG");
+ z->img_comp[which].ha = q & 15; if (z->img_comp[which].ha > 3) return stbi__err("bad AC huff","Corrupt JPEG");
+ z->order[i] = which;
+ }
+
+ {
+ int aa;
+ z->spec_start = stbi__get8(z->s);
+ z->spec_end = stbi__get8(z->s); // should be 63, but might be 0
+ aa = stbi__get8(z->s);
+ z->succ_high = (aa >> 4);
+ z->succ_low = (aa & 15);
+ if (z->progressive) {
+ if (z->spec_start > 63 || z->spec_end > 63 || z->spec_start > z->spec_end || z->succ_high > 13 || z->succ_low > 13)
+ return stbi__err("bad SOS", "Corrupt JPEG");
+ } else {
+ if (z->spec_start != 0) return stbi__err("bad SOS","Corrupt JPEG");
+ if (z->succ_high != 0 || z->succ_low != 0) return stbi__err("bad SOS","Corrupt JPEG");
+ z->spec_end = 63;
+ }
+ }
+
+ return 1;
+}
+
+static int stbi__free_jpeg_components(stbi__jpeg *z, int ncomp, int why)
+{
+ int i;
+ for (i=0; i < ncomp; ++i) {
+ if (z->img_comp[i].raw_data) {
+ STBI_FREE(z->img_comp[i].raw_data);
+ z->img_comp[i].raw_data = NULL;
+ z->img_comp[i].data = NULL;
+ }
+ if (z->img_comp[i].raw_coeff) {
+ STBI_FREE(z->img_comp[i].raw_coeff);
+ z->img_comp[i].raw_coeff = 0;
+ z->img_comp[i].coeff = 0;
+ }
+ if (z->img_comp[i].linebuf) {
+ STBI_FREE(z->img_comp[i].linebuf);
+ z->img_comp[i].linebuf = NULL;
+ }
+ }
+ return why;
+}
+
+static int stbi__process_frame_header(stbi__jpeg *z, int scan)
+{
+ stbi__context *s = z->s;
+ int Lf,p,i,q, h_max=1,v_max=1,c;
+ Lf = stbi__get16be(s); if (Lf < 11) return stbi__err("bad SOF len","Corrupt JPEG"); // JPEG
+ p = stbi__get8(s); if (p != 8) return stbi__err("only 8-bit","JPEG format not supported: 8-bit only"); // JPEG baseline
+ s->img_y = stbi__get16be(s); if (s->img_y == 0) return stbi__err("no header height", "JPEG format not supported: delayed height"); // Legal, but we don't handle it--but neither does IJG
+ s->img_x = stbi__get16be(s); if (s->img_x == 0) return stbi__err("0 width","Corrupt JPEG"); // JPEG requires
+ if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)");
+ if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)");
+ c = stbi__get8(s);
+ if (c != 3 && c != 1 && c != 4) return stbi__err("bad component count","Corrupt JPEG");
+ s->img_n = c;
+ for (i=0; i < c; ++i) {
+ z->img_comp[i].data = NULL;
+ z->img_comp[i].linebuf = NULL;
+ }
+
+ if (Lf != 8+3*s->img_n) return stbi__err("bad SOF len","Corrupt JPEG");
+
+ z->rgb = 0;
+ for (i=0; i < s->img_n; ++i) {
+ static const unsigned char rgb[3] = { 'R', 'G', 'B' };
+ z->img_comp[i].id = stbi__get8(s);
+ if (s->img_n == 3 && z->img_comp[i].id == rgb[i])
+ ++z->rgb;
+ q = stbi__get8(s);
+ z->img_comp[i].h = (q >> 4); if (!z->img_comp[i].h || z->img_comp[i].h > 4) return stbi__err("bad H","Corrupt JPEG");
+ z->img_comp[i].v = q & 15; if (!z->img_comp[i].v || z->img_comp[i].v > 4) return stbi__err("bad V","Corrupt JPEG");
+ z->img_comp[i].tq = stbi__get8(s); if (z->img_comp[i].tq > 3) return stbi__err("bad TQ","Corrupt JPEG");
+ }
+
+ if (scan != STBI__SCAN_load) return 1;
+
+ if (!stbi__mad3sizes_valid(s->img_x, s->img_y, s->img_n, 0)) return stbi__err("too large", "Image too large to decode");
+
+ for (i=0; i < s->img_n; ++i) {
+ if (z->img_comp[i].h > h_max) h_max = z->img_comp[i].h;
+ if (z->img_comp[i].v > v_max) v_max = z->img_comp[i].v;
+ }
+
+ // check that plane subsampling factors are integer ratios; our resamplers can't deal with fractional ratios
+ // and I've never seen a non-corrupted JPEG file actually use them
+ for (i=0; i < s->img_n; ++i) {
+ if (h_max % z->img_comp[i].h != 0) return stbi__err("bad H","Corrupt JPEG");
+ if (v_max % z->img_comp[i].v != 0) return stbi__err("bad V","Corrupt JPEG");
+ }
+
+ // compute interleaved mcu info
+ z->img_h_max = h_max;
+ z->img_v_max = v_max;
+ z->img_mcu_w = h_max * 8;
+ z->img_mcu_h = v_max * 8;
+ // these sizes can't be more than 17 bits
+ z->img_mcu_x = (s->img_x + z->img_mcu_w-1) / z->img_mcu_w;
+ z->img_mcu_y = (s->img_y + z->img_mcu_h-1) / z->img_mcu_h;
+
+ for (i=0; i < s->img_n; ++i) {
+ // number of effective pixels (e.g. for non-interleaved MCU)
+ z->img_comp[i].x = (s->img_x * z->img_comp[i].h + h_max-1) / h_max;
+ z->img_comp[i].y = (s->img_y * z->img_comp[i].v + v_max-1) / v_max;
+ // to simplify generation, we'll allocate enough memory to decode
+ // the bogus oversized data from using interleaved MCUs and their
+ // big blocks (e.g. a 16x16 iMCU on an image of width 33); we won't
+ // discard the extra data until colorspace conversion
+ //
+ // img_mcu_x, img_mcu_y: <=17 bits; comp[i].h and .v are <=4 (checked earlier)
+ // so these muls can't overflow with 32-bit ints (which we require)
+ z->img_comp[i].w2 = z->img_mcu_x * z->img_comp[i].h * 8;
+ z->img_comp[i].h2 = z->img_mcu_y * z->img_comp[i].v * 8;
+ z->img_comp[i].coeff = 0;
+ z->img_comp[i].raw_coeff = 0;
+ z->img_comp[i].linebuf = NULL;
+ z->img_comp[i].raw_data = stbi__malloc_mad2(z->img_comp[i].w2, z->img_comp[i].h2, 15);
+ if (z->img_comp[i].raw_data == NULL)
+ return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory"));
+ // align blocks for idct using mmx/sse
+ z->img_comp[i].data = (stbi_uc*) (((size_t) z->img_comp[i].raw_data + 15) & ~15);
+ if (z->progressive) {
+ // w2, h2 are multiples of 8 (see above)
+ z->img_comp[i].coeff_w = z->img_comp[i].w2 / 8;
+ z->img_comp[i].coeff_h = z->img_comp[i].h2 / 8;
+ z->img_comp[i].raw_coeff = stbi__malloc_mad3(z->img_comp[i].w2, z->img_comp[i].h2, sizeof(short), 15);
+ if (z->img_comp[i].raw_coeff == NULL)
+ return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory"));
+ z->img_comp[i].coeff = (short*) (((size_t) z->img_comp[i].raw_coeff + 15) & ~15);
+ }
+ }
+
+ return 1;
+}
+
+// use comparisons since in some cases we handle more than one case (e.g. SOF)
+#define stbi__DNL(x) ((x) == 0xdc)
+#define stbi__SOI(x) ((x) == 0xd8)
+#define stbi__EOI(x) ((x) == 0xd9)
+#define stbi__SOF(x) ((x) == 0xc0 || (x) == 0xc1 || (x) == 0xc2)
+#define stbi__SOS(x) ((x) == 0xda)
+
+#define stbi__SOF_progressive(x) ((x) == 0xc2)
+
+static int stbi__decode_jpeg_header(stbi__jpeg *z, int scan)
+{
+ int m;
+ z->jfif = 0;
+ z->app14_color_transform = -1; // valid values are 0,1,2
+ z->marker = STBI__MARKER_none; // initialize cached marker to empty
+ m = stbi__get_marker(z);
+ if (!stbi__SOI(m)) return stbi__err("no SOI","Corrupt JPEG");
+ if (scan == STBI__SCAN_type) return 1;
+ m = stbi__get_marker(z);
+ while (!stbi__SOF(m)) {
+ if (!stbi__process_marker(z,m)) return 0;
+ m = stbi__get_marker(z);
+ while (m == STBI__MARKER_none) {
+ // some files have extra padding after their blocks, so ok, we'll scan
+ if (stbi__at_eof(z->s)) return stbi__err("no SOF", "Corrupt JPEG");
+ m = stbi__get_marker(z);
+ }
+ }
+ z->progressive = stbi__SOF_progressive(m);
+ if (!stbi__process_frame_header(z, scan)) return 0;
+ return 1;
+}
+
+static stbi_uc stbi__skip_jpeg_junk_at_end(stbi__jpeg *j)
+{
+ // some JPEGs have junk at end, skip over it but if we find what looks
+ // like a valid marker, resume there
+ while (!stbi__at_eof(j->s)) {
+ stbi_uc x = stbi__get8(j->s);
+ while (x == 0xff) { // might be a marker
+ if (stbi__at_eof(j->s)) return STBI__MARKER_none;
+ x = stbi__get8(j->s);
+ if (x != 0x00 && x != 0xff) {
+ // not a stuffed zero or lead-in to another marker, looks
+ // like an actual marker, return it
+ return x;
+ }
+ // stuffed zero has x=0 now which ends the loop, meaning we go
+ // back to regular scan loop.
+ // repeated 0xff keeps trying to read the next byte of the marker.
+ }
+ }
+ return STBI__MARKER_none;
+}
+
+// decode image to YCbCr format
+static int stbi__decode_jpeg_image(stbi__jpeg *j)
+{
+ int m;
+ for (m = 0; m < 4; m++) {
+ j->img_comp[m].raw_data = NULL;
+ j->img_comp[m].raw_coeff = NULL;
+ }
+ j->restart_interval = 0;
+ if (!stbi__decode_jpeg_header(j, STBI__SCAN_load)) return 0;
+ m = stbi__get_marker(j);
+ while (!stbi__EOI(m)) {
+ if (stbi__SOS(m)) {
+ if (!stbi__process_scan_header(j)) return 0;
+ if (!stbi__parse_entropy_coded_data(j)) return 0;
+ if (j->marker == STBI__MARKER_none ) {
+ j->marker = stbi__skip_jpeg_junk_at_end(j);
+ // if we reach eof without hitting a marker, stbi__get_marker() below will fail and we'll eventually return 0
+ }
+ m = stbi__get_marker(j);
+ if (STBI__RESTART(m))
+ m = stbi__get_marker(j);
+ } else if (stbi__DNL(m)) {
+ int Ld = stbi__get16be(j->s);
+ stbi__uint32 NL = stbi__get16be(j->s);
+ if (Ld != 4) return stbi__err("bad DNL len", "Corrupt JPEG");
+ if (NL != j->s->img_y) return stbi__err("bad DNL height", "Corrupt JPEG");
+ m = stbi__get_marker(j);
+ } else {
+ if (!stbi__process_marker(j, m)) return 1;
+ m = stbi__get_marker(j);
+ }
+ }
+ if (j->progressive)
+ stbi__jpeg_finish(j);
+ return 1;
+}
+
+// static jfif-centered resampling (across block boundaries)
+
+typedef stbi_uc *(*resample_row_func)(stbi_uc *out, stbi_uc *in0, stbi_uc *in1,
+ int w, int hs);
+
+#define stbi__div4(x) ((stbi_uc) ((x) >> 2))
+
+static stbi_uc *resample_row_1(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)
+{
+ STBI_NOTUSED(out);
+ STBI_NOTUSED(in_far);
+ STBI_NOTUSED(w);
+ STBI_NOTUSED(hs);
+ return in_near;
+}
+
+static stbi_uc* stbi__resample_row_v_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)
+{
+ // need to generate two samples vertically for every one in input
+ int i;
+ STBI_NOTUSED(hs);
+ for (i=0; i < w; ++i)
+ out[i] = stbi__div4(3*in_near[i] + in_far[i] + 2);
+ return out;
+}
+
+static stbi_uc* stbi__resample_row_h_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)
+{
+ // need to generate two samples horizontally for every one in input
+ int i;
+ stbi_uc *input = in_near;
+
+ if (w == 1) {
+ // if only one sample, can't do any interpolation
+ out[0] = out[1] = input[0];
+ return out;
+ }
+
+ out[0] = input[0];
+ out[1] = stbi__div4(input[0]*3 + input[1] + 2);
+ for (i=1; i < w-1; ++i) {
+ int n = 3*input[i]+2;
+ out[i*2+0] = stbi__div4(n+input[i-1]);
+ out[i*2+1] = stbi__div4(n+input[i+1]);
+ }
+ out[i*2+0] = stbi__div4(input[w-2]*3 + input[w-1] + 2);
+ out[i*2+1] = input[w-1];
+
+ STBI_NOTUSED(in_far);
+ STBI_NOTUSED(hs);
+
+ return out;
+}
+
+#define stbi__div16(x) ((stbi_uc) ((x) >> 4))
+
+static stbi_uc *stbi__resample_row_hv_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)
+{
+ // need to generate 2x2 samples for every one in input
+ int i,t0,t1;
+ if (w == 1) {
+ out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2);
+ return out;
+ }
+
+ t1 = 3*in_near[0] + in_far[0];
+ out[0] = stbi__div4(t1+2);
+ for (i=1; i < w; ++i) {
+ t0 = t1;
+ t1 = 3*in_near[i]+in_far[i];
+ out[i*2-1] = stbi__div16(3*t0 + t1 + 8);
+ out[i*2 ] = stbi__div16(3*t1 + t0 + 8);
+ }
+ out[w*2-1] = stbi__div4(t1+2);
+
+ STBI_NOTUSED(hs);
+
+ return out;
+}
+
+#if defined(STBI_SSE2) || defined(STBI_NEON)
+static stbi_uc *stbi__resample_row_hv_2_simd(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)
+{
+ // need to generate 2x2 samples for every one in input
+ int i=0,t0,t1;
+
+ if (w == 1) {
+ out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2);
+ return out;
+ }
+
+ t1 = 3*in_near[0] + in_far[0];
+ // process groups of 8 pixels for as long as we can.
+ // note we can't handle the last pixel in a row in this loop
+ // because we need to handle the filter boundary conditions.
+ for (; i < ((w-1) & ~7); i += 8) {
+#if defined(STBI_SSE2)
+ // load and perform the vertical filtering pass
+ // this uses 3*x + y = 4*x + (y - x)
+ __m128i zero = _mm_setzero_si128();
+ __m128i farb = _mm_loadl_epi64((__m128i *) (in_far + i));
+ __m128i nearb = _mm_loadl_epi64((__m128i *) (in_near + i));
+ __m128i farw = _mm_unpacklo_epi8(farb, zero);
+ __m128i nearw = _mm_unpacklo_epi8(nearb, zero);
+ __m128i diff = _mm_sub_epi16(farw, nearw);
+ __m128i nears = _mm_slli_epi16(nearw, 2);
+ __m128i curr = _mm_add_epi16(nears, diff); // current row
+
+ // horizontal filter works the same based on shifted vers of current
+ // row. "prev" is current row shifted right by 1 pixel; we need to
+ // insert the previous pixel value (from t1).
+ // "next" is current row shifted left by 1 pixel, with first pixel
+ // of next block of 8 pixels added in.
+ __m128i prv0 = _mm_slli_si128(curr, 2);
+ __m128i nxt0 = _mm_srli_si128(curr, 2);
+ __m128i prev = _mm_insert_epi16(prv0, t1, 0);
+ __m128i next = _mm_insert_epi16(nxt0, 3*in_near[i+8] + in_far[i+8], 7);
+
+ // horizontal filter, polyphase implementation since it's convenient:
+ // even pixels = 3*cur + prev = cur*4 + (prev - cur)
+ // odd pixels = 3*cur + next = cur*4 + (next - cur)
+ // note the shared term.
+ __m128i bias = _mm_set1_epi16(8);
+ __m128i curs = _mm_slli_epi16(curr, 2);
+ __m128i prvd = _mm_sub_epi16(prev, curr);
+ __m128i nxtd = _mm_sub_epi16(next, curr);
+ __m128i curb = _mm_add_epi16(curs, bias);
+ __m128i even = _mm_add_epi16(prvd, curb);
+ __m128i odd = _mm_add_epi16(nxtd, curb);
+
+ // interleave even and odd pixels, then undo scaling.
+ __m128i int0 = _mm_unpacklo_epi16(even, odd);
+ __m128i int1 = _mm_unpackhi_epi16(even, odd);
+ __m128i de0 = _mm_srli_epi16(int0, 4);
+ __m128i de1 = _mm_srli_epi16(int1, 4);
+
+ // pack and write output
+ __m128i outv = _mm_packus_epi16(de0, de1);
+ _mm_storeu_si128((__m128i *) (out + i*2), outv);
+#elif defined(STBI_NEON)
+ // load and perform the vertical filtering pass
+ // this uses 3*x + y = 4*x + (y - x)
+ uint8x8_t farb = vld1_u8(in_far + i);
+ uint8x8_t nearb = vld1_u8(in_near + i);
+ int16x8_t diff = vreinterpretq_s16_u16(vsubl_u8(farb, nearb));
+ int16x8_t nears = vreinterpretq_s16_u16(vshll_n_u8(nearb, 2));
+ int16x8_t curr = vaddq_s16(nears, diff); // current row
+
+ // horizontal filter works the same based on shifted vers of current
+ // row. "prev" is current row shifted right by 1 pixel; we need to
+ // insert the previous pixel value (from t1).
+ // "next" is current row shifted left by 1 pixel, with first pixel
+ // of next block of 8 pixels added in.
+ int16x8_t prv0 = vextq_s16(curr, curr, 7);
+ int16x8_t nxt0 = vextq_s16(curr, curr, 1);
+ int16x8_t prev = vsetq_lane_s16(t1, prv0, 0);
+ int16x8_t next = vsetq_lane_s16(3*in_near[i+8] + in_far[i+8], nxt0, 7);
+
+ // horizontal filter, polyphase implementation since it's convenient:
+ // even pixels = 3*cur + prev = cur*4 + (prev - cur)
+ // odd pixels = 3*cur + next = cur*4 + (next - cur)
+ // note the shared term.
+ int16x8_t curs = vshlq_n_s16(curr, 2);
+ int16x8_t prvd = vsubq_s16(prev, curr);
+ int16x8_t nxtd = vsubq_s16(next, curr);
+ int16x8_t even = vaddq_s16(curs, prvd);
+ int16x8_t odd = vaddq_s16(curs, nxtd);
+
+ // undo scaling and round, then store with even/odd phases interleaved
+ uint8x8x2_t o;
+ o.val[0] = vqrshrun_n_s16(even, 4);
+ o.val[1] = vqrshrun_n_s16(odd, 4);
+ vst2_u8(out + i*2, o);
+#endif
+
+ // "previous" value for next iter
+ t1 = 3*in_near[i+7] + in_far[i+7];
+ }
+
+ t0 = t1;
+ t1 = 3*in_near[i] + in_far[i];
+ out[i*2] = stbi__div16(3*t1 + t0 + 8);
+
+ for (++i; i < w; ++i) {
+ t0 = t1;
+ t1 = 3*in_near[i]+in_far[i];
+ out[i*2-1] = stbi__div16(3*t0 + t1 + 8);
+ out[i*2 ] = stbi__div16(3*t1 + t0 + 8);
+ }
+ out[w*2-1] = stbi__div4(t1+2);
+
+ STBI_NOTUSED(hs);
+
+ return out;
+}
+#endif
+
+static stbi_uc *stbi__resample_row_generic(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)
+{
+ // resample with nearest-neighbor
+ int i,j;
+ STBI_NOTUSED(in_far);
+ for (i=0; i < w; ++i)
+ for (j=0; j < hs; ++j)
+ out[i*hs+j] = in_near[i];
+ return out;
+}
+
+// this is a reduced-precision calculation of YCbCr-to-RGB introduced
+// to make sure the code produces the same results in both SIMD and scalar
+#define stbi__float2fixed(x) (((int) ((x) * 4096.0f + 0.5f)) << 8)
+static void stbi__YCbCr_to_RGB_row(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step)
+{
+ int i;
+ for (i=0; i < count; ++i) {
+ int y_fixed = (y[i] << 20) + (1<<19); // rounding
+ int r,g,b;
+ int cr = pcr[i] - 128;
+ int cb = pcb[i] - 128;
+ r = y_fixed + cr* stbi__float2fixed(1.40200f);
+ g = y_fixed + (cr*-stbi__float2fixed(0.71414f)) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000);
+ b = y_fixed + cb* stbi__float2fixed(1.77200f);
+ r >>= 20;
+ g >>= 20;
+ b >>= 20;
+ if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; }
+ if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; }
+ if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; }
+ out[0] = (stbi_uc)r;
+ out[1] = (stbi_uc)g;
+ out[2] = (stbi_uc)b;
+ out[3] = 255;
+ out += step;
+ }
+}
+
+#if defined(STBI_SSE2) || defined(STBI_NEON)
+static void stbi__YCbCr_to_RGB_simd(stbi_uc *out, stbi_uc const *y, stbi_uc const *pcb, stbi_uc const *pcr, int count, int step)
+{
+ int i = 0;
+
+#ifdef STBI_SSE2
+ // step == 3 is pretty ugly on the final interleave, and i'm not convinced
+ // it's useful in practice (you wouldn't use it for textures, for example).
+ // so just accelerate step == 4 case.
+ if (step == 4) {
+ // this is a fairly straightforward implementation and not super-optimized.
+ __m128i signflip = _mm_set1_epi8(-0x80);
+ __m128i cr_const0 = _mm_set1_epi16( (short) ( 1.40200f*4096.0f+0.5f));
+ __m128i cr_const1 = _mm_set1_epi16( - (short) ( 0.71414f*4096.0f+0.5f));
+ __m128i cb_const0 = _mm_set1_epi16( - (short) ( 0.34414f*4096.0f+0.5f));
+ __m128i cb_const1 = _mm_set1_epi16( (short) ( 1.77200f*4096.0f+0.5f));
+ __m128i y_bias = _mm_set1_epi8((char) (unsigned char) 128);
+ __m128i xw = _mm_set1_epi16(255); // alpha channel
+
+ for (; i+7 < count; i += 8) {
+ // load
+ __m128i y_bytes = _mm_loadl_epi64((__m128i *) (y+i));
+ __m128i cr_bytes = _mm_loadl_epi64((__m128i *) (pcr+i));
+ __m128i cb_bytes = _mm_loadl_epi64((__m128i *) (pcb+i));
+ __m128i cr_biased = _mm_xor_si128(cr_bytes, signflip); // -128
+ __m128i cb_biased = _mm_xor_si128(cb_bytes, signflip); // -128
+
+ // unpack to short (and left-shift cr, cb by 8)
+ __m128i yw = _mm_unpacklo_epi8(y_bias, y_bytes);
+ __m128i crw = _mm_unpacklo_epi8(_mm_setzero_si128(), cr_biased);
+ __m128i cbw = _mm_unpacklo_epi8(_mm_setzero_si128(), cb_biased);
+
+ // color transform
+ __m128i yws = _mm_srli_epi16(yw, 4);
+ __m128i cr0 = _mm_mulhi_epi16(cr_const0, crw);
+ __m128i cb0 = _mm_mulhi_epi16(cb_const0, cbw);
+ __m128i cb1 = _mm_mulhi_epi16(cbw, cb_const1);
+ __m128i cr1 = _mm_mulhi_epi16(crw, cr_const1);
+ __m128i rws = _mm_add_epi16(cr0, yws);
+ __m128i gwt = _mm_add_epi16(cb0, yws);
+ __m128i bws = _mm_add_epi16(yws, cb1);
+ __m128i gws = _mm_add_epi16(gwt, cr1);
+
+ // descale
+ __m128i rw = _mm_srai_epi16(rws, 4);
+ __m128i bw = _mm_srai_epi16(bws, 4);
+ __m128i gw = _mm_srai_epi16(gws, 4);
+
+ // back to byte, set up for transpose
+ __m128i brb = _mm_packus_epi16(rw, bw);
+ __m128i gxb = _mm_packus_epi16(gw, xw);
+
+ // transpose to interleave channels
+ __m128i t0 = _mm_unpacklo_epi8(brb, gxb);
+ __m128i t1 = _mm_unpackhi_epi8(brb, gxb);
+ __m128i o0 = _mm_unpacklo_epi16(t0, t1);
+ __m128i o1 = _mm_unpackhi_epi16(t0, t1);
+
+ // store
+ _mm_storeu_si128((__m128i *) (out + 0), o0);
+ _mm_storeu_si128((__m128i *) (out + 16), o1);
+ out += 32;
+ }
+ }
+#endif
+
+#ifdef STBI_NEON
+ // in this version, step=3 support would be easy to add. but is there demand?
+ if (step == 4) {
+ // this is a fairly straightforward implementation and not super-optimized.
+ uint8x8_t signflip = vdup_n_u8(0x80);
+ int16x8_t cr_const0 = vdupq_n_s16( (short) ( 1.40200f*4096.0f+0.5f));
+ int16x8_t cr_const1 = vdupq_n_s16( - (short) ( 0.71414f*4096.0f+0.5f));
+ int16x8_t cb_const0 = vdupq_n_s16( - (short) ( 0.34414f*4096.0f+0.5f));
+ int16x8_t cb_const1 = vdupq_n_s16( (short) ( 1.77200f*4096.0f+0.5f));
+
+ for (; i+7 < count; i += 8) {
+ // load
+ uint8x8_t y_bytes = vld1_u8(y + i);
+ uint8x8_t cr_bytes = vld1_u8(pcr + i);
+ uint8x8_t cb_bytes = vld1_u8(pcb + i);
+ int8x8_t cr_biased = vreinterpret_s8_u8(vsub_u8(cr_bytes, signflip));
+ int8x8_t cb_biased = vreinterpret_s8_u8(vsub_u8(cb_bytes, signflip));
+
+ // expand to s16
+ int16x8_t yws = vreinterpretq_s16_u16(vshll_n_u8(y_bytes, 4));
+ int16x8_t crw = vshll_n_s8(cr_biased, 7);
+ int16x8_t cbw = vshll_n_s8(cb_biased, 7);
+
+ // color transform
+ int16x8_t cr0 = vqdmulhq_s16(crw, cr_const0);
+ int16x8_t cb0 = vqdmulhq_s16(cbw, cb_const0);
+ int16x8_t cr1 = vqdmulhq_s16(crw, cr_const1);
+ int16x8_t cb1 = vqdmulhq_s16(cbw, cb_const1);
+ int16x8_t rws = vaddq_s16(yws, cr0);
+ int16x8_t gws = vaddq_s16(vaddq_s16(yws, cb0), cr1);
+ int16x8_t bws = vaddq_s16(yws, cb1);
+
+ // undo scaling, round, convert to byte
+ uint8x8x4_t o;
+ o.val[0] = vqrshrun_n_s16(rws, 4);
+ o.val[1] = vqrshrun_n_s16(gws, 4);
+ o.val[2] = vqrshrun_n_s16(bws, 4);
+ o.val[3] = vdup_n_u8(255);
+
+ // store, interleaving r/g/b/a
+ vst4_u8(out, o);
+ out += 8*4;
+ }
+ }
+#endif
+
+ for (; i < count; ++i) {
+ int y_fixed = (y[i] << 20) + (1<<19); // rounding
+ int r,g,b;
+ int cr = pcr[i] - 128;
+ int cb = pcb[i] - 128;
+ r = y_fixed + cr* stbi__float2fixed(1.40200f);
+ g = y_fixed + cr*-stbi__float2fixed(0.71414f) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000);
+ b = y_fixed + cb* stbi__float2fixed(1.77200f);
+ r >>= 20;
+ g >>= 20;
+ b >>= 20;
+ if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; }
+ if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; }
+ if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; }
+ out[0] = (stbi_uc)r;
+ out[1] = (stbi_uc)g;
+ out[2] = (stbi_uc)b;
+ out[3] = 255;
+ out += step;
+ }
+}
+#endif
+
+// set up the kernels
+static void stbi__setup_jpeg(stbi__jpeg *j)
+{
+ j->idct_block_kernel = stbi__idct_block;
+ j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_row;
+ j->resample_row_hv_2_kernel = stbi__resample_row_hv_2;
+
+#ifdef STBI_SSE2
+ if (stbi__sse2_available()) {
+ j->idct_block_kernel = stbi__idct_simd;
+ j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd;
+ j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd;
+ }
+#endif
+
+#ifdef STBI_NEON
+ j->idct_block_kernel = stbi__idct_simd;
+ j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd;
+ j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd;
+#endif
+}
+
+// clean up the temporary component buffers
+static void stbi__cleanup_jpeg(stbi__jpeg *j)
+{
+ stbi__free_jpeg_components(j, j->s->img_n, 0);
+}
+
+typedef struct
+{
+ resample_row_func resample;
+ stbi_uc *line0,*line1;
+ int hs,vs; // expansion factor in each axis
+ int w_lores; // horizontal pixels pre-expansion
+ int ystep; // how far through vertical expansion we are
+ int ypos; // which pre-expansion row we're on
+} stbi__resample;
+
+// fast 0..255 * 0..255 => 0..255 rounded multiplication
+static stbi_uc stbi__blinn_8x8(stbi_uc x, stbi_uc y)
+{
+ unsigned int t = x*y + 128;
+ return (stbi_uc) ((t + (t >>8)) >> 8);
+}
+
+static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp, int req_comp)
+{
+ int n, decode_n, is_rgb;
+ z->s->img_n = 0; // make stbi__cleanup_jpeg safe
+
+ // validate req_comp
+ if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error");
+
+ // load a jpeg image from whichever source, but leave in YCbCr format
+ if (!stbi__decode_jpeg_image(z)) { stbi__cleanup_jpeg(z); return NULL; }
+
+ // determine actual number of components to generate
+ n = req_comp ? req_comp : z->s->img_n >= 3 ? 3 : 1;
+
+ is_rgb = z->s->img_n == 3 && (z->rgb == 3 || (z->app14_color_transform == 0 && !z->jfif));
+
+ if (z->s->img_n == 3 && n < 3 && !is_rgb)
+ decode_n = 1;
+ else
+ decode_n = z->s->img_n;
+
+ // nothing to do if no components requested; check this now to avoid
+ // accessing uninitialized coutput[0] later
+ if (decode_n <= 0) { stbi__cleanup_jpeg(z); return NULL; }
+
+ // resample and color-convert
+ {
+ int k;
+ unsigned int i,j;
+ stbi_uc *output;
+ stbi_uc *coutput[4] = { NULL, NULL, NULL, NULL };
+
+ stbi__resample res_comp[4];
+
+ for (k=0; k < decode_n; ++k) {
+ stbi__resample *r = &res_comp[k];
+
+ // allocate line buffer big enough for upsampling off the edges
+ // with upsample factor of 4
+ z->img_comp[k].linebuf = (stbi_uc *) stbi__malloc(z->s->img_x + 3);
+ if (!z->img_comp[k].linebuf) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); }
+
+ r->hs = z->img_h_max / z->img_comp[k].h;
+ r->vs = z->img_v_max / z->img_comp[k].v;
+ r->ystep = r->vs >> 1;
+ r->w_lores = (z->s->img_x + r->hs-1) / r->hs;
+ r->ypos = 0;
+ r->line0 = r->line1 = z->img_comp[k].data;
+
+ if (r->hs == 1 && r->vs == 1) r->resample = resample_row_1;
+ else if (r->hs == 1 && r->vs == 2) r->resample = stbi__resample_row_v_2;
+ else if (r->hs == 2 && r->vs == 1) r->resample = stbi__resample_row_h_2;
+ else if (r->hs == 2 && r->vs == 2) r->resample = z->resample_row_hv_2_kernel;
+ else r->resample = stbi__resample_row_generic;
+ }
+
+ // can't error after this so, this is safe
+ output = (stbi_uc *) stbi__malloc_mad3(n, z->s->img_x, z->s->img_y, 1);
+ if (!output) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); }
+
+ // now go ahead and resample
+ for (j=0; j < z->s->img_y; ++j) {
+ stbi_uc *out = output + n * z->s->img_x * j;
+ for (k=0; k < decode_n; ++k) {
+ stbi__resample *r = &res_comp[k];
+ int y_bot = r->ystep >= (r->vs >> 1);
+ coutput[k] = r->resample(z->img_comp[k].linebuf,
+ y_bot ? r->line1 : r->line0,
+ y_bot ? r->line0 : r->line1,
+ r->w_lores, r->hs);
+ if (++r->ystep >= r->vs) {
+ r->ystep = 0;
+ r->line0 = r->line1;
+ if (++r->ypos < z->img_comp[k].y)
+ r->line1 += z->img_comp[k].w2;
+ }
+ }
+ if (n >= 3) {
+ stbi_uc *y = coutput[0];
+ if (z->s->img_n == 3) {
+ if (is_rgb) {
+ for (i=0; i < z->s->img_x; ++i) {
+ out[0] = y[i];
+ out[1] = coutput[1][i];
+ out[2] = coutput[2][i];
+ out[3] = 255;
+ out += n;
+ }
+ } else {
+ z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n);
+ }
+ } else if (z->s->img_n == 4) {
+ if (z->app14_color_transform == 0) { // CMYK
+ for (i=0; i < z->s->img_x; ++i) {
+ stbi_uc m = coutput[3][i];
+ out[0] = stbi__blinn_8x8(coutput[0][i], m);
+ out[1] = stbi__blinn_8x8(coutput[1][i], m);
+ out[2] = stbi__blinn_8x8(coutput[2][i], m);
+ out[3] = 255;
+ out += n;
+ }
+ } else if (z->app14_color_transform == 2) { // YCCK
+ z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n);
+ for (i=0; i < z->s->img_x; ++i) {
+ stbi_uc m = coutput[3][i];
+ out[0] = stbi__blinn_8x8(255 - out[0], m);
+ out[1] = stbi__blinn_8x8(255 - out[1], m);
+ out[2] = stbi__blinn_8x8(255 - out[2], m);
+ out += n;
+ }
+ } else { // YCbCr + alpha? Ignore the fourth channel for now
+ z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n);
+ }
+ } else
+ for (i=0; i < z->s->img_x; ++i) {
+ out[0] = out[1] = out[2] = y[i];
+ out[3] = 255; // not used if n==3
+ out += n;
+ }
+ } else {
+ if (is_rgb) {
+ if (n == 1)
+ for (i=0; i < z->s->img_x; ++i)
+ *out++ = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]);
+ else {
+ for (i=0; i < z->s->img_x; ++i, out += 2) {
+ out[0] = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]);
+ out[1] = 255;
+ }
+ }
+ } else if (z->s->img_n == 4 && z->app14_color_transform == 0) {
+ for (i=0; i < z->s->img_x; ++i) {
+ stbi_uc m = coutput[3][i];
+ stbi_uc r = stbi__blinn_8x8(coutput[0][i], m);
+ stbi_uc g = stbi__blinn_8x8(coutput[1][i], m);
+ stbi_uc b = stbi__blinn_8x8(coutput[2][i], m);
+ out[0] = stbi__compute_y(r, g, b);
+ out[1] = 255;
+ out += n;
+ }
+ } else if (z->s->img_n == 4 && z->app14_color_transform == 2) {
+ for (i=0; i < z->s->img_x; ++i) {
+ out[0] = stbi__blinn_8x8(255 - coutput[0][i], coutput[3][i]);
+ out[1] = 255;
+ out += n;
+ }
+ } else {
+ stbi_uc *y = coutput[0];
+ if (n == 1)
+ for (i=0; i < z->s->img_x; ++i) out[i] = y[i];
+ else
+ for (i=0; i < z->s->img_x; ++i) { *out++ = y[i]; *out++ = 255; }
+ }
+ }
+ }
+ stbi__cleanup_jpeg(z);
+ *out_x = z->s->img_x;
+ *out_y = z->s->img_y;
+ if (comp) *comp = z->s->img_n >= 3 ? 3 : 1; // report original components, not output
+ return output;
+ }
+}
+
+static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)
+{
+ unsigned char* result;
+ stbi__jpeg* j = (stbi__jpeg*) stbi__malloc(sizeof(stbi__jpeg));
+ if (!j) return stbi__errpuc("outofmem", "Out of memory");
+ memset(j, 0, sizeof(stbi__jpeg));
+ STBI_NOTUSED(ri);
+ j->s = s;
+ stbi__setup_jpeg(j);
+ result = load_jpeg_image(j, x,y,comp,req_comp);
+ STBI_FREE(j);
+ return result;
+}
+
+static int stbi__jpeg_test(stbi__context *s)
+{
+ int r;
+ stbi__jpeg* j = (stbi__jpeg*)stbi__malloc(sizeof(stbi__jpeg));
+ if (!j) return stbi__err("outofmem", "Out of memory");
+ memset(j, 0, sizeof(stbi__jpeg));
+ j->s = s;
+ stbi__setup_jpeg(j);
+ r = stbi__decode_jpeg_header(j, STBI__SCAN_type);
+ stbi__rewind(s);
+ STBI_FREE(j);
+ return r;
+}
+
+static int stbi__jpeg_info_raw(stbi__jpeg *j, int *x, int *y, int *comp)
+{
+ if (!stbi__decode_jpeg_header(j, STBI__SCAN_header)) {
+ stbi__rewind( j->s );
+ return 0;
+ }
+ if (x) *x = j->s->img_x;
+ if (y) *y = j->s->img_y;
+ if (comp) *comp = j->s->img_n >= 3 ? 3 : 1;
+ return 1;
+}
+
+static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp)
+{
+ int result;
+ stbi__jpeg* j = (stbi__jpeg*) (stbi__malloc(sizeof(stbi__jpeg)));
+ if (!j) return stbi__err("outofmem", "Out of memory");
+ memset(j, 0, sizeof(stbi__jpeg));
+ j->s = s;
+ result = stbi__jpeg_info_raw(j, x, y, comp);
+ STBI_FREE(j);
+ return result;
+}
+#endif
+
+// public domain zlib decode v0.2 Sean Barrett 2006-11-18
+// simple implementation
+// - all input must be provided in an upfront buffer
+// - all output is written to a single output buffer (can malloc/realloc)
+// performance
+// - fast huffman
+
+#ifndef STBI_NO_ZLIB
+
+// fast-way is faster to check than jpeg huffman, but slow way is slower
+#define STBI__ZFAST_BITS 9 // accelerate all cases in default tables
+#define STBI__ZFAST_MASK ((1 << STBI__ZFAST_BITS) - 1)
+#define STBI__ZNSYMS 288 // number of symbols in literal/length alphabet
+
+// zlib-style huffman encoding
+// (jpegs packs from left, zlib from right, so can't share code)
+typedef struct
+{
+ stbi__uint16 fast[1 << STBI__ZFAST_BITS];
+ stbi__uint16 firstcode[16];
+ int maxcode[17];
+ stbi__uint16 firstsymbol[16];
+ stbi_uc size[STBI__ZNSYMS];
+ stbi__uint16 value[STBI__ZNSYMS];
+} stbi__zhuffman;
+
+stbi_inline static int stbi__bitreverse16(int n)
+{
+ n = ((n & 0xAAAA) >> 1) | ((n & 0x5555) << 1);
+ n = ((n & 0xCCCC) >> 2) | ((n & 0x3333) << 2);
+ n = ((n & 0xF0F0) >> 4) | ((n & 0x0F0F) << 4);
+ n = ((n & 0xFF00) >> 8) | ((n & 0x00FF) << 8);
+ return n;
+}
+
+stbi_inline static int stbi__bit_reverse(int v, int bits)
+{
+ STBI_ASSERT(bits <= 16);
+ // to bit reverse n bits, reverse 16 and shift
+ // e.g. 11 bits, bit reverse and shift away 5
+ return stbi__bitreverse16(v) >> (16-bits);
+}
+
+static int stbi__zbuild_huffman(stbi__zhuffman *z, const stbi_uc *sizelist, int num)
+{
+ int i,k=0;
+ int code, next_code[16], sizes[17];
+
+ // DEFLATE spec for generating codes
+ memset(sizes, 0, sizeof(sizes));
+ memset(z->fast, 0, sizeof(z->fast));
+ for (i=0; i < num; ++i)
+ ++sizes[sizelist[i]];
+ sizes[0] = 0;
+ for (i=1; i < 16; ++i)
+ if (sizes[i] > (1 << i))
+ return stbi__err("bad sizes", "Corrupt PNG");
+ code = 0;
+ for (i=1; i < 16; ++i) {
+ next_code[i] = code;
+ z->firstcode[i] = (stbi__uint16) code;
+ z->firstsymbol[i] = (stbi__uint16) k;
+ code = (code + sizes[i]);
+ if (sizes[i])
+ if (code-1 >= (1 << i)) return stbi__err("bad codelengths","Corrupt PNG");
+ z->maxcode[i] = code << (16-i); // preshift for inner loop
+ code <<= 1;
+ k += sizes[i];
+ }
+ z->maxcode[16] = 0x10000; // sentinel
+ for (i=0; i < num; ++i) {
+ int s = sizelist[i];
+ if (s) {
+ int c = next_code[s] - z->firstcode[s] + z->firstsymbol[s];
+ stbi__uint16 fastv = (stbi__uint16) ((s << 9) | i);
+ z->size [c] = (stbi_uc ) s;
+ z->value[c] = (stbi__uint16) i;
+ if (s <= STBI__ZFAST_BITS) {
+ int j = stbi__bit_reverse(next_code[s],s);
+ while (j < (1 << STBI__ZFAST_BITS)) {
+ z->fast[j] = fastv;
+ j += (1 << s);
+ }
+ }
+ ++next_code[s];
+ }
+ }
+ return 1;
+}
+
+// zlib-from-memory implementation for PNG reading
+// because PNG allows splitting the zlib stream arbitrarily,
+// and it's annoying structurally to have PNG call ZLIB call PNG,
+// we require PNG read all the IDATs and combine them into a single
+// memory buffer
+
+typedef struct
+{
+ stbi_uc *zbuffer, *zbuffer_end;
+ int num_bits;
+ int hit_zeof_once;
+ stbi__uint32 code_buffer;
+
+ char *zout;
+ char *zout_start;
+ char *zout_end;
+ int z_expandable;
+
+ stbi__zhuffman z_length, z_distance;
+} stbi__zbuf;
+
+stbi_inline static int stbi__zeof(stbi__zbuf *z)
+{
+ return (z->zbuffer >= z->zbuffer_end);
+}
+
+stbi_inline static stbi_uc stbi__zget8(stbi__zbuf *z)
+{
+ return stbi__zeof(z) ? 0 : *z->zbuffer++;
+}
+
+static void stbi__fill_bits(stbi__zbuf *z)
+{
+ do {
+ if (z->code_buffer >= (1U << z->num_bits)) {
+ z->zbuffer = z->zbuffer_end; /* treat this as EOF so we fail. */
+ return;
+ }
+ z->code_buffer |= (unsigned int) stbi__zget8(z) << z->num_bits;
+ z->num_bits += 8;
+ } while (z->num_bits <= 24);
+}
+
+stbi_inline static unsigned int stbi__zreceive(stbi__zbuf *z, int n)
+{
+ unsigned int k;
+ if (z->num_bits < n) stbi__fill_bits(z);
+ k = z->code_buffer & ((1 << n) - 1);
+ z->code_buffer >>= n;
+ z->num_bits -= n;
+ return k;
+}
+
+static int stbi__zhuffman_decode_slowpath(stbi__zbuf *a, stbi__zhuffman *z)
+{
+ int b,s,k;
+ // not resolved by fast table, so compute it the slow way
+ // use jpeg approach, which requires MSbits at top
+ k = stbi__bit_reverse(a->code_buffer, 16);
+ for (s=STBI__ZFAST_BITS+1; ; ++s)
+ if (k < z->maxcode[s])
+ break;
+ if (s >= 16) return -1; // invalid code!
+ // code size is s, so:
+ b = (k >> (16-s)) - z->firstcode[s] + z->firstsymbol[s];
+ if (b >= STBI__ZNSYMS) return -1; // some data was corrupt somewhere!
+ if (z->size[b] != s) return -1; // was originally an assert, but report failure instead.
+ a->code_buffer >>= s;
+ a->num_bits -= s;
+ return z->value[b];
+}
+
+stbi_inline static int stbi__zhuffman_decode(stbi__zbuf *a, stbi__zhuffman *z)
+{
+ int b,s;
+ if (a->num_bits < 16) {
+ if (stbi__zeof(a)) {
+ if (!a->hit_zeof_once) {
+ // This is the first time we hit eof, insert 16 extra padding btis
+ // to allow us to keep going; if we actually consume any of them
+ // though, that is invalid data. This is caught later.
+ a->hit_zeof_once = 1;
+ a->num_bits += 16; // add 16 implicit zero bits
+ } else {
+ // We already inserted our extra 16 padding bits and are again
+ // out, this stream is actually prematurely terminated.
+ return -1;
+ }
+ } else {
+ stbi__fill_bits(a);
+ }
+ }
+ b = z->fast[a->code_buffer & STBI__ZFAST_MASK];
+ if (b) {
+ s = b >> 9;
+ a->code_buffer >>= s;
+ a->num_bits -= s;
+ return b & 511;
+ }
+ return stbi__zhuffman_decode_slowpath(a, z);
+}
+
+static int stbi__zexpand(stbi__zbuf *z, char *zout, int n) // need to make room for n bytes
+{
+ char *q;
+ unsigned int cur, limit, old_limit;
+ z->zout = zout;
+ if (!z->z_expandable) return stbi__err("output buffer limit","Corrupt PNG");
+ cur = (unsigned int) (z->zout - z->zout_start);
+ limit = old_limit = (unsigned) (z->zout_end - z->zout_start);
+ if (UINT_MAX - cur < (unsigned) n) return stbi__err("outofmem", "Out of memory");
+ while (cur + n > limit) {
+ if(limit > UINT_MAX / 2) return stbi__err("outofmem", "Out of memory");
+ limit *= 2;
+ }
+ q = (char *) STBI_REALLOC_SIZED(z->zout_start, old_limit, limit);
+ STBI_NOTUSED(old_limit);
+ if (q == NULL) return stbi__err("outofmem", "Out of memory");
+ z->zout_start = q;
+ z->zout = q + cur;
+ z->zout_end = q + limit;
+ return 1;
+}
+
+static const int stbi__zlength_base[31] = {
+ 3,4,5,6,7,8,9,10,11,13,
+ 15,17,19,23,27,31,35,43,51,59,
+ 67,83,99,115,131,163,195,227,258,0,0 };
+
+static const int stbi__zlength_extra[31]=
+{ 0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0 };
+
+static const int stbi__zdist_base[32] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,
+257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577,0,0};
+
+static const int stbi__zdist_extra[32] =
+{ 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13};
+
+static int stbi__parse_huffman_block(stbi__zbuf *a)
+{
+ char *zout = a->zout;
+ for(;;) {
+ int z = stbi__zhuffman_decode(a, &a->z_length);
+ if (z < 256) {
+ if (z < 0) return stbi__err("bad huffman code","Corrupt PNG"); // error in huffman codes
+ if (zout >= a->zout_end) {
+ if (!stbi__zexpand(a, zout, 1)) return 0;
+ zout = a->zout;
+ }
+ *zout++ = (char) z;
+ } else {
+ stbi_uc *p;
+ int len,dist;
+ if (z == 256) {
+ a->zout = zout;
+ if (a->hit_zeof_once && a->num_bits < 16) {
+ // The first time we hit zeof, we inserted 16 extra zero bits into our bit
+ // buffer so the decoder can just do its speculative decoding. But if we
+ // actually consumed any of those bits (which is the case when num_bits < 16),
+ // the stream actually read past the end so it is malformed.
+ return stbi__err("unexpected end","Corrupt PNG");
+ }
+ return 1;
+ }
+ if (z >= 286) return stbi__err("bad huffman code","Corrupt PNG"); // per DEFLATE, length codes 286 and 287 must not appear in compressed data
+ z -= 257;
+ len = stbi__zlength_base[z];
+ if (stbi__zlength_extra[z]) len += stbi__zreceive(a, stbi__zlength_extra[z]);
+ z = stbi__zhuffman_decode(a, &a->z_distance);
+ if (z < 0 || z >= 30) return stbi__err("bad huffman code","Corrupt PNG"); // per DEFLATE, distance codes 30 and 31 must not appear in compressed data
+ dist = stbi__zdist_base[z];
+ if (stbi__zdist_extra[z]) dist += stbi__zreceive(a, stbi__zdist_extra[z]);
+ if (zout - a->zout_start < dist) return stbi__err("bad dist","Corrupt PNG");
+ if (len > a->zout_end - zout) {
+ if (!stbi__zexpand(a, zout, len)) return 0;
+ zout = a->zout;
+ }
+ p = (stbi_uc *) (zout - dist);
+ if (dist == 1) { // run of one byte; common in images.
+ stbi_uc v = *p;
+ if (len) { do *zout++ = v; while (--len); }
+ } else {
+ if (len) { do *zout++ = *p++; while (--len); }
+ }
+ }
+ }
+}
+
+static int stbi__compute_huffman_codes(stbi__zbuf *a)
+{
+ static const stbi_uc length_dezigzag[19] = { 16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15 };
+ stbi__zhuffman z_codelength;
+ stbi_uc lencodes[286+32+137];//padding for maximum single op
+ stbi_uc codelength_sizes[19];
+ int i,n;
+
+ int hlit = stbi__zreceive(a,5) + 257;
+ int hdist = stbi__zreceive(a,5) + 1;
+ int hclen = stbi__zreceive(a,4) + 4;
+ int ntot = hlit + hdist;
+
+ memset(codelength_sizes, 0, sizeof(codelength_sizes));
+ for (i=0; i < hclen; ++i) {
+ int s = stbi__zreceive(a,3);
+ codelength_sizes[length_dezigzag[i]] = (stbi_uc) s;
+ }
+ if (!stbi__zbuild_huffman(&z_codelength, codelength_sizes, 19)) return 0;
+
+ n = 0;
+ while (n < ntot) {
+ int c = stbi__zhuffman_decode(a, &z_codelength);
+ if (c < 0 || c >= 19) return stbi__err("bad codelengths", "Corrupt PNG");
+ if (c < 16)
+ lencodes[n++] = (stbi_uc) c;
+ else {
+ stbi_uc fill = 0;
+ if (c == 16) {
+ c = stbi__zreceive(a,2)+3;
+ if (n == 0) return stbi__err("bad codelengths", "Corrupt PNG");
+ fill = lencodes[n-1];
+ } else if (c == 17) {
+ c = stbi__zreceive(a,3)+3;
+ } else if (c == 18) {
+ c = stbi__zreceive(a,7)+11;
+ } else {
+ return stbi__err("bad codelengths", "Corrupt PNG");
+ }
+ if (ntot - n < c) return stbi__err("bad codelengths", "Corrupt PNG");
+ memset(lencodes+n, fill, c);
+ n += c;
+ }
+ }
+ if (n != ntot) return stbi__err("bad codelengths","Corrupt PNG");
+ if (!stbi__zbuild_huffman(&a->z_length, lencodes, hlit)) return 0;
+ if (!stbi__zbuild_huffman(&a->z_distance, lencodes+hlit, hdist)) return 0;
+ return 1;
+}
+
+static int stbi__parse_uncompressed_block(stbi__zbuf *a)
+{
+ stbi_uc header[4];
+ int len,nlen,k;
+ if (a->num_bits & 7)
+ stbi__zreceive(a, a->num_bits & 7); // discard
+ // drain the bit-packed data into header
+ k = 0;
+ while (a->num_bits > 0) {
+ header[k++] = (stbi_uc) (a->code_buffer & 255); // suppress MSVC run-time check
+ a->code_buffer >>= 8;
+ a->num_bits -= 8;
+ }
+ if (a->num_bits < 0) return stbi__err("zlib corrupt","Corrupt PNG");
+ // now fill header the normal way
+ while (k < 4)
+ header[k++] = stbi__zget8(a);
+ len = header[1] * 256 + header[0];
+ nlen = header[3] * 256 + header[2];
+ if (nlen != (len ^ 0xffff)) return stbi__err("zlib corrupt","Corrupt PNG");
+ if (a->zbuffer + len > a->zbuffer_end) return stbi__err("read past buffer","Corrupt PNG");
+ if (a->zout + len > a->zout_end)
+ if (!stbi__zexpand(a, a->zout, len)) return 0;
+ memcpy(a->zout, a->zbuffer, len);
+ a->zbuffer += len;
+ a->zout += len;
+ return 1;
+}
+
+static int stbi__parse_zlib_header(stbi__zbuf *a)
+{
+ int cmf = stbi__zget8(a);
+ int cm = cmf & 15;
+ /* int cinfo = cmf >> 4; */
+ int flg = stbi__zget8(a);
+ if (stbi__zeof(a)) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec
+ if ((cmf*256+flg) % 31 != 0) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec
+ if (flg & 32) return stbi__err("no preset dict","Corrupt PNG"); // preset dictionary not allowed in png
+ if (cm != 8) return stbi__err("bad compression","Corrupt PNG"); // DEFLATE required for png
+ // window = 1 << (8 + cinfo)... but who cares, we fully buffer output
+ return 1;
+}
+
+static const stbi_uc stbi__zdefault_length[STBI__ZNSYMS] =
+{
+ 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,
+ 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,
+ 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,
+ 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,
+ 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
+ 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
+ 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
+ 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8
+};
+static const stbi_uc stbi__zdefault_distance[32] =
+{
+ 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5
+};
+/*
+Init algorithm:
+{
+ int i; // use <= to match clearly with spec
+ for (i=0; i <= 143; ++i) stbi__zdefault_length[i] = 8;
+ for ( ; i <= 255; ++i) stbi__zdefault_length[i] = 9;
+ for ( ; i <= 279; ++i) stbi__zdefault_length[i] = 7;
+ for ( ; i <= 287; ++i) stbi__zdefault_length[i] = 8;
+
+ for (i=0; i <= 31; ++i) stbi__zdefault_distance[i] = 5;
+}
+*/
+
+static int stbi__parse_zlib(stbi__zbuf *a, int parse_header)
+{
+ int final, type;
+ if (parse_header)
+ if (!stbi__parse_zlib_header(a)) return 0;
+ a->num_bits = 0;
+ a->code_buffer = 0;
+ a->hit_zeof_once = 0;
+ do {
+ final = stbi__zreceive(a,1);
+ type = stbi__zreceive(a,2);
+ if (type == 0) {
+ if (!stbi__parse_uncompressed_block(a)) return 0;
+ } else if (type == 3) {
+ return 0;
+ } else {
+ if (type == 1) {
+ // use fixed code lengths
+ if (!stbi__zbuild_huffman(&a->z_length , stbi__zdefault_length , STBI__ZNSYMS)) return 0;
+ if (!stbi__zbuild_huffman(&a->z_distance, stbi__zdefault_distance, 32)) return 0;
+ } else {
+ if (!stbi__compute_huffman_codes(a)) return 0;
+ }
+ if (!stbi__parse_huffman_block(a)) return 0;
+ }
+ } while (!final);
+ return 1;
+}
+
+static int stbi__do_zlib(stbi__zbuf *a, char *obuf, int olen, int exp, int parse_header)
+{
+ a->zout_start = obuf;
+ a->zout = obuf;
+ a->zout_end = obuf + olen;
+ a->z_expandable = exp;
+
+ return stbi__parse_zlib(a, parse_header);
+}
+
+STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen)
+{
+ stbi__zbuf a;
+ char *p = (char *) stbi__malloc(initial_size);
+ if (p == NULL) return NULL;
+ a.zbuffer = (stbi_uc *) buffer;
+ a.zbuffer_end = (stbi_uc *) buffer + len;
+ if (stbi__do_zlib(&a, p, initial_size, 1, 1)) {
+ if (outlen) *outlen = (int) (a.zout - a.zout_start);
+ return a.zout_start;
+ } else {
+ STBI_FREE(a.zout_start);
+ return NULL;
+ }
+}
+
+STBIDEF char *stbi_zlib_decode_malloc(char const *buffer, int len, int *outlen)
+{
+ return stbi_zlib_decode_malloc_guesssize(buffer, len, 16384, outlen);
+}
+
+STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header)
+{
+ stbi__zbuf a;
+ char *p = (char *) stbi__malloc(initial_size);
+ if (p == NULL) return NULL;
+ a.zbuffer = (stbi_uc *) buffer;
+ a.zbuffer_end = (stbi_uc *) buffer + len;
+ if (stbi__do_zlib(&a, p, initial_size, 1, parse_header)) {
+ if (outlen) *outlen = (int) (a.zout - a.zout_start);
+ return a.zout_start;
+ } else {
+ STBI_FREE(a.zout_start);
+ return NULL;
+ }
+}
+
+STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, char const *ibuffer, int ilen)
+{
+ stbi__zbuf a;
+ a.zbuffer = (stbi_uc *) ibuffer;
+ a.zbuffer_end = (stbi_uc *) ibuffer + ilen;
+ if (stbi__do_zlib(&a, obuffer, olen, 0, 1))
+ return (int) (a.zout - a.zout_start);
+ else
+ return -1;
+}
+
+STBIDEF char *stbi_zlib_decode_noheader_malloc(char const *buffer, int len, int *outlen)
+{
+ stbi__zbuf a;
+ char *p = (char *) stbi__malloc(16384);
+ if (p == NULL) return NULL;
+ a.zbuffer = (stbi_uc *) buffer;
+ a.zbuffer_end = (stbi_uc *) buffer+len;
+ if (stbi__do_zlib(&a, p, 16384, 1, 0)) {
+ if (outlen) *outlen = (int) (a.zout - a.zout_start);
+ return a.zout_start;
+ } else {
+ STBI_FREE(a.zout_start);
+ return NULL;
+ }
+}
+
+STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen)
+{
+ stbi__zbuf a;
+ a.zbuffer = (stbi_uc *) ibuffer;
+ a.zbuffer_end = (stbi_uc *) ibuffer + ilen;
+ if (stbi__do_zlib(&a, obuffer, olen, 0, 0))
+ return (int) (a.zout - a.zout_start);
+ else
+ return -1;
+}
+#endif
+
+// public domain "baseline" PNG decoder v0.10 Sean Barrett 2006-11-18
+// simple implementation
+// - only 8-bit samples
+// - no CRC checking
+// - allocates lots of intermediate memory
+// - avoids problem of streaming data between subsystems
+// - avoids explicit window management
+// performance
+// - uses stb_zlib, a PD zlib implementation with fast huffman decoding
+
+#ifndef STBI_NO_PNG
+typedef struct
+{
+ stbi__uint32 length;
+ stbi__uint32 type;
+} stbi__pngchunk;
+
+static stbi__pngchunk stbi__get_chunk_header(stbi__context *s)
+{
+ stbi__pngchunk c;
+ c.length = stbi__get32be(s);
+ c.type = stbi__get32be(s);
+ return c;
+}
+
+static int stbi__check_png_header(stbi__context *s)
+{
+ static const stbi_uc png_sig[8] = { 137,80,78,71,13,10,26,10 };
+ int i;
+ for (i=0; i < 8; ++i)
+ if (stbi__get8(s) != png_sig[i]) return stbi__err("bad png sig","Not a PNG");
+ return 1;
+}
+
+typedef struct
+{
+ stbi__context *s;
+ stbi_uc *idata, *expanded, *out;
+ int depth;
+} stbi__png;
+
+
+enum {
+ STBI__F_none=0,
+ STBI__F_sub=1,
+ STBI__F_up=2,
+ STBI__F_avg=3,
+ STBI__F_paeth=4,
+ // synthetic filter used for first scanline to avoid needing a dummy row of 0s
+ STBI__F_avg_first
+};
+
+static stbi_uc first_row_filter[5] =
+{
+ STBI__F_none,
+ STBI__F_sub,
+ STBI__F_none,
+ STBI__F_avg_first,
+ STBI__F_sub // Paeth with b=c=0 turns out to be equivalent to sub
+};
+
+static int stbi__paeth(int a, int b, int c)
+{
+ // This formulation looks very different from the reference in the PNG spec, but is
+ // actually equivalent and has favorable data dependencies and admits straightforward
+ // generation of branch-free code, which helps performance significantly.
+ int thresh = c*3 - (a + b);
+ int lo = a < b ? a : b;
+ int hi = a < b ? b : a;
+ int t0 = (hi <= thresh) ? lo : c;
+ int t1 = (thresh <= lo) ? hi : t0;
+ return t1;
+}
+
+static const stbi_uc stbi__depth_scale_table[9] = { 0, 0xff, 0x55, 0, 0x11, 0,0,0, 0x01 };
+
+// adds an extra all-255 alpha channel
+// dest == src is legal
+// img_n must be 1 or 3
+static void stbi__create_png_alpha_expand8(stbi_uc *dest, stbi_uc *src, stbi__uint32 x, int img_n)
+{
+ int i;
+ // must process data backwards since we allow dest==src
+ if (img_n == 1) {
+ for (i=x-1; i >= 0; --i) {
+ dest[i*2+1] = 255;
+ dest[i*2+0] = src[i];
+ }
+ } else {
+ STBI_ASSERT(img_n == 3);
+ for (i=x-1; i >= 0; --i) {
+ dest[i*4+3] = 255;
+ dest[i*4+2] = src[i*3+2];
+ dest[i*4+1] = src[i*3+1];
+ dest[i*4+0] = src[i*3+0];
+ }
+ }
+}
+
+// create the png data from post-deflated data
+static int stbi__create_png_image_raw(stbi__png *a, stbi_uc *raw, stbi__uint32 raw_len, int out_n, stbi__uint32 x, stbi__uint32 y, int depth, int color)
+{
+ int bytes = (depth == 16 ? 2 : 1);
+ stbi__context *s = a->s;
+ stbi__uint32 i,j,stride = x*out_n*bytes;
+ stbi__uint32 img_len, img_width_bytes;
+ stbi_uc *filter_buf;
+ int all_ok = 1;
+ int k;
+ int img_n = s->img_n; // copy it into a local for later
+
+ int output_bytes = out_n*bytes;
+ int filter_bytes = img_n*bytes;
+ int width = x;
+
+ STBI_ASSERT(out_n == s->img_n || out_n == s->img_n+1);
+ a->out = (stbi_uc *) stbi__malloc_mad3(x, y, output_bytes, 0); // extra bytes to write off the end into
+ if (!a->out) return stbi__err("outofmem", "Out of memory");
+
+ // note: error exits here don't need to clean up a->out individually,
+ // stbi__do_png always does on error.
+ if (!stbi__mad3sizes_valid(img_n, x, depth, 7)) return stbi__err("too large", "Corrupt PNG");
+ img_width_bytes = (((img_n * x * depth) + 7) >> 3);
+ if (!stbi__mad2sizes_valid(img_width_bytes, y, img_width_bytes)) return stbi__err("too large", "Corrupt PNG");
+ img_len = (img_width_bytes + 1) * y;
+
+ // we used to check for exact match between raw_len and img_len on non-interlaced PNGs,
+ // but issue #276 reported a PNG in the wild that had extra data at the end (all zeros),
+ // so just check for raw_len < img_len always.
+ if (raw_len < img_len) return stbi__err("not enough pixels","Corrupt PNG");
+
+ // Allocate two scan lines worth of filter workspace buffer.
+ filter_buf = (stbi_uc *) stbi__malloc_mad2(img_width_bytes, 2, 0);
+ if (!filter_buf) return stbi__err("outofmem", "Out of memory");
+
+ // Filtering for low-bit-depth images
+ if (depth < 8) {
+ filter_bytes = 1;
+ width = img_width_bytes;
+ }
+
+ for (j=0; j < y; ++j) {
+ // cur/prior filter buffers alternate
+ stbi_uc *cur = filter_buf + (j & 1)*img_width_bytes;
+ stbi_uc *prior = filter_buf + (~j & 1)*img_width_bytes;
+ stbi_uc *dest = a->out + stride*j;
+ int nk = width * filter_bytes;
+ int filter = *raw++;
+
+ // check filter type
+ if (filter > 4) {
+ all_ok = stbi__err("invalid filter","Corrupt PNG");
+ break;
+ }
+
+ // if first row, use special filter that doesn't sample previous row
+ if (j == 0) filter = first_row_filter[filter];
+
+ // perform actual filtering
+ switch (filter) {
+ case STBI__F_none:
+ memcpy(cur, raw, nk);
+ break;
+ case STBI__F_sub:
+ memcpy(cur, raw, filter_bytes);
+ for (k = filter_bytes; k < nk; ++k)
+ cur[k] = STBI__BYTECAST(raw[k] + cur[k-filter_bytes]);
+ break;
+ case STBI__F_up:
+ for (k = 0; k < nk; ++k)
+ cur[k] = STBI__BYTECAST(raw[k] + prior[k]);
+ break;
+ case STBI__F_avg:
+ for (k = 0; k < filter_bytes; ++k)
+ cur[k] = STBI__BYTECAST(raw[k] + (prior[k]>>1));
+ for (k = filter_bytes; k < nk; ++k)
+ cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k-filter_bytes])>>1));
+ break;
+ case STBI__F_paeth:
+ for (k = 0; k < filter_bytes; ++k)
+ cur[k] = STBI__BYTECAST(raw[k] + prior[k]); // prior[k] == stbi__paeth(0,prior[k],0)
+ for (k = filter_bytes; k < nk; ++k)
+ cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes], prior[k], prior[k-filter_bytes]));
+ break;
+ case STBI__F_avg_first:
+ memcpy(cur, raw, filter_bytes);
+ for (k = filter_bytes; k < nk; ++k)
+ cur[k] = STBI__BYTECAST(raw[k] + (cur[k-filter_bytes] >> 1));
+ break;
+ }
+
+ raw += nk;
+
+ // expand decoded bits in cur to dest, also adding an extra alpha channel if desired
+ if (depth < 8) {
+ stbi_uc scale = (color == 0) ? stbi__depth_scale_table[depth] : 1; // scale grayscale values to 0..255 range
+ stbi_uc *in = cur;
+ stbi_uc *out = dest;
+ stbi_uc inb = 0;
+ stbi__uint32 nsmp = x*img_n;
+
+ // expand bits to bytes first
+ if (depth == 4) {
+ for (i=0; i < nsmp; ++i) {
+ if ((i & 1) == 0) inb = *in++;
+ *out++ = scale * (inb >> 4);
+ inb <<= 4;
+ }
+ } else if (depth == 2) {
+ for (i=0; i < nsmp; ++i) {
+ if ((i & 3) == 0) inb = *in++;
+ *out++ = scale * (inb >> 6);
+ inb <<= 2;
+ }
+ } else {
+ STBI_ASSERT(depth == 1);
+ for (i=0; i < nsmp; ++i) {
+ if ((i & 7) == 0) inb = *in++;
+ *out++ = scale * (inb >> 7);
+ inb <<= 1;
+ }
+ }
+
+ // insert alpha=255 values if desired
+ if (img_n != out_n)
+ stbi__create_png_alpha_expand8(dest, dest, x, img_n);
+ } else if (depth == 8) {
+ if (img_n == out_n)
+ memcpy(dest, cur, x*img_n);
+ else
+ stbi__create_png_alpha_expand8(dest, cur, x, img_n);
+ } else if (depth == 16) {
+ // convert the image data from big-endian to platform-native
+ stbi__uint16 *dest16 = (stbi__uint16*)dest;
+ stbi__uint32 nsmp = x*img_n;
+
+ if (img_n == out_n) {
+ for (i = 0; i < nsmp; ++i, ++dest16, cur += 2)
+ *dest16 = (cur[0] << 8) | cur[1];
+ } else {
+ STBI_ASSERT(img_n+1 == out_n);
+ if (img_n == 1) {
+ for (i = 0; i < x; ++i, dest16 += 2, cur += 2) {
+ dest16[0] = (cur[0] << 8) | cur[1];
+ dest16[1] = 0xffff;
+ }
+ } else {
+ STBI_ASSERT(img_n == 3);
+ for (i = 0; i < x; ++i, dest16 += 4, cur += 6) {
+ dest16[0] = (cur[0] << 8) | cur[1];
+ dest16[1] = (cur[2] << 8) | cur[3];
+ dest16[2] = (cur[4] << 8) | cur[5];
+ dest16[3] = 0xffff;
+ }
+ }
+ }
+ }
+ }
+
+ STBI_FREE(filter_buf);
+ if (!all_ok) return 0;
+
+ return 1;
+}
+
+static int stbi__create_png_image(stbi__png *a, stbi_uc *image_data, stbi__uint32 image_data_len, int out_n, int depth, int color, int interlaced)
+{
+ int bytes = (depth == 16 ? 2 : 1);
+ int out_bytes = out_n * bytes;
+ stbi_uc *final;
+ int p;
+ if (!interlaced)
+ return stbi__create_png_image_raw(a, image_data, image_data_len, out_n, a->s->img_x, a->s->img_y, depth, color);
+
+ // de-interlacing
+ final = (stbi_uc *) stbi__malloc_mad3(a->s->img_x, a->s->img_y, out_bytes, 0);
+ if (!final) return stbi__err("outofmem", "Out of memory");
+ for (p=0; p < 7; ++p) {
+ int xorig[] = { 0,4,0,2,0,1,0 };
+ int yorig[] = { 0,0,4,0,2,0,1 };
+ int xspc[] = { 8,8,4,4,2,2,1 };
+ int yspc[] = { 8,8,8,4,4,2,2 };
+ int i,j,x,y;
+ // pass1_x[4] = 0, pass1_x[5] = 1, pass1_x[12] = 1
+ x = (a->s->img_x - xorig[p] + xspc[p]-1) / xspc[p];
+ y = (a->s->img_y - yorig[p] + yspc[p]-1) / yspc[p];
+ if (x && y) {
+ stbi__uint32 img_len = ((((a->s->img_n * x * depth) + 7) >> 3) + 1) * y;
+ if (!stbi__create_png_image_raw(a, image_data, image_data_len, out_n, x, y, depth, color)) {
+ STBI_FREE(final);
+ return 0;
+ }
+ for (j=0; j < y; ++j) {
+ for (i=0; i < x; ++i) {
+ int out_y = j*yspc[p]+yorig[p];
+ int out_x = i*xspc[p]+xorig[p];
+ memcpy(final + out_y*a->s->img_x*out_bytes + out_x*out_bytes,
+ a->out + (j*x+i)*out_bytes, out_bytes);
+ }
+ }
+ STBI_FREE(a->out);
+ image_data += img_len;
+ image_data_len -= img_len;
+ }
+ }
+ a->out = final;
+
+ return 1;
+}
+
+static int stbi__compute_transparency(stbi__png *z, stbi_uc tc[3], int out_n)
+{
+ stbi__context *s = z->s;
+ stbi__uint32 i, pixel_count = s->img_x * s->img_y;
+ stbi_uc *p = z->out;
+
+ // compute color-based transparency, assuming we've
+ // already got 255 as the alpha value in the output
+ STBI_ASSERT(out_n == 2 || out_n == 4);
+
+ if (out_n == 2) {
+ for (i=0; i < pixel_count; ++i) {
+ p[1] = (p[0] == tc[0] ? 0 : 255);
+ p += 2;
+ }
+ } else {
+ for (i=0; i < pixel_count; ++i) {
+ if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2])
+ p[3] = 0;
+ p += 4;
+ }
+ }
+ return 1;
+}
+
+static int stbi__compute_transparency16(stbi__png *z, stbi__uint16 tc[3], int out_n)
+{
+ stbi__context *s = z->s;
+ stbi__uint32 i, pixel_count = s->img_x * s->img_y;
+ stbi__uint16 *p = (stbi__uint16*) z->out;
+
+ // compute color-based transparency, assuming we've
+ // already got 65535 as the alpha value in the output
+ STBI_ASSERT(out_n == 2 || out_n == 4);
+
+ if (out_n == 2) {
+ for (i = 0; i < pixel_count; ++i) {
+ p[1] = (p[0] == tc[0] ? 0 : 65535);
+ p += 2;
+ }
+ } else {
+ for (i = 0; i < pixel_count; ++i) {
+ if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2])
+ p[3] = 0;
+ p += 4;
+ }
+ }
+ return 1;
+}
+
+static int stbi__expand_png_palette(stbi__png *a, stbi_uc *palette, int len, int pal_img_n)
+{
+ stbi__uint32 i, pixel_count = a->s->img_x * a->s->img_y;
+ stbi_uc *p, *temp_out, *orig = a->out;
+
+ p = (stbi_uc *) stbi__malloc_mad2(pixel_count, pal_img_n, 0);
+ if (p == NULL) return stbi__err("outofmem", "Out of memory");
+
+ // between here and free(out) below, exitting would leak
+ temp_out = p;
+
+ if (pal_img_n == 3) {
+ for (i=0; i < pixel_count; ++i) {
+ int n = orig[i]*4;
+ p[0] = palette[n ];
+ p[1] = palette[n+1];
+ p[2] = palette[n+2];
+ p += 3;
+ }
+ } else {
+ for (i=0; i < pixel_count; ++i) {
+ int n = orig[i]*4;
+ p[0] = palette[n ];
+ p[1] = palette[n+1];
+ p[2] = palette[n+2];
+ p[3] = palette[n+3];
+ p += 4;
+ }
+ }
+ STBI_FREE(a->out);
+ a->out = temp_out;
+
+ STBI_NOTUSED(len);
+
+ return 1;
+}
+
+static int stbi__unpremultiply_on_load_global = 0;
+static int stbi__de_iphone_flag_global = 0;
+
+STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply)
+{
+ stbi__unpremultiply_on_load_global = flag_true_if_should_unpremultiply;
+}
+
+STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert)
+{
+ stbi__de_iphone_flag_global = flag_true_if_should_convert;
+}
+
+#ifndef STBI_THREAD_LOCAL
+#define stbi__unpremultiply_on_load stbi__unpremultiply_on_load_global
+#define stbi__de_iphone_flag stbi__de_iphone_flag_global
+#else
+static STBI_THREAD_LOCAL int stbi__unpremultiply_on_load_local, stbi__unpremultiply_on_load_set;
+static STBI_THREAD_LOCAL int stbi__de_iphone_flag_local, stbi__de_iphone_flag_set;
+
+STBIDEF void stbi_set_unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply)
+{
+ stbi__unpremultiply_on_load_local = flag_true_if_should_unpremultiply;
+ stbi__unpremultiply_on_load_set = 1;
+}
+
+STBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert)
+{
+ stbi__de_iphone_flag_local = flag_true_if_should_convert;
+ stbi__de_iphone_flag_set = 1;
+}
+
+#define stbi__unpremultiply_on_load (stbi__unpremultiply_on_load_set \
+ ? stbi__unpremultiply_on_load_local \
+ : stbi__unpremultiply_on_load_global)
+#define stbi__de_iphone_flag (stbi__de_iphone_flag_set \
+ ? stbi__de_iphone_flag_local \
+ : stbi__de_iphone_flag_global)
+#endif // STBI_THREAD_LOCAL
+
+static void stbi__de_iphone(stbi__png *z)
+{
+ stbi__context *s = z->s;
+ stbi__uint32 i, pixel_count = s->img_x * s->img_y;
+ stbi_uc *p = z->out;
+
+ if (s->img_out_n == 3) { // convert bgr to rgb
+ for (i=0; i < pixel_count; ++i) {
+ stbi_uc t = p[0];
+ p[0] = p[2];
+ p[2] = t;
+ p += 3;
+ }
+ } else {
+ STBI_ASSERT(s->img_out_n == 4);
+ if (stbi__unpremultiply_on_load) {
+ // convert bgr to rgb and unpremultiply
+ for (i=0; i < pixel_count; ++i) {
+ stbi_uc a = p[3];
+ stbi_uc t = p[0];
+ if (a) {
+ stbi_uc half = a / 2;
+ p[0] = (p[2] * 255 + half) / a;
+ p[1] = (p[1] * 255 + half) / a;
+ p[2] = ( t * 255 + half) / a;
+ } else {
+ p[0] = p[2];
+ p[2] = t;
+ }
+ p += 4;
+ }
+ } else {
+ // convert bgr to rgb
+ for (i=0; i < pixel_count; ++i) {
+ stbi_uc t = p[0];
+ p[0] = p[2];
+ p[2] = t;
+ p += 4;
+ }
+ }
+ }
+}
+
+#define STBI__PNG_TYPE(a,b,c,d) (((unsigned) (a) << 24) + ((unsigned) (b) << 16) + ((unsigned) (c) << 8) + (unsigned) (d))
+
+static int stbi__parse_png_file(stbi__png *z, int scan, int req_comp)
+{
+ stbi_uc palette[1024], pal_img_n=0;
+ stbi_uc has_trans=0, tc[3]={0};
+ stbi__uint16 tc16[3];
+ stbi__uint32 ioff=0, idata_limit=0, i, pal_len=0;
+ int first=1,k,interlace=0, color=0, is_iphone=0;
+ stbi__context *s = z->s;
+
+ z->expanded = NULL;
+ z->idata = NULL;
+ z->out = NULL;
+
+ if (!stbi__check_png_header(s)) return 0;
+
+ if (scan == STBI__SCAN_type) return 1;
+
+ for (;;) {
+ stbi__pngchunk c = stbi__get_chunk_header(s);
+ switch (c.type) {
+ case STBI__PNG_TYPE('C','g','B','I'):
+ is_iphone = 1;
+ stbi__skip(s, c.length);
+ break;
+ case STBI__PNG_TYPE('I','H','D','R'): {
+ int comp,filter;
+ if (!first) return stbi__err("multiple IHDR","Corrupt PNG");
+ first = 0;
+ if (c.length != 13) return stbi__err("bad IHDR len","Corrupt PNG");
+ s->img_x = stbi__get32be(s);
+ s->img_y = stbi__get32be(s);
+ if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)");
+ if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)");
+ z->depth = stbi__get8(s); if (z->depth != 1 && z->depth != 2 && z->depth != 4 && z->depth != 8 && z->depth != 16) return stbi__err("1/2/4/8/16-bit only","PNG not supported: 1/2/4/8/16-bit only");
+ color = stbi__get8(s); if (color > 6) return stbi__err("bad ctype","Corrupt PNG");
+ if (color == 3 && z->depth == 16) return stbi__err("bad ctype","Corrupt PNG");
+ if (color == 3) pal_img_n = 3; else if (color & 1) return stbi__err("bad ctype","Corrupt PNG");
+ comp = stbi__get8(s); if (comp) return stbi__err("bad comp method","Corrupt PNG");
+ filter= stbi__get8(s); if (filter) return stbi__err("bad filter method","Corrupt PNG");
+ interlace = stbi__get8(s); if (interlace>1) return stbi__err("bad interlace method","Corrupt PNG");
+ if (!s->img_x || !s->img_y) return stbi__err("0-pixel image","Corrupt PNG");
+ if (!pal_img_n) {
+ s->img_n = (color & 2 ? 3 : 1) + (color & 4 ? 1 : 0);
+ if ((1 << 30) / s->img_x / s->img_n < s->img_y) return stbi__err("too large", "Image too large to decode");
+ } else {
+ // if paletted, then pal_n is our final components, and
+ // img_n is # components to decompress/filter.
+ s->img_n = 1;
+ if ((1 << 30) / s->img_x / 4 < s->img_y) return stbi__err("too large","Corrupt PNG");
+ }
+ // even with SCAN_header, have to scan to see if we have a tRNS
+ break;
+ }
+
+ case STBI__PNG_TYPE('P','L','T','E'): {
+ if (first) return stbi__err("first not IHDR", "Corrupt PNG");
+ if (c.length > 256*3) return stbi__err("invalid PLTE","Corrupt PNG");
+ pal_len = c.length / 3;
+ if (pal_len * 3 != c.length) return stbi__err("invalid PLTE","Corrupt PNG");
+ for (i=0; i < pal_len; ++i) {
+ palette[i*4+0] = stbi__get8(s);
+ palette[i*4+1] = stbi__get8(s);
+ palette[i*4+2] = stbi__get8(s);
+ palette[i*4+3] = 255;
+ }
+ break;
+ }
+
+ case STBI__PNG_TYPE('t','R','N','S'): {
+ if (first) return stbi__err("first not IHDR", "Corrupt PNG");
+ if (z->idata) return stbi__err("tRNS after IDAT","Corrupt PNG");
+ if (pal_img_n) {
+ if (scan == STBI__SCAN_header) { s->img_n = 4; return 1; }
+ if (pal_len == 0) return stbi__err("tRNS before PLTE","Corrupt PNG");
+ if (c.length > pal_len) return stbi__err("bad tRNS len","Corrupt PNG");
+ pal_img_n = 4;
+ for (i=0; i < c.length; ++i)
+ palette[i*4+3] = stbi__get8(s);
+ } else {
+ if (!(s->img_n & 1)) return stbi__err("tRNS with alpha","Corrupt PNG");
+ if (c.length != (stbi__uint32) s->img_n*2) return stbi__err("bad tRNS len","Corrupt PNG");
+ has_trans = 1;
+ // non-paletted with tRNS = constant alpha. if header-scanning, we can stop now.
+ if (scan == STBI__SCAN_header) { ++s->img_n; return 1; }
+ if (z->depth == 16) {
+ for (k = 0; k < s->img_n && k < 3; ++k) // extra loop test to suppress false GCC warning
+ tc16[k] = (stbi__uint16)stbi__get16be(s); // copy the values as-is
+ } else {
+ for (k = 0; k < s->img_n && k < 3; ++k)
+ tc[k] = (stbi_uc)(stbi__get16be(s) & 255) * stbi__depth_scale_table[z->depth]; // non 8-bit images will be larger
+ }
+ }
+ break;
+ }
+
+ case STBI__PNG_TYPE('I','D','A','T'): {
+ if (first) return stbi__err("first not IHDR", "Corrupt PNG");
+ if (pal_img_n && !pal_len) return stbi__err("no PLTE","Corrupt PNG");
+ if (scan == STBI__SCAN_header) {
+ // header scan definitely stops at first IDAT
+ if (pal_img_n)
+ s->img_n = pal_img_n;
+ return 1;
+ }
+ if (c.length > (1u << 30)) return stbi__err("IDAT size limit", "IDAT section larger than 2^30 bytes");
+ if ((int)(ioff + c.length) < (int)ioff) return 0;
+ if (ioff + c.length > idata_limit) {
+ stbi__uint32 idata_limit_old = idata_limit;
+ stbi_uc *p;
+ if (idata_limit == 0) idata_limit = c.length > 4096 ? c.length : 4096;
+ while (ioff + c.length > idata_limit)
+ idata_limit *= 2;
+ STBI_NOTUSED(idata_limit_old);
+ p = (stbi_uc *) STBI_REALLOC_SIZED(z->idata, idata_limit_old, idata_limit); if (p == NULL) return stbi__err("outofmem", "Out of memory");
+ z->idata = p;
+ }
+ if (!stbi__getn(s, z->idata+ioff,c.length)) return stbi__err("outofdata","Corrupt PNG");
+ ioff += c.length;
+ break;
+ }
+
+ case STBI__PNG_TYPE('I','E','N','D'): {
+ stbi__uint32 raw_len, bpl;
+ if (first) return stbi__err("first not IHDR", "Corrupt PNG");
+ if (scan != STBI__SCAN_load) return 1;
+ if (z->idata == NULL) return stbi__err("no IDAT","Corrupt PNG");
+ // initial guess for decoded data size to avoid unnecessary reallocs
+ bpl = (s->img_x * z->depth + 7) / 8; // bytes per line, per component
+ raw_len = bpl * s->img_y * s->img_n /* pixels */ + s->img_y /* filter mode per row */;
+ z->expanded = (stbi_uc *) stbi_zlib_decode_malloc_guesssize_headerflag((char *) z->idata, ioff, raw_len, (int *) &raw_len, !is_iphone);
+ if (z->expanded == NULL) return 0; // zlib should set error
+ STBI_FREE(z->idata); z->idata = NULL;
+ if ((req_comp == s->img_n+1 && req_comp != 3 && !pal_img_n) || has_trans)
+ s->img_out_n = s->img_n+1;
+ else
+ s->img_out_n = s->img_n;
+ if (!stbi__create_png_image(z, z->expanded, raw_len, s->img_out_n, z->depth, color, interlace)) return 0;
+ if (has_trans) {
+ if (z->depth == 16) {
+ if (!stbi__compute_transparency16(z, tc16, s->img_out_n)) return 0;
+ } else {
+ if (!stbi__compute_transparency(z, tc, s->img_out_n)) return 0;
+ }
+ }
+ if (is_iphone && stbi__de_iphone_flag && s->img_out_n > 2)
+ stbi__de_iphone(z);
+ if (pal_img_n) {
+ // pal_img_n == 3 or 4
+ s->img_n = pal_img_n; // record the actual colors we had
+ s->img_out_n = pal_img_n;
+ if (req_comp >= 3) s->img_out_n = req_comp;
+ if (!stbi__expand_png_palette(z, palette, pal_len, s->img_out_n))
+ return 0;
+ } else if (has_trans) {
+ // non-paletted image with tRNS -> source image has (constant) alpha
+ ++s->img_n;
+ }
+ STBI_FREE(z->expanded); z->expanded = NULL;
+ // end of PNG chunk, read and skip CRC
+ stbi__get32be(s);
+ return 1;
+ }
+
+ default:
+ // if critical, fail
+ if (first) return stbi__err("first not IHDR", "Corrupt PNG");
+ if ((c.type & (1 << 29)) == 0) {
+ #ifndef STBI_NO_FAILURE_STRINGS
+ // not threadsafe
+ static char invalid_chunk[] = "XXXX PNG chunk not known";
+ invalid_chunk[0] = STBI__BYTECAST(c.type >> 24);
+ invalid_chunk[1] = STBI__BYTECAST(c.type >> 16);
+ invalid_chunk[2] = STBI__BYTECAST(c.type >> 8);
+ invalid_chunk[3] = STBI__BYTECAST(c.type >> 0);
+ #endif
+ return stbi__err(invalid_chunk, "PNG not supported: unknown PNG chunk type");
+ }
+ stbi__skip(s, c.length);
+ break;
+ }
+ // end of PNG chunk, read and skip CRC
+ stbi__get32be(s);
+ }
+}
+
+static void *stbi__do_png(stbi__png *p, int *x, int *y, int *n, int req_comp, stbi__result_info *ri)
+{
+ void *result=NULL;
+ if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error");
+ if (stbi__parse_png_file(p, STBI__SCAN_load, req_comp)) {
+ if (p->depth <= 8)
+ ri->bits_per_channel = 8;
+ else if (p->depth == 16)
+ ri->bits_per_channel = 16;
+ else
+ return stbi__errpuc("bad bits_per_channel", "PNG not supported: unsupported color depth");
+ result = p->out;
+ p->out = NULL;
+ if (req_comp && req_comp != p->s->img_out_n) {
+ if (ri->bits_per_channel == 8)
+ result = stbi__convert_format((unsigned char *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y);
+ else
+ result = stbi__convert_format16((stbi__uint16 *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y);
+ p->s->img_out_n = req_comp;
+ if (result == NULL) return result;
+ }
+ *x = p->s->img_x;
+ *y = p->s->img_y;
+ if (n) *n = p->s->img_n;
+ }
+ STBI_FREE(p->out); p->out = NULL;
+ STBI_FREE(p->expanded); p->expanded = NULL;
+ STBI_FREE(p->idata); p->idata = NULL;
+
+ return result;
+}
+
+static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)
+{
+ stbi__png p;
+ p.s = s;
+ return stbi__do_png(&p, x,y,comp,req_comp, ri);
+}
+
+static int stbi__png_test(stbi__context *s)
+{
+ int r;
+ r = stbi__check_png_header(s);
+ stbi__rewind(s);
+ return r;
+}
+
+static int stbi__png_info_raw(stbi__png *p, int *x, int *y, int *comp)
+{
+ if (!stbi__parse_png_file(p, STBI__SCAN_header, 0)) {
+ stbi__rewind( p->s );
+ return 0;
+ }
+ if (x) *x = p->s->img_x;
+ if (y) *y = p->s->img_y;
+ if (comp) *comp = p->s->img_n;
+ return 1;
+}
+
+static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp)
+{
+ stbi__png p;
+ p.s = s;
+ return stbi__png_info_raw(&p, x, y, comp);
+}
+
+static int stbi__png_is16(stbi__context *s)
+{
+ stbi__png p;
+ p.s = s;
+ if (!stbi__png_info_raw(&p, NULL, NULL, NULL))
+ return 0;
+ if (p.depth != 16) {
+ stbi__rewind(p.s);
+ return 0;
+ }
+ return 1;
+}
+#endif
+
+// Microsoft/Windows BMP image
+
+#ifndef STBI_NO_BMP
+static int stbi__bmp_test_raw(stbi__context *s)
+{
+ int r;
+ int sz;
+ if (stbi__get8(s) != 'B') return 0;
+ if (stbi__get8(s) != 'M') return 0;
+ stbi__get32le(s); // discard filesize
+ stbi__get16le(s); // discard reserved
+ stbi__get16le(s); // discard reserved
+ stbi__get32le(s); // discard data offset
+ sz = stbi__get32le(s);
+ r = (sz == 12 || sz == 40 || sz == 56 || sz == 108 || sz == 124);
+ return r;
+}
+
+static int stbi__bmp_test(stbi__context *s)
+{
+ int r = stbi__bmp_test_raw(s);
+ stbi__rewind(s);
+ return r;
+}
+
+
+// returns 0..31 for the highest set bit
+static int stbi__high_bit(unsigned int z)
+{
+ int n=0;
+ if (z == 0) return -1;
+ if (z >= 0x10000) { n += 16; z >>= 16; }
+ if (z >= 0x00100) { n += 8; z >>= 8; }
+ if (z >= 0x00010) { n += 4; z >>= 4; }
+ if (z >= 0x00004) { n += 2; z >>= 2; }
+ if (z >= 0x00002) { n += 1;/* >>= 1;*/ }
+ return n;
+}
+
+static int stbi__bitcount(unsigned int a)
+{
+ a = (a & 0x55555555) + ((a >> 1) & 0x55555555); // max 2
+ a = (a & 0x33333333) + ((a >> 2) & 0x33333333); // max 4
+ a = (a + (a >> 4)) & 0x0f0f0f0f; // max 8 per 4, now 8 bits
+ a = (a + (a >> 8)); // max 16 per 8 bits
+ a = (a + (a >> 16)); // max 32 per 8 bits
+ return a & 0xff;
+}
+
+// extract an arbitrarily-aligned N-bit value (N=bits)
+// from v, and then make it 8-bits long and fractionally
+// extend it to full full range.
+static int stbi__shiftsigned(unsigned int v, int shift, int bits)
+{
+ static unsigned int mul_table[9] = {
+ 0,
+ 0xff/*0b11111111*/, 0x55/*0b01010101*/, 0x49/*0b01001001*/, 0x11/*0b00010001*/,
+ 0x21/*0b00100001*/, 0x41/*0b01000001*/, 0x81/*0b10000001*/, 0x01/*0b00000001*/,
+ };
+ static unsigned int shift_table[9] = {
+ 0, 0,0,1,0,2,4,6,0,
+ };
+ if (shift < 0)
+ v <<= -shift;
+ else
+ v >>= shift;
+ STBI_ASSERT(v < 256);
+ v >>= (8-bits);
+ STBI_ASSERT(bits >= 0 && bits <= 8);
+ return (int) ((unsigned) v * mul_table[bits]) >> shift_table[bits];
+}
+
+typedef struct
+{
+ int bpp, offset, hsz;
+ unsigned int mr,mg,mb,ma, all_a;
+ int extra_read;
+} stbi__bmp_data;
+
+static int stbi__bmp_set_mask_defaults(stbi__bmp_data *info, int compress)
+{
+ // BI_BITFIELDS specifies masks explicitly, don't override
+ if (compress == 3)
+ return 1;
+
+ if (compress == 0) {
+ if (info->bpp == 16) {
+ info->mr = 31u << 10;
+ info->mg = 31u << 5;
+ info->mb = 31u << 0;
+ } else if (info->bpp == 32) {
+ info->mr = 0xffu << 16;
+ info->mg = 0xffu << 8;
+ info->mb = 0xffu << 0;
+ info->ma = 0xffu << 24;
+ info->all_a = 0; // if all_a is 0 at end, then we loaded alpha channel but it was all 0
+ } else {
+ // otherwise, use defaults, which is all-0
+ info->mr = info->mg = info->mb = info->ma = 0;
+ }
+ return 1;
+ }
+ return 0; // error
+}
+
+static void *stbi__bmp_parse_header(stbi__context *s, stbi__bmp_data *info)
+{
+ int hsz;
+ if (stbi__get8(s) != 'B' || stbi__get8(s) != 'M') return stbi__errpuc("not BMP", "Corrupt BMP");
+ stbi__get32le(s); // discard filesize
+ stbi__get16le(s); // discard reserved
+ stbi__get16le(s); // discard reserved
+ info->offset = stbi__get32le(s);
+ info->hsz = hsz = stbi__get32le(s);
+ info->mr = info->mg = info->mb = info->ma = 0;
+ info->extra_read = 14;
+
+ if (info->offset < 0) return stbi__errpuc("bad BMP", "bad BMP");
+
+ if (hsz != 12 && hsz != 40 && hsz != 56 && hsz != 108 && hsz != 124) return stbi__errpuc("unknown BMP", "BMP type not supported: unknown");
+ if (hsz == 12) {
+ s->img_x = stbi__get16le(s);
+ s->img_y = stbi__get16le(s);
+ } else {
+ s->img_x = stbi__get32le(s);
+ s->img_y = stbi__get32le(s);
+ }
+ if (stbi__get16le(s) != 1) return stbi__errpuc("bad BMP", "bad BMP");
+ info->bpp = stbi__get16le(s);
+ if (hsz != 12) {
+ int compress = stbi__get32le(s);
+ if (compress == 1 || compress == 2) return stbi__errpuc("BMP RLE", "BMP type not supported: RLE");
+ if (compress >= 4) return stbi__errpuc("BMP JPEG/PNG", "BMP type not supported: unsupported compression"); // this includes PNG/JPEG modes
+ if (compress == 3 && info->bpp != 16 && info->bpp != 32) return stbi__errpuc("bad BMP", "bad BMP"); // bitfields requires 16 or 32 bits/pixel
+ stbi__get32le(s); // discard sizeof
+ stbi__get32le(s); // discard hres
+ stbi__get32le(s); // discard vres
+ stbi__get32le(s); // discard colorsused
+ stbi__get32le(s); // discard max important
+ if (hsz == 40 || hsz == 56) {
+ if (hsz == 56) {
+ stbi__get32le(s);
+ stbi__get32le(s);
+ stbi__get32le(s);
+ stbi__get32le(s);
+ }
+ if (info->bpp == 16 || info->bpp == 32) {
+ if (compress == 0) {
+ stbi__bmp_set_mask_defaults(info, compress);
+ } else if (compress == 3) {
+ info->mr = stbi__get32le(s);
+ info->mg = stbi__get32le(s);
+ info->mb = stbi__get32le(s);
+ info->extra_read += 12;
+ // not documented, but generated by photoshop and handled by mspaint
+ if (info->mr == info->mg && info->mg == info->mb) {
+ // ?!?!?
+ return stbi__errpuc("bad BMP", "bad BMP");
+ }
+ } else
+ return stbi__errpuc("bad BMP", "bad BMP");
+ }
+ } else {
+ // V4/V5 header
+ int i;
+ if (hsz != 108 && hsz != 124)
+ return stbi__errpuc("bad BMP", "bad BMP");
+ info->mr = stbi__get32le(s);
+ info->mg = stbi__get32le(s);
+ info->mb = stbi__get32le(s);
+ info->ma = stbi__get32le(s);
+ if (compress != 3) // override mr/mg/mb unless in BI_BITFIELDS mode, as per docs
+ stbi__bmp_set_mask_defaults(info, compress);
+ stbi__get32le(s); // discard color space
+ for (i=0; i < 12; ++i)
+ stbi__get32le(s); // discard color space parameters
+ if (hsz == 124) {
+ stbi__get32le(s); // discard rendering intent
+ stbi__get32le(s); // discard offset of profile data
+ stbi__get32le(s); // discard size of profile data
+ stbi__get32le(s); // discard reserved
+ }
+ }
+ }
+ return (void *) 1;
+}
+
+
+static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)
+{
+ stbi_uc *out;
+ unsigned int mr=0,mg=0,mb=0,ma=0, all_a;
+ stbi_uc pal[256][4];
+ int psize=0,i,j,width;
+ int flip_vertically, pad, target;
+ stbi__bmp_data info;
+ STBI_NOTUSED(ri);
+
+ info.all_a = 255;
+ if (stbi__bmp_parse_header(s, &info) == NULL)
+ return NULL; // error code already set
+
+ flip_vertically = ((int) s->img_y) > 0;
+ s->img_y = abs((int) s->img_y);
+
+ if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)");
+ if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)");
+
+ mr = info.mr;
+ mg = info.mg;
+ mb = info.mb;
+ ma = info.ma;
+ all_a = info.all_a;
+
+ if (info.hsz == 12) {
+ if (info.bpp < 24)
+ psize = (info.offset - info.extra_read - 24) / 3;
+ } else {
+ if (info.bpp < 16)
+ psize = (info.offset - info.extra_read - info.hsz) >> 2;
+ }
+ if (psize == 0) {
+ // accept some number of extra bytes after the header, but if the offset points either to before
+ // the header ends or implies a large amount of extra data, reject the file as malformed
+ int bytes_read_so_far = s->callback_already_read + (int)(s->img_buffer - s->img_buffer_original);
+ int header_limit = 1024; // max we actually read is below 256 bytes currently.
+ int extra_data_limit = 256*4; // what ordinarily goes here is a palette; 256 entries*4 bytes is its max size.
+ if (bytes_read_so_far <= 0 || bytes_read_so_far > header_limit) {
+ return stbi__errpuc("bad header", "Corrupt BMP");
+ }
+ // we established that bytes_read_so_far is positive and sensible.
+ // the first half of this test rejects offsets that are either too small positives, or
+ // negative, and guarantees that info.offset >= bytes_read_so_far > 0. this in turn
+ // ensures the number computed in the second half of the test can't overflow.
+ if (info.offset < bytes_read_so_far || info.offset - bytes_read_so_far > extra_data_limit) {
+ return stbi__errpuc("bad offset", "Corrupt BMP");
+ } else {
+ stbi__skip(s, info.offset - bytes_read_so_far);
+ }
+ }
+
+ if (info.bpp == 24 && ma == 0xff000000)
+ s->img_n = 3;
+ else
+ s->img_n = ma ? 4 : 3;
+ if (req_comp && req_comp >= 3) // we can directly decode 3 or 4
+ target = req_comp;
+ else
+ target = s->img_n; // if they want monochrome, we'll post-convert
+
+ // sanity-check size
+ if (!stbi__mad3sizes_valid(target, s->img_x, s->img_y, 0))
+ return stbi__errpuc("too large", "Corrupt BMP");
+
+ out = (stbi_uc *) stbi__malloc_mad3(target, s->img_x, s->img_y, 0);
+ if (!out) return stbi__errpuc("outofmem", "Out of memory");
+ if (info.bpp < 16) {
+ int z=0;
+ if (psize == 0 || psize > 256) { STBI_FREE(out); return stbi__errpuc("invalid", "Corrupt BMP"); }
+ for (i=0; i < psize; ++i) {
+ pal[i][2] = stbi__get8(s);
+ pal[i][1] = stbi__get8(s);
+ pal[i][0] = stbi__get8(s);
+ if (info.hsz != 12) stbi__get8(s);
+ pal[i][3] = 255;
+ }
+ stbi__skip(s, info.offset - info.extra_read - info.hsz - psize * (info.hsz == 12 ? 3 : 4));
+ if (info.bpp == 1) width = (s->img_x + 7) >> 3;
+ else if (info.bpp == 4) width = (s->img_x + 1) >> 1;
+ else if (info.bpp == 8) width = s->img_x;
+ else { STBI_FREE(out); return stbi__errpuc("bad bpp", "Corrupt BMP"); }
+ pad = (-width)&3;
+ if (info.bpp == 1) {
+ for (j=0; j < (int) s->img_y; ++j) {
+ int bit_offset = 7, v = stbi__get8(s);
+ for (i=0; i < (int) s->img_x; ++i) {
+ int color = (v>>bit_offset)&0x1;
+ out[z++] = pal[color][0];
+ out[z++] = pal[color][1];
+ out[z++] = pal[color][2];
+ if (target == 4) out[z++] = 255;
+ if (i+1 == (int) s->img_x) break;
+ if((--bit_offset) < 0) {
+ bit_offset = 7;
+ v = stbi__get8(s);
+ }
+ }
+ stbi__skip(s, pad);
+ }
+ } else {
+ for (j=0; j < (int) s->img_y; ++j) {
+ for (i=0; i < (int) s->img_x; i += 2) {
+ int v=stbi__get8(s),v2=0;
+ if (info.bpp == 4) {
+ v2 = v & 15;
+ v >>= 4;
+ }
+ out[z++] = pal[v][0];
+ out[z++] = pal[v][1];
+ out[z++] = pal[v][2];
+ if (target == 4) out[z++] = 255;
+ if (i+1 == (int) s->img_x) break;
+ v = (info.bpp == 8) ? stbi__get8(s) : v2;
+ out[z++] = pal[v][0];
+ out[z++] = pal[v][1];
+ out[z++] = pal[v][2];
+ if (target == 4) out[z++] = 255;
+ }
+ stbi__skip(s, pad);
+ }
+ }
+ } else {
+ int rshift=0,gshift=0,bshift=0,ashift=0,rcount=0,gcount=0,bcount=0,acount=0;
+ int z = 0;
+ int easy=0;
+ stbi__skip(s, info.offset - info.extra_read - info.hsz);
+ if (info.bpp == 24) width = 3 * s->img_x;
+ else if (info.bpp == 16) width = 2*s->img_x;
+ else /* bpp = 32 and pad = 0 */ width=0;
+ pad = (-width) & 3;
+ if (info.bpp == 24) {
+ easy = 1;
+ } else if (info.bpp == 32) {
+ if (mb == 0xff && mg == 0xff00 && mr == 0x00ff0000 && ma == 0xff000000)
+ easy = 2;
+ }
+ if (!easy) {
+ if (!mr || !mg || !mb) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); }
+ // right shift amt to put high bit in position #7
+ rshift = stbi__high_bit(mr)-7; rcount = stbi__bitcount(mr);
+ gshift = stbi__high_bit(mg)-7; gcount = stbi__bitcount(mg);
+ bshift = stbi__high_bit(mb)-7; bcount = stbi__bitcount(mb);
+ ashift = stbi__high_bit(ma)-7; acount = stbi__bitcount(ma);
+ if (rcount > 8 || gcount > 8 || bcount > 8 || acount > 8) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); }
+ }
+ for (j=0; j < (int) s->img_y; ++j) {
+ if (easy) {
+ for (i=0; i < (int) s->img_x; ++i) {
+ unsigned char a;
+ out[z+2] = stbi__get8(s);
+ out[z+1] = stbi__get8(s);
+ out[z+0] = stbi__get8(s);
+ z += 3;
+ a = (easy == 2 ? stbi__get8(s) : 255);
+ all_a |= a;
+ if (target == 4) out[z++] = a;
+ }
+ } else {
+ int bpp = info.bpp;
+ for (i=0; i < (int) s->img_x; ++i) {
+ stbi__uint32 v = (bpp == 16 ? (stbi__uint32) stbi__get16le(s) : stbi__get32le(s));
+ unsigned int a;
+ out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mr, rshift, rcount));
+ out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mg, gshift, gcount));
+ out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mb, bshift, bcount));
+ a = (ma ? stbi__shiftsigned(v & ma, ashift, acount) : 255);
+ all_a |= a;
+ if (target == 4) out[z++] = STBI__BYTECAST(a);
+ }
+ }
+ stbi__skip(s, pad);
+ }
+ }
+
+ // if alpha channel is all 0s, replace with all 255s
+ if (target == 4 && all_a == 0)
+ for (i=4*s->img_x*s->img_y-1; i >= 0; i -= 4)
+ out[i] = 255;
+
+ if (flip_vertically) {
+ stbi_uc t;
+ for (j=0; j < (int) s->img_y>>1; ++j) {
+ stbi_uc *p1 = out + j *s->img_x*target;
+ stbi_uc *p2 = out + (s->img_y-1-j)*s->img_x*target;
+ for (i=0; i < (int) s->img_x*target; ++i) {
+ t = p1[i]; p1[i] = p2[i]; p2[i] = t;
+ }
+ }
+ }
+
+ if (req_comp && req_comp != target) {
+ out = stbi__convert_format(out, target, req_comp, s->img_x, s->img_y);
+ if (out == NULL) return out; // stbi__convert_format frees input on failure
+ }
+
+ *x = s->img_x;
+ *y = s->img_y;
+ if (comp) *comp = s->img_n;
+ return out;
+}
+#endif
+
+// Targa Truevision - TGA
+// by Jonathan Dummer
+#ifndef STBI_NO_TGA
+// returns STBI_rgb or whatever, 0 on error
+static int stbi__tga_get_comp(int bits_per_pixel, int is_grey, int* is_rgb16)
+{
+ // only RGB or RGBA (incl. 16bit) or grey allowed
+ if (is_rgb16) *is_rgb16 = 0;
+ switch(bits_per_pixel) {
+ case 8: return STBI_grey;
+ case 16: if(is_grey) return STBI_grey_alpha;
+ // fallthrough
+ case 15: if(is_rgb16) *is_rgb16 = 1;
+ return STBI_rgb;
+ case 24: // fallthrough
+ case 32: return bits_per_pixel/8;
+ default: return 0;
+ }
+}
+
+static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp)
+{
+ int tga_w, tga_h, tga_comp, tga_image_type, tga_bits_per_pixel, tga_colormap_bpp;
+ int sz, tga_colormap_type;
+ stbi__get8(s); // discard Offset
+ tga_colormap_type = stbi__get8(s); // colormap type
+ if( tga_colormap_type > 1 ) {
+ stbi__rewind(s);
+ return 0; // only RGB or indexed allowed
+ }
+ tga_image_type = stbi__get8(s); // image type
+ if ( tga_colormap_type == 1 ) { // colormapped (paletted) image
+ if (tga_image_type != 1 && tga_image_type != 9) {
+ stbi__rewind(s);
+ return 0;
+ }
+ stbi__skip(s,4); // skip index of first colormap entry and number of entries
+ sz = stbi__get8(s); // check bits per palette color entry
+ if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) {
+ stbi__rewind(s);
+ return 0;
+ }
+ stbi__skip(s,4); // skip image x and y origin
+ tga_colormap_bpp = sz;
+ } else { // "normal" image w/o colormap - only RGB or grey allowed, +/- RLE
+ if ( (tga_image_type != 2) && (tga_image_type != 3) && (tga_image_type != 10) && (tga_image_type != 11) ) {
+ stbi__rewind(s);
+ return 0; // only RGB or grey allowed, +/- RLE
+ }
+ stbi__skip(s,9); // skip colormap specification and image x/y origin
+ tga_colormap_bpp = 0;
+ }
+ tga_w = stbi__get16le(s);
+ if( tga_w < 1 ) {
+ stbi__rewind(s);
+ return 0; // test width
+ }
+ tga_h = stbi__get16le(s);
+ if( tga_h < 1 ) {
+ stbi__rewind(s);
+ return 0; // test height
+ }
+ tga_bits_per_pixel = stbi__get8(s); // bits per pixel
+ stbi__get8(s); // ignore alpha bits
+ if (tga_colormap_bpp != 0) {
+ if((tga_bits_per_pixel != 8) && (tga_bits_per_pixel != 16)) {
+ // when using a colormap, tga_bits_per_pixel is the size of the indexes
+ // I don't think anything but 8 or 16bit indexes makes sense
+ stbi__rewind(s);
+ return 0;
+ }
+ tga_comp = stbi__tga_get_comp(tga_colormap_bpp, 0, NULL);
+ } else {
+ tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3) || (tga_image_type == 11), NULL);
+ }
+ if(!tga_comp) {
+ stbi__rewind(s);
+ return 0;
+ }
+ if (x) *x = tga_w;
+ if (y) *y = tga_h;
+ if (comp) *comp = tga_comp;
+ return 1; // seems to have passed everything
+}
+
+static int stbi__tga_test(stbi__context *s)
+{
+ int res = 0;
+ int sz, tga_color_type;
+ stbi__get8(s); // discard Offset
+ tga_color_type = stbi__get8(s); // color type
+ if ( tga_color_type > 1 ) goto errorEnd; // only RGB or indexed allowed
+ sz = stbi__get8(s); // image type
+ if ( tga_color_type == 1 ) { // colormapped (paletted) image
+ if (sz != 1 && sz != 9) goto errorEnd; // colortype 1 demands image type 1 or 9
+ stbi__skip(s,4); // skip index of first colormap entry and number of entries
+ sz = stbi__get8(s); // check bits per palette color entry
+ if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd;
+ stbi__skip(s,4); // skip image x and y origin
+ } else { // "normal" image w/o colormap
+ if ( (sz != 2) && (sz != 3) && (sz != 10) && (sz != 11) ) goto errorEnd; // only RGB or grey allowed, +/- RLE
+ stbi__skip(s,9); // skip colormap specification and image x/y origin
+ }
+ if ( stbi__get16le(s) < 1 ) goto errorEnd; // test width
+ if ( stbi__get16le(s) < 1 ) goto errorEnd; // test height
+ sz = stbi__get8(s); // bits per pixel
+ if ( (tga_color_type == 1) && (sz != 8) && (sz != 16) ) goto errorEnd; // for colormapped images, bpp is size of an index
+ if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd;
+
+ res = 1; // if we got this far, everything's good and we can return 1 instead of 0
+
+errorEnd:
+ stbi__rewind(s);
+ return res;
+}
+
+// read 16bit value and convert to 24bit RGB
+static void stbi__tga_read_rgb16(stbi__context *s, stbi_uc* out)
+{
+ stbi__uint16 px = (stbi__uint16)stbi__get16le(s);
+ stbi__uint16 fiveBitMask = 31;
+ // we have 3 channels with 5bits each
+ int r = (px >> 10) & fiveBitMask;
+ int g = (px >> 5) & fiveBitMask;
+ int b = px & fiveBitMask;
+ // Note that this saves the data in RGB(A) order, so it doesn't need to be swapped later
+ out[0] = (stbi_uc)((r * 255)/31);
+ out[1] = (stbi_uc)((g * 255)/31);
+ out[2] = (stbi_uc)((b * 255)/31);
+
+ // some people claim that the most significant bit might be used for alpha
+ // (possibly if an alpha-bit is set in the "image descriptor byte")
+ // but that only made 16bit test images completely translucent..
+ // so let's treat all 15 and 16bit TGAs as RGB with no alpha.
+}
+
+static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)
+{
+ // read in the TGA header stuff
+ int tga_offset = stbi__get8(s);
+ int tga_indexed = stbi__get8(s);
+ int tga_image_type = stbi__get8(s);
+ int tga_is_RLE = 0;
+ int tga_palette_start = stbi__get16le(s);
+ int tga_palette_len = stbi__get16le(s);
+ int tga_palette_bits = stbi__get8(s);
+ int tga_x_origin = stbi__get16le(s);
+ int tga_y_origin = stbi__get16le(s);
+ int tga_width = stbi__get16le(s);
+ int tga_height = stbi__get16le(s);
+ int tga_bits_per_pixel = stbi__get8(s);
+ int tga_comp, tga_rgb16=0;
+ int tga_inverted = stbi__get8(s);
+ // int tga_alpha_bits = tga_inverted & 15; // the 4 lowest bits - unused (useless?)
+ // image data
+ unsigned char *tga_data;
+ unsigned char *tga_palette = NULL;
+ int i, j;
+ unsigned char raw_data[4] = {0};
+ int RLE_count = 0;
+ int RLE_repeating = 0;
+ int read_next_pixel = 1;
+ STBI_NOTUSED(ri);
+ STBI_NOTUSED(tga_x_origin); // @TODO
+ STBI_NOTUSED(tga_y_origin); // @TODO
+
+ if (tga_height > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)");
+ if (tga_width > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)");
+
+ // do a tiny bit of precessing
+ if ( tga_image_type >= 8 )
+ {
+ tga_image_type -= 8;
+ tga_is_RLE = 1;
+ }
+ tga_inverted = 1 - ((tga_inverted >> 5) & 1);
+
+ // If I'm paletted, then I'll use the number of bits from the palette
+ if ( tga_indexed ) tga_comp = stbi__tga_get_comp(tga_palette_bits, 0, &tga_rgb16);
+ else tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3), &tga_rgb16);
+
+ if(!tga_comp) // shouldn't really happen, stbi__tga_test() should have ensured basic consistency
+ return stbi__errpuc("bad format", "Can't find out TGA pixelformat");
+
+ // tga info
+ *x = tga_width;
+ *y = tga_height;
+ if (comp) *comp = tga_comp;
+
+ if (!stbi__mad3sizes_valid(tga_width, tga_height, tga_comp, 0))
+ return stbi__errpuc("too large", "Corrupt TGA");
+
+ tga_data = (unsigned char*)stbi__malloc_mad3(tga_width, tga_height, tga_comp, 0);
+ if (!tga_data) return stbi__errpuc("outofmem", "Out of memory");
+
+ // skip to the data's starting position (offset usually = 0)
+ stbi__skip(s, tga_offset );
+
+ if ( !tga_indexed && !tga_is_RLE && !tga_rgb16 ) {
+ for (i=0; i < tga_height; ++i) {
+ int row = tga_inverted ? tga_height -i - 1 : i;
+ stbi_uc *tga_row = tga_data + row*tga_width*tga_comp;
+ stbi__getn(s, tga_row, tga_width * tga_comp);
+ }
+ } else {
+ // do I need to load a palette?
+ if ( tga_indexed)
+ {
+ if (tga_palette_len == 0) { /* you have to have at least one entry! */
+ STBI_FREE(tga_data);
+ return stbi__errpuc("bad palette", "Corrupt TGA");
+ }
+
+ // any data to skip? (offset usually = 0)
+ stbi__skip(s, tga_palette_start );
+ // load the palette
+ tga_palette = (unsigned char*)stbi__malloc_mad2(tga_palette_len, tga_comp, 0);
+ if (!tga_palette) {
+ STBI_FREE(tga_data);
+ return stbi__errpuc("outofmem", "Out of memory");
+ }
+ if (tga_rgb16) {
+ stbi_uc *pal_entry = tga_palette;
+ STBI_ASSERT(tga_comp == STBI_rgb);
+ for (i=0; i < tga_palette_len; ++i) {
+ stbi__tga_read_rgb16(s, pal_entry);
+ pal_entry += tga_comp;
+ }
+ } else if (!stbi__getn(s, tga_palette, tga_palette_len * tga_comp)) {
+ STBI_FREE(tga_data);
+ STBI_FREE(tga_palette);
+ return stbi__errpuc("bad palette", "Corrupt TGA");
+ }
+ }
+ // load the data
+ for (i=0; i < tga_width * tga_height; ++i)
+ {
+ // if I'm in RLE mode, do I need to get a RLE stbi__pngchunk?
+ if ( tga_is_RLE )
+ {
+ if ( RLE_count == 0 )
+ {
+ // yep, get the next byte as a RLE command
+ int RLE_cmd = stbi__get8(s);
+ RLE_count = 1 + (RLE_cmd & 127);
+ RLE_repeating = RLE_cmd >> 7;
+ read_next_pixel = 1;
+ } else if ( !RLE_repeating )
+ {
+ read_next_pixel = 1;
+ }
+ } else
+ {
+ read_next_pixel = 1;
+ }
+ // OK, if I need to read a pixel, do it now
+ if ( read_next_pixel )
+ {
+ // load however much data we did have
+ if ( tga_indexed )
+ {
+ // read in index, then perform the lookup
+ int pal_idx = (tga_bits_per_pixel == 8) ? stbi__get8(s) : stbi__get16le(s);
+ if ( pal_idx >= tga_palette_len ) {
+ // invalid index
+ pal_idx = 0;
+ }
+ pal_idx *= tga_comp;
+ for (j = 0; j < tga_comp; ++j) {
+ raw_data[j] = tga_palette[pal_idx+j];
+ }
+ } else if(tga_rgb16) {
+ STBI_ASSERT(tga_comp == STBI_rgb);
+ stbi__tga_read_rgb16(s, raw_data);
+ } else {
+ // read in the data raw
+ for (j = 0; j < tga_comp; ++j) {
+ raw_data[j] = stbi__get8(s);
+ }
+ }
+ // clear the reading flag for the next pixel
+ read_next_pixel = 0;
+ } // end of reading a pixel
+
+ // copy data
+ for (j = 0; j < tga_comp; ++j)
+ tga_data[i*tga_comp+j] = raw_data[j];
+
+ // in case we're in RLE mode, keep counting down
+ --RLE_count;
+ }
+ // do I need to invert the image?
+ if ( tga_inverted )
+ {
+ for (j = 0; j*2 < tga_height; ++j)
+ {
+ int index1 = j * tga_width * tga_comp;
+ int index2 = (tga_height - 1 - j) * tga_width * tga_comp;
+ for (i = tga_width * tga_comp; i > 0; --i)
+ {
+ unsigned char temp = tga_data[index1];
+ tga_data[index1] = tga_data[index2];
+ tga_data[index2] = temp;
+ ++index1;
+ ++index2;
+ }
+ }
+ }
+ // clear my palette, if I had one
+ if ( tga_palette != NULL )
+ {
+ STBI_FREE( tga_palette );
+ }
+ }
+
+ // swap RGB - if the source data was RGB16, it already is in the right order
+ if (tga_comp >= 3 && !tga_rgb16)
+ {
+ unsigned char* tga_pixel = tga_data;
+ for (i=0; i < tga_width * tga_height; ++i)
+ {
+ unsigned char temp = tga_pixel[0];
+ tga_pixel[0] = tga_pixel[2];
+ tga_pixel[2] = temp;
+ tga_pixel += tga_comp;
+ }
+ }
+
+ // convert to target component count
+ if (req_comp && req_comp != tga_comp)
+ tga_data = stbi__convert_format(tga_data, tga_comp, req_comp, tga_width, tga_height);
+
+ // the things I do to get rid of an error message, and yet keep
+ // Microsoft's C compilers happy... [8^(
+ tga_palette_start = tga_palette_len = tga_palette_bits =
+ tga_x_origin = tga_y_origin = 0;
+ STBI_NOTUSED(tga_palette_start);
+ // OK, done
+ return tga_data;
+}
+#endif
+
+// *************************************************************************************************
+// Photoshop PSD loader -- PD by Thatcher Ulrich, integration by Nicolas Schulz, tweaked by STB
+
+#ifndef STBI_NO_PSD
+static int stbi__psd_test(stbi__context *s)
+{
+ int r = (stbi__get32be(s) == 0x38425053);
+ stbi__rewind(s);
+ return r;
+}
+
+static int stbi__psd_decode_rle(stbi__context *s, stbi_uc *p, int pixelCount)
+{
+ int count, nleft, len;
+
+ count = 0;
+ while ((nleft = pixelCount - count) > 0) {
+ len = stbi__get8(s);
+ if (len == 128) {
+ // No-op.
+ } else if (len < 128) {
+ // Copy next len+1 bytes literally.
+ len++;
+ if (len > nleft) return 0; // corrupt data
+ count += len;
+ while (len) {
+ *p = stbi__get8(s);
+ p += 4;
+ len--;
+ }
+ } else if (len > 128) {
+ stbi_uc val;
+ // Next -len+1 bytes in the dest are replicated from next source byte.
+ // (Interpret len as a negative 8-bit int.)
+ len = 257 - len;
+ if (len > nleft) return 0; // corrupt data
+ val = stbi__get8(s);
+ count += len;
+ while (len) {
+ *p = val;
+ p += 4;
+ len--;
+ }
+ }
+ }
+
+ return 1;
+}
+
+static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc)
+{
+ int pixelCount;
+ int channelCount, compression;
+ int channel, i;
+ int bitdepth;
+ int w,h;
+ stbi_uc *out;
+ STBI_NOTUSED(ri);
+
+ // Check identifier
+ if (stbi__get32be(s) != 0x38425053) // "8BPS"
+ return stbi__errpuc("not PSD", "Corrupt PSD image");
+
+ // Check file type version.
+ if (stbi__get16be(s) != 1)
+ return stbi__errpuc("wrong version", "Unsupported version of PSD image");
+
+ // Skip 6 reserved bytes.
+ stbi__skip(s, 6 );
+
+ // Read the number of channels (R, G, B, A, etc).
+ channelCount = stbi__get16be(s);
+ if (channelCount < 0 || channelCount > 16)
+ return stbi__errpuc("wrong channel count", "Unsupported number of channels in PSD image");
+
+ // Read the rows and columns of the image.
+ h = stbi__get32be(s);
+ w = stbi__get32be(s);
+
+ if (h > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)");
+ if (w > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)");
+
+ // Make sure the depth is 8 bits.
+ bitdepth = stbi__get16be(s);
+ if (bitdepth != 8 && bitdepth != 16)
+ return stbi__errpuc("unsupported bit depth", "PSD bit depth is not 8 or 16 bit");
+
+ // Make sure the color mode is RGB.
+ // Valid options are:
+ // 0: Bitmap
+ // 1: Grayscale
+ // 2: Indexed color
+ // 3: RGB color
+ // 4: CMYK color
+ // 7: Multichannel
+ // 8: Duotone
+ // 9: Lab color
+ if (stbi__get16be(s) != 3)
+ return stbi__errpuc("wrong color format", "PSD is not in RGB color format");
+
+ // Skip the Mode Data. (It's the palette for indexed color; other info for other modes.)
+ stbi__skip(s,stbi__get32be(s) );
+
+ // Skip the image resources. (resolution, pen tool paths, etc)
+ stbi__skip(s, stbi__get32be(s) );
+
+ // Skip the reserved data.
+ stbi__skip(s, stbi__get32be(s) );
+
+ // Find out if the data is compressed.
+ // Known values:
+ // 0: no compression
+ // 1: RLE compressed
+ compression = stbi__get16be(s);
+ if (compression > 1)
+ return stbi__errpuc("bad compression", "PSD has an unknown compression format");
+
+ // Check size
+ if (!stbi__mad3sizes_valid(4, w, h, 0))
+ return stbi__errpuc("too large", "Corrupt PSD");
+
+ // Create the destination image.
+
+ if (!compression && bitdepth == 16 && bpc == 16) {
+ out = (stbi_uc *) stbi__malloc_mad3(8, w, h, 0);
+ ri->bits_per_channel = 16;
+ } else
+ out = (stbi_uc *) stbi__malloc(4 * w*h);
+
+ if (!out) return stbi__errpuc("outofmem", "Out of memory");
+ pixelCount = w*h;
+
+ // Initialize the data to zero.
+ //memset( out, 0, pixelCount * 4 );
+
+ // Finally, the image data.
+ if (compression) {
+ // RLE as used by .PSD and .TIFF
+ // Loop until you get the number of unpacked bytes you are expecting:
+ // Read the next source byte into n.
+ // If n is between 0 and 127 inclusive, copy the next n+1 bytes literally.
+ // Else if n is between -127 and -1 inclusive, copy the next byte -n+1 times.
+ // Else if n is 128, noop.
+ // Endloop
+
+ // The RLE-compressed data is preceded by a 2-byte data count for each row in the data,
+ // which we're going to just skip.
+ stbi__skip(s, h * channelCount * 2 );
+
+ // Read the RLE data by channel.
+ for (channel = 0; channel < 4; channel++) {
+ stbi_uc *p;
+
+ p = out+channel;
+ if (channel >= channelCount) {
+ // Fill this channel with default data.
+ for (i = 0; i < pixelCount; i++, p += 4)
+ *p = (channel == 3 ? 255 : 0);
+ } else {
+ // Read the RLE data.
+ if (!stbi__psd_decode_rle(s, p, pixelCount)) {
+ STBI_FREE(out);
+ return stbi__errpuc("corrupt", "bad RLE data");
+ }
+ }
+ }
+
+ } else {
+ // We're at the raw image data. It's each channel in order (Red, Green, Blue, Alpha, ...)
+ // where each channel consists of an 8-bit (or 16-bit) value for each pixel in the image.
+
+ // Read the data by channel.
+ for (channel = 0; channel < 4; channel++) {
+ if (channel >= channelCount) {
+ // Fill this channel with default data.
+ if (bitdepth == 16 && bpc == 16) {
+ stbi__uint16 *q = ((stbi__uint16 *) out) + channel;
+ stbi__uint16 val = channel == 3 ? 65535 : 0;
+ for (i = 0; i < pixelCount; i++, q += 4)
+ *q = val;
+ } else {
+ stbi_uc *p = out+channel;
+ stbi_uc val = channel == 3 ? 255 : 0;
+ for (i = 0; i < pixelCount; i++, p += 4)
+ *p = val;
+ }
+ } else {
+ if (ri->bits_per_channel == 16) { // output bpc
+ stbi__uint16 *q = ((stbi__uint16 *) out) + channel;
+ for (i = 0; i < pixelCount; i++, q += 4)
+ *q = (stbi__uint16) stbi__get16be(s);
+ } else {
+ stbi_uc *p = out+channel;
+ if (bitdepth == 16) { // input bpc
+ for (i = 0; i < pixelCount; i++, p += 4)
+ *p = (stbi_uc) (stbi__get16be(s) >> 8);
+ } else {
+ for (i = 0; i < pixelCount; i++, p += 4)
+ *p = stbi__get8(s);
+ }
+ }
+ }
+ }
+ }
+
+ // remove weird white matte from PSD
+ if (channelCount >= 4) {
+ if (ri->bits_per_channel == 16) {
+ for (i=0; i < w*h; ++i) {
+ stbi__uint16 *pixel = (stbi__uint16 *) out + 4*i;
+ if (pixel[3] != 0 && pixel[3] != 65535) {
+ float a = pixel[3] / 65535.0f;
+ float ra = 1.0f / a;
+ float inv_a = 65535.0f * (1 - ra);
+ pixel[0] = (stbi__uint16) (pixel[0]*ra + inv_a);
+ pixel[1] = (stbi__uint16) (pixel[1]*ra + inv_a);
+ pixel[2] = (stbi__uint16) (pixel[2]*ra + inv_a);
+ }
+ }
+ } else {
+ for (i=0; i < w*h; ++i) {
+ unsigned char *pixel = out + 4*i;
+ if (pixel[3] != 0 && pixel[3] != 255) {
+ float a = pixel[3] / 255.0f;
+ float ra = 1.0f / a;
+ float inv_a = 255.0f * (1 - ra);
+ pixel[0] = (unsigned char) (pixel[0]*ra + inv_a);
+ pixel[1] = (unsigned char) (pixel[1]*ra + inv_a);
+ pixel[2] = (unsigned char) (pixel[2]*ra + inv_a);
+ }
+ }
+ }
+ }
+
+ // convert to desired output format
+ if (req_comp && req_comp != 4) {
+ if (ri->bits_per_channel == 16)
+ out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, 4, req_comp, w, h);
+ else
+ out = stbi__convert_format(out, 4, req_comp, w, h);
+ if (out == NULL) return out; // stbi__convert_format frees input on failure
+ }
+
+ if (comp) *comp = 4;
+ *y = h;
+ *x = w;
+
+ return out;
+}
+#endif
+
+// *************************************************************************************************
+// Softimage PIC loader
+// by Tom Seddon
+//
+// See http://softimage.wiki.softimage.com/index.php/INFO:_PIC_file_format
+// See http://ozviz.wasp.uwa.edu.au/~pbourke/dataformats/softimagepic/
+
+#ifndef STBI_NO_PIC
+static int stbi__pic_is4(stbi__context *s,const char *str)
+{
+ int i;
+ for (i=0; i<4; ++i)
+ if (stbi__get8(s) != (stbi_uc)str[i])
+ return 0;
+
+ return 1;
+}
+
+static int stbi__pic_test_core(stbi__context *s)
+{
+ int i;
+
+ if (!stbi__pic_is4(s,"\x53\x80\xF6\x34"))
+ return 0;
+
+ for(i=0;i<84;++i)
+ stbi__get8(s);
+
+ if (!stbi__pic_is4(s,"PICT"))
+ return 0;
+
+ return 1;
+}
+
+typedef struct
+{
+ stbi_uc size,type,channel;
+} stbi__pic_packet;
+
+static stbi_uc *stbi__readval(stbi__context *s, int channel, stbi_uc *dest)
+{
+ int mask=0x80, i;
+
+ for (i=0; i<4; ++i, mask>>=1) {
+ if (channel & mask) {
+ if (stbi__at_eof(s)) return stbi__errpuc("bad file","PIC file too short");
+ dest[i]=stbi__get8(s);
+ }
+ }
+
+ return dest;
+}
+
+static void stbi__copyval(int channel,stbi_uc *dest,const stbi_uc *src)
+{
+ int mask=0x80,i;
+
+ for (i=0;i<4; ++i, mask>>=1)
+ if (channel&mask)
+ dest[i]=src[i];
+}
+
+static stbi_uc *stbi__pic_load_core(stbi__context *s,int width,int height,int *comp, stbi_uc *result)
+{
+ int act_comp=0,num_packets=0,y,chained;
+ stbi__pic_packet packets[10];
+
+ // this will (should...) cater for even some bizarre stuff like having data
+ // for the same channel in multiple packets.
+ do {
+ stbi__pic_packet *packet;
+
+ if (num_packets==sizeof(packets)/sizeof(packets[0]))
+ return stbi__errpuc("bad format","too many packets");
+
+ packet = &packets[num_packets++];
+
+ chained = stbi__get8(s);
+ packet->size = stbi__get8(s);
+ packet->type = stbi__get8(s);
+ packet->channel = stbi__get8(s);
+
+ act_comp |= packet->channel;
+
+ if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (reading packets)");
+ if (packet->size != 8) return stbi__errpuc("bad format","packet isn't 8bpp");
+ } while (chained);
+
+ *comp = (act_comp & 0x10 ? 4 : 3); // has alpha channel?
+
+ for(y=0; y<height; ++y) {
+ int packet_idx;
+
+ for(packet_idx=0; packet_idx < num_packets; ++packet_idx) {
+ stbi__pic_packet *packet = &packets[packet_idx];
+ stbi_uc *dest = result+y*width*4;
+
+ switch (packet->type) {
+ default:
+ return stbi__errpuc("bad format","packet has bad compression type");
+
+ case 0: {//uncompressed
+ int x;
+
+ for(x=0;x<width;++x, dest+=4)
+ if (!stbi__readval(s,packet->channel,dest))
+ return 0;
+ break;
+ }
+
+ case 1://Pure RLE
+ {
+ int left=width, i;
+
+ while (left>0) {
+ stbi_uc count,value[4];
+
+ count=stbi__get8(s);
+ if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pure read count)");
+
+ if (count > left)
+ count = (stbi_uc) left;
+
+ if (!stbi__readval(s,packet->channel,value)) return 0;
+
+ for(i=0; i<count; ++i,dest+=4)
+ stbi__copyval(packet->channel,dest,value);
+ left -= count;
+ }
+ }
+ break;
+
+ case 2: {//Mixed RLE
+ int left=width;
+ while (left>0) {
+ int count = stbi__get8(s), i;
+ if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (mixed read count)");
+
+ if (count >= 128) { // Repeated
+ stbi_uc value[4];
+
+ if (count==128)
+ count = stbi__get16be(s);
+ else
+ count -= 127;
+ if (count > left)
+ return stbi__errpuc("bad file","scanline overrun");
+
+ if (!stbi__readval(s,packet->channel,value))
+ return 0;
+
+ for(i=0;i<count;++i, dest += 4)
+ stbi__copyval(packet->channel,dest,value);
+ } else { // Raw
+ ++count;
+ if (count>left) return stbi__errpuc("bad file","scanline overrun");
+
+ for(i=0;i<count;++i, dest+=4)
+ if (!stbi__readval(s,packet->channel,dest))
+ return 0;
+ }
+ left-=count;
+ }
+ break;
+ }
+ }
+ }
+ }
+
+ return result;
+}
+
+static void *stbi__pic_load(stbi__context *s,int *px,int *py,int *comp,int req_comp, stbi__result_info *ri)
+{
+ stbi_uc *result;
+ int i, x,y, internal_comp;
+ STBI_NOTUSED(ri);
+
+ if (!comp) comp = &internal_comp;
+
+ for (i=0; i<92; ++i)
+ stbi__get8(s);
+
+ x = stbi__get16be(s);
+ y = stbi__get16be(s);
+
+ if (y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)");
+ if (x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)");
+
+ if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pic header)");
+ if (!stbi__mad3sizes_valid(x, y, 4, 0)) return stbi__errpuc("too large", "PIC image too large to decode");
+
+ stbi__get32be(s); //skip `ratio'
+ stbi__get16be(s); //skip `fields'
+ stbi__get16be(s); //skip `pad'
+
+ // intermediate buffer is RGBA
+ result = (stbi_uc *) stbi__malloc_mad3(x, y, 4, 0);
+ if (!result) return stbi__errpuc("outofmem", "Out of memory");
+ memset(result, 0xff, x*y*4);
+
+ if (!stbi__pic_load_core(s,x,y,comp, result)) {
+ STBI_FREE(result);
+ result=0;
+ }
+ *px = x;
+ *py = y;
+ if (req_comp == 0) req_comp = *comp;
+ result=stbi__convert_format(result,4,req_comp,x,y);
+
+ return result;
+}
+
+static int stbi__pic_test(stbi__context *s)
+{
+ int r = stbi__pic_test_core(s);
+ stbi__rewind(s);
+ return r;
+}
+#endif
+
+// *************************************************************************************************
+// GIF loader -- public domain by Jean-Marc Lienher -- simplified/shrunk by stb
+
+#ifndef STBI_NO_GIF
+typedef struct
+{
+ stbi__int16 prefix;
+ stbi_uc first;
+ stbi_uc suffix;
+} stbi__gif_lzw;
+
+typedef struct
+{
+ int w,h;
+ stbi_uc *out; // output buffer (always 4 components)
+ stbi_uc *background; // The current "background" as far as a gif is concerned
+ stbi_uc *history;
+ int flags, bgindex, ratio, transparent, eflags;
+ stbi_uc pal[256][4];
+ stbi_uc lpal[256][4];
+ stbi__gif_lzw codes[8192];
+ stbi_uc *color_table;
+ int parse, step;
+ int lflags;
+ int start_x, start_y;
+ int max_x, max_y;
+ int cur_x, cur_y;
+ int line_size;
+ int delay;
+} stbi__gif;
+
+static int stbi__gif_test_raw(stbi__context *s)
+{
+ int sz;
+ if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') return 0;
+ sz = stbi__get8(s);
+ if (sz != '9' && sz != '7') return 0;
+ if (stbi__get8(s) != 'a') return 0;
+ return 1;
+}
+
+static int stbi__gif_test(stbi__context *s)
+{
+ int r = stbi__gif_test_raw(s);
+ stbi__rewind(s);
+ return r;
+}
+
+static void stbi__gif_parse_colortable(stbi__context *s, stbi_uc pal[256][4], int num_entries, int transp)
+{
+ int i;
+ for (i=0; i < num_entries; ++i) {
+ pal[i][2] = stbi__get8(s);
+ pal[i][1] = stbi__get8(s);
+ pal[i][0] = stbi__get8(s);
+ pal[i][3] = transp == i ? 0 : 255;
+ }
+}
+
+static int stbi__gif_header(stbi__context *s, stbi__gif *g, int *comp, int is_info)
+{
+ stbi_uc version;
+ if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8')
+ return stbi__err("not GIF", "Corrupt GIF");
+
+ version = stbi__get8(s);
+ if (version != '7' && version != '9') return stbi__err("not GIF", "Corrupt GIF");
+ if (stbi__get8(s) != 'a') return stbi__err("not GIF", "Corrupt GIF");
+
+ stbi__g_failure_reason = "";
+ g->w = stbi__get16le(s);
+ g->h = stbi__get16le(s);
+ g->flags = stbi__get8(s);
+ g->bgindex = stbi__get8(s);
+ g->ratio = stbi__get8(s);
+ g->transparent = -1;
+
+ if (g->w > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)");
+ if (g->h > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)");
+
+ if (comp != 0) *comp = 4; // can't actually tell whether it's 3 or 4 until we parse the comments
+
+ if (is_info) return 1;
+
+ if (g->flags & 0x80)
+ stbi__gif_parse_colortable(s,g->pal, 2 << (g->flags & 7), -1);
+
+ return 1;
+}
+
+static int stbi__gif_info_raw(stbi__context *s, int *x, int *y, int *comp)
+{
+ stbi__gif* g = (stbi__gif*) stbi__malloc(sizeof(stbi__gif));
+ if (!g) return stbi__err("outofmem", "Out of memory");
+ if (!stbi__gif_header(s, g, comp, 1)) {
+ STBI_FREE(g);
+ stbi__rewind( s );
+ return 0;
+ }
+ if (x) *x = g->w;
+ if (y) *y = g->h;
+ STBI_FREE(g);
+ return 1;
+}
+
+static void stbi__out_gif_code(stbi__gif *g, stbi__uint16 code)
+{
+ stbi_uc *p, *c;
+ int idx;
+
+ // recurse to decode the prefixes, since the linked-list is backwards,
+ // and working backwards through an interleaved image would be nasty
+ if (g->codes[code].prefix >= 0)
+ stbi__out_gif_code(g, g->codes[code].prefix);
+
+ if (g->cur_y >= g->max_y) return;
+
+ idx = g->cur_x + g->cur_y;
+ p = &g->out[idx];
+ g->history[idx / 4] = 1;
+
+ c = &g->color_table[g->codes[code].suffix * 4];
+ if (c[3] > 128) { // don't render transparent pixels;
+ p[0] = c[2];
+ p[1] = c[1];
+ p[2] = c[0];
+ p[3] = c[3];
+ }
+ g->cur_x += 4;
+
+ if (g->cur_x >= g->max_x) {
+ g->cur_x = g->start_x;
+ g->cur_y += g->step;
+
+ while (g->cur_y >= g->max_y && g->parse > 0) {
+ g->step = (1 << g->parse) * g->line_size;
+ g->cur_y = g->start_y + (g->step >> 1);
+ --g->parse;
+ }
+ }
+}
+
+static stbi_uc *stbi__process_gif_raster(stbi__context *s, stbi__gif *g)
+{
+ stbi_uc lzw_cs;
+ stbi__int32 len, init_code;
+ stbi__uint32 first;
+ stbi__int32 codesize, codemask, avail, oldcode, bits, valid_bits, clear;
+ stbi__gif_lzw *p;
+
+ lzw_cs = stbi__get8(s);
+ if (lzw_cs > 12) return NULL;
+ clear = 1 << lzw_cs;
+ first = 1;
+ codesize = lzw_cs + 1;
+ codemask = (1 << codesize) - 1;
+ bits = 0;
+ valid_bits = 0;
+ for (init_code = 0; init_code < clear; init_code++) {
+ g->codes[init_code].prefix = -1;
+ g->codes[init_code].first = (stbi_uc) init_code;
+ g->codes[init_code].suffix = (stbi_uc) init_code;
+ }
+
+ // support no starting clear code
+ avail = clear+2;
+ oldcode = -1;
+
+ len = 0;
+ for(;;) {
+ if (valid_bits < codesize) {
+ if (len == 0) {
+ len = stbi__get8(s); // start new block
+ if (len == 0)
+ return g->out;
+ }
+ --len;
+ bits |= (stbi__int32) stbi__get8(s) << valid_bits;
+ valid_bits += 8;
+ } else {
+ stbi__int32 code = bits & codemask;
+ bits >>= codesize;
+ valid_bits -= codesize;
+ // @OPTIMIZE: is there some way we can accelerate the non-clear path?
+ if (code == clear) { // clear code
+ codesize = lzw_cs + 1;
+ codemask = (1 << codesize) - 1;
+ avail = clear + 2;
+ oldcode = -1;
+ first = 0;
+ } else if (code == clear + 1) { // end of stream code
+ stbi__skip(s, len);
+ while ((len = stbi__get8(s)) > 0)
+ stbi__skip(s,len);
+ return g->out;
+ } else if (code <= avail) {
+ if (first) {
+ return stbi__errpuc("no clear code", "Corrupt GIF");
+ }
+
+ if (oldcode >= 0) {
+ p = &g->codes[avail++];
+ if (avail > 8192) {
+ return stbi__errpuc("too many codes", "Corrupt GIF");
+ }
+
+ p->prefix = (stbi__int16) oldcode;
+ p->first = g->codes[oldcode].first;
+ p->suffix = (code == avail) ? p->first : g->codes[code].first;
+ } else if (code == avail)
+ return stbi__errpuc("illegal code in raster", "Corrupt GIF");
+
+ stbi__out_gif_code(g, (stbi__uint16) code);
+
+ if ((avail & codemask) == 0 && avail <= 0x0FFF) {
+ codesize++;
+ codemask = (1 << codesize) - 1;
+ }
+
+ oldcode = code;
+ } else {
+ return stbi__errpuc("illegal code in raster", "Corrupt GIF");
+ }
+ }
+ }
+}
+
+// this function is designed to support animated gifs, although stb_image doesn't support it
+// two back is the image from two frames ago, used for a very specific disposal format
+static stbi_uc *stbi__gif_load_next(stbi__context *s, stbi__gif *g, int *comp, int req_comp, stbi_uc *two_back)
+{
+ int dispose;
+ int first_frame;
+ int pi;
+ int pcount;
+ STBI_NOTUSED(req_comp);
+
+ // on first frame, any non-written pixels get the background colour (non-transparent)
+ first_frame = 0;
+ if (g->out == 0) {
+ if (!stbi__gif_header(s, g, comp,0)) return 0; // stbi__g_failure_reason set by stbi__gif_header
+ if (!stbi__mad3sizes_valid(4, g->w, g->h, 0))
+ return stbi__errpuc("too large", "GIF image is too large");
+ pcount = g->w * g->h;
+ g->out = (stbi_uc *) stbi__malloc(4 * pcount);
+ g->background = (stbi_uc *) stbi__malloc(4 * pcount);
+ g->history = (stbi_uc *) stbi__malloc(pcount);
+ if (!g->out || !g->background || !g->history)
+ return stbi__errpuc("outofmem", "Out of memory");
+
+ // image is treated as "transparent" at the start - ie, nothing overwrites the current background;
+ // background colour is only used for pixels that are not rendered first frame, after that "background"
+ // color refers to the color that was there the previous frame.
+ memset(g->out, 0x00, 4 * pcount);
+ memset(g->background, 0x00, 4 * pcount); // state of the background (starts transparent)
+ memset(g->history, 0x00, pcount); // pixels that were affected previous frame
+ first_frame = 1;
+ } else {
+ // second frame - how do we dispose of the previous one?
+ dispose = (g->eflags & 0x1C) >> 2;
+ pcount = g->w * g->h;
+
+ if ((dispose == 3) && (two_back == 0)) {
+ dispose = 2; // if I don't have an image to revert back to, default to the old background
+ }
+
+ if (dispose == 3) { // use previous graphic
+ for (pi = 0; pi < pcount; ++pi) {
+ if (g->history[pi]) {
+ memcpy( &g->out[pi * 4], &two_back[pi * 4], 4 );
+ }
+ }
+ } else if (dispose == 2) {
+ // restore what was changed last frame to background before that frame;
+ for (pi = 0; pi < pcount; ++pi) {
+ if (g->history[pi]) {
+ memcpy( &g->out[pi * 4], &g->background[pi * 4], 4 );
+ }
+ }
+ } else {
+ // This is a non-disposal case eithe way, so just
+ // leave the pixels as is, and they will become the new background
+ // 1: do not dispose
+ // 0: not specified.
+ }
+
+ // background is what out is after the undoing of the previou frame;
+ memcpy( g->background, g->out, 4 * g->w * g->h );
+ }
+
+ // clear my history;
+ memset( g->history, 0x00, g->w * g->h ); // pixels that were affected previous frame
+
+ for (;;) {
+ int tag = stbi__get8(s);
+ switch (tag) {
+ case 0x2C: /* Image Descriptor */
+ {
+ stbi__int32 x, y, w, h;
+ stbi_uc *o;
+
+ x = stbi__get16le(s);
+ y = stbi__get16le(s);
+ w = stbi__get16le(s);
+ h = stbi__get16le(s);
+ if (((x + w) > (g->w)) || ((y + h) > (g->h)))
+ return stbi__errpuc("bad Image Descriptor", "Corrupt GIF");
+
+ g->line_size = g->w * 4;
+ g->start_x = x * 4;
+ g->start_y = y * g->line_size;
+ g->max_x = g->start_x + w * 4;
+ g->max_y = g->start_y + h * g->line_size;
+ g->cur_x = g->start_x;
+ g->cur_y = g->start_y;
+
+ // if the width of the specified rectangle is 0, that means
+ // we may not see *any* pixels or the image is malformed;
+ // to make sure this is caught, move the current y down to
+ // max_y (which is what out_gif_code checks).
+ if (w == 0)
+ g->cur_y = g->max_y;
+
+ g->lflags = stbi__get8(s);
+
+ if (g->lflags & 0x40) {
+ g->step = 8 * g->line_size; // first interlaced spacing
+ g->parse = 3;
+ } else {
+ g->step = g->line_size;
+ g->parse = 0;
+ }
+
+ if (g->lflags & 0x80) {
+ stbi__gif_parse_colortable(s,g->lpal, 2 << (g->lflags & 7), g->eflags & 0x01 ? g->transparent : -1);
+ g->color_table = (stbi_uc *) g->lpal;
+ } else if (g->flags & 0x80) {
+ g->color_table = (stbi_uc *) g->pal;
+ } else
+ return stbi__errpuc("missing color table", "Corrupt GIF");
+
+ o = stbi__process_gif_raster(s, g);
+ if (!o) return NULL;
+
+ // if this was the first frame,
+ pcount = g->w * g->h;
+ if (first_frame && (g->bgindex > 0)) {
+ // if first frame, any pixel not drawn to gets the background color
+ for (pi = 0; pi < pcount; ++pi) {
+ if (g->history[pi] == 0) {
+ g->pal[g->bgindex][3] = 255; // just in case it was made transparent, undo that; It will be reset next frame if need be;
+ memcpy( &g->out[pi * 4], &g->pal[g->bgindex], 4 );
+ }
+ }
+ }
+
+ return o;
+ }
+
+ case 0x21: // Comment Extension.
+ {
+ int len;
+ int ext = stbi__get8(s);
+ if (ext == 0xF9) { // Graphic Control Extension.
+ len = stbi__get8(s);
+ if (len == 4) {
+ g->eflags = stbi__get8(s);
+ g->delay = 10 * stbi__get16le(s); // delay - 1/100th of a second, saving as 1/1000ths.
+
+ // unset old transparent
+ if (g->transparent >= 0) {
+ g->pal[g->transparent][3] = 255;
+ }
+ if (g->eflags & 0x01) {
+ g->transparent = stbi__get8(s);
+ if (g->transparent >= 0) {
+ g->pal[g->transparent][3] = 0;
+ }
+ } else {
+ // don't need transparent
+ stbi__skip(s, 1);
+ g->transparent = -1;
+ }
+ } else {
+ stbi__skip(s, len);
+ break;
+ }
+ }
+ while ((len = stbi__get8(s)) != 0) {
+ stbi__skip(s, len);
+ }
+ break;
+ }
+
+ case 0x3B: // gif stream termination code
+ return (stbi_uc *) s; // using '1' causes warning on some compilers
+
+ default:
+ return stbi__errpuc("unknown code", "Corrupt GIF");
+ }
+ }
+}
+
+static void *stbi__load_gif_main_outofmem(stbi__gif *g, stbi_uc *out, int **delays)
+{
+ STBI_FREE(g->out);
+ STBI_FREE(g->history);
+ STBI_FREE(g->background);
+
+ if (out) STBI_FREE(out);
+ if (delays && *delays) STBI_FREE(*delays);
+ return stbi__errpuc("outofmem", "Out of memory");
+}
+
+static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp)
+{
+ if (stbi__gif_test(s)) {
+ int layers = 0;
+ stbi_uc *u = 0;
+ stbi_uc *out = 0;
+ stbi_uc *two_back = 0;
+ stbi__gif g;
+ int stride;
+ int out_size = 0;
+ int delays_size = 0;
+
+ STBI_NOTUSED(out_size);
+ STBI_NOTUSED(delays_size);
+
+ memset(&g, 0, sizeof(g));
+ if (delays) {
+ *delays = 0;
+ }
+
+ do {
+ u = stbi__gif_load_next(s, &g, comp, req_comp, two_back);
+ if (u == (stbi_uc *) s) u = 0; // end of animated gif marker
+
+ if (u) {
+ *x = g.w;
+ *y = g.h;
+ ++layers;
+ stride = g.w * g.h * 4;
+
+ if (out) {
+ void *tmp = (stbi_uc*) STBI_REALLOC_SIZED( out, out_size, layers * stride );
+ if (!tmp)
+ return stbi__load_gif_main_outofmem(&g, out, delays);
+ else {
+ out = (stbi_uc*) tmp;
+ out_size = layers * stride;
+ }
+
+ if (delays) {
+ int *new_delays = (int*) STBI_REALLOC_SIZED( *delays, delays_size, sizeof(int) * layers );
+ if (!new_delays)
+ return stbi__load_gif_main_outofmem(&g, out, delays);
+ *delays = new_delays;
+ delays_size = layers * sizeof(int);
+ }
+ } else {
+ out = (stbi_uc*)stbi__malloc( layers * stride );
+ if (!out)
+ return stbi__load_gif_main_outofmem(&g, out, delays);
+ out_size = layers * stride;
+ if (delays) {
+ *delays = (int*) stbi__malloc( layers * sizeof(int) );
+ if (!*delays)
+ return stbi__load_gif_main_outofmem(&g, out, delays);
+ delays_size = layers * sizeof(int);
+ }
+ }
+ memcpy( out + ((layers - 1) * stride), u, stride );
+ if (layers >= 2) {
+ two_back = out - 2 * stride;
+ }
+
+ if (delays) {
+ (*delays)[layers - 1U] = g.delay;
+ }
+ }
+ } while (u != 0);
+
+ // free temp buffer;
+ STBI_FREE(g.out);
+ STBI_FREE(g.history);
+ STBI_FREE(g.background);
+
+ // do the final conversion after loading everything;
+ if (req_comp && req_comp != 4)
+ out = stbi__convert_format(out, 4, req_comp, layers * g.w, g.h);
+
+ *z = layers;
+ return out;
+ } else {
+ return stbi__errpuc("not GIF", "Image was not as a gif type.");
+ }
+}
+
+static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)
+{
+ stbi_uc *u = 0;
+ stbi__gif g;
+ memset(&g, 0, sizeof(g));
+ STBI_NOTUSED(ri);
+
+ u = stbi__gif_load_next(s, &g, comp, req_comp, 0);
+ if (u == (stbi_uc *) s) u = 0; // end of animated gif marker
+ if (u) {
+ *x = g.w;
+ *y = g.h;
+
+ // moved conversion to after successful load so that the same
+ // can be done for multiple frames.
+ if (req_comp && req_comp != 4)
+ u = stbi__convert_format(u, 4, req_comp, g.w, g.h);
+ } else if (g.out) {
+ // if there was an error and we allocated an image buffer, free it!
+ STBI_FREE(g.out);
+ }
+
+ // free buffers needed for multiple frame loading;
+ STBI_FREE(g.history);
+ STBI_FREE(g.background);
+
+ return u;
+}
+
+static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp)
+{
+ return stbi__gif_info_raw(s,x,y,comp);
+}
+#endif
+
+// *************************************************************************************************
+// Radiance RGBE HDR loader
+// originally by Nicolas Schulz
+#ifndef STBI_NO_HDR
+static int stbi__hdr_test_core(stbi__context *s, const char *signature)
+{
+ int i;
+ for (i=0; signature[i]; ++i)
+ if (stbi__get8(s) != signature[i])
+ return 0;
+ stbi__rewind(s);
+ return 1;
+}
+
+static int stbi__hdr_test(stbi__context* s)
+{
+ int r = stbi__hdr_test_core(s, "#?RADIANCE\n");
+ stbi__rewind(s);
+ if(!r) {
+ r = stbi__hdr_test_core(s, "#?RGBE\n");
+ stbi__rewind(s);
+ }
+ return r;
+}
+
+#define STBI__HDR_BUFLEN 1024
+static char *stbi__hdr_gettoken(stbi__context *z, char *buffer)
+{
+ int len=0;
+ char c = '\0';
+
+ c = (char) stbi__get8(z);
+
+ while (!stbi__at_eof(z) && c != '\n') {
+ buffer[len++] = c;
+ if (len == STBI__HDR_BUFLEN-1) {
+ // flush to end of line
+ while (!stbi__at_eof(z) && stbi__get8(z) != '\n')
+ ;
+ break;
+ }
+ c = (char) stbi__get8(z);
+ }
+
+ buffer[len] = 0;
+ return buffer;
+}
+
+static void stbi__hdr_convert(float *output, stbi_uc *input, int req_comp)
+{
+ if ( input[3] != 0 ) {
+ float f1;
+ // Exponent
+ f1 = (float) ldexp(1.0f, input[3] - (int)(128 + 8));
+ if (req_comp <= 2)
+ output[0] = (input[0] + input[1] + input[2]) * f1 / 3;
+ else {
+ output[0] = input[0] * f1;
+ output[1] = input[1] * f1;
+ output[2] = input[2] * f1;
+ }
+ if (req_comp == 2) output[1] = 1;
+ if (req_comp == 4) output[3] = 1;
+ } else {
+ switch (req_comp) {
+ case 4: output[3] = 1; /* fallthrough */
+ case 3: output[0] = output[1] = output[2] = 0;
+ break;
+ case 2: output[1] = 1; /* fallthrough */
+ case 1: output[0] = 0;
+ break;
+ }
+ }
+}
+
+static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)
+{
+ char buffer[STBI__HDR_BUFLEN];
+ char *token;
+ int valid = 0;
+ int width, height;
+ stbi_uc *scanline;
+ float *hdr_data;
+ int len;
+ unsigned char count, value;
+ int i, j, k, c1,c2, z;
+ const char *headerToken;
+ STBI_NOTUSED(ri);
+
+ // Check identifier
+ headerToken = stbi__hdr_gettoken(s,buffer);
+ if (strcmp(headerToken, "#?RADIANCE") != 0 && strcmp(headerToken, "#?RGBE") != 0)
+ return stbi__errpf("not HDR", "Corrupt HDR image");
+
+ // Parse header
+ for(;;) {
+ token = stbi__hdr_gettoken(s,buffer);
+ if (token[0] == 0) break;
+ if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1;
+ }
+
+ if (!valid) return stbi__errpf("unsupported format", "Unsupported HDR format");
+
+ // Parse width and height
+ // can't use sscanf() if we're not using stdio!
+ token = stbi__hdr_gettoken(s,buffer);
+ if (strncmp(token, "-Y ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format");
+ token += 3;
+ height = (int) strtol(token, &token, 10);
+ while (*token == ' ') ++token;
+ if (strncmp(token, "+X ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format");
+ token += 3;
+ width = (int) strtol(token, NULL, 10);
+
+ if (height > STBI_MAX_DIMENSIONS) return stbi__errpf("too large","Very large image (corrupt?)");
+ if (width > STBI_MAX_DIMENSIONS) return stbi__errpf("too large","Very large image (corrupt?)");
+
+ *x = width;
+ *y = height;
+
+ if (comp) *comp = 3;
+ if (req_comp == 0) req_comp = 3;
+
+ if (!stbi__mad4sizes_valid(width, height, req_comp, sizeof(float), 0))
+ return stbi__errpf("too large", "HDR image is too large");
+
+ // Read data
+ hdr_data = (float *) stbi__malloc_mad4(width, height, req_comp, sizeof(float), 0);
+ if (!hdr_data)
+ return stbi__errpf("outofmem", "Out of memory");
+
+ // Load image data
+ // image data is stored as some number of sca
+ if ( width < 8 || width >= 32768) {
+ // Read flat data
+ for (j=0; j < height; ++j) {
+ for (i=0; i < width; ++i) {
+ stbi_uc rgbe[4];
+ main_decode_loop:
+ stbi__getn(s, rgbe, 4);
+ stbi__hdr_convert(hdr_data + j * width * req_comp + i * req_comp, rgbe, req_comp);
+ }
+ }
+ } else {
+ // Read RLE-encoded data
+ scanline = NULL;
+
+ for (j = 0; j < height; ++j) {
+ c1 = stbi__get8(s);
+ c2 = stbi__get8(s);
+ len = stbi__get8(s);
+ if (c1 != 2 || c2 != 2 || (len & 0x80)) {
+ // not run-length encoded, so we have to actually use THIS data as a decoded
+ // pixel (note this can't be a valid pixel--one of RGB must be >= 128)
+ stbi_uc rgbe[4];
+ rgbe[0] = (stbi_uc) c1;
+ rgbe[1] = (stbi_uc) c2;
+ rgbe[2] = (stbi_uc) len;
+ rgbe[3] = (stbi_uc) stbi__get8(s);
+ stbi__hdr_convert(hdr_data, rgbe, req_comp);
+ i = 1;
+ j = 0;
+ STBI_FREE(scanline);
+ goto main_decode_loop; // yes, this makes no sense
+ }
+ len <<= 8;
+ len |= stbi__get8(s);
+ if (len != width) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("invalid decoded scanline length", "corrupt HDR"); }
+ if (scanline == NULL) {
+ scanline = (stbi_uc *) stbi__malloc_mad2(width, 4, 0);
+ if (!scanline) {
+ STBI_FREE(hdr_data);
+ return stbi__errpf("outofmem", "Out of memory");
+ }
+ }
+
+ for (k = 0; k < 4; ++k) {
+ int nleft;
+ i = 0;
+ while ((nleft = width - i) > 0) {
+ count = stbi__get8(s);
+ if (count > 128) {
+ // Run
+ value = stbi__get8(s);
+ count -= 128;
+ if ((count == 0) || (count > nleft)) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); }
+ for (z = 0; z < count; ++z)
+ scanline[i++ * 4 + k] = value;
+ } else {
+ // Dump
+ if ((count == 0) || (count > nleft)) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); }
+ for (z = 0; z < count; ++z)
+ scanline[i++ * 4 + k] = stbi__get8(s);
+ }
+ }
+ }
+ for (i=0; i < width; ++i)
+ stbi__hdr_convert(hdr_data+(j*width + i)*req_comp, scanline + i*4, req_comp);
+ }
+ if (scanline)
+ STBI_FREE(scanline);
+ }
+
+ return hdr_data;
+}
+
+static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp)
+{
+ char buffer[STBI__HDR_BUFLEN];
+ char *token;
+ int valid = 0;
+ int dummy;
+
+ if (!x) x = &dummy;
+ if (!y) y = &dummy;
+ if (!comp) comp = &dummy;
+
+ if (stbi__hdr_test(s) == 0) {
+ stbi__rewind( s );
+ return 0;
+ }
+
+ for(;;) {
+ token = stbi__hdr_gettoken(s,buffer);
+ if (token[0] == 0) break;
+ if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1;
+ }
+
+ if (!valid) {
+ stbi__rewind( s );
+ return 0;
+ }
+ token = stbi__hdr_gettoken(s,buffer);
+ if (strncmp(token, "-Y ", 3)) {
+ stbi__rewind( s );
+ return 0;
+ }
+ token += 3;
+ *y = (int) strtol(token, &token, 10);
+ while (*token == ' ') ++token;
+ if (strncmp(token, "+X ", 3)) {
+ stbi__rewind( s );
+ return 0;
+ }
+ token += 3;
+ *x = (int) strtol(token, NULL, 10);
+ *comp = 3;
+ return 1;
+}
+#endif // STBI_NO_HDR
+
+#ifndef STBI_NO_BMP
+static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp)
+{
+ void *p;
+ stbi__bmp_data info;
+
+ info.all_a = 255;
+ p = stbi__bmp_parse_header(s, &info);
+ if (p == NULL) {
+ stbi__rewind( s );
+ return 0;
+ }
+ if (x) *x = s->img_x;
+ if (y) *y = s->img_y;
+ if (comp) {
+ if (info.bpp == 24 && info.ma == 0xff000000)
+ *comp = 3;
+ else
+ *comp = info.ma ? 4 : 3;
+ }
+ return 1;
+}
+#endif
+
+#ifndef STBI_NO_PSD
+static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp)
+{
+ int channelCount, dummy, depth;
+ if (!x) x = &dummy;
+ if (!y) y = &dummy;
+ if (!comp) comp = &dummy;
+ if (stbi__get32be(s) != 0x38425053) {
+ stbi__rewind( s );
+ return 0;
+ }
+ if (stbi__get16be(s) != 1) {
+ stbi__rewind( s );
+ return 0;
+ }
+ stbi__skip(s, 6);
+ channelCount = stbi__get16be(s);
+ if (channelCount < 0 || channelCount > 16) {
+ stbi__rewind( s );
+ return 0;
+ }
+ *y = stbi__get32be(s);
+ *x = stbi__get32be(s);
+ depth = stbi__get16be(s);
+ if (depth != 8 && depth != 16) {
+ stbi__rewind( s );
+ return 0;
+ }
+ if (stbi__get16be(s) != 3) {
+ stbi__rewind( s );
+ return 0;
+ }
+ *comp = 4;
+ return 1;
+}
+
+static int stbi__psd_is16(stbi__context *s)
+{
+ int channelCount, depth;
+ if (stbi__get32be(s) != 0x38425053) {
+ stbi__rewind( s );
+ return 0;
+ }
+ if (stbi__get16be(s) != 1) {
+ stbi__rewind( s );
+ return 0;
+ }
+ stbi__skip(s, 6);
+ channelCount = stbi__get16be(s);
+ if (channelCount < 0 || channelCount > 16) {
+ stbi__rewind( s );
+ return 0;
+ }
+ STBI_NOTUSED(stbi__get32be(s));
+ STBI_NOTUSED(stbi__get32be(s));
+ depth = stbi__get16be(s);
+ if (depth != 16) {
+ stbi__rewind( s );
+ return 0;
+ }
+ return 1;
+}
+#endif
+
+#ifndef STBI_NO_PIC
+static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp)
+{
+ int act_comp=0,num_packets=0,chained,dummy;
+ stbi__pic_packet packets[10];
+
+ if (!x) x = &dummy;
+ if (!y) y = &dummy;
+ if (!comp) comp = &dummy;
+
+ if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) {
+ stbi__rewind(s);
+ return 0;
+ }
+
+ stbi__skip(s, 88);
+
+ *x = stbi__get16be(s);
+ *y = stbi__get16be(s);
+ if (stbi__at_eof(s)) {
+ stbi__rewind( s);
+ return 0;
+ }
+ if ( (*x) != 0 && (1 << 28) / (*x) < (*y)) {
+ stbi__rewind( s );
+ return 0;
+ }
+
+ stbi__skip(s, 8);
+
+ do {
+ stbi__pic_packet *packet;
+
+ if (num_packets==sizeof(packets)/sizeof(packets[0]))
+ return 0;
+
+ packet = &packets[num_packets++];
+ chained = stbi__get8(s);
+ packet->size = stbi__get8(s);
+ packet->type = stbi__get8(s);
+ packet->channel = stbi__get8(s);
+ act_comp |= packet->channel;
+
+ if (stbi__at_eof(s)) {
+ stbi__rewind( s );
+ return 0;
+ }
+ if (packet->size != 8) {
+ stbi__rewind( s );
+ return 0;
+ }
+ } while (chained);
+
+ *comp = (act_comp & 0x10 ? 4 : 3);
+
+ return 1;
+}
+#endif
+
+// *************************************************************************************************
+// Portable Gray Map and Portable Pixel Map loader
+// by Ken Miller
+//
+// PGM: http://netpbm.sourceforge.net/doc/pgm.html
+// PPM: http://netpbm.sourceforge.net/doc/ppm.html
+//
+// Known limitations:
+// Does not support comments in the header section
+// Does not support ASCII image data (formats P2 and P3)
+
+#ifndef STBI_NO_PNM
+
+static int stbi__pnm_test(stbi__context *s)
+{
+ char p, t;
+ p = (char) stbi__get8(s);
+ t = (char) stbi__get8(s);
+ if (p != 'P' || (t != '5' && t != '6')) {
+ stbi__rewind( s );
+ return 0;
+ }
+ return 1;
+}
+
+static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)
+{
+ stbi_uc *out;
+ STBI_NOTUSED(ri);
+
+ ri->bits_per_channel = stbi__pnm_info(s, (int *)&s->img_x, (int *)&s->img_y, (int *)&s->img_n);
+ if (ri->bits_per_channel == 0)
+ return 0;
+
+ if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)");
+ if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)");
+
+ *x = s->img_x;
+ *y = s->img_y;
+ if (comp) *comp = s->img_n;
+
+ if (!stbi__mad4sizes_valid(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0))
+ return stbi__errpuc("too large", "PNM too large");
+
+ out = (stbi_uc *) stbi__malloc_mad4(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0);
+ if (!out) return stbi__errpuc("outofmem", "Out of memory");
+ if (!stbi__getn(s, out, s->img_n * s->img_x * s->img_y * (ri->bits_per_channel / 8))) {
+ STBI_FREE(out);
+ return stbi__errpuc("bad PNM", "PNM file truncated");
+ }
+
+ if (req_comp && req_comp != s->img_n) {
+ if (ri->bits_per_channel == 16) {
+ out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, s->img_n, req_comp, s->img_x, s->img_y);
+ } else {
+ out = stbi__convert_format(out, s->img_n, req_comp, s->img_x, s->img_y);
+ }
+ if (out == NULL) return out; // stbi__convert_format frees input on failure
+ }
+ return out;
+}
+
+static int stbi__pnm_isspace(char c)
+{
+ return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r';
+}
+
+static void stbi__pnm_skip_whitespace(stbi__context *s, char *c)
+{
+ for (;;) {
+ while (!stbi__at_eof(s) && stbi__pnm_isspace(*c))
+ *c = (char) stbi__get8(s);
+
+ if (stbi__at_eof(s) || *c != '#')
+ break;
+
+ while (!stbi__at_eof(s) && *c != '\n' && *c != '\r' )
+ *c = (char) stbi__get8(s);
+ }
+}
+
+static int stbi__pnm_isdigit(char c)
+{
+ return c >= '0' && c <= '9';
+}
+
+static int stbi__pnm_getinteger(stbi__context *s, char *c)
+{
+ int value = 0;
+
+ while (!stbi__at_eof(s) && stbi__pnm_isdigit(*c)) {
+ value = value*10 + (*c - '0');
+ *c = (char) stbi__get8(s);
+ if((value > 214748364) || (value == 214748364 && *c > '7'))
+ return stbi__err("integer parse overflow", "Parsing an integer in the PPM header overflowed a 32-bit int");
+ }
+
+ return value;
+}
+
+static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp)
+{
+ int maxv, dummy;
+ char c, p, t;
+
+ if (!x) x = &dummy;
+ if (!y) y = &dummy;
+ if (!comp) comp = &dummy;
+
+ stbi__rewind(s);
+
+ // Get identifier
+ p = (char) stbi__get8(s);
+ t = (char) stbi__get8(s);
+ if (p != 'P' || (t != '5' && t != '6')) {
+ stbi__rewind(s);
+ return 0;
+ }
+
+ *comp = (t == '6') ? 3 : 1; // '5' is 1-component .pgm; '6' is 3-component .ppm
+
+ c = (char) stbi__get8(s);
+ stbi__pnm_skip_whitespace(s, &c);
+
+ *x = stbi__pnm_getinteger(s, &c); // read width
+ if(*x == 0)
+ return stbi__err("invalid width", "PPM image header had zero or overflowing width");
+ stbi__pnm_skip_whitespace(s, &c);
+
+ *y = stbi__pnm_getinteger(s, &c); // read height
+ if (*y == 0)
+ return stbi__err("invalid width", "PPM image header had zero or overflowing width");
+ stbi__pnm_skip_whitespace(s, &c);
+
+ maxv = stbi__pnm_getinteger(s, &c); // read max value
+ if (maxv > 65535)
+ return stbi__err("max value > 65535", "PPM image supports only 8-bit and 16-bit images");
+ else if (maxv > 255)
+ return 16;
+ else
+ return 8;
+}
+
+static int stbi__pnm_is16(stbi__context *s)
+{
+ if (stbi__pnm_info(s, NULL, NULL, NULL) == 16)
+ return 1;
+ return 0;
+}
+#endif
+
+static int stbi__info_main(stbi__context *s, int *x, int *y, int *comp)
+{
+ #ifndef STBI_NO_JPEG
+ if (stbi__jpeg_info(s, x, y, comp)) return 1;
+ #endif
+
+ #ifndef STBI_NO_PNG
+ if (stbi__png_info(s, x, y, comp)) return 1;
+ #endif
+
+ #ifndef STBI_NO_GIF
+ if (stbi__gif_info(s, x, y, comp)) return 1;
+ #endif
+
+ #ifndef STBI_NO_BMP
+ if (stbi__bmp_info(s, x, y, comp)) return 1;
+ #endif
+
+ #ifndef STBI_NO_PSD
+ if (stbi__psd_info(s, x, y, comp)) return 1;
+ #endif
+
+ #ifndef STBI_NO_PIC
+ if (stbi__pic_info(s, x, y, comp)) return 1;
+ #endif
+
+ #ifndef STBI_NO_PNM
+ if (stbi__pnm_info(s, x, y, comp)) return 1;
+ #endif
+
+ #ifndef STBI_NO_HDR
+ if (stbi__hdr_info(s, x, y, comp)) return 1;
+ #endif
+
+ // test tga last because it's a crappy test!
+ #ifndef STBI_NO_TGA
+ if (stbi__tga_info(s, x, y, comp))
+ return 1;
+ #endif
+ return stbi__err("unknown image type", "Image not of any known type, or corrupt");
+}
+
+static int stbi__is_16_main(stbi__context *s)
+{
+ #ifndef STBI_NO_PNG
+ if (stbi__png_is16(s)) return 1;
+ #endif
+
+ #ifndef STBI_NO_PSD
+ if (stbi__psd_is16(s)) return 1;
+ #endif
+
+ #ifndef STBI_NO_PNM
+ if (stbi__pnm_is16(s)) return 1;
+ #endif
+ return 0;
+}
+
+#ifndef STBI_NO_STDIO
+STBIDEF int stbi_info(char const *filename, int *x, int *y, int *comp)
+{
+ FILE *f = stbi__fopen(filename, "rb");
+ int result;
+ if (!f) return stbi__err("can't fopen", "Unable to open file");
+ result = stbi_info_from_file(f, x, y, comp);
+ fclose(f);
+ return result;
+}
+
+STBIDEF int stbi_info_from_file(FILE *f, int *x, int *y, int *comp)
+{
+ int r;
+ stbi__context s;
+ long pos = ftell(f);
+ stbi__start_file(&s, f);
+ r = stbi__info_main(&s,x,y,comp);
+ fseek(f,pos,SEEK_SET);
+ return r;
+}
+
+STBIDEF int stbi_is_16_bit(char const *filename)
+{
+ FILE *f = stbi__fopen(filename, "rb");
+ int result;
+ if (!f) return stbi__err("can't fopen", "Unable to open file");
+ result = stbi_is_16_bit_from_file(f);
+ fclose(f);
+ return result;
+}
+
+STBIDEF int stbi_is_16_bit_from_file(FILE *f)
+{
+ int r;
+ stbi__context s;
+ long pos = ftell(f);
+ stbi__start_file(&s, f);
+ r = stbi__is_16_main(&s);
+ fseek(f,pos,SEEK_SET);
+ return r;
+}
+#endif // !STBI_NO_STDIO
+
+STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp)
+{
+ stbi__context s;
+ stbi__start_mem(&s,buffer,len);
+ return stbi__info_main(&s,x,y,comp);
+}
+
+STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *c, void *user, int *x, int *y, int *comp)
+{
+ stbi__context s;
+ stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user);
+ return stbi__info_main(&s,x,y,comp);
+}
+
+STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len)
+{
+ stbi__context s;
+ stbi__start_mem(&s,buffer,len);
+ return stbi__is_16_main(&s);
+}
+
+STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *c, void *user)
+{
+ stbi__context s;
+ stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user);
+ return stbi__is_16_main(&s);
+}
+
+#endif // STB_IMAGE_IMPLEMENTATION
+
+/*
+ revision history:
+ 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs
+ 2.19 (2018-02-11) fix warning
+ 2.18 (2018-01-30) fix warnings
+ 2.17 (2018-01-29) change sbti__shiftsigned to avoid clang -O2 bug
+ 1-bit BMP
+ *_is_16_bit api
+ avoid warnings
+ 2.16 (2017-07-23) all functions have 16-bit variants;
+ STBI_NO_STDIO works again;
+ compilation fixes;
+ fix rounding in unpremultiply;
+ optimize vertical flip;
+ disable raw_len validation;
+ documentation fixes
+ 2.15 (2017-03-18) fix png-1,2,4 bug; now all Imagenet JPGs decode;
+ warning fixes; disable run-time SSE detection on gcc;
+ uniform handling of optional "return" values;
+ thread-safe initialization of zlib tables
+ 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs
+ 2.13 (2016-11-29) add 16-bit API, only supported for PNG right now
+ 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes
+ 2.11 (2016-04-02) allocate large structures on the stack
+ remove white matting for transparent PSD
+ fix reported channel count for PNG & BMP
+ re-enable SSE2 in non-gcc 64-bit
+ support RGB-formatted JPEG
+ read 16-bit PNGs (only as 8-bit)
+ 2.10 (2016-01-22) avoid warning introduced in 2.09 by STBI_REALLOC_SIZED
+ 2.09 (2016-01-16) allow comments in PNM files
+ 16-bit-per-pixel TGA (not bit-per-component)
+ info() for TGA could break due to .hdr handling
+ info() for BMP to shares code instead of sloppy parse
+ can use STBI_REALLOC_SIZED if allocator doesn't support realloc
+ code cleanup
+ 2.08 (2015-09-13) fix to 2.07 cleanup, reading RGB PSD as RGBA
+ 2.07 (2015-09-13) fix compiler warnings
+ partial animated GIF support
+ limited 16-bpc PSD support
+ #ifdef unused functions
+ bug with < 92 byte PIC,PNM,HDR,TGA
+ 2.06 (2015-04-19) fix bug where PSD returns wrong '*comp' value
+ 2.05 (2015-04-19) fix bug in progressive JPEG handling, fix warning
+ 2.04 (2015-04-15) try to re-enable SIMD on MinGW 64-bit
+ 2.03 (2015-04-12) extra corruption checking (mmozeiko)
+ stbi_set_flip_vertically_on_load (nguillemot)
+ fix NEON support; fix mingw support
+ 2.02 (2015-01-19) fix incorrect assert, fix warning
+ 2.01 (2015-01-17) fix various warnings; suppress SIMD on gcc 32-bit without -msse2
+ 2.00b (2014-12-25) fix STBI_MALLOC in progressive JPEG
+ 2.00 (2014-12-25) optimize JPG, including x86 SSE2 & NEON SIMD (ryg)
+ progressive JPEG (stb)
+ PGM/PPM support (Ken Miller)
+ STBI_MALLOC,STBI_REALLOC,STBI_FREE
+ GIF bugfix -- seemingly never worked
+ STBI_NO_*, STBI_ONLY_*
+ 1.48 (2014-12-14) fix incorrectly-named assert()
+ 1.47 (2014-12-14) 1/2/4-bit PNG support, both direct and paletted (Omar Cornut & stb)
+ optimize PNG (ryg)
+ fix bug in interlaced PNG with user-specified channel count (stb)
+ 1.46 (2014-08-26)
+ fix broken tRNS chunk (colorkey-style transparency) in non-paletted PNG
+ 1.45 (2014-08-16)
+ fix MSVC-ARM internal compiler error by wrapping malloc
+ 1.44 (2014-08-07)
+ various warning fixes from Ronny Chevalier
+ 1.43 (2014-07-15)
+ fix MSVC-only compiler problem in code changed in 1.42
+ 1.42 (2014-07-09)
+ don't define _CRT_SECURE_NO_WARNINGS (affects user code)
+ fixes to stbi__cleanup_jpeg path
+ added STBI_ASSERT to avoid requiring assert.h
+ 1.41 (2014-06-25)
+ fix search&replace from 1.36 that messed up comments/error messages
+ 1.40 (2014-06-22)
+ fix gcc struct-initialization warning
+ 1.39 (2014-06-15)
+ fix to TGA optimization when req_comp != number of components in TGA;
+ fix to GIF loading because BMP wasn't rewinding (whoops, no GIFs in my test suite)
+ add support for BMP version 5 (more ignored fields)
+ 1.38 (2014-06-06)
+ suppress MSVC warnings on integer casts truncating values
+ fix accidental rename of 'skip' field of I/O
+ 1.37 (2014-06-04)
+ remove duplicate typedef
+ 1.36 (2014-06-03)
+ convert to header file single-file library
+ if de-iphone isn't set, load iphone images color-swapped instead of returning NULL
+ 1.35 (2014-05-27)
+ various warnings
+ fix broken STBI_SIMD path
+ fix bug where stbi_load_from_file no longer left file pointer in correct place
+ fix broken non-easy path for 32-bit BMP (possibly never used)
+ TGA optimization by Arseny Kapoulkine
+ 1.34 (unknown)
+ use STBI_NOTUSED in stbi__resample_row_generic(), fix one more leak in tga failure case
+ 1.33 (2011-07-14)
+ make stbi_is_hdr work in STBI_NO_HDR (as specified), minor compiler-friendly improvements
+ 1.32 (2011-07-13)
+ support for "info" function for all supported filetypes (SpartanJ)
+ 1.31 (2011-06-20)
+ a few more leak fixes, bug in PNG handling (SpartanJ)
+ 1.30 (2011-06-11)
+ added ability to load files via callbacks to accomidate custom input streams (Ben Wenger)
+ removed deprecated format-specific test/load functions
+ removed support for installable file formats (stbi_loader) -- would have been broken for IO callbacks anyway
+ error cases in bmp and tga give messages and don't leak (Raymond Barbiero, grisha)
+ fix inefficiency in decoding 32-bit BMP (David Woo)
+ 1.29 (2010-08-16)
+ various warning fixes from Aurelien Pocheville
+ 1.28 (2010-08-01)
+ fix bug in GIF palette transparency (SpartanJ)
+ 1.27 (2010-08-01)
+ cast-to-stbi_uc to fix warnings
+ 1.26 (2010-07-24)
+ fix bug in file buffering for PNG reported by SpartanJ
+ 1.25 (2010-07-17)
+ refix trans_data warning (Won Chun)
+ 1.24 (2010-07-12)
+ perf improvements reading from files on platforms with lock-heavy fgetc()
+ minor perf improvements for jpeg
+ deprecated type-specific functions so we'll get feedback if they're needed
+ attempt to fix trans_data warning (Won Chun)
+ 1.23 fixed bug in iPhone support
+ 1.22 (2010-07-10)
+ removed image *writing* support
+ stbi_info support from Jetro Lauha
+ GIF support from Jean-Marc Lienher
+ iPhone PNG-extensions from James Brown
+ warning-fixes from Nicolas Schulz and Janez Zemva (i.stbi__err. Janez (U+017D)emva)
+ 1.21 fix use of 'stbi_uc' in header (reported by jon blow)
+ 1.20 added support for Softimage PIC, by Tom Seddon
+ 1.19 bug in interlaced PNG corruption check (found by ryg)
+ 1.18 (2008-08-02)
+ fix a threading bug (local mutable static)
+ 1.17 support interlaced PNG
+ 1.16 major bugfix - stbi__convert_format converted one too many pixels
+ 1.15 initialize some fields for thread safety
+ 1.14 fix threadsafe conversion bug
+ header-file-only version (#define STBI_HEADER_FILE_ONLY before including)
+ 1.13 threadsafe
+ 1.12 const qualifiers in the API
+ 1.11 Support installable IDCT, colorspace conversion routines
+ 1.10 Fixes for 64-bit (don't use "unsigned long")
+ optimized upsampling by Fabian "ryg" Giesen
+ 1.09 Fix format-conversion for PSD code (bad global variables!)
+ 1.08 Thatcher Ulrich's PSD code integrated by Nicolas Schulz
+ 1.07 attempt to fix C++ warning/errors again
+ 1.06 attempt to fix C++ warning/errors again
+ 1.05 fix TGA loading to return correct *comp and use good luminance calc
+ 1.04 default float alpha is 1, not 255; use 'void *' for stbi_image_free
+ 1.03 bugfixes to STBI_NO_STDIO, STBI_NO_HDR
+ 1.02 support for (subset of) HDR files, float interface for preferred access to them
+ 1.01 fix bug: possible bug in handling right-side up bmps... not sure
+ fix bug: the stbi__bmp_load() and stbi__tga_load() functions didn't work at all
+ 1.00 interface to zlib that skips zlib header
+ 0.99 correct handling of alpha in palette
+ 0.98 TGA loader by lonesock; dynamically add loaders (untested)
+ 0.97 jpeg errors on too large a file; also catch another malloc failure
+ 0.96 fix detection of invalid v value - particleman@mollyrocket forum
+ 0.95 during header scan, seek to markers in case of padding
+ 0.94 STBI_NO_STDIO to disable stdio usage; rename all #defines the same
+ 0.93 handle jpegtran output; verbose errors
+ 0.92 read 4,8,16,24,32-bit BMP files of several formats
+ 0.91 output 24-bit Windows 3.0 BMP files
+ 0.90 fix a few more warnings; bump version number to approach 1.0
+ 0.61 bugfixes due to Marc LeBlanc, Christopher Lloyd
+ 0.60 fix compiling as c++
+ 0.59 fix warnings: merge Dave Moore's -Wall fixes
+ 0.58 fix bug: zlib uncompressed mode len/nlen was wrong endian
+ 0.57 fix bug: jpg last huffman symbol before marker was >9 bits but less than 16 available
+ 0.56 fix bug: zlib uncompressed mode len vs. nlen
+ 0.55 fix bug: restart_interval not initialized to 0
+ 0.54 allow NULL for 'int *comp'
+ 0.53 fix bug in png 3->4; speedup png decoding
+ 0.52 png handles req_comp=3,4 directly; minor cleanup; jpeg comments
+ 0.51 obey req_comp requests, 1-component jpegs return as 1-component,
+ on 'test' only check type, not whether we support this variant
+ 0.50 (2006-11-19)
+ first released version
+*/
+
+
+/*
+------------------------------------------------------------------------------
+This software is available under 2 licenses -- choose whichever you prefer.
+------------------------------------------------------------------------------
+ALTERNATIVE A - MIT License
+Copyright (c) 2017 Sean Barrett
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+------------------------------------------------------------------------------
+ALTERNATIVE B - Public Domain (www.unlicense.org)
+This is free and unencumbered software released into the public domain.
+Anyone is free to copy, modify, publish, use, compile, sell, or distribute this
+software, either in source code form or as a compiled binary, for any purpose,
+commercial or non-commercial, and by any means.
+In jurisdictions that recognize copyright laws, the author or authors of this
+software dedicate any and all copyright interest in the software to the public
+domain. We make this dedication for the benefit of the public at large and to
+the detriment of our heirs and successors. We intend this dedication to be an
+overt act of relinquishment in perpetuity of all present and future rights to
+this software under copyright law.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+------------------------------------------------------------------------------
+*/
diff --git a/llama/sync.sh b/llama/sync.sh
new file mode 100755
index 00000000..353847c4
--- /dev/null
+++ b/llama/sync.sh
@@ -0,0 +1,138 @@
+#!/bin/bash
+
+# Run in the llama directory
+
+set -e
+
+# Set the source directory
+# TODO in the future: src_dir=$1
+src_dir=../llm/llama.cpp
+
+if [ -z "$src_dir" ]; then
+ echo "Usage: $0 LLAMA_CPP_DIR"
+ exit 1
+fi
+
+# Set the destination directory
+dst_dir=$(pwd)
+
+# TODO remove once we no longer use the submodule
+if [ -z "${OLLAMA_SKIP_PATCHING}" ]; then
+ (cd ../ && git submodule init && git submodule update --force ./llm/llama.cpp)
+
+ # apply patches
+ for patch in $dst_dir/patches/*.diff; do
+ echo "Applying $patch"
+ git -C $src_dir apply "$patch"
+ done
+else
+ echo "Skipping patching"
+fi
+
+# llama.cpp
+cp $src_dir/src/unicode.cpp $dst_dir/unicode.cpp
+cp $src_dir/src/unicode.h $dst_dir/unicode.h
+cp $src_dir/src/unicode-data.cpp $dst_dir/unicode-data.cpp
+cp $src_dir/src/unicode-data.h $dst_dir/unicode-data.h
+cp $src_dir/src/llama.cpp $dst_dir/llama.cpp
+cp $src_dir/src/llama-impl.h $dst_dir/llama-impl.h
+cp $src_dir/src/llama-vocab.cpp $dst_dir/llama-vocab.cpp
+cp $src_dir/src/llama-vocab.h $dst_dir/llama-vocab.h
+cp $src_dir/src/llama-grammar.cpp $dst_dir/llama-grammar.cpp
+cp $src_dir/src/llama-grammar.h $dst_dir/llama-grammar.h
+cp $src_dir/src/llama-sampling.cpp $dst_dir/llama-sampling.cpp
+cp $src_dir/src/llama-sampling.h $dst_dir/llama-sampling.h
+cp $src_dir/include/llama.h $dst_dir/llama.h
+cp $src_dir/ggml/src/llamafile/sgemm.cpp $dst_dir/sgemm.cpp
+cp $src_dir/ggml/src/llamafile/sgemm.h $dst_dir/sgemm.h
+mkdir -p $dst_dir/llamafile
+cp $src_dir/ggml/src/llamafile/sgemm.h $dst_dir/llamafile/sgemm.h
+
+# ggml
+cp $src_dir/ggml/src/ggml.c $dst_dir/ggml.c
+cp $src_dir/ggml/include/ggml.h $dst_dir/ggml.h
+cp $src_dir/ggml/src/ggml-quants.c $dst_dir/ggml-quants.c
+cp $src_dir/ggml/src/ggml-quants.h $dst_dir/ggml-quants.h
+cp $src_dir/ggml/src/ggml-metal.metal $dst_dir/ggml-metal.metal
+cp $src_dir/ggml/include/ggml-metal.h $dst_dir/ggml-metal.h
+cp $src_dir/ggml/src/ggml-metal.m $dst_dir/ggml-metal_darwin_arm64.m
+cp $src_dir/ggml/src/ggml-impl.h $dst_dir/ggml-impl.h
+cp $src_dir/ggml/include/ggml-cuda.h $dst_dir/ggml-cuda.h
+cp $src_dir/ggml/src/ggml-cuda.cu $dst_dir/ggml-cuda.cu
+cp $src_dir/ggml/src/ggml-common.h $dst_dir/ggml-common.h
+cp $src_dir/ggml/include/ggml-backend.h $dst_dir/ggml-backend.h
+cp $src_dir/ggml/src/ggml-backend.c $dst_dir/ggml-backend.c
+cp $src_dir/ggml/src/ggml-backend-impl.h $dst_dir/ggml-backend-impl.h
+cp $src_dir/ggml/include/ggml-alloc.h $dst_dir/ggml-alloc.h
+cp $src_dir/ggml/src/ggml-alloc.c $dst_dir/ggml-alloc.c
+cp $src_dir/ggml/src/ggml-aarch64.h $dst_dir/ggml-aarch64.h
+cp $src_dir/ggml/src/ggml-aarch64.c $dst_dir/ggml-aarch64.c
+cp $src_dir/ggml/include/ggml-blas.h $dst_dir/ggml-blas.h
+cp $src_dir/ggml/src/ggml-blas.cpp $dst_dir/ggml-blas.cpp
+
+# ggml-cuda
+mkdir -p $dst_dir/ggml-cuda/template-instances
+mkdir -p $dst_dir/ggml-cuda/vendors
+cp $src_dir/ggml/src/ggml-cuda/*.cu $dst_dir/ggml-cuda/
+cp $src_dir/ggml/src/ggml-cuda/*.cuh $dst_dir/ggml-cuda/
+cp $src_dir/ggml/src/ggml-cuda/template-instances/*.cu $dst_dir/ggml-cuda/template-instances/
+cp $src_dir/ggml/src/ggml-cuda/vendors/*.h $dst_dir/ggml-cuda/vendors/
+
+# llava
+cp $src_dir/examples/llava/clip.cpp $dst_dir/clip.cpp
+cp $src_dir/examples/llava/clip.h $dst_dir/clip.h
+cp $src_dir/examples/llava/llava.cpp $dst_dir/llava.cpp
+cp $src_dir/examples/llava/llava.h $dst_dir/llava.h
+cp $src_dir/common/log.h $dst_dir/log.h
+cp $src_dir/common/stb_image.h $dst_dir/stb_image.h
+
+# These files are mostly used by the llava code
+# and shouldn't be necessary once we use clip.cpp directly
+cp $src_dir/common/common.cpp $dst_dir/common.cpp
+cp $src_dir/common/common.h $dst_dir/common.h
+cp $src_dir/common/sampling.cpp $dst_dir/sampling.cpp
+cp $src_dir/common/sampling.h $dst_dir/sampling.h
+cp $src_dir/common/grammar-parser.cpp $dst_dir/grammar-parser.cpp
+cp $src_dir/common/grammar-parser.h $dst_dir/grammar-parser.h
+cp $src_dir/common/json.hpp $dst_dir/json.hpp
+cp $src_dir/common/json-schema-to-grammar.cpp $dst_dir/json-schema-to-grammar.cpp
+cp $src_dir/common/json-schema-to-grammar.h $dst_dir/json-schema-to-grammar.h
+cp $src_dir/common/base64.hpp $dst_dir/base64.hpp
+cat <<EOF > $dst_dir/build-info.cpp
+int LLAMA_BUILD_NUMBER = 0;
+char const *LLAMA_COMMIT = "$sha1";
+char const *LLAMA_COMPILER = "";
+char const *LLAMA_BUILD_TARGET = "";
+EOF
+
+# add licenses
+sha1=$(git -C $src_dir rev-parse @)
+
+TEMP_LICENSE=$(mktemp)
+cleanup() {
+ rm -f $TEMP_LICENSE
+}
+trap cleanup 0
+
+cat <<EOF | sed 's/ *$//' >$TEMP_LICENSE
+/**
+ * llama.cpp - commit $sha1 - do not edit this file
+ *
+$(sed 's/^/ * /' <$src_dir/LICENSE)
+ */
+
+EOF
+
+LICENSE_FILES=$(find $dst_dir -type f \( -name "*.c" -o -name "*.h" -o -name "*.cpp" -o -name "*.m" -o -name "*.metal" -o -name "*.cu" -o -name "*.cuh" \))
+EXCLUDED_FILES=("sgemm.cpp" "sgemm.h" "sampling_ext.cpp" "sampling_ext.h" "stb_image.h" "json.hpp" "llama_darwin.c")
+
+for IN in $LICENSE_FILES; do
+ for EXCLUDED in "${EXCLUDED_FILES[@]}"; do
+ if [[ "$IN" == *"$EXCLUDED" ]]; then
+ continue 2
+ fi
+ done
+ TMP=$(mktemp)
+ cat $TEMP_LICENSE $IN >$TMP
+ mv $TMP $IN
+done
diff --git a/llama/unicode-data.cpp b/llama/unicode-data.cpp
new file mode 100644
index 00000000..71de6795
--- /dev/null
+++ b/llama/unicode-data.cpp
@@ -0,0 +1,7058 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// generated with scripts/gen-unicode-data.py
+
+#include "unicode-data.h"
+
+#include <cstdint>
+#include <vector>
+#include <unordered_map>
+#include <unordered_set>
+
+const std::vector<std::pair<uint32_t, uint16_t>> unicode_ranges_flags = { // start, flags // last=next_start-1
+{0x000000, 0x0080},
+{0x000020, 0x0008},
+{0x000021, 0x0020},
+{0x000024, 0x0040},
+{0x000025, 0x0020},
+{0x00002B, 0x0040},
+{0x00002C, 0x0020},
+{0x000030, 0x0002},
+{0x00003A, 0x0020},
+{0x00003C, 0x0040},
+{0x00003F, 0x0020},
+{0x000041, 0x0004},
+{0x00005B, 0x0020},
+{0x00005E, 0x0040},
+{0x00005F, 0x0020},
+{0x000060, 0x0040},
+{0x000061, 0x0004},
+{0x00007B, 0x0020},
+{0x00007C, 0x0040},
+{0x00007D, 0x0020},
+{0x00007E, 0x0040},
+{0x00007F, 0x0080},
+{0x0000A0, 0x0008},
+{0x0000A1, 0x0020},
+{0x0000A2, 0x0040},
+{0x0000A7, 0x0020},
+{0x0000A8, 0x0040},
+{0x0000AA, 0x0004},
+{0x0000AB, 0x0020},
+{0x0000AC, 0x0040},
+{0x0000AD, 0x0080},
+{0x0000AE, 0x0040},
+{0x0000B2, 0x0002},
+{0x0000B4, 0x0040},
+{0x0000B5, 0x0004},
+{0x0000B6, 0x0020},
+{0x0000B8, 0x0040},
+{0x0000B9, 0x0002},
+{0x0000BA, 0x0004},
+{0x0000BB, 0x0020},
+{0x0000BC, 0x0002},
+{0x0000BF, 0x0020},
+{0x0000C0, 0x0004},
+{0x0000D7, 0x0040},
+{0x0000D8, 0x0004},
+{0x0000F7, 0x0040},
+{0x0000F8, 0x0004},
+{0x0002C2, 0x0040},
+{0x0002C6, 0x0004},
+{0x0002D2, 0x0040},
+{0x0002E0, 0x0004},
+{0x0002E5, 0x0040},
+{0x0002EC, 0x0004},
+{0x0002ED, 0x0040},
+{0x0002EE, 0x0004},
+{0x0002EF, 0x0040},
+{0x000300, 0x0010},
+{0x000370, 0x0004},
+{0x000375, 0x0040},
+{0x000376, 0x0004},
+{0x000378, 0x0001},
+{0x00037A, 0x0004},
+{0x00037E, 0x0020},
+{0x00037F, 0x0004},
+{0x000380, 0x0001},
+{0x000384, 0x0040},
+{0x000386, 0x0004},
+{0x000387, 0x0020},
+{0x000388, 0x0004},
+{0x00038B, 0x0001},
+{0x00038C, 0x0004},
+{0x00038D, 0x0001},
+{0x00038E, 0x0004},
+{0x0003A2, 0x0001},
+{0x0003A3, 0x0004},
+{0x0003F6, 0x0040},
+{0x0003F7, 0x0004},
+{0x000482, 0x0040},
+{0x000483, 0x0010},
+{0x00048A, 0x0004},
+{0x000530, 0x0001},
+{0x000531, 0x0004},
+{0x000557, 0x0001},
+{0x000559, 0x0004},
+{0x00055A, 0x0020},
+{0x000560, 0x0004},
+{0x000589, 0x0020},
+{0x00058B, 0x0001},
+{0x00058D, 0x0040},
+{0x000590, 0x0001},
+{0x000591, 0x0010},
+{0x0005BE, 0x0020},
+{0x0005BF, 0x0010},
+{0x0005C0, 0x0020},
+{0x0005C1, 0x0010},
+{0x0005C3, 0x0020},
+{0x0005C4, 0x0010},
+{0x0005C6, 0x0020},
+{0x0005C7, 0x0010},
+{0x0005C8, 0x0001},
+{0x0005D0, 0x0004},
+{0x0005EB, 0x0001},
+{0x0005EF, 0x0004},
+{0x0005F3, 0x0020},
+{0x0005F5, 0x0001},
+{0x000600, 0x0080},
+{0x000606, 0x0040},
+{0x000609, 0x0020},
+{0x00060B, 0x0040},
+{0x00060C, 0x0020},
+{0x00060E, 0x0040},
+{0x000610, 0x0010},
+{0x00061B, 0x0020},
+{0x00061C, 0x0080},
+{0x00061D, 0x0020},
+{0x000620, 0x0004},
+{0x00064B, 0x0010},
+{0x000660, 0x0002},
+{0x00066A, 0x0020},
+{0x00066E, 0x0004},
+{0x000670, 0x0010},
+{0x000671, 0x0004},
+{0x0006D4, 0x0020},
+{0x0006D5, 0x0004},
+{0x0006D6, 0x0010},
+{0x0006DD, 0x0080},
+{0x0006DE, 0x0040},
+{0x0006DF, 0x0010},
+{0x0006E5, 0x0004},
+{0x0006E7, 0x0010},
+{0x0006E9, 0x0040},
+{0x0006EA, 0x0010},
+{0x0006EE, 0x0004},
+{0x0006F0, 0x0002},
+{0x0006FA, 0x0004},
+{0x0006FD, 0x0040},
+{0x0006FF, 0x0004},
+{0x000700, 0x0020},
+{0x00070E, 0x0001},
+{0x00070F, 0x0080},
+{0x000710, 0x0004},
+{0x000711, 0x0010},
+{0x000712, 0x0004},
+{0x000730, 0x0010},
+{0x00074B, 0x0001},
+{0x00074D, 0x0004},
+{0x0007A6, 0x0010},
+{0x0007B1, 0x0004},
+{0x0007B2, 0x0001},
+{0x0007C0, 0x0002},
+{0x0007CA, 0x0004},
+{0x0007EB, 0x0010},
+{0x0007F4, 0x0004},
+{0x0007F6, 0x0040},
+{0x0007F7, 0x0020},
+{0x0007FA, 0x0004},
+{0x0007FB, 0x0001},
+{0x0007FD, 0x0010},
+{0x0007FE, 0x0040},
+{0x000800, 0x0004},
+{0x000816, 0x0010},
+{0x00081A, 0x0004},
+{0x00081B, 0x0010},
+{0x000824, 0x0004},
+{0x000825, 0x0010},
+{0x000828, 0x0004},
+{0x000829, 0x0010},
+{0x00082E, 0x0001},
+{0x000830, 0x0020},
+{0x00083F, 0x0001},
+{0x000840, 0x0004},
+{0x000859, 0x0010},
+{0x00085C, 0x0001},
+{0x00085E, 0x0020},
+{0x00085F, 0x0001},
+{0x000860, 0x0004},
+{0x00086B, 0x0001},
+{0x000870, 0x0004},
+{0x000888, 0x0040},
+{0x000889, 0x0004},
+{0x00088F, 0x0001},
+{0x000890, 0x0080},
+{0x000892, 0x0001},
+{0x000898, 0x0010},
+{0x0008A0, 0x0004},
+{0x0008CA, 0x0010},
+{0x0008E2, 0x0080},
+{0x0008E3, 0x0010},
+{0x000904, 0x0004},
+{0x00093A, 0x0010},
+{0x00093D, 0x0004},
+{0x00093E, 0x0010},
+{0x000950, 0x0004},
+{0x000951, 0x0010},
+{0x000958, 0x0004},
+{0x000962, 0x0010},
+{0x000964, 0x0020},
+{0x000966, 0x0002},
+{0x000970, 0x0020},
+{0x000971, 0x0004},
+{0x000981, 0x0010},
+{0x000984, 0x0001},
+{0x000985, 0x0004},
+{0x00098D, 0x0001},
+{0x00098F, 0x0004},
+{0x000991, 0x0001},
+{0x000993, 0x0004},
+{0x0009A9, 0x0001},
+{0x0009AA, 0x0004},
+{0x0009B1, 0x0001},
+{0x0009B2, 0x0004},
+{0x0009B3, 0x0001},
+{0x0009B6, 0x0004},
+{0x0009BA, 0x0001},
+{0x0009BC, 0x0010},
+{0x0009BD, 0x0004},
+{0x0009BE, 0x0010},
+{0x0009C5, 0x0001},
+{0x0009C7, 0x0010},
+{0x0009C9, 0x0001},
+{0x0009CB, 0x0010},
+{0x0009CE, 0x0004},
+{0x0009CF, 0x0001},
+{0x0009D7, 0x0010},
+{0x0009D8, 0x0001},
+{0x0009DC, 0x0004},
+{0x0009DE, 0x0001},
+{0x0009DF, 0x0004},
+{0x0009E2, 0x0010},
+{0x0009E4, 0x0001},
+{0x0009E6, 0x0002},
+{0x0009F0, 0x0004},
+{0x0009F2, 0x0040},
+{0x0009F4, 0x0002},
+{0x0009FA, 0x0040},
+{0x0009FC, 0x0004},
+{0x0009FD, 0x0020},
+{0x0009FE, 0x0010},
+{0x0009FF, 0x0001},
+{0x000A01, 0x0010},
+{0x000A04, 0x0001},
+{0x000A05, 0x0004},
+{0x000A0B, 0x0001},
+{0x000A0F, 0x0004},
+{0x000A11, 0x0001},
+{0x000A13, 0x0004},
+{0x000A29, 0x0001},
+{0x000A2A, 0x0004},
+{0x000A31, 0x0001},
+{0x000A32, 0x0004},
+{0x000A34, 0x0001},
+{0x000A35, 0x0004},
+{0x000A37, 0x0001},
+{0x000A38, 0x0004},
+{0x000A3A, 0x0001},
+{0x000A3C, 0x0010},
+{0x000A3D, 0x0001},
+{0x000A3E, 0x0010},
+{0x000A43, 0x0001},
+{0x000A47, 0x0010},
+{0x000A49, 0x0001},
+{0x000A4B, 0x0010},
+{0x000A4E, 0x0001},
+{0x000A51, 0x0010},
+{0x000A52, 0x0001},
+{0x000A59, 0x0004},
+{0x000A5D, 0x0001},
+{0x000A5E, 0x0004},
+{0x000A5F, 0x0001},
+{0x000A66, 0x0002},
+{0x000A70, 0x0010},
+{0x000A72, 0x0004},
+{0x000A75, 0x0010},
+{0x000A76, 0x0020},
+{0x000A77, 0x0001},
+{0x000A81, 0x0010},
+{0x000A84, 0x0001},
+{0x000A85, 0x0004},
+{0x000A8E, 0x0001},
+{0x000A8F, 0x0004},
+{0x000A92, 0x0001},
+{0x000A93, 0x0004},
+{0x000AA9, 0x0001},
+{0x000AAA, 0x0004},
+{0x000AB1, 0x0001},
+{0x000AB2, 0x0004},
+{0x000AB4, 0x0001},
+{0x000AB5, 0x0004},
+{0x000ABA, 0x0001},
+{0x000ABC, 0x0010},
+{0x000ABD, 0x0004},
+{0x000ABE, 0x0010},
+{0x000AC6, 0x0001},
+{0x000AC7, 0x0010},
+{0x000ACA, 0x0001},
+{0x000ACB, 0x0010},
+{0x000ACE, 0x0001},
+{0x000AD0, 0x0004},
+{0x000AD1, 0x0001},
+{0x000AE0, 0x0004},
+{0x000AE2, 0x0010},
+{0x000AE4, 0x0001},
+{0x000AE6, 0x0002},
+{0x000AF0, 0x0020},
+{0x000AF1, 0x0040},
+{0x000AF2, 0x0001},
+{0x000AF9, 0x0004},
+{0x000AFA, 0x0010},
+{0x000B00, 0x0001},
+{0x000B01, 0x0010},
+{0x000B04, 0x0001},
+{0x000B05, 0x0004},
+{0x000B0D, 0x0001},
+{0x000B0F, 0x0004},
+{0x000B11, 0x0001},
+{0x000B13, 0x0004},
+{0x000B29, 0x0001},
+{0x000B2A, 0x0004},
+{0x000B31, 0x0001},
+{0x000B32, 0x0004},
+{0x000B34, 0x0001},
+{0x000B35, 0x0004},
+{0x000B3A, 0x0001},
+{0x000B3C, 0x0010},
+{0x000B3D, 0x0004},
+{0x000B3E, 0x0010},
+{0x000B45, 0x0001},
+{0x000B47, 0x0010},
+{0x000B49, 0x0001},
+{0x000B4B, 0x0010},
+{0x000B4E, 0x0001},
+{0x000B55, 0x0010},
+{0x000B58, 0x0001},
+{0x000B5C, 0x0004},
+{0x000B5E, 0x0001},
+{0x000B5F, 0x0004},
+{0x000B62, 0x0010},
+{0x000B64, 0x0001},
+{0x000B66, 0x0002},
+{0x000B70, 0x0040},
+{0x000B71, 0x0004},
+{0x000B72, 0x0002},
+{0x000B78, 0x0001},
+{0x000B82, 0x0010},
+{0x000B83, 0x0004},
+{0x000B84, 0x0001},
+{0x000B85, 0x0004},
+{0x000B8B, 0x0001},
+{0x000B8E, 0x0004},
+{0x000B91, 0x0001},
+{0x000B92, 0x0004},
+{0x000B96, 0x0001},
+{0x000B99, 0x0004},
+{0x000B9B, 0x0001},
+{0x000B9C, 0x0004},
+{0x000B9D, 0x0001},
+{0x000B9E, 0x0004},
+{0x000BA0, 0x0001},
+{0x000BA3, 0x0004},
+{0x000BA5, 0x0001},
+{0x000BA8, 0x0004},
+{0x000BAB, 0x0001},
+{0x000BAE, 0x0004},
+{0x000BBA, 0x0001},
+{0x000BBE, 0x0010},
+{0x000BC3, 0x0001},
+{0x000BC6, 0x0010},
+{0x000BC9, 0x0001},
+{0x000BCA, 0x0010},
+{0x000BCE, 0x0001},
+{0x000BD0, 0x0004},
+{0x000BD1, 0x0001},
+{0x000BD7, 0x0010},
+{0x000BD8, 0x0001},
+{0x000BE6, 0x0002},
+{0x000BF3, 0x0040},
+{0x000BFB, 0x0001},
+{0x000C00, 0x0010},
+{0x000C05, 0x0004},
+{0x000C0D, 0x0001},
+{0x000C0E, 0x0004},
+{0x000C11, 0x0001},
+{0x000C12, 0x0004},
+{0x000C29, 0x0001},
+{0x000C2A, 0x0004},
+{0x000C3A, 0x0001},
+{0x000C3C, 0x0010},
+{0x000C3D, 0x0004},
+{0x000C3E, 0x0010},
+{0x000C45, 0x0001},
+{0x000C46, 0x0010},
+{0x000C49, 0x0001},
+{0x000C4A, 0x0010},
+{0x000C4E, 0x0001},
+{0x000C55, 0x0010},
+{0x000C57, 0x0001},
+{0x000C58, 0x0004},
+{0x000C5B, 0x0001},
+{0x000C5D, 0x0004},
+{0x000C5E, 0x0001},
+{0x000C60, 0x0004},
+{0x000C62, 0x0010},
+{0x000C64, 0x0001},
+{0x000C66, 0x0002},
+{0x000C70, 0x0001},
+{0x000C77, 0x0020},
+{0x000C78, 0x0002},
+{0x000C7F, 0x0040},
+{0x000C80, 0x0004},
+{0x000C81, 0x0010},
+{0x000C84, 0x0020},
+{0x000C85, 0x0004},
+{0x000C8D, 0x0001},
+{0x000C8E, 0x0004},
+{0x000C91, 0x0001},
+{0x000C92, 0x0004},
+{0x000CA9, 0x0001},
+{0x000CAA, 0x0004},
+{0x000CB4, 0x0001},
+{0x000CB5, 0x0004},
+{0x000CBA, 0x0001},
+{0x000CBC, 0x0010},
+{0x000CBD, 0x0004},
+{0x000CBE, 0x0010},
+{0x000CC5, 0x0001},
+{0x000CC6, 0x0010},
+{0x000CC9, 0x0001},
+{0x000CCA, 0x0010},
+{0x000CCE, 0x0001},
+{0x000CD5, 0x0010},
+{0x000CD7, 0x0001},
+{0x000CDD, 0x0004},
+{0x000CDF, 0x0001},
+{0x000CE0, 0x0004},
+{0x000CE2, 0x0010},
+{0x000CE4, 0x0001},
+{0x000CE6, 0x0002},
+{0x000CF0, 0x0001},
+{0x000CF1, 0x0004},
+{0x000CF3, 0x0010},
+{0x000CF4, 0x0001},
+{0x000D00, 0x0010},
+{0x000D04, 0x0004},
+{0x000D0D, 0x0001},
+{0x000D0E, 0x0004},
+{0x000D11, 0x0001},
+{0x000D12, 0x0004},
+{0x000D3B, 0x0010},
+{0x000D3D, 0x0004},
+{0x000D3E, 0x0010},
+{0x000D45, 0x0001},
+{0x000D46, 0x0010},
+{0x000D49, 0x0001},
+{0x000D4A, 0x0010},
+{0x000D4E, 0x0004},
+{0x000D4F, 0x0040},
+{0x000D50, 0x0001},
+{0x000D54, 0x0004},
+{0x000D57, 0x0010},
+{0x000D58, 0x0002},
+{0x000D5F, 0x0004},
+{0x000D62, 0x0010},
+{0x000D64, 0x0001},
+{0x000D66, 0x0002},
+{0x000D79, 0x0040},
+{0x000D7A, 0x0004},
+{0x000D80, 0x0001},
+{0x000D81, 0x0010},
+{0x000D84, 0x0001},
+{0x000D85, 0x0004},
+{0x000D97, 0x0001},
+{0x000D9A, 0x0004},
+{0x000DB2, 0x0001},
+{0x000DB3, 0x0004},
+{0x000DBC, 0x0001},
+{0x000DBD, 0x0004},
+{0x000DBE, 0x0001},
+{0x000DC0, 0x0004},
+{0x000DC7, 0x0001},
+{0x000DCA, 0x0010},
+{0x000DCB, 0x0001},
+{0x000DCF, 0x0010},
+{0x000DD5, 0x0001},
+{0x000DD6, 0x0010},
+{0x000DD7, 0x0001},
+{0x000DD8, 0x0010},
+{0x000DE0, 0x0001},
+{0x000DE6, 0x0002},
+{0x000DF0, 0x0001},
+{0x000DF2, 0x0010},
+{0x000DF4, 0x0020},
+{0x000DF5, 0x0001},
+{0x000E01, 0x0004},
+{0x000E31, 0x0010},
+{0x000E32, 0x0004},
+{0x000E34, 0x0010},
+{0x000E3B, 0x0001},
+{0x000E3F, 0x0040},
+{0x000E40, 0x0004},
+{0x000E47, 0x0010},
+{0x000E4F, 0x0020},
+{0x000E50, 0x0002},
+{0x000E5A, 0x0020},
+{0x000E5C, 0x0001},
+{0x000E81, 0x0004},
+{0x000E83, 0x0001},
+{0x000E84, 0x0004},
+{0x000E85, 0x0001},
+{0x000E86, 0x0004},
+{0x000E8B, 0x0001},
+{0x000E8C, 0x0004},
+{0x000EA4, 0x0001},
+{0x000EA5, 0x0004},
+{0x000EA6, 0x0001},
+{0x000EA7, 0x0004},
+{0x000EB1, 0x0010},
+{0x000EB2, 0x0004},
+{0x000EB4, 0x0010},
+{0x000EBD, 0x0004},
+{0x000EBE, 0x0001},
+{0x000EC0, 0x0004},
+{0x000EC5, 0x0001},
+{0x000EC6, 0x0004},
+{0x000EC7, 0x0001},
+{0x000EC8, 0x0010},
+{0x000ECF, 0x0001},
+{0x000ED0, 0x0002},
+{0x000EDA, 0x0001},
+{0x000EDC, 0x0004},
+{0x000EE0, 0x0001},
+{0x000F00, 0x0004},
+{0x000F01, 0x0040},
+{0x000F04, 0x0020},
+{0x000F13, 0x0040},
+{0x000F14, 0x0020},
+{0x000F15, 0x0040},
+{0x000F18, 0x0010},
+{0x000F1A, 0x0040},
+{0x000F20, 0x0002},
+{0x000F34, 0x0040},
+{0x000F35, 0x0010},
+{0x000F36, 0x0040},
+{0x000F37, 0x0010},
+{0x000F38, 0x0040},
+{0x000F39, 0x0010},
+{0x000F3A, 0x0020},
+{0x000F3E, 0x0010},
+{0x000F40, 0x0004},
+{0x000F48, 0x0001},
+{0x000F49, 0x0004},
+{0x000F6D, 0x0001},
+{0x000F71, 0x0010},
+{0x000F85, 0x0020},
+{0x000F86, 0x0010},
+{0x000F88, 0x0004},
+{0x000F8D, 0x0010},
+{0x000F98, 0x0001},
+{0x000F99, 0x0010},
+{0x000FBD, 0x0001},
+{0x000FBE, 0x0040},
+{0x000FC6, 0x0010},
+{0x000FC7, 0x0040},
+{0x000FCD, 0x0001},
+{0x000FCE, 0x0040},
+{0x000FD0, 0x0020},
+{0x000FD5, 0x0040},
+{0x000FD9, 0x0020},
+{0x000FDB, 0x0001},
+{0x001000, 0x0004},
+{0x00102B, 0x0010},
+{0x00103F, 0x0004},
+{0x001040, 0x0002},
+{0x00104A, 0x0020},
+{0x001050, 0x0004},
+{0x001056, 0x0010},
+{0x00105A, 0x0004},
+{0x00105E, 0x0010},
+{0x001061, 0x0004},
+{0x001062, 0x0010},
+{0x001065, 0x0004},
+{0x001067, 0x0010},
+{0x00106E, 0x0004},
+{0x001071, 0x0010},
+{0x001075, 0x0004},
+{0x001082, 0x0010},
+{0x00108E, 0x0004},
+{0x00108F, 0x0010},
+{0x001090, 0x0002},
+{0x00109A, 0x0010},
+{0x00109E, 0x0040},
+{0x0010A0, 0x0004},
+{0x0010C6, 0x0001},
+{0x0010C7, 0x0004},
+{0x0010C8, 0x0001},
+{0x0010CD, 0x0004},
+{0x0010CE, 0x0001},
+{0x0010D0, 0x0004},
+{0x0010FB, 0x0020},
+{0x0010FC, 0x0004},
+{0x001249, 0x0001},
+{0x00124A, 0x0004},
+{0x00124E, 0x0001},
+{0x001250, 0x0004},
+{0x001257, 0x0001},
+{0x001258, 0x0004},
+{0x001259, 0x0001},
+{0x00125A, 0x0004},
+{0x00125E, 0x0001},
+{0x001260, 0x0004},
+{0x001289, 0x0001},
+{0x00128A, 0x0004},
+{0x00128E, 0x0001},
+{0x001290, 0x0004},
+{0x0012B1, 0x0001},
+{0x0012B2, 0x0004},
+{0x0012B6, 0x0001},
+{0x0012B8, 0x0004},
+{0x0012BF, 0x0001},
+{0x0012C0, 0x0004},
+{0x0012C1, 0x0001},
+{0x0012C2, 0x0004},
+{0x0012C6, 0x0001},
+{0x0012C8, 0x0004},
+{0x0012D7, 0x0001},
+{0x0012D8, 0x0004},
+{0x001311, 0x0001},
+{0x001312, 0x0004},
+{0x001316, 0x0001},
+{0x001318, 0x0004},
+{0x00135B, 0x0001},
+{0x00135D, 0x0010},
+{0x001360, 0x0020},
+{0x001369, 0x0002},
+{0x00137D, 0x0001},
+{0x001380, 0x0004},
+{0x001390, 0x0040},
+{0x00139A, 0x0001},
+{0x0013A0, 0x0004},
+{0x0013F6, 0x0001},
+{0x0013F8, 0x0004},
+{0x0013FE, 0x0001},
+{0x001400, 0x0020},
+{0x001401, 0x0004},
+{0x00166D, 0x0040},
+{0x00166E, 0x0020},
+{0x00166F, 0x0004},
+{0x001680, 0x0008},
+{0x001681, 0x0004},
+{0x00169B, 0x0020},
+{0x00169D, 0x0001},
+{0x0016A0, 0x0004},
+{0x0016EB, 0x0020},
+{0x0016EE, 0x0002},
+{0x0016F1, 0x0004},
+{0x0016F9, 0x0001},
+{0x001700, 0x0004},
+{0x001712, 0x0010},
+{0x001716, 0x0001},
+{0x00171F, 0x0004},
+{0x001732, 0x0010},
+{0x001735, 0x0020},
+{0x001737, 0x0001},
+{0x001740, 0x0004},
+{0x001752, 0x0010},
+{0x001754, 0x0001},
+{0x001760, 0x0004},
+{0x00176D, 0x0001},
+{0x00176E, 0x0004},
+{0x001771, 0x0001},
+{0x001772, 0x0010},
+{0x001774, 0x0001},
+{0x001780, 0x0004},
+{0x0017B4, 0x0010},
+{0x0017D4, 0x0020},
+{0x0017D7, 0x0004},
+{0x0017D8, 0x0020},
+{0x0017DB, 0x0040},
+{0x0017DC, 0x0004},
+{0x0017DD, 0x0010},
+{0x0017DE, 0x0001},
+{0x0017E0, 0x0002},
+{0x0017EA, 0x0001},
+{0x0017F0, 0x0002},
+{0x0017FA, 0x0001},
+{0x001800, 0x0020},
+{0x00180B, 0x0010},
+{0x00180E, 0x0080},
+{0x00180F, 0x0010},
+{0x001810, 0x0002},
+{0x00181A, 0x0001},
+{0x001820, 0x0004},
+{0x001879, 0x0001},
+{0x001880, 0x0004},
+{0x001885, 0x0010},
+{0x001887, 0x0004},
+{0x0018A9, 0x0010},
+{0x0018AA, 0x0004},
+{0x0018AB, 0x0001},
+{0x0018B0, 0x0004},
+{0x0018F6, 0x0001},
+{0x001900, 0x0004},
+{0x00191F, 0x0001},
+{0x001920, 0x0010},
+{0x00192C, 0x0001},
+{0x001930, 0x0010},
+{0x00193C, 0x0001},
+{0x001940, 0x0040},
+{0x001941, 0x0001},
+{0x001944, 0x0020},
+{0x001946, 0x0002},
+{0x001950, 0x0004},
+{0x00196E, 0x0001},
+{0x001970, 0x0004},
+{0x001975, 0x0001},
+{0x001980, 0x0004},
+{0x0019AC, 0x0001},
+{0x0019B0, 0x0004},
+{0x0019CA, 0x0001},
+{0x0019D0, 0x0002},
+{0x0019DB, 0x0001},
+{0x0019DE, 0x0040},
+{0x001A00, 0x0004},
+{0x001A17, 0x0010},
+{0x001A1C, 0x0001},
+{0x001A1E, 0x0020},
+{0x001A20, 0x0004},
+{0x001A55, 0x0010},
+{0x001A5F, 0x0001},
+{0x001A60, 0x0010},
+{0x001A7D, 0x0001},
+{0x001A7F, 0x0010},
+{0x001A80, 0x0002},
+{0x001A8A, 0x0001},
+{0x001A90, 0x0002},
+{0x001A9A, 0x0001},
+{0x001AA0, 0x0020},
+{0x001AA7, 0x0004},
+{0x001AA8, 0x0020},
+{0x001AAE, 0x0001},
+{0x001AB0, 0x0010},
+{0x001ACF, 0x0001},
+{0x001B00, 0x0010},
+{0x001B05, 0x0004},
+{0x001B34, 0x0010},
+{0x001B45, 0x0004},
+{0x001B4D, 0x0001},
+{0x001B50, 0x0002},
+{0x001B5A, 0x0020},
+{0x001B61, 0x0040},
+{0x001B6B, 0x0010},
+{0x001B74, 0x0040},
+{0x001B7D, 0x0020},
+{0x001B7F, 0x0001},
+{0x001B80, 0x0010},
+{0x001B83, 0x0004},
+{0x001BA1, 0x0010},
+{0x001BAE, 0x0004},
+{0x001BB0, 0x0002},
+{0x001BBA, 0x0004},
+{0x001BE6, 0x0010},
+{0x001BF4, 0x0001},
+{0x001BFC, 0x0020},
+{0x001C00, 0x0004},
+{0x001C24, 0x0010},
+{0x001C38, 0x0001},
+{0x001C3B, 0x0020},
+{0x001C40, 0x0002},
+{0x001C4A, 0x0001},
+{0x001C4D, 0x0004},
+{0x001C50, 0x0002},
+{0x001C5A, 0x0004},
+{0x001C7E, 0x0020},
+{0x001C80, 0x0004},
+{0x001C89, 0x0001},
+{0x001C90, 0x0004},
+{0x001CBB, 0x0001},
+{0x001CBD, 0x0004},
+{0x001CC0, 0x0020},
+{0x001CC8, 0x0001},
+{0x001CD0, 0x0010},
+{0x001CD3, 0x0020},
+{0x001CD4, 0x0010},
+{0x001CE9, 0x0004},
+{0x001CED, 0x0010},
+{0x001CEE, 0x0004},
+{0x001CF4, 0x0010},
+{0x001CF5, 0x0004},
+{0x001CF7, 0x0010},
+{0x001CFA, 0x0004},
+{0x001CFB, 0x0001},
+{0x001D00, 0x0004},
+{0x001DC0, 0x0010},
+{0x001E00, 0x0004},
+{0x001F16, 0x0001},
+{0x001F18, 0x0004},
+{0x001F1E, 0x0001},
+{0x001F20, 0x0004},
+{0x001F46, 0x0001},
+{0x001F48, 0x0004},
+{0x001F4E, 0x0001},
+{0x001F50, 0x0004},
+{0x001F58, 0x0001},
+{0x001F59, 0x0004},
+{0x001F5A, 0x0001},
+{0x001F5B, 0x0004},
+{0x001F5C, 0x0001},
+{0x001F5D, 0x0004},
+{0x001F5E, 0x0001},
+{0x001F5F, 0x0004},
+{0x001F7E, 0x0001},
+{0x001F80, 0x0004},
+{0x001FB5, 0x0001},
+{0x001FB6, 0x0004},
+{0x001FBD, 0x0040},
+{0x001FBE, 0x0004},
+{0x001FBF, 0x0040},
+{0x001FC2, 0x0004},
+{0x001FC5, 0x0001},
+{0x001FC6, 0x0004},
+{0x001FCD, 0x0040},
+{0x001FD0, 0x0004},
+{0x001FD4, 0x0001},
+{0x001FD6, 0x0004},
+{0x001FDC, 0x0001},
+{0x001FDD, 0x0040},
+{0x001FE0, 0x0004},
+{0x001FED, 0x0040},
+{0x001FF0, 0x0001},
+{0x001FF2, 0x0004},
+{0x001FF5, 0x0001},
+{0x001FF6, 0x0004},
+{0x001FFD, 0x0040},
+{0x001FFF, 0x0001},
+{0x002000, 0x0008},
+{0x00200B, 0x0080},
+{0x002010, 0x0020},
+{0x002028, 0x0008},
+{0x00202A, 0x0080},
+{0x00202F, 0x0008},
+{0x002030, 0x0020},
+{0x002044, 0x0040},
+{0x002045, 0x0020},
+{0x002052, 0x0040},
+{0x002053, 0x0020},
+{0x00205F, 0x0008},
+{0x002060, 0x0080},
+{0x002065, 0x0001},
+{0x002066, 0x0080},
+{0x002070, 0x0002},
+{0x002071, 0x0004},
+{0x002072, 0x0001},
+{0x002074, 0x0002},
+{0x00207A, 0x0040},
+{0x00207D, 0x0020},
+{0x00207F, 0x0004},
+{0x002080, 0x0002},
+{0x00208A, 0x0040},
+{0x00208D, 0x0020},
+{0x00208F, 0x0001},
+{0x002090, 0x0004},
+{0x00209D, 0x0001},
+{0x0020A0, 0x0040},
+{0x0020C1, 0x0001},
+{0x0020D0, 0x0010},
+{0x0020F1, 0x0001},
+{0x002100, 0x0040},
+{0x002102, 0x0004},
+{0x002103, 0x0040},
+{0x002107, 0x0004},
+{0x002108, 0x0040},
+{0x00210A, 0x0004},
+{0x002114, 0x0040},
+{0x002115, 0x0004},
+{0x002116, 0x0040},
+{0x002119, 0x0004},
+{0x00211E, 0x0040},
+{0x002124, 0x0004},
+{0x002125, 0x0040},
+{0x002126, 0x0004},
+{0x002127, 0x0040},
+{0x002128, 0x0004},
+{0x002129, 0x0040},
+{0x00212A, 0x0004},
+{0x00212E, 0x0040},
+{0x00212F, 0x0004},
+{0x00213A, 0x0040},
+{0x00213C, 0x0004},
+{0x002140, 0x0040},
+{0x002145, 0x0004},
+{0x00214A, 0x0040},
+{0x00214E, 0x0004},
+{0x00214F, 0x0040},
+{0x002150, 0x0002},
+{0x002183, 0x0004},
+{0x002185, 0x0002},
+{0x00218A, 0x0040},
+{0x00218C, 0x0001},
+{0x002190, 0x0040},
+{0x002308, 0x0020},
+{0x00230C, 0x0040},
+{0x002329, 0x0020},
+{0x00232B, 0x0040},
+{0x002427, 0x0001},
+{0x002440, 0x0040},
+{0x00244B, 0x0001},
+{0x002460, 0x0002},
+{0x00249C, 0x0040},
+{0x0024EA, 0x0002},
+{0x002500, 0x0040},
+{0x002768, 0x0020},
+{0x002776, 0x0002},
+{0x002794, 0x0040},
+{0x0027C5, 0x0020},
+{0x0027C7, 0x0040},
+{0x0027E6, 0x0020},
+{0x0027F0, 0x0040},
+{0x002983, 0x0020},
+{0x002999, 0x0040},
+{0x0029D8, 0x0020},
+{0x0029DC, 0x0040},
+{0x0029FC, 0x0020},
+{0x0029FE, 0x0040},
+{0x002B74, 0x0001},
+{0x002B76, 0x0040},
+{0x002B96, 0x0001},
+{0x002B97, 0x0040},
+{0x002C00, 0x0004},
+{0x002CE5, 0x0040},
+{0x002CEB, 0x0004},
+{0x002CEF, 0x0010},
+{0x002CF2, 0x0004},
+{0x002CF4, 0x0001},
+{0x002CF9, 0x0020},
+{0x002CFD, 0x0002},
+{0x002CFE, 0x0020},
+{0x002D00, 0x0004},
+{0x002D26, 0x0001},
+{0x002D27, 0x0004},
+{0x002D28, 0x0001},
+{0x002D2D, 0x0004},
+{0x002D2E, 0x0001},
+{0x002D30, 0x0004},
+{0x002D68, 0x0001},
+{0x002D6F, 0x0004},
+{0x002D70, 0x0020},
+{0x002D71, 0x0001},
+{0x002D7F, 0x0010},
+{0x002D80, 0x0004},
+{0x002D97, 0x0001},
+{0x002DA0, 0x0004},
+{0x002DA7, 0x0001},
+{0x002DA8, 0x0004},
+{0x002DAF, 0x0001},
+{0x002DB0, 0x0004},
+{0x002DB7, 0x0001},
+{0x002DB8, 0x0004},
+{0x002DBF, 0x0001},
+{0x002DC0, 0x0004},
+{0x002DC7, 0x0001},
+{0x002DC8, 0x0004},
+{0x002DCF, 0x0001},
+{0x002DD0, 0x0004},
+{0x002DD7, 0x0001},
+{0x002DD8, 0x0004},
+{0x002DDF, 0x0001},
+{0x002DE0, 0x0010},
+{0x002E00, 0x0020},
+{0x002E2F, 0x0004},
+{0x002E30, 0x0020},
+{0x002E50, 0x0040},
+{0x002E52, 0x0020},
+{0x002E5E, 0x0001},
+{0x002E80, 0x0040},
+{0x002E9A, 0x0001},
+{0x002E9B, 0x0040},
+{0x002EF4, 0x0001},
+{0x002F00, 0x0040},
+{0x002FD6, 0x0001},
+{0x002FF0, 0x0040},
+{0x003000, 0x0008},
+{0x003001, 0x0020},
+{0x003004, 0x0040},
+{0x003005, 0x0004},
+{0x003007, 0x0002},
+{0x003008, 0x0020},
+{0x003012, 0x0040},
+{0x003014, 0x0020},
+{0x003020, 0x0040},
+{0x003021, 0x0002},
+{0x00302A, 0x0010},
+{0x003030, 0x0020},
+{0x003031, 0x0004},
+{0x003036, 0x0040},
+{0x003038, 0x0002},
+{0x00303B, 0x0004},
+{0x00303D, 0x0020},
+{0x00303E, 0x0040},
+{0x003040, 0x0001},
+{0x003041, 0x0004},
+{0x003097, 0x0001},
+{0x003099, 0x0010},
+{0x00309B, 0x0040},
+{0x00309D, 0x0004},
+{0x0030A0, 0x0020},
+{0x0030A1, 0x0004},
+{0x0030FB, 0x0020},
+{0x0030FC, 0x0004},
+{0x003100, 0x0001},
+{0x003105, 0x0004},
+{0x003130, 0x0001},
+{0x003131, 0x0004},
+{0x00318F, 0x0001},
+{0x003190, 0x0040},
+{0x003192, 0x0002},
+{0x003196, 0x0040},
+{0x0031A0, 0x0004},
+{0x0031C0, 0x0040},
+{0x0031E4, 0x0001},
+{0x0031EF, 0x0040},
+{0x0031F0, 0x0004},
+{0x003200, 0x0040},
+{0x00321F, 0x0001},
+{0x003220, 0x0002},
+{0x00322A, 0x0040},
+{0x003248, 0x0002},
+{0x003250, 0x0040},
+{0x003251, 0x0002},
+{0x003260, 0x0040},
+{0x003280, 0x0002},
+{0x00328A, 0x0040},
+{0x0032B1, 0x0002},
+{0x0032C0, 0x0040},
+{0x003400, 0x0004},
+{0x004DC0, 0x0040},
+{0x004E00, 0x0004},
+{0x00A48D, 0x0001},
+{0x00A490, 0x0040},
+{0x00A4C7, 0x0001},
+{0x00A4D0, 0x0004},
+{0x00A4FE, 0x0020},
+{0x00A500, 0x0004},
+{0x00A60D, 0x0020},
+{0x00A610, 0x0004},
+{0x00A620, 0x0002},
+{0x00A62A, 0x0004},
+{0x00A62C, 0x0001},
+{0x00A640, 0x0004},
+{0x00A66F, 0x0010},
+{0x00A673, 0x0020},
+{0x00A674, 0x0010},
+{0x00A67E, 0x0020},
+{0x00A67F, 0x0004},
+{0x00A69E, 0x0010},
+{0x00A6A0, 0x0004},
+{0x00A6E6, 0x0002},
+{0x00A6F0, 0x0010},
+{0x00A6F2, 0x0020},
+{0x00A6F8, 0x0001},
+{0x00A700, 0x0040},
+{0x00A717, 0x0004},
+{0x00A720, 0x0040},
+{0x00A722, 0x0004},
+{0x00A789, 0x0040},
+{0x00A78B, 0x0004},
+{0x00A7CB, 0x0001},
+{0x00A7D0, 0x0004},
+{0x00A7D2, 0x0001},
+{0x00A7D3, 0x0004},
+{0x00A7D4, 0x0001},
+{0x00A7D5, 0x0004},
+{0x00A7DA, 0x0001},
+{0x00A7F2, 0x0004},
+{0x00A802, 0x0010},
+{0x00A803, 0x0004},
+{0x00A806, 0x0010},
+{0x00A807, 0x0004},
+{0x00A80B, 0x0010},
+{0x00A80C, 0x0004},
+{0x00A823, 0x0010},
+{0x00A828, 0x0040},
+{0x00A82C, 0x0010},
+{0x00A82D, 0x0001},
+{0x00A830, 0x0002},
+{0x00A836, 0x0040},
+{0x00A83A, 0x0001},
+{0x00A840, 0x0004},
+{0x00A874, 0x0020},
+{0x00A878, 0x0001},
+{0x00A880, 0x0010},
+{0x00A882, 0x0004},
+{0x00A8B4, 0x0010},
+{0x00A8C6, 0x0001},
+{0x00A8CE, 0x0020},
+{0x00A8D0, 0x0002},
+{0x00A8DA, 0x0001},
+{0x00A8E0, 0x0010},
+{0x00A8F2, 0x0004},
+{0x00A8F8, 0x0020},
+{0x00A8FB, 0x0004},
+{0x00A8FC, 0x0020},
+{0x00A8FD, 0x0004},
+{0x00A8FF, 0x0010},
+{0x00A900, 0x0002},
+{0x00A90A, 0x0004},
+{0x00A926, 0x0010},
+{0x00A92E, 0x0020},
+{0x00A930, 0x0004},
+{0x00A947, 0x0010},
+{0x00A954, 0x0001},
+{0x00A95F, 0x0020},
+{0x00A960, 0x0004},
+{0x00A97D, 0x0001},
+{0x00A980, 0x0010},
+{0x00A984, 0x0004},
+{0x00A9B3, 0x0010},
+{0x00A9C1, 0x0020},
+{0x00A9CE, 0x0001},
+{0x00A9CF, 0x0004},
+{0x00A9D0, 0x0002},
+{0x00A9DA, 0x0001},
+{0x00A9DE, 0x0020},
+{0x00A9E0, 0x0004},
+{0x00A9E5, 0x0010},
+{0x00A9E6, 0x0004},
+{0x00A9F0, 0x0002},
+{0x00A9FA, 0x0004},
+{0x00A9FF, 0x0001},
+{0x00AA00, 0x0004},
+{0x00AA29, 0x0010},
+{0x00AA37, 0x0001},
+{0x00AA40, 0x0004},
+{0x00AA43, 0x0010},
+{0x00AA44, 0x0004},
+{0x00AA4C, 0x0010},
+{0x00AA4E, 0x0001},
+{0x00AA50, 0x0002},
+{0x00AA5A, 0x0001},
+{0x00AA5C, 0x0020},
+{0x00AA60, 0x0004},
+{0x00AA77, 0x0040},
+{0x00AA7A, 0x0004},
+{0x00AA7B, 0x0010},
+{0x00AA7E, 0x0004},
+{0x00AAB0, 0x0010},
+{0x00AAB1, 0x0004},
+{0x00AAB2, 0x0010},
+{0x00AAB5, 0x0004},
+{0x00AAB7, 0x0010},
+{0x00AAB9, 0x0004},
+{0x00AABE, 0x0010},
+{0x00AAC0, 0x0004},
+{0x00AAC1, 0x0010},
+{0x00AAC2, 0x0004},
+{0x00AAC3, 0x0001},
+{0x00AADB, 0x0004},
+{0x00AADE, 0x0020},
+{0x00AAE0, 0x0004},
+{0x00AAEB, 0x0010},
+{0x00AAF0, 0x0020},
+{0x00AAF2, 0x0004},
+{0x00AAF5, 0x0010},
+{0x00AAF7, 0x0001},
+{0x00AB01, 0x0004},
+{0x00AB07, 0x0001},
+{0x00AB09, 0x0004},
+{0x00AB0F, 0x0001},
+{0x00AB11, 0x0004},
+{0x00AB17, 0x0001},
+{0x00AB20, 0x0004},
+{0x00AB27, 0x0001},
+{0x00AB28, 0x0004},
+{0x00AB2F, 0x0001},
+{0x00AB30, 0x0004},
+{0x00AB5B, 0x0040},
+{0x00AB5C, 0x0004},
+{0x00AB6A, 0x0040},
+{0x00AB6C, 0x0001},
+{0x00AB70, 0x0004},
+{0x00ABE3, 0x0010},
+{0x00ABEB, 0x0020},
+{0x00ABEC, 0x0010},
+{0x00ABEE, 0x0001},
+{0x00ABF0, 0x0002},
+{0x00ABFA, 0x0001},
+{0x00AC00, 0x0004},
+{0x00D7A4, 0x0001},
+{0x00D7B0, 0x0004},
+{0x00D7C7, 0x0001},
+{0x00D7CB, 0x0004},
+{0x00D7FC, 0x0001},
+{0x00D800, 0x0080},
+{0x00F900, 0x0004},
+{0x00FA6E, 0x0001},
+{0x00FA70, 0x0004},
+{0x00FADA, 0x0001},
+{0x00FB00, 0x0004},
+{0x00FB07, 0x0001},
+{0x00FB13, 0x0004},
+{0x00FB18, 0x0001},
+{0x00FB1D, 0x0004},
+{0x00FB1E, 0x0010},
+{0x00FB1F, 0x0004},
+{0x00FB29, 0x0040},
+{0x00FB2A, 0x0004},
+{0x00FB37, 0x0001},
+{0x00FB38, 0x0004},
+{0x00FB3D, 0x0001},
+{0x00FB3E, 0x0004},
+{0x00FB3F, 0x0001},
+{0x00FB40, 0x0004},
+{0x00FB42, 0x0001},
+{0x00FB43, 0x0004},
+{0x00FB45, 0x0001},
+{0x00FB46, 0x0004},
+{0x00FBB2, 0x0040},
+{0x00FBC3, 0x0001},
+{0x00FBD3, 0x0004},
+{0x00FD3E, 0x0020},
+{0x00FD40, 0x0040},
+{0x00FD50, 0x0004},
+{0x00FD90, 0x0001},
+{0x00FD92, 0x0004},
+{0x00FDC8, 0x0001},
+{0x00FDCF, 0x0040},
+{0x00FDD0, 0x0001},
+{0x00FDF0, 0x0004},
+{0x00FDFC, 0x0040},
+{0x00FE00, 0x0010},
+{0x00FE10, 0x0020},
+{0x00FE1A, 0x0001},
+{0x00FE20, 0x0010},
+{0x00FE30, 0x0020},
+{0x00FE53, 0x0001},
+{0x00FE54, 0x0020},
+{0x00FE62, 0x0040},
+{0x00FE63, 0x0020},
+{0x00FE64, 0x0040},
+{0x00FE67, 0x0001},
+{0x00FE68, 0x0020},
+{0x00FE69, 0x0040},
+{0x00FE6A, 0x0020},
+{0x00FE6C, 0x0001},
+{0x00FE70, 0x0004},
+{0x00FE75, 0x0001},
+{0x00FE76, 0x0004},
+{0x00FEFD, 0x0001},
+{0x00FEFF, 0x0080},
+{0x00FF00, 0x0001},
+{0x00FF01, 0x0020},
+{0x00FF04, 0x0040},
+{0x00FF05, 0x0020},
+{0x00FF0B, 0x0040},
+{0x00FF0C, 0x0020},
+{0x00FF10, 0x0002},
+{0x00FF1A, 0x0020},
+{0x00FF1C, 0x0040},
+{0x00FF1F, 0x0020},
+{0x00FF21, 0x0004},
+{0x00FF3B, 0x0020},
+{0x00FF3E, 0x0040},
+{0x00FF3F, 0x0020},
+{0x00FF40, 0x0040},
+{0x00FF41, 0x0004},
+{0x00FF5B, 0x0020},
+{0x00FF5C, 0x0040},
+{0x00FF5D, 0x0020},
+{0x00FF5E, 0x0040},
+{0x00FF5F, 0x0020},
+{0x00FF66, 0x0004},
+{0x00FFBF, 0x0001},
+{0x00FFC2, 0x0004},
+{0x00FFC8, 0x0001},
+{0x00FFCA, 0x0004},
+{0x00FFD0, 0x0001},
+{0x00FFD2, 0x0004},
+{0x00FFD8, 0x0001},
+{0x00FFDA, 0x0004},
+{0x00FFDD, 0x0001},
+{0x00FFE0, 0x0040},
+{0x00FFE7, 0x0001},
+{0x00FFE8, 0x0040},
+{0x00FFEF, 0x0001},
+{0x00FFF9, 0x0080},
+{0x00FFFC, 0x0040},
+{0x00FFFE, 0x0001},
+{0x010000, 0x0004},
+{0x01000C, 0x0001},
+{0x01000D, 0x0004},
+{0x010027, 0x0001},
+{0x010028, 0x0004},
+{0x01003B, 0x0001},
+{0x01003C, 0x0004},
+{0x01003E, 0x0001},
+{0x01003F, 0x0004},
+{0x01004E, 0x0001},
+{0x010050, 0x0004},
+{0x01005E, 0x0001},
+{0x010080, 0x0004},
+{0x0100FB, 0x0001},
+{0x010100, 0x0020},
+{0x010103, 0x0001},
+{0x010107, 0x0002},
+{0x010134, 0x0001},
+{0x010137, 0x0040},
+{0x010140, 0x0002},
+{0x010179, 0x0040},
+{0x01018A, 0x0002},
+{0x01018C, 0x0040},
+{0x01018F, 0x0001},
+{0x010190, 0x0040},
+{0x01019D, 0x0001},
+{0x0101A0, 0x0040},
+{0x0101A1, 0x0001},
+{0x0101D0, 0x0040},
+{0x0101FD, 0x0010},
+{0x0101FE, 0x0001},
+{0x010280, 0x0004},
+{0x01029D, 0x0001},
+{0x0102A0, 0x0004},
+{0x0102D1, 0x0001},
+{0x0102E0, 0x0010},
+{0x0102E1, 0x0002},
+{0x0102FC, 0x0001},
+{0x010300, 0x0004},
+{0x010320, 0x0002},
+{0x010324, 0x0001},
+{0x01032D, 0x0004},
+{0x010341, 0x0002},
+{0x010342, 0x0004},
+{0x01034A, 0x0002},
+{0x01034B, 0x0001},
+{0x010350, 0x0004},
+{0x010376, 0x0010},
+{0x01037B, 0x0001},
+{0x010380, 0x0004},
+{0x01039E, 0x0001},
+{0x01039F, 0x0020},
+{0x0103A0, 0x0004},
+{0x0103C4, 0x0001},
+{0x0103C8, 0x0004},
+{0x0103D0, 0x0020},
+{0x0103D1, 0x0002},
+{0x0103D6, 0x0001},
+{0x010400, 0x0004},
+{0x01049E, 0x0001},
+{0x0104A0, 0x0002},
+{0x0104AA, 0x0001},
+{0x0104B0, 0x0004},
+{0x0104D4, 0x0001},
+{0x0104D8, 0x0004},
+{0x0104FC, 0x0001},
+{0x010500, 0x0004},
+{0x010528, 0x0001},
+{0x010530, 0x0004},
+{0x010564, 0x0001},
+{0x01056F, 0x0020},
+{0x010570, 0x0004},
+{0x01057B, 0x0001},
+{0x01057C, 0x0004},
+{0x01058B, 0x0001},
+{0x01058C, 0x0004},
+{0x010593, 0x0001},
+{0x010594, 0x0004},
+{0x010596, 0x0001},
+{0x010597, 0x0004},
+{0x0105A2, 0x0001},
+{0x0105A3, 0x0004},
+{0x0105B2, 0x0001},
+{0x0105B3, 0x0004},
+{0x0105BA, 0x0001},
+{0x0105BB, 0x0004},
+{0x0105BD, 0x0001},
+{0x010600, 0x0004},
+{0x010737, 0x0001},
+{0x010740, 0x0004},
+{0x010756, 0x0001},
+{0x010760, 0x0004},
+{0x010768, 0x0001},
+{0x010780, 0x0004},
+{0x010786, 0x0001},
+{0x010787, 0x0004},
+{0x0107B1, 0x0001},
+{0x0107B2, 0x0004},
+{0x0107BB, 0x0001},
+{0x010800, 0x0004},
+{0x010806, 0x0001},
+{0x010808, 0x0004},
+{0x010809, 0x0001},
+{0x01080A, 0x0004},
+{0x010836, 0x0001},
+{0x010837, 0x0004},
+{0x010839, 0x0001},
+{0x01083C, 0x0004},
+{0x01083D, 0x0001},
+{0x01083F, 0x0004},
+{0x010856, 0x0001},
+{0x010857, 0x0020},
+{0x010858, 0x0002},
+{0x010860, 0x0004},
+{0x010877, 0x0040},
+{0x010879, 0x0002},
+{0x010880, 0x0004},
+{0x01089F, 0x0001},
+{0x0108A7, 0x0002},
+{0x0108B0, 0x0001},
+{0x0108E0, 0x0004},
+{0x0108F3, 0x0001},
+{0x0108F4, 0x0004},
+{0x0108F6, 0x0001},
+{0x0108FB, 0x0002},
+{0x010900, 0x0004},
+{0x010916, 0x0002},
+{0x01091C, 0x0001},
+{0x01091F, 0x0020},
+{0x010920, 0x0004},
+{0x01093A, 0x0001},
+{0x01093F, 0x0020},
+{0x010940, 0x0001},
+{0x010980, 0x0004},
+{0x0109B8, 0x0001},
+{0x0109BC, 0x0002},
+{0x0109BE, 0x0004},
+{0x0109C0, 0x0002},
+{0x0109D0, 0x0001},
+{0x0109D2, 0x0002},
+{0x010A00, 0x0004},
+{0x010A01, 0x0010},
+{0x010A04, 0x0001},
+{0x010A05, 0x0010},
+{0x010A07, 0x0001},
+{0x010A0C, 0x0010},
+{0x010A10, 0x0004},
+{0x010A14, 0x0001},
+{0x010A15, 0x0004},
+{0x010A18, 0x0001},
+{0x010A19, 0x0004},
+{0x010A36, 0x0001},
+{0x010A38, 0x0010},
+{0x010A3B, 0x0001},
+{0x010A3F, 0x0010},
+{0x010A40, 0x0002},
+{0x010A49, 0x0001},
+{0x010A50, 0x0020},
+{0x010A59, 0x0001},
+{0x010A60, 0x0004},
+{0x010A7D, 0x0002},
+{0x010A7F, 0x0020},
+{0x010A80, 0x0004},
+{0x010A9D, 0x0002},
+{0x010AA0, 0x0001},
+{0x010AC0, 0x0004},
+{0x010AC8, 0x0040},
+{0x010AC9, 0x0004},
+{0x010AE5, 0x0010},
+{0x010AE7, 0x0001},
+{0x010AEB, 0x0002},
+{0x010AF0, 0x0020},
+{0x010AF7, 0x0001},
+{0x010B00, 0x0004},
+{0x010B36, 0x0001},
+{0x010B39, 0x0020},
+{0x010B40, 0x0004},
+{0x010B56, 0x0001},
+{0x010B58, 0x0002},
+{0x010B60, 0x0004},
+{0x010B73, 0x0001},
+{0x010B78, 0x0002},
+{0x010B80, 0x0004},
+{0x010B92, 0x0001},
+{0x010B99, 0x0020},
+{0x010B9D, 0x0001},
+{0x010BA9, 0x0002},
+{0x010BB0, 0x0001},
+{0x010C00, 0x0004},
+{0x010C49, 0x0001},
+{0x010C80, 0x0004},
+{0x010CB3, 0x0001},
+{0x010CC0, 0x0004},
+{0x010CF3, 0x0001},
+{0x010CFA, 0x0002},
+{0x010D00, 0x0004},
+{0x010D24, 0x0010},
+{0x010D28, 0x0001},
+{0x010D30, 0x0002},
+{0x010D3A, 0x0001},
+{0x010E60, 0x0002},
+{0x010E7F, 0x0001},
+{0x010E80, 0x0004},
+{0x010EAA, 0x0001},
+{0x010EAB, 0x0010},
+{0x010EAD, 0x0020},
+{0x010EAE, 0x0001},
+{0x010EB0, 0x0004},
+{0x010EB2, 0x0001},
+{0x010EFD, 0x0010},
+{0x010F00, 0x0004},
+{0x010F1D, 0x0002},
+{0x010F27, 0x0004},
+{0x010F28, 0x0001},
+{0x010F30, 0x0004},
+{0x010F46, 0x0010},
+{0x010F51, 0x0002},
+{0x010F55, 0x0020},
+{0x010F5A, 0x0001},
+{0x010F70, 0x0004},
+{0x010F82, 0x0010},
+{0x010F86, 0x0020},
+{0x010F8A, 0x0001},
+{0x010FB0, 0x0004},
+{0x010FC5, 0x0002},
+{0x010FCC, 0x0001},
+{0x010FE0, 0x0004},
+{0x010FF7, 0x0001},
+{0x011000, 0x0010},
+{0x011003, 0x0004},
+{0x011038, 0x0010},
+{0x011047, 0x0020},
+{0x01104E, 0x0001},
+{0x011052, 0x0002},
+{0x011070, 0x0010},
+{0x011071, 0x0004},
+{0x011073, 0x0010},
+{0x011075, 0x0004},
+{0x011076, 0x0001},
+{0x01107F, 0x0010},
+{0x011083, 0x0004},
+{0x0110B0, 0x0010},
+{0x0110BB, 0x0020},
+{0x0110BD, 0x0080},
+{0x0110BE, 0x0020},
+{0x0110C2, 0x0010},
+{0x0110C3, 0x0001},
+{0x0110CD, 0x0080},
+{0x0110CE, 0x0001},
+{0x0110D0, 0x0004},
+{0x0110E9, 0x0001},
+{0x0110F0, 0x0002},
+{0x0110FA, 0x0001},
+{0x011100, 0x0010},
+{0x011103, 0x0004},
+{0x011127, 0x0010},
+{0x011135, 0x0001},
+{0x011136, 0x0002},
+{0x011140, 0x0020},
+{0x011144, 0x0004},
+{0x011145, 0x0010},
+{0x011147, 0x0004},
+{0x011148, 0x0001},
+{0x011150, 0x0004},
+{0x011173, 0x0010},
+{0x011174, 0x0020},
+{0x011176, 0x0004},
+{0x011177, 0x0001},
+{0x011180, 0x0010},
+{0x011183, 0x0004},
+{0x0111B3, 0x0010},
+{0x0111C1, 0x0004},
+{0x0111C5, 0x0020},
+{0x0111C9, 0x0010},
+{0x0111CD, 0x0020},
+{0x0111CE, 0x0010},
+{0x0111D0, 0x0002},
+{0x0111DA, 0x0004},
+{0x0111DB, 0x0020},
+{0x0111DC, 0x0004},
+{0x0111DD, 0x0020},
+{0x0111E0, 0x0001},
+{0x0111E1, 0x0002},
+{0x0111F5, 0x0001},
+{0x011200, 0x0004},
+{0x011212, 0x0001},
+{0x011213, 0x0004},
+{0x01122C, 0x0010},
+{0x011238, 0x0020},
+{0x01123E, 0x0010},
+{0x01123F, 0x0004},
+{0x011241, 0x0010},
+{0x011242, 0x0001},
+{0x011280, 0x0004},
+{0x011287, 0x0001},
+{0x011288, 0x0004},
+{0x011289, 0x0001},
+{0x01128A, 0x0004},
+{0x01128E, 0x0001},
+{0x01128F, 0x0004},
+{0x01129E, 0x0001},
+{0x01129F, 0x0004},
+{0x0112A9, 0x0020},
+{0x0112AA, 0x0001},
+{0x0112B0, 0x0004},
+{0x0112DF, 0x0010},
+{0x0112EB, 0x0001},
+{0x0112F0, 0x0002},
+{0x0112FA, 0x0001},
+{0x011300, 0x0010},
+{0x011304, 0x0001},
+{0x011305, 0x0004},
+{0x01130D, 0x0001},
+{0x01130F, 0x0004},
+{0x011311, 0x0001},
+{0x011313, 0x0004},
+{0x011329, 0x0001},
+{0x01132A, 0x0004},
+{0x011331, 0x0001},
+{0x011332, 0x0004},
+{0x011334, 0x0001},
+{0x011335, 0x0004},
+{0x01133A, 0x0001},
+{0x01133B, 0x0010},
+{0x01133D, 0x0004},
+{0x01133E, 0x0010},
+{0x011345, 0x0001},
+{0x011347, 0x0010},
+{0x011349, 0x0001},
+{0x01134B, 0x0010},
+{0x01134E, 0x0001},
+{0x011350, 0x0004},
+{0x011351, 0x0001},
+{0x011357, 0x0010},
+{0x011358, 0x0001},
+{0x01135D, 0x0004},
+{0x011362, 0x0010},
+{0x011364, 0x0001},
+{0x011366, 0x0010},
+{0x01136D, 0x0001},
+{0x011370, 0x0010},
+{0x011375, 0x0001},
+{0x011400, 0x0004},
+{0x011435, 0x0010},
+{0x011447, 0x0004},
+{0x01144B, 0x0020},
+{0x011450, 0x0002},
+{0x01145A, 0x0020},
+{0x01145C, 0x0001},
+{0x01145D, 0x0020},
+{0x01145E, 0x0010},
+{0x01145F, 0x0004},
+{0x011462, 0x0001},
+{0x011480, 0x0004},
+{0x0114B0, 0x0010},
+{0x0114C4, 0x0004},
+{0x0114C6, 0x0020},
+{0x0114C7, 0x0004},
+{0x0114C8, 0x0001},
+{0x0114D0, 0x0002},
+{0x0114DA, 0x0001},
+{0x011580, 0x0004},
+{0x0115AF, 0x0010},
+{0x0115B6, 0x0001},
+{0x0115B8, 0x0010},
+{0x0115C1, 0x0020},
+{0x0115D8, 0x0004},
+{0x0115DC, 0x0010},
+{0x0115DE, 0x0001},
+{0x011600, 0x0004},
+{0x011630, 0x0010},
+{0x011641, 0x0020},
+{0x011644, 0x0004},
+{0x011645, 0x0001},
+{0x011650, 0x0002},
+{0x01165A, 0x0001},
+{0x011660, 0x0020},
+{0x01166D, 0x0001},
+{0x011680, 0x0004},
+{0x0116AB, 0x0010},
+{0x0116B8, 0x0004},
+{0x0116B9, 0x0020},
+{0x0116BA, 0x0001},
+{0x0116C0, 0x0002},
+{0x0116CA, 0x0001},
+{0x011700, 0x0004},
+{0x01171B, 0x0001},
+{0x01171D, 0x0010},
+{0x01172C, 0x0001},
+{0x011730, 0x0002},
+{0x01173C, 0x0020},
+{0x01173F, 0x0040},
+{0x011740, 0x0004},
+{0x011747, 0x0001},
+{0x011800, 0x0004},
+{0x01182C, 0x0010},
+{0x01183B, 0x0020},
+{0x01183C, 0x0001},
+{0x0118A0, 0x0004},
+{0x0118E0, 0x0002},
+{0x0118F3, 0x0001},
+{0x0118FF, 0x0004},
+{0x011907, 0x0001},
+{0x011909, 0x0004},
+{0x01190A, 0x0001},
+{0x01190C, 0x0004},
+{0x011914, 0x0001},
+{0x011915, 0x0004},
+{0x011917, 0x0001},
+{0x011918, 0x0004},
+{0x011930, 0x0010},
+{0x011936, 0x0001},
+{0x011937, 0x0010},
+{0x011939, 0x0001},
+{0x01193B, 0x0010},
+{0x01193F, 0x0004},
+{0x011940, 0x0010},
+{0x011941, 0x0004},
+{0x011942, 0x0010},
+{0x011944, 0x0020},
+{0x011947, 0x0001},
+{0x011950, 0x0002},
+{0x01195A, 0x0001},
+{0x0119A0, 0x0004},
+{0x0119A8, 0x0001},
+{0x0119AA, 0x0004},
+{0x0119D1, 0x0010},
+{0x0119D8, 0x0001},
+{0x0119DA, 0x0010},
+{0x0119E1, 0x0004},
+{0x0119E2, 0x0020},
+{0x0119E3, 0x0004},
+{0x0119E4, 0x0010},
+{0x0119E5, 0x0001},
+{0x011A00, 0x0004},
+{0x011A01, 0x0010},
+{0x011A0B, 0x0004},
+{0x011A33, 0x0010},
+{0x011A3A, 0x0004},
+{0x011A3B, 0x0010},
+{0x011A3F, 0x0020},
+{0x011A47, 0x0010},
+{0x011A48, 0x0001},
+{0x011A50, 0x0004},
+{0x011A51, 0x0010},
+{0x011A5C, 0x0004},
+{0x011A8A, 0x0010},
+{0x011A9A, 0x0020},
+{0x011A9D, 0x0004},
+{0x011A9E, 0x0020},
+{0x011AA3, 0x0001},
+{0x011AB0, 0x0004},
+{0x011AF9, 0x0001},
+{0x011B00, 0x0020},
+{0x011B0A, 0x0001},
+{0x011C00, 0x0004},
+{0x011C09, 0x0001},
+{0x011C0A, 0x0004},
+{0x011C2F, 0x0010},
+{0x011C37, 0x0001},
+{0x011C38, 0x0010},
+{0x011C40, 0x0004},
+{0x011C41, 0x0020},
+{0x011C46, 0x0001},
+{0x011C50, 0x0002},
+{0x011C6D, 0x0001},
+{0x011C70, 0x0020},
+{0x011C72, 0x0004},
+{0x011C90, 0x0001},
+{0x011C92, 0x0010},
+{0x011CA8, 0x0001},
+{0x011CA9, 0x0010},
+{0x011CB7, 0x0001},
+{0x011D00, 0x0004},
+{0x011D07, 0x0001},
+{0x011D08, 0x0004},
+{0x011D0A, 0x0001},
+{0x011D0B, 0x0004},
+{0x011D31, 0x0010},
+{0x011D37, 0x0001},
+{0x011D3A, 0x0010},
+{0x011D3B, 0x0001},
+{0x011D3C, 0x0010},
+{0x011D3E, 0x0001},
+{0x011D3F, 0x0010},
+{0x011D46, 0x0004},
+{0x011D47, 0x0010},
+{0x011D48, 0x0001},
+{0x011D50, 0x0002},
+{0x011D5A, 0x0001},
+{0x011D60, 0x0004},
+{0x011D66, 0x0001},
+{0x011D67, 0x0004},
+{0x011D69, 0x0001},
+{0x011D6A, 0x0004},
+{0x011D8A, 0x0010},
+{0x011D8F, 0x0001},
+{0x011D90, 0x0010},
+{0x011D92, 0x0001},
+{0x011D93, 0x0010},
+{0x011D98, 0x0004},
+{0x011D99, 0x0001},
+{0x011DA0, 0x0002},
+{0x011DAA, 0x0001},
+{0x011EE0, 0x0004},
+{0x011EF3, 0x0010},
+{0x011EF7, 0x0020},
+{0x011EF9, 0x0001},
+{0x011F00, 0x0010},
+{0x011F02, 0x0004},
+{0x011F03, 0x0010},
+{0x011F04, 0x0004},
+{0x011F11, 0x0001},
+{0x011F12, 0x0004},
+{0x011F34, 0x0010},
+{0x011F3B, 0x0001},
+{0x011F3E, 0x0010},
+{0x011F43, 0x0020},
+{0x011F50, 0x0002},
+{0x011F5A, 0x0001},
+{0x011FB0, 0x0004},
+{0x011FB1, 0x0001},
+{0x011FC0, 0x0002},
+{0x011FD5, 0x0040},
+{0x011FF2, 0x0001},
+{0x011FFF, 0x0020},
+{0x012000, 0x0004},
+{0x01239A, 0x0001},
+{0x012400, 0x0002},
+{0x01246F, 0x0001},
+{0x012470, 0x0020},
+{0x012475, 0x0001},
+{0x012480, 0x0004},
+{0x012544, 0x0001},
+{0x012F90, 0x0004},
+{0x012FF1, 0x0020},
+{0x012FF3, 0x0001},
+{0x013000, 0x0004},
+{0x013430, 0x0080},
+{0x013440, 0x0010},
+{0x013441, 0x0004},
+{0x013447, 0x0010},
+{0x013456, 0x0001},
+{0x014400, 0x0004},
+{0x014647, 0x0001},
+{0x016800, 0x0004},
+{0x016A39, 0x0001},
+{0x016A40, 0x0004},
+{0x016A5F, 0x0001},
+{0x016A60, 0x0002},
+{0x016A6A, 0x0001},
+{0x016A6E, 0x0020},
+{0x016A70, 0x0004},
+{0x016ABF, 0x0001},
+{0x016AC0, 0x0002},
+{0x016ACA, 0x0001},
+{0x016AD0, 0x0004},
+{0x016AEE, 0x0001},
+{0x016AF0, 0x0010},
+{0x016AF5, 0x0020},
+{0x016AF6, 0x0001},
+{0x016B00, 0x0004},
+{0x016B30, 0x0010},
+{0x016B37, 0x0020},
+{0x016B3C, 0x0040},
+{0x016B40, 0x0004},
+{0x016B44, 0x0020},
+{0x016B45, 0x0040},
+{0x016B46, 0x0001},
+{0x016B50, 0x0002},
+{0x016B5A, 0x0001},
+{0x016B5B, 0x0002},
+{0x016B62, 0x0001},
+{0x016B63, 0x0004},
+{0x016B78, 0x0001},
+{0x016B7D, 0x0004},
+{0x016B90, 0x0001},
+{0x016E40, 0x0004},
+{0x016E80, 0x0002},
+{0x016E97, 0x0020},
+{0x016E9B, 0x0001},
+{0x016F00, 0x0004},
+{0x016F4B, 0x0001},
+{0x016F4F, 0x0010},
+{0x016F50, 0x0004},
+{0x016F51, 0x0010},
+{0x016F88, 0x0001},
+{0x016F8F, 0x0010},
+{0x016F93, 0x0004},
+{0x016FA0, 0x0001},
+{0x016FE0, 0x0004},
+{0x016FE2, 0x0020},
+{0x016FE3, 0x0004},
+{0x016FE4, 0x0010},
+{0x016FE5, 0x0001},
+{0x016FF0, 0x0010},
+{0x016FF2, 0x0001},
+{0x017000, 0x0004},
+{0x0187F8, 0x0001},
+{0x018800, 0x0004},
+{0x018CD6, 0x0001},
+{0x018D00, 0x0004},
+{0x018D09, 0x0001},
+{0x01AFF0, 0x0004},
+{0x01AFF4, 0x0001},
+{0x01AFF5, 0x0004},
+{0x01AFFC, 0x0001},
+{0x01AFFD, 0x0004},
+{0x01AFFF, 0x0001},
+{0x01B000, 0x0004},
+{0x01B123, 0x0001},
+{0x01B132, 0x0004},
+{0x01B133, 0x0001},
+{0x01B150, 0x0004},
+{0x01B153, 0x0001},
+{0x01B155, 0x0004},
+{0x01B156, 0x0001},
+{0x01B164, 0x0004},
+{0x01B168, 0x0001},
+{0x01B170, 0x0004},
+{0x01B2FC, 0x0001},
+{0x01BC00, 0x0004},
+{0x01BC6B, 0x0001},
+{0x01BC70, 0x0004},
+{0x01BC7D, 0x0001},
+{0x01BC80, 0x0004},
+{0x01BC89, 0x0001},
+{0x01BC90, 0x0004},
+{0x01BC9A, 0x0001},
+{0x01BC9C, 0x0040},
+{0x01BC9D, 0x0010},
+{0x01BC9F, 0x0020},
+{0x01BCA0, 0x0080},
+{0x01BCA4, 0x0001},
+{0x01CF00, 0x0010},
+{0x01CF2E, 0x0001},
+{0x01CF30, 0x0010},
+{0x01CF47, 0x0001},
+{0x01CF50, 0x0040},
+{0x01CFC4, 0x0001},
+{0x01D000, 0x0040},
+{0x01D0F6, 0x0001},
+{0x01D100, 0x0040},
+{0x01D127, 0x0001},
+{0x01D129, 0x0040},
+{0x01D165, 0x0010},
+{0x01D16A, 0x0040},
+{0x01D16D, 0x0010},
+{0x01D173, 0x0080},
+{0x01D17B, 0x0010},
+{0x01D183, 0x0040},
+{0x01D185, 0x0010},
+{0x01D18C, 0x0040},
+{0x01D1AA, 0x0010},
+{0x01D1AE, 0x0040},
+{0x01D1EB, 0x0001},
+{0x01D200, 0x0040},
+{0x01D242, 0x0010},
+{0x01D245, 0x0040},
+{0x01D246, 0x0001},
+{0x01D2C0, 0x0002},
+{0x01D2D4, 0x0001},
+{0x01D2E0, 0x0002},
+{0x01D2F4, 0x0001},
+{0x01D300, 0x0040},
+{0x01D357, 0x0001},
+{0x01D360, 0x0002},
+{0x01D379, 0x0001},
+{0x01D400, 0x0004},
+{0x01D455, 0x0001},
+{0x01D456, 0x0004},
+{0x01D49D, 0x0001},
+{0x01D49E, 0x0004},
+{0x01D4A0, 0x0001},
+{0x01D4A2, 0x0004},
+{0x01D4A3, 0x0001},
+{0x01D4A5, 0x0004},
+{0x01D4A7, 0x0001},
+{0x01D4A9, 0x0004},
+{0x01D4AD, 0x0001},
+{0x01D4AE, 0x0004},
+{0x01D4BA, 0x0001},
+{0x01D4BB, 0x0004},
+{0x01D4BC, 0x0001},
+{0x01D4BD, 0x0004},
+{0x01D4C4, 0x0001},
+{0x01D4C5, 0x0004},
+{0x01D506, 0x0001},
+{0x01D507, 0x0004},
+{0x01D50B, 0x0001},
+{0x01D50D, 0x0004},
+{0x01D515, 0x0001},
+{0x01D516, 0x0004},
+{0x01D51D, 0x0001},
+{0x01D51E, 0x0004},
+{0x01D53A, 0x0001},
+{0x01D53B, 0x0004},
+{0x01D53F, 0x0001},
+{0x01D540, 0x0004},
+{0x01D545, 0x0001},
+{0x01D546, 0x0004},
+{0x01D547, 0x0001},
+{0x01D54A, 0x0004},
+{0x01D551, 0x0001},
+{0x01D552, 0x0004},
+{0x01D6A6, 0x0001},
+{0x01D6A8, 0x0004},
+{0x01D6C1, 0x0040},
+{0x01D6C2, 0x0004},
+{0x01D6DB, 0x0040},
+{0x01D6DC, 0x0004},
+{0x01D6FB, 0x0040},
+{0x01D6FC, 0x0004},
+{0x01D715, 0x0040},
+{0x01D716, 0x0004},
+{0x01D735, 0x0040},
+{0x01D736, 0x0004},
+{0x01D74F, 0x0040},
+{0x01D750, 0x0004},
+{0x01D76F, 0x0040},
+{0x01D770, 0x0004},
+{0x01D789, 0x0040},
+{0x01D78A, 0x0004},
+{0x01D7A9, 0x0040},
+{0x01D7AA, 0x0004},
+{0x01D7C3, 0x0040},
+{0x01D7C4, 0x0004},
+{0x01D7CC, 0x0001},
+{0x01D7CE, 0x0002},
+{0x01D800, 0x0040},
+{0x01DA00, 0x0010},
+{0x01DA37, 0x0040},
+{0x01DA3B, 0x0010},
+{0x01DA6D, 0x0040},
+{0x01DA75, 0x0010},
+{0x01DA76, 0x0040},
+{0x01DA84, 0x0010},
+{0x01DA85, 0x0040},
+{0x01DA87, 0x0020},
+{0x01DA8C, 0x0001},
+{0x01DA9B, 0x0010},
+{0x01DAA0, 0x0001},
+{0x01DAA1, 0x0010},
+{0x01DAB0, 0x0001},
+{0x01DF00, 0x0004},
+{0x01DF1F, 0x0001},
+{0x01DF25, 0x0004},
+{0x01DF2B, 0x0001},
+{0x01E000, 0x0010},
+{0x01E007, 0x0001},
+{0x01E008, 0x0010},
+{0x01E019, 0x0001},
+{0x01E01B, 0x0010},
+{0x01E022, 0x0001},
+{0x01E023, 0x0010},
+{0x01E025, 0x0001},
+{0x01E026, 0x0010},
+{0x01E02B, 0x0001},
+{0x01E030, 0x0004},
+{0x01E06E, 0x0001},
+{0x01E08F, 0x0010},
+{0x01E090, 0x0001},
+{0x01E100, 0x0004},
+{0x01E12D, 0x0001},
+{0x01E130, 0x0010},
+{0x01E137, 0x0004},
+{0x01E13E, 0x0001},
+{0x01E140, 0x0002},
+{0x01E14A, 0x0001},
+{0x01E14E, 0x0004},
+{0x01E14F, 0x0040},
+{0x01E150, 0x0001},
+{0x01E290, 0x0004},
+{0x01E2AE, 0x0010},
+{0x01E2AF, 0x0001},
+{0x01E2C0, 0x0004},
+{0x01E2EC, 0x0010},
+{0x01E2F0, 0x0002},
+{0x01E2FA, 0x0001},
+{0x01E2FF, 0x0040},
+{0x01E300, 0x0001},
+{0x01E4D0, 0x0004},
+{0x01E4EC, 0x0010},
+{0x01E4F0, 0x0002},
+{0x01E4FA, 0x0001},
+{0x01E7E0, 0x0004},
+{0x01E7E7, 0x0001},
+{0x01E7E8, 0x0004},
+{0x01E7EC, 0x0001},
+{0x01E7ED, 0x0004},
+{0x01E7EF, 0x0001},
+{0x01E7F0, 0x0004},
+{0x01E7FF, 0x0001},
+{0x01E800, 0x0004},
+{0x01E8C5, 0x0001},
+{0x01E8C7, 0x0002},
+{0x01E8D0, 0x0010},
+{0x01E8D7, 0x0001},
+{0x01E900, 0x0004},
+{0x01E944, 0x0010},
+{0x01E94B, 0x0004},
+{0x01E94C, 0x0001},
+{0x01E950, 0x0002},
+{0x01E95A, 0x0001},
+{0x01E95E, 0x0020},
+{0x01E960, 0x0001},
+{0x01EC71, 0x0002},
+{0x01ECAC, 0x0040},
+{0x01ECAD, 0x0002},
+{0x01ECB0, 0x0040},
+{0x01ECB1, 0x0002},
+{0x01ECB5, 0x0001},
+{0x01ED01, 0x0002},
+{0x01ED2E, 0x0040},
+{0x01ED2F, 0x0002},
+{0x01ED3E, 0x0001},
+{0x01EE00, 0x0004},
+{0x01EE04, 0x0001},
+{0x01EE05, 0x0004},
+{0x01EE20, 0x0001},
+{0x01EE21, 0x0004},
+{0x01EE23, 0x0001},
+{0x01EE24, 0x0004},
+{0x01EE25, 0x0001},
+{0x01EE27, 0x0004},
+{0x01EE28, 0x0001},
+{0x01EE29, 0x0004},
+{0x01EE33, 0x0001},
+{0x01EE34, 0x0004},
+{0x01EE38, 0x0001},
+{0x01EE39, 0x0004},
+{0x01EE3A, 0x0001},
+{0x01EE3B, 0x0004},
+{0x01EE3C, 0x0001},
+{0x01EE42, 0x0004},
+{0x01EE43, 0x0001},
+{0x01EE47, 0x0004},
+{0x01EE48, 0x0001},
+{0x01EE49, 0x0004},
+{0x01EE4A, 0x0001},
+{0x01EE4B, 0x0004},
+{0x01EE4C, 0x0001},
+{0x01EE4D, 0x0004},
+{0x01EE50, 0x0001},
+{0x01EE51, 0x0004},
+{0x01EE53, 0x0001},
+{0x01EE54, 0x0004},
+{0x01EE55, 0x0001},
+{0x01EE57, 0x0004},
+{0x01EE58, 0x0001},
+{0x01EE59, 0x0004},
+{0x01EE5A, 0x0001},
+{0x01EE5B, 0x0004},
+{0x01EE5C, 0x0001},
+{0x01EE5D, 0x0004},
+{0x01EE5E, 0x0001},
+{0x01EE5F, 0x0004},
+{0x01EE60, 0x0001},
+{0x01EE61, 0x0004},
+{0x01EE63, 0x0001},
+{0x01EE64, 0x0004},
+{0x01EE65, 0x0001},
+{0x01EE67, 0x0004},
+{0x01EE6B, 0x0001},
+{0x01EE6C, 0x0004},
+{0x01EE73, 0x0001},
+{0x01EE74, 0x0004},
+{0x01EE78, 0x0001},
+{0x01EE79, 0x0004},
+{0x01EE7D, 0x0001},
+{0x01EE7E, 0x0004},
+{0x01EE7F, 0x0001},
+{0x01EE80, 0x0004},
+{0x01EE8A, 0x0001},
+{0x01EE8B, 0x0004},
+{0x01EE9C, 0x0001},
+{0x01EEA1, 0x0004},
+{0x01EEA4, 0x0001},
+{0x01EEA5, 0x0004},
+{0x01EEAA, 0x0001},
+{0x01EEAB, 0x0004},
+{0x01EEBC, 0x0001},
+{0x01EEF0, 0x0040},
+{0x01EEF2, 0x0001},
+{0x01F000, 0x0040},
+{0x01F02C, 0x0001},
+{0x01F030, 0x0040},
+{0x01F094, 0x0001},
+{0x01F0A0, 0x0040},
+{0x01F0AF, 0x0001},
+{0x01F0B1, 0x0040},
+{0x01F0C0, 0x0001},
+{0x01F0C1, 0x0040},
+{0x01F0D0, 0x0001},
+{0x01F0D1, 0x0040},
+{0x01F0F6, 0x0001},
+{0x01F100, 0x0002},
+{0x01F10D, 0x0040},
+{0x01F1AE, 0x0001},
+{0x01F1E6, 0x0040},
+{0x01F203, 0x0001},
+{0x01F210, 0x0040},
+{0x01F23C, 0x0001},
+{0x01F240, 0x0040},
+{0x01F249, 0x0001},
+{0x01F250, 0x0040},
+{0x01F252, 0x0001},
+{0x01F260, 0x0040},
+{0x01F266, 0x0001},
+{0x01F300, 0x0040},
+{0x01F6D8, 0x0001},
+{0x01F6DC, 0x0040},
+{0x01F6ED, 0x0001},
+{0x01F6F0, 0x0040},
+{0x01F6FD, 0x0001},
+{0x01F700, 0x0040},
+{0x01F777, 0x0001},
+{0x01F77B, 0x0040},
+{0x01F7DA, 0x0001},
+{0x01F7E0, 0x0040},
+{0x01F7EC, 0x0001},
+{0x01F7F0, 0x0040},
+{0x01F7F1, 0x0001},
+{0x01F800, 0x0040},
+{0x01F80C, 0x0001},
+{0x01F810, 0x0040},
+{0x01F848, 0x0001},
+{0x01F850, 0x0040},
+{0x01F85A, 0x0001},
+{0x01F860, 0x0040},
+{0x01F888, 0x0001},
+{0x01F890, 0x0040},
+{0x01F8AE, 0x0001},
+{0x01F8B0, 0x0040},
+{0x01F8B2, 0x0001},
+{0x01F900, 0x0040},
+{0x01FA54, 0x0001},
+{0x01FA60, 0x0040},
+{0x01FA6E, 0x0001},
+{0x01FA70, 0x0040},
+{0x01FA7D, 0x0001},
+{0x01FA80, 0x0040},
+{0x01FA89, 0x0001},
+{0x01FA90, 0x0040},
+{0x01FABE, 0x0001},
+{0x01FABF, 0x0040},
+{0x01FAC6, 0x0001},
+{0x01FACE, 0x0040},
+{0x01FADC, 0x0001},
+{0x01FAE0, 0x0040},
+{0x01FAE9, 0x0001},
+{0x01FAF0, 0x0040},
+{0x01FAF9, 0x0001},
+{0x01FB00, 0x0040},
+{0x01FB93, 0x0001},
+{0x01FB94, 0x0040},
+{0x01FBCB, 0x0001},
+{0x01FBF0, 0x0002},
+{0x01FBFA, 0x0001},
+{0x020000, 0x0004},
+{0x02A6E0, 0x0001},
+{0x02A700, 0x0004},
+{0x02B73A, 0x0001},
+{0x02B740, 0x0004},
+{0x02B81E, 0x0001},
+{0x02B820, 0x0004},
+{0x02CEA2, 0x0001},
+{0x02CEB0, 0x0004},
+{0x02EBE1, 0x0001},
+{0x02EBF0, 0x0004},
+{0x02EE5E, 0x0001},
+{0x02F800, 0x0004},
+{0x02FA1E, 0x0001},
+{0x030000, 0x0004},
+{0x03134B, 0x0001},
+{0x031350, 0x0004},
+{0x0323B0, 0x0001},
+{0x0E0001, 0x0080},
+{0x0E0002, 0x0001},
+{0x0E0020, 0x0080},
+{0x0E0080, 0x0001},
+{0x0E0100, 0x0010},
+{0x0E01F0, 0x0001},
+{0x0F0000, 0x0080},
+{0x0FFFFE, 0x0001},
+{0x100000, 0x0080},
+{0x10FFFE, 0x0001},
+{0x110000, 0x0000},
+};
+
+const std::unordered_set<uint32_t> unicode_set_whitespace = {
+0x000009,
+0x00000A,
+0x00000B,
+0x00000C,
+0x00000D,
+0x000020,
+0x000085,
+0x0000A0,
+0x001680,
+0x002000,
+0x002001,
+0x002002,
+0x002003,
+0x002004,
+0x002005,
+0x002006,
+0x002007,
+0x002008,
+0x002009,
+0x00200A,
+0x002028,
+0x002029,
+0x00202F,
+0x00205F,
+0x003000,
+};
+
+const std::unordered_map<uint32_t, uint32_t> unicode_map_lowercase = {
+{0x000041, 0x000061},
+{0x000042, 0x000062},
+{0x000043, 0x000063},
+{0x000044, 0x000064},
+{0x000045, 0x000065},
+{0x000046, 0x000066},
+{0x000047, 0x000067},
+{0x000048, 0x000068},
+{0x000049, 0x000069},
+{0x00004A, 0x00006A},
+{0x00004B, 0x00006B},
+{0x00004C, 0x00006C},
+{0x00004D, 0x00006D},
+{0x00004E, 0x00006E},
+{0x00004F, 0x00006F},
+{0x000050, 0x000070},
+{0x000051, 0x000071},
+{0x000052, 0x000072},
+{0x000053, 0x000073},
+{0x000054, 0x000074},
+{0x000055, 0x000075},
+{0x000056, 0x000076},
+{0x000057, 0x000077},
+{0x000058, 0x000078},
+{0x000059, 0x000079},
+{0x00005A, 0x00007A},
+{0x0000C0, 0x0000E0},
+{0x0000C1, 0x0000E1},
+{0x0000C2, 0x0000E2},
+{0x0000C3, 0x0000E3},
+{0x0000C4, 0x0000E4},
+{0x0000C5, 0x0000E5},
+{0x0000C6, 0x0000E6},
+{0x0000C7, 0x0000E7},
+{0x0000C8, 0x0000E8},
+{0x0000C9, 0x0000E9},
+{0x0000CA, 0x0000EA},
+{0x0000CB, 0x0000EB},
+{0x0000CC, 0x0000EC},
+{0x0000CD, 0x0000ED},
+{0x0000CE, 0x0000EE},
+{0x0000CF, 0x0000EF},
+{0x0000D0, 0x0000F0},
+{0x0000D1, 0x0000F1},
+{0x0000D2, 0x0000F2},
+{0x0000D3, 0x0000F3},
+{0x0000D4, 0x0000F4},
+{0x0000D5, 0x0000F5},
+{0x0000D6, 0x0000F6},
+{0x0000D8, 0x0000F8},
+{0x0000D9, 0x0000F9},
+{0x0000DA, 0x0000FA},
+{0x0000DB, 0x0000FB},
+{0x0000DC, 0x0000FC},
+{0x0000DD, 0x0000FD},
+{0x0000DE, 0x0000FE},
+{0x000100, 0x000101},
+{0x000102, 0x000103},
+{0x000104, 0x000105},
+{0x000106, 0x000107},
+{0x000108, 0x000109},
+{0x00010A, 0x00010B},
+{0x00010C, 0x00010D},
+{0x00010E, 0x00010F},
+{0x000110, 0x000111},
+{0x000112, 0x000113},
+{0x000114, 0x000115},
+{0x000116, 0x000117},
+{0x000118, 0x000119},
+{0x00011A, 0x00011B},
+{0x00011C, 0x00011D},
+{0x00011E, 0x00011F},
+{0x000120, 0x000121},
+{0x000122, 0x000123},
+{0x000124, 0x000125},
+{0x000126, 0x000127},
+{0x000128, 0x000129},
+{0x00012A, 0x00012B},
+{0x00012C, 0x00012D},
+{0x00012E, 0x00012F},
+{0x000130, 0x000069},
+{0x000132, 0x000133},
+{0x000134, 0x000135},
+{0x000136, 0x000137},
+{0x000139, 0x00013A},
+{0x00013B, 0x00013C},
+{0x00013D, 0x00013E},
+{0x00013F, 0x000140},
+{0x000141, 0x000142},
+{0x000143, 0x000144},
+{0x000145, 0x000146},
+{0x000147, 0x000148},
+{0x00014A, 0x00014B},
+{0x00014C, 0x00014D},
+{0x00014E, 0x00014F},
+{0x000150, 0x000151},
+{0x000152, 0x000153},
+{0x000154, 0x000155},
+{0x000156, 0x000157},
+{0x000158, 0x000159},
+{0x00015A, 0x00015B},
+{0x00015C, 0x00015D},
+{0x00015E, 0x00015F},
+{0x000160, 0x000161},
+{0x000162, 0x000163},
+{0x000164, 0x000165},
+{0x000166, 0x000167},
+{0x000168, 0x000169},
+{0x00016A, 0x00016B},
+{0x00016C, 0x00016D},
+{0x00016E, 0x00016F},
+{0x000170, 0x000171},
+{0x000172, 0x000173},
+{0x000174, 0x000175},
+{0x000176, 0x000177},
+{0x000178, 0x0000FF},
+{0x000179, 0x00017A},
+{0x00017B, 0x00017C},
+{0x00017D, 0x00017E},
+{0x000181, 0x000253},
+{0x000182, 0x000183},
+{0x000184, 0x000185},
+{0x000186, 0x000254},
+{0x000187, 0x000188},
+{0x000189, 0x000256},
+{0x00018A, 0x000257},
+{0x00018B, 0x00018C},
+{0x00018E, 0x0001DD},
+{0x00018F, 0x000259},
+{0x000190, 0x00025B},
+{0x000191, 0x000192},
+{0x000193, 0x000260},
+{0x000194, 0x000263},
+{0x000196, 0x000269},
+{0x000197, 0x000268},
+{0x000198, 0x000199},
+{0x00019C, 0x00026F},
+{0x00019D, 0x000272},
+{0x00019F, 0x000275},
+{0x0001A0, 0x0001A1},
+{0x0001A2, 0x0001A3},
+{0x0001A4, 0x0001A5},
+{0x0001A6, 0x000280},
+{0x0001A7, 0x0001A8},
+{0x0001A9, 0x000283},
+{0x0001AC, 0x0001AD},
+{0x0001AE, 0x000288},
+{0x0001AF, 0x0001B0},
+{0x0001B1, 0x00028A},
+{0x0001B2, 0x00028B},
+{0x0001B3, 0x0001B4},
+{0x0001B5, 0x0001B6},
+{0x0001B7, 0x000292},
+{0x0001B8, 0x0001B9},
+{0x0001BC, 0x0001BD},
+{0x0001C4, 0x0001C6},
+{0x0001C5, 0x0001C6},
+{0x0001C7, 0x0001C9},
+{0x0001C8, 0x0001C9},
+{0x0001CA, 0x0001CC},
+{0x0001CB, 0x0001CC},
+{0x0001CD, 0x0001CE},
+{0x0001CF, 0x0001D0},
+{0x0001D1, 0x0001D2},
+{0x0001D3, 0x0001D4},
+{0x0001D5, 0x0001D6},
+{0x0001D7, 0x0001D8},
+{0x0001D9, 0x0001DA},
+{0x0001DB, 0x0001DC},
+{0x0001DE, 0x0001DF},
+{0x0001E0, 0x0001E1},
+{0x0001E2, 0x0001E3},
+{0x0001E4, 0x0001E5},
+{0x0001E6, 0x0001E7},
+{0x0001E8, 0x0001E9},
+{0x0001EA, 0x0001EB},
+{0x0001EC, 0x0001ED},
+{0x0001EE, 0x0001EF},
+{0x0001F1, 0x0001F3},
+{0x0001F2, 0x0001F3},
+{0x0001F4, 0x0001F5},
+{0x0001F6, 0x000195},
+{0x0001F7, 0x0001BF},
+{0x0001F8, 0x0001F9},
+{0x0001FA, 0x0001FB},
+{0x0001FC, 0x0001FD},
+{0x0001FE, 0x0001FF},
+{0x000200, 0x000201},
+{0x000202, 0x000203},
+{0x000204, 0x000205},
+{0x000206, 0x000207},
+{0x000208, 0x000209},
+{0x00020A, 0x00020B},
+{0x00020C, 0x00020D},
+{0x00020E, 0x00020F},
+{0x000210, 0x000211},
+{0x000212, 0x000213},
+{0x000214, 0x000215},
+{0x000216, 0x000217},
+{0x000218, 0x000219},
+{0x00021A, 0x00021B},
+{0x00021C, 0x00021D},
+{0x00021E, 0x00021F},
+{0x000220, 0x00019E},
+{0x000222, 0x000223},
+{0x000224, 0x000225},
+{0x000226, 0x000227},
+{0x000228, 0x000229},
+{0x00022A, 0x00022B},
+{0x00022C, 0x00022D},
+{0x00022E, 0x00022F},
+{0x000230, 0x000231},
+{0x000232, 0x000233},
+{0x00023A, 0x002C65},
+{0x00023B, 0x00023C},
+{0x00023D, 0x00019A},
+{0x00023E, 0x002C66},
+{0x000241, 0x000242},
+{0x000243, 0x000180},
+{0x000244, 0x000289},
+{0x000245, 0x00028C},
+{0x000246, 0x000247},
+{0x000248, 0x000249},
+{0x00024A, 0x00024B},
+{0x00024C, 0x00024D},
+{0x00024E, 0x00024F},
+{0x000370, 0x000371},
+{0x000372, 0x000373},
+{0x000376, 0x000377},
+{0x00037F, 0x0003F3},
+{0x000386, 0x0003AC},
+{0x000388, 0x0003AD},
+{0x000389, 0x0003AE},
+{0x00038A, 0x0003AF},
+{0x00038C, 0x0003CC},
+{0x00038E, 0x0003CD},
+{0x00038F, 0x0003CE},
+{0x000391, 0x0003B1},
+{0x000392, 0x0003B2},
+{0x000393, 0x0003B3},
+{0x000394, 0x0003B4},
+{0x000395, 0x0003B5},
+{0x000396, 0x0003B6},
+{0x000397, 0x0003B7},
+{0x000398, 0x0003B8},
+{0x000399, 0x0003B9},
+{0x00039A, 0x0003BA},
+{0x00039B, 0x0003BB},
+{0x00039C, 0x0003BC},
+{0x00039D, 0x0003BD},
+{0x00039E, 0x0003BE},
+{0x00039F, 0x0003BF},
+{0x0003A0, 0x0003C0},
+{0x0003A1, 0x0003C1},
+{0x0003A3, 0x0003C3},
+{0x0003A4, 0x0003C4},
+{0x0003A5, 0x0003C5},
+{0x0003A6, 0x0003C6},
+{0x0003A7, 0x0003C7},
+{0x0003A8, 0x0003C8},
+{0x0003A9, 0x0003C9},
+{0x0003AA, 0x0003CA},
+{0x0003AB, 0x0003CB},
+{0x0003CF, 0x0003D7},
+{0x0003D8, 0x0003D9},
+{0x0003DA, 0x0003DB},
+{0x0003DC, 0x0003DD},
+{0x0003DE, 0x0003DF},
+{0x0003E0, 0x0003E1},
+{0x0003E2, 0x0003E3},
+{0x0003E4, 0x0003E5},
+{0x0003E6, 0x0003E7},
+{0x0003E8, 0x0003E9},
+{0x0003EA, 0x0003EB},
+{0x0003EC, 0x0003ED},
+{0x0003EE, 0x0003EF},
+{0x0003F4, 0x0003B8},
+{0x0003F7, 0x0003F8},
+{0x0003F9, 0x0003F2},
+{0x0003FA, 0x0003FB},
+{0x0003FD, 0x00037B},
+{0x0003FE, 0x00037C},
+{0x0003FF, 0x00037D},
+{0x000400, 0x000450},
+{0x000401, 0x000451},
+{0x000402, 0x000452},
+{0x000403, 0x000453},
+{0x000404, 0x000454},
+{0x000405, 0x000455},
+{0x000406, 0x000456},
+{0x000407, 0x000457},
+{0x000408, 0x000458},
+{0x000409, 0x000459},
+{0x00040A, 0x00045A},
+{0x00040B, 0x00045B},
+{0x00040C, 0x00045C},
+{0x00040D, 0x00045D},
+{0x00040E, 0x00045E},
+{0x00040F, 0x00045F},
+{0x000410, 0x000430},
+{0x000411, 0x000431},
+{0x000412, 0x000432},
+{0x000413, 0x000433},
+{0x000414, 0x000434},
+{0x000415, 0x000435},
+{0x000416, 0x000436},
+{0x000417, 0x000437},
+{0x000418, 0x000438},
+{0x000419, 0x000439},
+{0x00041A, 0x00043A},
+{0x00041B, 0x00043B},
+{0x00041C, 0x00043C},
+{0x00041D, 0x00043D},
+{0x00041E, 0x00043E},
+{0x00041F, 0x00043F},
+{0x000420, 0x000440},
+{0x000421, 0x000441},
+{0x000422, 0x000442},
+{0x000423, 0x000443},
+{0x000424, 0x000444},
+{0x000425, 0x000445},
+{0x000426, 0x000446},
+{0x000427, 0x000447},
+{0x000428, 0x000448},
+{0x000429, 0x000449},
+{0x00042A, 0x00044A},
+{0x00042B, 0x00044B},
+{0x00042C, 0x00044C},
+{0x00042D, 0x00044D},
+{0x00042E, 0x00044E},
+{0x00042F, 0x00044F},
+{0x000460, 0x000461},
+{0x000462, 0x000463},
+{0x000464, 0x000465},
+{0x000466, 0x000467},
+{0x000468, 0x000469},
+{0x00046A, 0x00046B},
+{0x00046C, 0x00046D},
+{0x00046E, 0x00046F},
+{0x000470, 0x000471},
+{0x000472, 0x000473},
+{0x000474, 0x000475},
+{0x000476, 0x000477},
+{0x000478, 0x000479},
+{0x00047A, 0x00047B},
+{0x00047C, 0x00047D},
+{0x00047E, 0x00047F},
+{0x000480, 0x000481},
+{0x00048A, 0x00048B},
+{0x00048C, 0x00048D},
+{0x00048E, 0x00048F},
+{0x000490, 0x000491},
+{0x000492, 0x000493},
+{0x000494, 0x000495},
+{0x000496, 0x000497},
+{0x000498, 0x000499},
+{0x00049A, 0x00049B},
+{0x00049C, 0x00049D},
+{0x00049E, 0x00049F},
+{0x0004A0, 0x0004A1},
+{0x0004A2, 0x0004A3},
+{0x0004A4, 0x0004A5},
+{0x0004A6, 0x0004A7},
+{0x0004A8, 0x0004A9},
+{0x0004AA, 0x0004AB},
+{0x0004AC, 0x0004AD},
+{0x0004AE, 0x0004AF},
+{0x0004B0, 0x0004B1},
+{0x0004B2, 0x0004B3},
+{0x0004B4, 0x0004B5},
+{0x0004B6, 0x0004B7},
+{0x0004B8, 0x0004B9},
+{0x0004BA, 0x0004BB},
+{0x0004BC, 0x0004BD},
+{0x0004BE, 0x0004BF},
+{0x0004C0, 0x0004CF},
+{0x0004C1, 0x0004C2},
+{0x0004C3, 0x0004C4},
+{0x0004C5, 0x0004C6},
+{0x0004C7, 0x0004C8},
+{0x0004C9, 0x0004CA},
+{0x0004CB, 0x0004CC},
+{0x0004CD, 0x0004CE},
+{0x0004D0, 0x0004D1},
+{0x0004D2, 0x0004D3},
+{0x0004D4, 0x0004D5},
+{0x0004D6, 0x0004D7},
+{0x0004D8, 0x0004D9},
+{0x0004DA, 0x0004DB},
+{0x0004DC, 0x0004DD},
+{0x0004DE, 0x0004DF},
+{0x0004E0, 0x0004E1},
+{0x0004E2, 0x0004E3},
+{0x0004E4, 0x0004E5},
+{0x0004E6, 0x0004E7},
+{0x0004E8, 0x0004E9},
+{0x0004EA, 0x0004EB},
+{0x0004EC, 0x0004ED},
+{0x0004EE, 0x0004EF},
+{0x0004F0, 0x0004F1},
+{0x0004F2, 0x0004F3},
+{0x0004F4, 0x0004F5},
+{0x0004F6, 0x0004F7},
+{0x0004F8, 0x0004F9},
+{0x0004FA, 0x0004FB},
+{0x0004FC, 0x0004FD},
+{0x0004FE, 0x0004FF},
+{0x000500, 0x000501},
+{0x000502, 0x000503},
+{0x000504, 0x000505},
+{0x000506, 0x000507},
+{0x000508, 0x000509},
+{0x00050A, 0x00050B},
+{0x00050C, 0x00050D},
+{0x00050E, 0x00050F},
+{0x000510, 0x000511},
+{0x000512, 0x000513},
+{0x000514, 0x000515},
+{0x000516, 0x000517},
+{0x000518, 0x000519},
+{0x00051A, 0x00051B},
+{0x00051C, 0x00051D},
+{0x00051E, 0x00051F},
+{0x000520, 0x000521},
+{0x000522, 0x000523},
+{0x000524, 0x000525},
+{0x000526, 0x000527},
+{0x000528, 0x000529},
+{0x00052A, 0x00052B},
+{0x00052C, 0x00052D},
+{0x00052E, 0x00052F},
+{0x000531, 0x000561},
+{0x000532, 0x000562},
+{0x000533, 0x000563},
+{0x000534, 0x000564},
+{0x000535, 0x000565},
+{0x000536, 0x000566},
+{0x000537, 0x000567},
+{0x000538, 0x000568},
+{0x000539, 0x000569},
+{0x00053A, 0x00056A},
+{0x00053B, 0x00056B},
+{0x00053C, 0x00056C},
+{0x00053D, 0x00056D},
+{0x00053E, 0x00056E},
+{0x00053F, 0x00056F},
+{0x000540, 0x000570},
+{0x000541, 0x000571},
+{0x000542, 0x000572},
+{0x000543, 0x000573},
+{0x000544, 0x000574},
+{0x000545, 0x000575},
+{0x000546, 0x000576},
+{0x000547, 0x000577},
+{0x000548, 0x000578},
+{0x000549, 0x000579},
+{0x00054A, 0x00057A},
+{0x00054B, 0x00057B},
+{0x00054C, 0x00057C},
+{0x00054D, 0x00057D},
+{0x00054E, 0x00057E},
+{0x00054F, 0x00057F},
+{0x000550, 0x000580},
+{0x000551, 0x000581},
+{0x000552, 0x000582},
+{0x000553, 0x000583},
+{0x000554, 0x000584},
+{0x000555, 0x000585},
+{0x000556, 0x000586},
+{0x0010A0, 0x002D00},
+{0x0010A1, 0x002D01},
+{0x0010A2, 0x002D02},
+{0x0010A3, 0x002D03},
+{0x0010A4, 0x002D04},
+{0x0010A5, 0x002D05},
+{0x0010A6, 0x002D06},
+{0x0010A7, 0x002D07},
+{0x0010A8, 0x002D08},
+{0x0010A9, 0x002D09},
+{0x0010AA, 0x002D0A},
+{0x0010AB, 0x002D0B},
+{0x0010AC, 0x002D0C},
+{0x0010AD, 0x002D0D},
+{0x0010AE, 0x002D0E},
+{0x0010AF, 0x002D0F},
+{0x0010B0, 0x002D10},
+{0x0010B1, 0x002D11},
+{0x0010B2, 0x002D12},
+{0x0010B3, 0x002D13},
+{0x0010B4, 0x002D14},
+{0x0010B5, 0x002D15},
+{0x0010B6, 0x002D16},
+{0x0010B7, 0x002D17},
+{0x0010B8, 0x002D18},
+{0x0010B9, 0x002D19},
+{0x0010BA, 0x002D1A},
+{0x0010BB, 0x002D1B},
+{0x0010BC, 0x002D1C},
+{0x0010BD, 0x002D1D},
+{0x0010BE, 0x002D1E},
+{0x0010BF, 0x002D1F},
+{0x0010C0, 0x002D20},
+{0x0010C1, 0x002D21},
+{0x0010C2, 0x002D22},
+{0x0010C3, 0x002D23},
+{0x0010C4, 0x002D24},
+{0x0010C5, 0x002D25},
+{0x0010C7, 0x002D27},
+{0x0010CD, 0x002D2D},
+{0x0013A0, 0x00AB70},
+{0x0013A1, 0x00AB71},
+{0x0013A2, 0x00AB72},
+{0x0013A3, 0x00AB73},
+{0x0013A4, 0x00AB74},
+{0x0013A5, 0x00AB75},
+{0x0013A6, 0x00AB76},
+{0x0013A7, 0x00AB77},
+{0x0013A8, 0x00AB78},
+{0x0013A9, 0x00AB79},
+{0x0013AA, 0x00AB7A},
+{0x0013AB, 0x00AB7B},
+{0x0013AC, 0x00AB7C},
+{0x0013AD, 0x00AB7D},
+{0x0013AE, 0x00AB7E},
+{0x0013AF, 0x00AB7F},
+{0x0013B0, 0x00AB80},
+{0x0013B1, 0x00AB81},
+{0x0013B2, 0x00AB82},
+{0x0013B3, 0x00AB83},
+{0x0013B4, 0x00AB84},
+{0x0013B5, 0x00AB85},
+{0x0013B6, 0x00AB86},
+{0x0013B7, 0x00AB87},
+{0x0013B8, 0x00AB88},
+{0x0013B9, 0x00AB89},
+{0x0013BA, 0x00AB8A},
+{0x0013BB, 0x00AB8B},
+{0x0013BC, 0x00AB8C},
+{0x0013BD, 0x00AB8D},
+{0x0013BE, 0x00AB8E},
+{0x0013BF, 0x00AB8F},
+{0x0013C0, 0x00AB90},
+{0x0013C1, 0x00AB91},
+{0x0013C2, 0x00AB92},
+{0x0013C3, 0x00AB93},
+{0x0013C4, 0x00AB94},
+{0x0013C5, 0x00AB95},
+{0x0013C6, 0x00AB96},
+{0x0013C7, 0x00AB97},
+{0x0013C8, 0x00AB98},
+{0x0013C9, 0x00AB99},
+{0x0013CA, 0x00AB9A},
+{0x0013CB, 0x00AB9B},
+{0x0013CC, 0x00AB9C},
+{0x0013CD, 0x00AB9D},
+{0x0013CE, 0x00AB9E},
+{0x0013CF, 0x00AB9F},
+{0x0013D0, 0x00ABA0},
+{0x0013D1, 0x00ABA1},
+{0x0013D2, 0x00ABA2},
+{0x0013D3, 0x00ABA3},
+{0x0013D4, 0x00ABA4},
+{0x0013D5, 0x00ABA5},
+{0x0013D6, 0x00ABA6},
+{0x0013D7, 0x00ABA7},
+{0x0013D8, 0x00ABA8},
+{0x0013D9, 0x00ABA9},
+{0x0013DA, 0x00ABAA},
+{0x0013DB, 0x00ABAB},
+{0x0013DC, 0x00ABAC},
+{0x0013DD, 0x00ABAD},
+{0x0013DE, 0x00ABAE},
+{0x0013DF, 0x00ABAF},
+{0x0013E0, 0x00ABB0},
+{0x0013E1, 0x00ABB1},
+{0x0013E2, 0x00ABB2},
+{0x0013E3, 0x00ABB3},
+{0x0013E4, 0x00ABB4},
+{0x0013E5, 0x00ABB5},
+{0x0013E6, 0x00ABB6},
+{0x0013E7, 0x00ABB7},
+{0x0013E8, 0x00ABB8},
+{0x0013E9, 0x00ABB9},
+{0x0013EA, 0x00ABBA},
+{0x0013EB, 0x00ABBB},
+{0x0013EC, 0x00ABBC},
+{0x0013ED, 0x00ABBD},
+{0x0013EE, 0x00ABBE},
+{0x0013EF, 0x00ABBF},
+{0x0013F0, 0x0013F8},
+{0x0013F1, 0x0013F9},
+{0x0013F2, 0x0013FA},
+{0x0013F3, 0x0013FB},
+{0x0013F4, 0x0013FC},
+{0x0013F5, 0x0013FD},
+{0x001C90, 0x0010D0},
+{0x001C91, 0x0010D1},
+{0x001C92, 0x0010D2},
+{0x001C93, 0x0010D3},
+{0x001C94, 0x0010D4},
+{0x001C95, 0x0010D5},
+{0x001C96, 0x0010D6},
+{0x001C97, 0x0010D7},
+{0x001C98, 0x0010D8},
+{0x001C99, 0x0010D9},
+{0x001C9A, 0x0010DA},
+{0x001C9B, 0x0010DB},
+{0x001C9C, 0x0010DC},
+{0x001C9D, 0x0010DD},
+{0x001C9E, 0x0010DE},
+{0x001C9F, 0x0010DF},
+{0x001CA0, 0x0010E0},
+{0x001CA1, 0x0010E1},
+{0x001CA2, 0x0010E2},
+{0x001CA3, 0x0010E3},
+{0x001CA4, 0x0010E4},
+{0x001CA5, 0x0010E5},
+{0x001CA6, 0x0010E6},
+{0x001CA7, 0x0010E7},
+{0x001CA8, 0x0010E8},
+{0x001CA9, 0x0010E9},
+{0x001CAA, 0x0010EA},
+{0x001CAB, 0x0010EB},
+{0x001CAC, 0x0010EC},
+{0x001CAD, 0x0010ED},
+{0x001CAE, 0x0010EE},
+{0x001CAF, 0x0010EF},
+{0x001CB0, 0x0010F0},
+{0x001CB1, 0x0010F1},
+{0x001CB2, 0x0010F2},
+{0x001CB3, 0x0010F3},
+{0x001CB4, 0x0010F4},
+{0x001CB5, 0x0010F5},
+{0x001CB6, 0x0010F6},
+{0x001CB7, 0x0010F7},
+{0x001CB8, 0x0010F8},
+{0x001CB9, 0x0010F9},
+{0x001CBA, 0x0010FA},
+{0x001CBD, 0x0010FD},
+{0x001CBE, 0x0010FE},
+{0x001CBF, 0x0010FF},
+{0x001E00, 0x001E01},
+{0x001E02, 0x001E03},
+{0x001E04, 0x001E05},
+{0x001E06, 0x001E07},
+{0x001E08, 0x001E09},
+{0x001E0A, 0x001E0B},
+{0x001E0C, 0x001E0D},
+{0x001E0E, 0x001E0F},
+{0x001E10, 0x001E11},
+{0x001E12, 0x001E13},
+{0x001E14, 0x001E15},
+{0x001E16, 0x001E17},
+{0x001E18, 0x001E19},
+{0x001E1A, 0x001E1B},
+{0x001E1C, 0x001E1D},
+{0x001E1E, 0x001E1F},
+{0x001E20, 0x001E21},
+{0x001E22, 0x001E23},
+{0x001E24, 0x001E25},
+{0x001E26, 0x001E27},
+{0x001E28, 0x001E29},
+{0x001E2A, 0x001E2B},
+{0x001E2C, 0x001E2D},
+{0x001E2E, 0x001E2F},
+{0x001E30, 0x001E31},
+{0x001E32, 0x001E33},
+{0x001E34, 0x001E35},
+{0x001E36, 0x001E37},
+{0x001E38, 0x001E39},
+{0x001E3A, 0x001E3B},
+{0x001E3C, 0x001E3D},
+{0x001E3E, 0x001E3F},
+{0x001E40, 0x001E41},
+{0x001E42, 0x001E43},
+{0x001E44, 0x001E45},
+{0x001E46, 0x001E47},
+{0x001E48, 0x001E49},
+{0x001E4A, 0x001E4B},
+{0x001E4C, 0x001E4D},
+{0x001E4E, 0x001E4F},
+{0x001E50, 0x001E51},
+{0x001E52, 0x001E53},
+{0x001E54, 0x001E55},
+{0x001E56, 0x001E57},
+{0x001E58, 0x001E59},
+{0x001E5A, 0x001E5B},
+{0x001E5C, 0x001E5D},
+{0x001E5E, 0x001E5F},
+{0x001E60, 0x001E61},
+{0x001E62, 0x001E63},
+{0x001E64, 0x001E65},
+{0x001E66, 0x001E67},
+{0x001E68, 0x001E69},
+{0x001E6A, 0x001E6B},
+{0x001E6C, 0x001E6D},
+{0x001E6E, 0x001E6F},
+{0x001E70, 0x001E71},
+{0x001E72, 0x001E73},
+{0x001E74, 0x001E75},
+{0x001E76, 0x001E77},
+{0x001E78, 0x001E79},
+{0x001E7A, 0x001E7B},
+{0x001E7C, 0x001E7D},
+{0x001E7E, 0x001E7F},
+{0x001E80, 0x001E81},
+{0x001E82, 0x001E83},
+{0x001E84, 0x001E85},
+{0x001E86, 0x001E87},
+{0x001E88, 0x001E89},
+{0x001E8A, 0x001E8B},
+{0x001E8C, 0x001E8D},
+{0x001E8E, 0x001E8F},
+{0x001E90, 0x001E91},
+{0x001E92, 0x001E93},
+{0x001E94, 0x001E95},
+{0x001E9E, 0x0000DF},
+{0x001EA0, 0x001EA1},
+{0x001EA2, 0x001EA3},
+{0x001EA4, 0x001EA5},
+{0x001EA6, 0x001EA7},
+{0x001EA8, 0x001EA9},
+{0x001EAA, 0x001EAB},
+{0x001EAC, 0x001EAD},
+{0x001EAE, 0x001EAF},
+{0x001EB0, 0x001EB1},
+{0x001EB2, 0x001EB3},
+{0x001EB4, 0x001EB5},
+{0x001EB6, 0x001EB7},
+{0x001EB8, 0x001EB9},
+{0x001EBA, 0x001EBB},
+{0x001EBC, 0x001EBD},
+{0x001EBE, 0x001EBF},
+{0x001EC0, 0x001EC1},
+{0x001EC2, 0x001EC3},
+{0x001EC4, 0x001EC5},
+{0x001EC6, 0x001EC7},
+{0x001EC8, 0x001EC9},
+{0x001ECA, 0x001ECB},
+{0x001ECC, 0x001ECD},
+{0x001ECE, 0x001ECF},
+{0x001ED0, 0x001ED1},
+{0x001ED2, 0x001ED3},
+{0x001ED4, 0x001ED5},
+{0x001ED6, 0x001ED7},
+{0x001ED8, 0x001ED9},
+{0x001EDA, 0x001EDB},
+{0x001EDC, 0x001EDD},
+{0x001EDE, 0x001EDF},
+{0x001EE0, 0x001EE1},
+{0x001EE2, 0x001EE3},
+{0x001EE4, 0x001EE5},
+{0x001EE6, 0x001EE7},
+{0x001EE8, 0x001EE9},
+{0x001EEA, 0x001EEB},
+{0x001EEC, 0x001EED},
+{0x001EEE, 0x001EEF},
+{0x001EF0, 0x001EF1},
+{0x001EF2, 0x001EF3},
+{0x001EF4, 0x001EF5},
+{0x001EF6, 0x001EF7},
+{0x001EF8, 0x001EF9},
+{0x001EFA, 0x001EFB},
+{0x001EFC, 0x001EFD},
+{0x001EFE, 0x001EFF},
+{0x001F08, 0x001F00},
+{0x001F09, 0x001F01},
+{0x001F0A, 0x001F02},
+{0x001F0B, 0x001F03},
+{0x001F0C, 0x001F04},
+{0x001F0D, 0x001F05},
+{0x001F0E, 0x001F06},
+{0x001F0F, 0x001F07},
+{0x001F18, 0x001F10},
+{0x001F19, 0x001F11},
+{0x001F1A, 0x001F12},
+{0x001F1B, 0x001F13},
+{0x001F1C, 0x001F14},
+{0x001F1D, 0x001F15},
+{0x001F28, 0x001F20},
+{0x001F29, 0x001F21},
+{0x001F2A, 0x001F22},
+{0x001F2B, 0x001F23},
+{0x001F2C, 0x001F24},
+{0x001F2D, 0x001F25},
+{0x001F2E, 0x001F26},
+{0x001F2F, 0x001F27},
+{0x001F38, 0x001F30},
+{0x001F39, 0x001F31},
+{0x001F3A, 0x001F32},
+{0x001F3B, 0x001F33},
+{0x001F3C, 0x001F34},
+{0x001F3D, 0x001F35},
+{0x001F3E, 0x001F36},
+{0x001F3F, 0x001F37},
+{0x001F48, 0x001F40},
+{0x001F49, 0x001F41},
+{0x001F4A, 0x001F42},
+{0x001F4B, 0x001F43},
+{0x001F4C, 0x001F44},
+{0x001F4D, 0x001F45},
+{0x001F59, 0x001F51},
+{0x001F5B, 0x001F53},
+{0x001F5D, 0x001F55},
+{0x001F5F, 0x001F57},
+{0x001F68, 0x001F60},
+{0x001F69, 0x001F61},
+{0x001F6A, 0x001F62},
+{0x001F6B, 0x001F63},
+{0x001F6C, 0x001F64},
+{0x001F6D, 0x001F65},
+{0x001F6E, 0x001F66},
+{0x001F6F, 0x001F67},
+{0x001F88, 0x001F80},
+{0x001F89, 0x001F81},
+{0x001F8A, 0x001F82},
+{0x001F8B, 0x001F83},
+{0x001F8C, 0x001F84},
+{0x001F8D, 0x001F85},
+{0x001F8E, 0x001F86},
+{0x001F8F, 0x001F87},
+{0x001F98, 0x001F90},
+{0x001F99, 0x001F91},
+{0x001F9A, 0x001F92},
+{0x001F9B, 0x001F93},
+{0x001F9C, 0x001F94},
+{0x001F9D, 0x001F95},
+{0x001F9E, 0x001F96},
+{0x001F9F, 0x001F97},
+{0x001FA8, 0x001FA0},
+{0x001FA9, 0x001FA1},
+{0x001FAA, 0x001FA2},
+{0x001FAB, 0x001FA3},
+{0x001FAC, 0x001FA4},
+{0x001FAD, 0x001FA5},
+{0x001FAE, 0x001FA6},
+{0x001FAF, 0x001FA7},
+{0x001FB8, 0x001FB0},
+{0x001FB9, 0x001FB1},
+{0x001FBA, 0x001F70},
+{0x001FBB, 0x001F71},
+{0x001FBC, 0x001FB3},
+{0x001FC8, 0x001F72},
+{0x001FC9, 0x001F73},
+{0x001FCA, 0x001F74},
+{0x001FCB, 0x001F75},
+{0x001FCC, 0x001FC3},
+{0x001FD8, 0x001FD0},
+{0x001FD9, 0x001FD1},
+{0x001FDA, 0x001F76},
+{0x001FDB, 0x001F77},
+{0x001FE8, 0x001FE0},
+{0x001FE9, 0x001FE1},
+{0x001FEA, 0x001F7A},
+{0x001FEB, 0x001F7B},
+{0x001FEC, 0x001FE5},
+{0x001FF8, 0x001F78},
+{0x001FF9, 0x001F79},
+{0x001FFA, 0x001F7C},
+{0x001FFB, 0x001F7D},
+{0x001FFC, 0x001FF3},
+{0x002126, 0x0003C9},
+{0x00212A, 0x00006B},
+{0x00212B, 0x0000E5},
+{0x002132, 0x00214E},
+{0x002160, 0x002170},
+{0x002161, 0x002171},
+{0x002162, 0x002172},
+{0x002163, 0x002173},
+{0x002164, 0x002174},
+{0x002165, 0x002175},
+{0x002166, 0x002176},
+{0x002167, 0x002177},
+{0x002168, 0x002178},
+{0x002169, 0x002179},
+{0x00216A, 0x00217A},
+{0x00216B, 0x00217B},
+{0x00216C, 0x00217C},
+{0x00216D, 0x00217D},
+{0x00216E, 0x00217E},
+{0x00216F, 0x00217F},
+{0x002183, 0x002184},
+{0x0024B6, 0x0024D0},
+{0x0024B7, 0x0024D1},
+{0x0024B8, 0x0024D2},
+{0x0024B9, 0x0024D3},
+{0x0024BA, 0x0024D4},
+{0x0024BB, 0x0024D5},
+{0x0024BC, 0x0024D6},
+{0x0024BD, 0x0024D7},
+{0x0024BE, 0x0024D8},
+{0x0024BF, 0x0024D9},
+{0x0024C0, 0x0024DA},
+{0x0024C1, 0x0024DB},
+{0x0024C2, 0x0024DC},
+{0x0024C3, 0x0024DD},
+{0x0024C4, 0x0024DE},
+{0x0024C5, 0x0024DF},
+{0x0024C6, 0x0024E0},
+{0x0024C7, 0x0024E1},
+{0x0024C8, 0x0024E2},
+{0x0024C9, 0x0024E3},
+{0x0024CA, 0x0024E4},
+{0x0024CB, 0x0024E5},
+{0x0024CC, 0x0024E6},
+{0x0024CD, 0x0024E7},
+{0x0024CE, 0x0024E8},
+{0x0024CF, 0x0024E9},
+{0x002C00, 0x002C30},
+{0x002C01, 0x002C31},
+{0x002C02, 0x002C32},
+{0x002C03, 0x002C33},
+{0x002C04, 0x002C34},
+{0x002C05, 0x002C35},
+{0x002C06, 0x002C36},
+{0x002C07, 0x002C37},
+{0x002C08, 0x002C38},
+{0x002C09, 0x002C39},
+{0x002C0A, 0x002C3A},
+{0x002C0B, 0x002C3B},
+{0x002C0C, 0x002C3C},
+{0x002C0D, 0x002C3D},
+{0x002C0E, 0x002C3E},
+{0x002C0F, 0x002C3F},
+{0x002C10, 0x002C40},
+{0x002C11, 0x002C41},
+{0x002C12, 0x002C42},
+{0x002C13, 0x002C43},
+{0x002C14, 0x002C44},
+{0x002C15, 0x002C45},
+{0x002C16, 0x002C46},
+{0x002C17, 0x002C47},
+{0x002C18, 0x002C48},
+{0x002C19, 0x002C49},
+{0x002C1A, 0x002C4A},
+{0x002C1B, 0x002C4B},
+{0x002C1C, 0x002C4C},
+{0x002C1D, 0x002C4D},
+{0x002C1E, 0x002C4E},
+{0x002C1F, 0x002C4F},
+{0x002C20, 0x002C50},
+{0x002C21, 0x002C51},
+{0x002C22, 0x002C52},
+{0x002C23, 0x002C53},
+{0x002C24, 0x002C54},
+{0x002C25, 0x002C55},
+{0x002C26, 0x002C56},
+{0x002C27, 0x002C57},
+{0x002C28, 0x002C58},
+{0x002C29, 0x002C59},
+{0x002C2A, 0x002C5A},
+{0x002C2B, 0x002C5B},
+{0x002C2C, 0x002C5C},
+{0x002C2D, 0x002C5D},
+{0x002C2E, 0x002C5E},
+{0x002C2F, 0x002C5F},
+{0x002C60, 0x002C61},
+{0x002C62, 0x00026B},
+{0x002C63, 0x001D7D},
+{0x002C64, 0x00027D},
+{0x002C67, 0x002C68},
+{0x002C69, 0x002C6A},
+{0x002C6B, 0x002C6C},
+{0x002C6D, 0x000251},
+{0x002C6E, 0x000271},
+{0x002C6F, 0x000250},
+{0x002C70, 0x000252},
+{0x002C72, 0x002C73},
+{0x002C75, 0x002C76},
+{0x002C7E, 0x00023F},
+{0x002C7F, 0x000240},
+{0x002C80, 0x002C81},
+{0x002C82, 0x002C83},
+{0x002C84, 0x002C85},
+{0x002C86, 0x002C87},
+{0x002C88, 0x002C89},
+{0x002C8A, 0x002C8B},
+{0x002C8C, 0x002C8D},
+{0x002C8E, 0x002C8F},
+{0x002C90, 0x002C91},
+{0x002C92, 0x002C93},
+{0x002C94, 0x002C95},
+{0x002C96, 0x002C97},
+{0x002C98, 0x002C99},
+{0x002C9A, 0x002C9B},
+{0x002C9C, 0x002C9D},
+{0x002C9E, 0x002C9F},
+{0x002CA0, 0x002CA1},
+{0x002CA2, 0x002CA3},
+{0x002CA4, 0x002CA5},
+{0x002CA6, 0x002CA7},
+{0x002CA8, 0x002CA9},
+{0x002CAA, 0x002CAB},
+{0x002CAC, 0x002CAD},
+{0x002CAE, 0x002CAF},
+{0x002CB0, 0x002CB1},
+{0x002CB2, 0x002CB3},
+{0x002CB4, 0x002CB5},
+{0x002CB6, 0x002CB7},
+{0x002CB8, 0x002CB9},
+{0x002CBA, 0x002CBB},
+{0x002CBC, 0x002CBD},
+{0x002CBE, 0x002CBF},
+{0x002CC0, 0x002CC1},
+{0x002CC2, 0x002CC3},
+{0x002CC4, 0x002CC5},
+{0x002CC6, 0x002CC7},
+{0x002CC8, 0x002CC9},
+{0x002CCA, 0x002CCB},
+{0x002CCC, 0x002CCD},
+{0x002CCE, 0x002CCF},
+{0x002CD0, 0x002CD1},
+{0x002CD2, 0x002CD3},
+{0x002CD4, 0x002CD5},
+{0x002CD6, 0x002CD7},
+{0x002CD8, 0x002CD9},
+{0x002CDA, 0x002CDB},
+{0x002CDC, 0x002CDD},
+{0x002CDE, 0x002CDF},
+{0x002CE0, 0x002CE1},
+{0x002CE2, 0x002CE3},
+{0x002CEB, 0x002CEC},
+{0x002CED, 0x002CEE},
+{0x002CF2, 0x002CF3},
+{0x00A640, 0x00A641},
+{0x00A642, 0x00A643},
+{0x00A644, 0x00A645},
+{0x00A646, 0x00A647},
+{0x00A648, 0x00A649},
+{0x00A64A, 0x00A64B},
+{0x00A64C, 0x00A64D},
+{0x00A64E, 0x00A64F},
+{0x00A650, 0x00A651},
+{0x00A652, 0x00A653},
+{0x00A654, 0x00A655},
+{0x00A656, 0x00A657},
+{0x00A658, 0x00A659},
+{0x00A65A, 0x00A65B},
+{0x00A65C, 0x00A65D},
+{0x00A65E, 0x00A65F},
+{0x00A660, 0x00A661},
+{0x00A662, 0x00A663},
+{0x00A664, 0x00A665},
+{0x00A666, 0x00A667},
+{0x00A668, 0x00A669},
+{0x00A66A, 0x00A66B},
+{0x00A66C, 0x00A66D},
+{0x00A680, 0x00A681},
+{0x00A682, 0x00A683},
+{0x00A684, 0x00A685},
+{0x00A686, 0x00A687},
+{0x00A688, 0x00A689},
+{0x00A68A, 0x00A68B},
+{0x00A68C, 0x00A68D},
+{0x00A68E, 0x00A68F},
+{0x00A690, 0x00A691},
+{0x00A692, 0x00A693},
+{0x00A694, 0x00A695},
+{0x00A696, 0x00A697},
+{0x00A698, 0x00A699},
+{0x00A69A, 0x00A69B},
+{0x00A722, 0x00A723},
+{0x00A724, 0x00A725},
+{0x00A726, 0x00A727},
+{0x00A728, 0x00A729},
+{0x00A72A, 0x00A72B},
+{0x00A72C, 0x00A72D},
+{0x00A72E, 0x00A72F},
+{0x00A732, 0x00A733},
+{0x00A734, 0x00A735},
+{0x00A736, 0x00A737},
+{0x00A738, 0x00A739},
+{0x00A73A, 0x00A73B},
+{0x00A73C, 0x00A73D},
+{0x00A73E, 0x00A73F},
+{0x00A740, 0x00A741},
+{0x00A742, 0x00A743},
+{0x00A744, 0x00A745},
+{0x00A746, 0x00A747},
+{0x00A748, 0x00A749},
+{0x00A74A, 0x00A74B},
+{0x00A74C, 0x00A74D},
+{0x00A74E, 0x00A74F},
+{0x00A750, 0x00A751},
+{0x00A752, 0x00A753},
+{0x00A754, 0x00A755},
+{0x00A756, 0x00A757},
+{0x00A758, 0x00A759},
+{0x00A75A, 0x00A75B},
+{0x00A75C, 0x00A75D},
+{0x00A75E, 0x00A75F},
+{0x00A760, 0x00A761},
+{0x00A762, 0x00A763},
+{0x00A764, 0x00A765},
+{0x00A766, 0x00A767},
+{0x00A768, 0x00A769},
+{0x00A76A, 0x00A76B},
+{0x00A76C, 0x00A76D},
+{0x00A76E, 0x00A76F},
+{0x00A779, 0x00A77A},
+{0x00A77B, 0x00A77C},
+{0x00A77D, 0x001D79},
+{0x00A77E, 0x00A77F},
+{0x00A780, 0x00A781},
+{0x00A782, 0x00A783},
+{0x00A784, 0x00A785},
+{0x00A786, 0x00A787},
+{0x00A78B, 0x00A78C},
+{0x00A78D, 0x000265},
+{0x00A790, 0x00A791},
+{0x00A792, 0x00A793},
+{0x00A796, 0x00A797},
+{0x00A798, 0x00A799},
+{0x00A79A, 0x00A79B},
+{0x00A79C, 0x00A79D},
+{0x00A79E, 0x00A79F},
+{0x00A7A0, 0x00A7A1},
+{0x00A7A2, 0x00A7A3},
+{0x00A7A4, 0x00A7A5},
+{0x00A7A6, 0x00A7A7},
+{0x00A7A8, 0x00A7A9},
+{0x00A7AA, 0x000266},
+{0x00A7AB, 0x00025C},
+{0x00A7AC, 0x000261},
+{0x00A7AD, 0x00026C},
+{0x00A7AE, 0x00026A},
+{0x00A7B0, 0x00029E},
+{0x00A7B1, 0x000287},
+{0x00A7B2, 0x00029D},
+{0x00A7B3, 0x00AB53},
+{0x00A7B4, 0x00A7B5},
+{0x00A7B6, 0x00A7B7},
+{0x00A7B8, 0x00A7B9},
+{0x00A7BA, 0x00A7BB},
+{0x00A7BC, 0x00A7BD},
+{0x00A7BE, 0x00A7BF},
+{0x00A7C0, 0x00A7C1},
+{0x00A7C2, 0x00A7C3},
+{0x00A7C4, 0x00A794},
+{0x00A7C5, 0x000282},
+{0x00A7C6, 0x001D8E},
+{0x00A7C7, 0x00A7C8},
+{0x00A7C9, 0x00A7CA},
+{0x00A7D0, 0x00A7D1},
+{0x00A7D6, 0x00A7D7},
+{0x00A7D8, 0x00A7D9},
+{0x00A7F5, 0x00A7F6},
+{0x00FF21, 0x00FF41},
+{0x00FF22, 0x00FF42},
+{0x00FF23, 0x00FF43},
+{0x00FF24, 0x00FF44},
+{0x00FF25, 0x00FF45},
+{0x00FF26, 0x00FF46},
+{0x00FF27, 0x00FF47},
+{0x00FF28, 0x00FF48},
+{0x00FF29, 0x00FF49},
+{0x00FF2A, 0x00FF4A},
+{0x00FF2B, 0x00FF4B},
+{0x00FF2C, 0x00FF4C},
+{0x00FF2D, 0x00FF4D},
+{0x00FF2E, 0x00FF4E},
+{0x00FF2F, 0x00FF4F},
+{0x00FF30, 0x00FF50},
+{0x00FF31, 0x00FF51},
+{0x00FF32, 0x00FF52},
+{0x00FF33, 0x00FF53},
+{0x00FF34, 0x00FF54},
+{0x00FF35, 0x00FF55},
+{0x00FF36, 0x00FF56},
+{0x00FF37, 0x00FF57},
+{0x00FF38, 0x00FF58},
+{0x00FF39, 0x00FF59},
+{0x00FF3A, 0x00FF5A},
+{0x010400, 0x010428},
+{0x010401, 0x010429},
+{0x010402, 0x01042A},
+{0x010403, 0x01042B},
+{0x010404, 0x01042C},
+{0x010405, 0x01042D},
+{0x010406, 0x01042E},
+{0x010407, 0x01042F},
+{0x010408, 0x010430},
+{0x010409, 0x010431},
+{0x01040A, 0x010432},
+{0x01040B, 0x010433},
+{0x01040C, 0x010434},
+{0x01040D, 0x010435},
+{0x01040E, 0x010436},
+{0x01040F, 0x010437},
+{0x010410, 0x010438},
+{0x010411, 0x010439},
+{0x010412, 0x01043A},
+{0x010413, 0x01043B},
+{0x010414, 0x01043C},
+{0x010415, 0x01043D},
+{0x010416, 0x01043E},
+{0x010417, 0x01043F},
+{0x010418, 0x010440},
+{0x010419, 0x010441},
+{0x01041A, 0x010442},
+{0x01041B, 0x010443},
+{0x01041C, 0x010444},
+{0x01041D, 0x010445},
+{0x01041E, 0x010446},
+{0x01041F, 0x010447},
+{0x010420, 0x010448},
+{0x010421, 0x010449},
+{0x010422, 0x01044A},
+{0x010423, 0x01044B},
+{0x010424, 0x01044C},
+{0x010425, 0x01044D},
+{0x010426, 0x01044E},
+{0x010427, 0x01044F},
+{0x0104B0, 0x0104D8},
+{0x0104B1, 0x0104D9},
+{0x0104B2, 0x0104DA},
+{0x0104B3, 0x0104DB},
+{0x0104B4, 0x0104DC},
+{0x0104B5, 0x0104DD},
+{0x0104B6, 0x0104DE},
+{0x0104B7, 0x0104DF},
+{0x0104B8, 0x0104E0},
+{0x0104B9, 0x0104E1},
+{0x0104BA, 0x0104E2},
+{0x0104BB, 0x0104E3},
+{0x0104BC, 0x0104E4},
+{0x0104BD, 0x0104E5},
+{0x0104BE, 0x0104E6},
+{0x0104BF, 0x0104E7},
+{0x0104C0, 0x0104E8},
+{0x0104C1, 0x0104E9},
+{0x0104C2, 0x0104EA},
+{0x0104C3, 0x0104EB},
+{0x0104C4, 0x0104EC},
+{0x0104C5, 0x0104ED},
+{0x0104C6, 0x0104EE},
+{0x0104C7, 0x0104EF},
+{0x0104C8, 0x0104F0},
+{0x0104C9, 0x0104F1},
+{0x0104CA, 0x0104F2},
+{0x0104CB, 0x0104F3},
+{0x0104CC, 0x0104F4},
+{0x0104CD, 0x0104F5},
+{0x0104CE, 0x0104F6},
+{0x0104CF, 0x0104F7},
+{0x0104D0, 0x0104F8},
+{0x0104D1, 0x0104F9},
+{0x0104D2, 0x0104FA},
+{0x0104D3, 0x0104FB},
+{0x010570, 0x010597},
+{0x010571, 0x010598},
+{0x010572, 0x010599},
+{0x010573, 0x01059A},
+{0x010574, 0x01059B},
+{0x010575, 0x01059C},
+{0x010576, 0x01059D},
+{0x010577, 0x01059E},
+{0x010578, 0x01059F},
+{0x010579, 0x0105A0},
+{0x01057A, 0x0105A1},
+{0x01057C, 0x0105A3},
+{0x01057D, 0x0105A4},
+{0x01057E, 0x0105A5},
+{0x01057F, 0x0105A6},
+{0x010580, 0x0105A7},
+{0x010581, 0x0105A8},
+{0x010582, 0x0105A9},
+{0x010583, 0x0105AA},
+{0x010584, 0x0105AB},
+{0x010585, 0x0105AC},
+{0x010586, 0x0105AD},
+{0x010587, 0x0105AE},
+{0x010588, 0x0105AF},
+{0x010589, 0x0105B0},
+{0x01058A, 0x0105B1},
+{0x01058C, 0x0105B3},
+{0x01058D, 0x0105B4},
+{0x01058E, 0x0105B5},
+{0x01058F, 0x0105B6},
+{0x010590, 0x0105B7},
+{0x010591, 0x0105B8},
+{0x010592, 0x0105B9},
+{0x010594, 0x0105BB},
+{0x010595, 0x0105BC},
+{0x010C80, 0x010CC0},
+{0x010C81, 0x010CC1},
+{0x010C82, 0x010CC2},
+{0x010C83, 0x010CC3},
+{0x010C84, 0x010CC4},
+{0x010C85, 0x010CC5},
+{0x010C86, 0x010CC6},
+{0x010C87, 0x010CC7},
+{0x010C88, 0x010CC8},
+{0x010C89, 0x010CC9},
+{0x010C8A, 0x010CCA},
+{0x010C8B, 0x010CCB},
+{0x010C8C, 0x010CCC},
+{0x010C8D, 0x010CCD},
+{0x010C8E, 0x010CCE},
+{0x010C8F, 0x010CCF},
+{0x010C90, 0x010CD0},
+{0x010C91, 0x010CD1},
+{0x010C92, 0x010CD2},
+{0x010C93, 0x010CD3},
+{0x010C94, 0x010CD4},
+{0x010C95, 0x010CD5},
+{0x010C96, 0x010CD6},
+{0x010C97, 0x010CD7},
+{0x010C98, 0x010CD8},
+{0x010C99, 0x010CD9},
+{0x010C9A, 0x010CDA},
+{0x010C9B, 0x010CDB},
+{0x010C9C, 0x010CDC},
+{0x010C9D, 0x010CDD},
+{0x010C9E, 0x010CDE},
+{0x010C9F, 0x010CDF},
+{0x010CA0, 0x010CE0},
+{0x010CA1, 0x010CE1},
+{0x010CA2, 0x010CE2},
+{0x010CA3, 0x010CE3},
+{0x010CA4, 0x010CE4},
+{0x010CA5, 0x010CE5},
+{0x010CA6, 0x010CE6},
+{0x010CA7, 0x010CE7},
+{0x010CA8, 0x010CE8},
+{0x010CA9, 0x010CE9},
+{0x010CAA, 0x010CEA},
+{0x010CAB, 0x010CEB},
+{0x010CAC, 0x010CEC},
+{0x010CAD, 0x010CED},
+{0x010CAE, 0x010CEE},
+{0x010CAF, 0x010CEF},
+{0x010CB0, 0x010CF0},
+{0x010CB1, 0x010CF1},
+{0x010CB2, 0x010CF2},
+{0x0118A0, 0x0118C0},
+{0x0118A1, 0x0118C1},
+{0x0118A2, 0x0118C2},
+{0x0118A3, 0x0118C3},
+{0x0118A4, 0x0118C4},
+{0x0118A5, 0x0118C5},
+{0x0118A6, 0x0118C6},
+{0x0118A7, 0x0118C7},
+{0x0118A8, 0x0118C8},
+{0x0118A9, 0x0118C9},
+{0x0118AA, 0x0118CA},
+{0x0118AB, 0x0118CB},
+{0x0118AC, 0x0118CC},
+{0x0118AD, 0x0118CD},
+{0x0118AE, 0x0118CE},
+{0x0118AF, 0x0118CF},
+{0x0118B0, 0x0118D0},
+{0x0118B1, 0x0118D1},
+{0x0118B2, 0x0118D2},
+{0x0118B3, 0x0118D3},
+{0x0118B4, 0x0118D4},
+{0x0118B5, 0x0118D5},
+{0x0118B6, 0x0118D6},
+{0x0118B7, 0x0118D7},
+{0x0118B8, 0x0118D8},
+{0x0118B9, 0x0118D9},
+{0x0118BA, 0x0118DA},
+{0x0118BB, 0x0118DB},
+{0x0118BC, 0x0118DC},
+{0x0118BD, 0x0118DD},
+{0x0118BE, 0x0118DE},
+{0x0118BF, 0x0118DF},
+{0x016E40, 0x016E60},
+{0x016E41, 0x016E61},
+{0x016E42, 0x016E62},
+{0x016E43, 0x016E63},
+{0x016E44, 0x016E64},
+{0x016E45, 0x016E65},
+{0x016E46, 0x016E66},
+{0x016E47, 0x016E67},
+{0x016E48, 0x016E68},
+{0x016E49, 0x016E69},
+{0x016E4A, 0x016E6A},
+{0x016E4B, 0x016E6B},
+{0x016E4C, 0x016E6C},
+{0x016E4D, 0x016E6D},
+{0x016E4E, 0x016E6E},
+{0x016E4F, 0x016E6F},
+{0x016E50, 0x016E70},
+{0x016E51, 0x016E71},
+{0x016E52, 0x016E72},
+{0x016E53, 0x016E73},
+{0x016E54, 0x016E74},
+{0x016E55, 0x016E75},
+{0x016E56, 0x016E76},
+{0x016E57, 0x016E77},
+{0x016E58, 0x016E78},
+{0x016E59, 0x016E79},
+{0x016E5A, 0x016E7A},
+{0x016E5B, 0x016E7B},
+{0x016E5C, 0x016E7C},
+{0x016E5D, 0x016E7D},
+{0x016E5E, 0x016E7E},
+{0x016E5F, 0x016E7F},
+{0x01E900, 0x01E922},
+{0x01E901, 0x01E923},
+{0x01E902, 0x01E924},
+{0x01E903, 0x01E925},
+{0x01E904, 0x01E926},
+{0x01E905, 0x01E927},
+{0x01E906, 0x01E928},
+{0x01E907, 0x01E929},
+{0x01E908, 0x01E92A},
+{0x01E909, 0x01E92B},
+{0x01E90A, 0x01E92C},
+{0x01E90B, 0x01E92D},
+{0x01E90C, 0x01E92E},
+{0x01E90D, 0x01E92F},
+{0x01E90E, 0x01E930},
+{0x01E90F, 0x01E931},
+{0x01E910, 0x01E932},
+{0x01E911, 0x01E933},
+{0x01E912, 0x01E934},
+{0x01E913, 0x01E935},
+{0x01E914, 0x01E936},
+{0x01E915, 0x01E937},
+{0x01E916, 0x01E938},
+{0x01E917, 0x01E939},
+{0x01E918, 0x01E93A},
+{0x01E919, 0x01E93B},
+{0x01E91A, 0x01E93C},
+{0x01E91B, 0x01E93D},
+{0x01E91C, 0x01E93E},
+{0x01E91D, 0x01E93F},
+{0x01E91E, 0x01E940},
+{0x01E91F, 0x01E941},
+{0x01E920, 0x01E942},
+{0x01E921, 0x01E943},
+};
+
+const std::unordered_map<uint32_t, uint32_t> unicode_map_uppercase = {
+{0x000061, 0x000041},
+{0x000062, 0x000042},
+{0x000063, 0x000043},
+{0x000064, 0x000044},
+{0x000065, 0x000045},
+{0x000066, 0x000046},
+{0x000067, 0x000047},
+{0x000068, 0x000048},
+{0x000069, 0x000049},
+{0x00006A, 0x00004A},
+{0x00006B, 0x00004B},
+{0x00006C, 0x00004C},
+{0x00006D, 0x00004D},
+{0x00006E, 0x00004E},
+{0x00006F, 0x00004F},
+{0x000070, 0x000050},
+{0x000071, 0x000051},
+{0x000072, 0x000052},
+{0x000073, 0x000053},
+{0x000074, 0x000054},
+{0x000075, 0x000055},
+{0x000076, 0x000056},
+{0x000077, 0x000057},
+{0x000078, 0x000058},
+{0x000079, 0x000059},
+{0x00007A, 0x00005A},
+{0x0000B5, 0x00039C},
+{0x0000E0, 0x0000C0},
+{0x0000E1, 0x0000C1},
+{0x0000E2, 0x0000C2},
+{0x0000E3, 0x0000C3},
+{0x0000E4, 0x0000C4},
+{0x0000E5, 0x0000C5},
+{0x0000E6, 0x0000C6},
+{0x0000E7, 0x0000C7},
+{0x0000E8, 0x0000C8},
+{0x0000E9, 0x0000C9},
+{0x0000EA, 0x0000CA},
+{0x0000EB, 0x0000CB},
+{0x0000EC, 0x0000CC},
+{0x0000ED, 0x0000CD},
+{0x0000EE, 0x0000CE},
+{0x0000EF, 0x0000CF},
+{0x0000F0, 0x0000D0},
+{0x0000F1, 0x0000D1},
+{0x0000F2, 0x0000D2},
+{0x0000F3, 0x0000D3},
+{0x0000F4, 0x0000D4},
+{0x0000F5, 0x0000D5},
+{0x0000F6, 0x0000D6},
+{0x0000F8, 0x0000D8},
+{0x0000F9, 0x0000D9},
+{0x0000FA, 0x0000DA},
+{0x0000FB, 0x0000DB},
+{0x0000FC, 0x0000DC},
+{0x0000FD, 0x0000DD},
+{0x0000FE, 0x0000DE},
+{0x0000FF, 0x000178},
+{0x000101, 0x000100},
+{0x000103, 0x000102},
+{0x000105, 0x000104},
+{0x000107, 0x000106},
+{0x000109, 0x000108},
+{0x00010B, 0x00010A},
+{0x00010D, 0x00010C},
+{0x00010F, 0x00010E},
+{0x000111, 0x000110},
+{0x000113, 0x000112},
+{0x000115, 0x000114},
+{0x000117, 0x000116},
+{0x000119, 0x000118},
+{0x00011B, 0x00011A},
+{0x00011D, 0x00011C},
+{0x00011F, 0x00011E},
+{0x000121, 0x000120},
+{0x000123, 0x000122},
+{0x000125, 0x000124},
+{0x000127, 0x000126},
+{0x000129, 0x000128},
+{0x00012B, 0x00012A},
+{0x00012D, 0x00012C},
+{0x00012F, 0x00012E},
+{0x000131, 0x000049},
+{0x000133, 0x000132},
+{0x000135, 0x000134},
+{0x000137, 0x000136},
+{0x00013A, 0x000139},
+{0x00013C, 0x00013B},
+{0x00013E, 0x00013D},
+{0x000140, 0x00013F},
+{0x000142, 0x000141},
+{0x000144, 0x000143},
+{0x000146, 0x000145},
+{0x000148, 0x000147},
+{0x00014B, 0x00014A},
+{0x00014D, 0x00014C},
+{0x00014F, 0x00014E},
+{0x000151, 0x000150},
+{0x000153, 0x000152},
+{0x000155, 0x000154},
+{0x000157, 0x000156},
+{0x000159, 0x000158},
+{0x00015B, 0x00015A},
+{0x00015D, 0x00015C},
+{0x00015F, 0x00015E},
+{0x000161, 0x000160},
+{0x000163, 0x000162},
+{0x000165, 0x000164},
+{0x000167, 0x000166},
+{0x000169, 0x000168},
+{0x00016B, 0x00016A},
+{0x00016D, 0x00016C},
+{0x00016F, 0x00016E},
+{0x000171, 0x000170},
+{0x000173, 0x000172},
+{0x000175, 0x000174},
+{0x000177, 0x000176},
+{0x00017A, 0x000179},
+{0x00017C, 0x00017B},
+{0x00017E, 0x00017D},
+{0x00017F, 0x000053},
+{0x000180, 0x000243},
+{0x000183, 0x000182},
+{0x000185, 0x000184},
+{0x000188, 0x000187},
+{0x00018C, 0x00018B},
+{0x000192, 0x000191},
+{0x000195, 0x0001F6},
+{0x000199, 0x000198},
+{0x00019A, 0x00023D},
+{0x00019E, 0x000220},
+{0x0001A1, 0x0001A0},
+{0x0001A3, 0x0001A2},
+{0x0001A5, 0x0001A4},
+{0x0001A8, 0x0001A7},
+{0x0001AD, 0x0001AC},
+{0x0001B0, 0x0001AF},
+{0x0001B4, 0x0001B3},
+{0x0001B6, 0x0001B5},
+{0x0001B9, 0x0001B8},
+{0x0001BD, 0x0001BC},
+{0x0001BF, 0x0001F7},
+{0x0001C5, 0x0001C4},
+{0x0001C6, 0x0001C4},
+{0x0001C8, 0x0001C7},
+{0x0001C9, 0x0001C7},
+{0x0001CB, 0x0001CA},
+{0x0001CC, 0x0001CA},
+{0x0001CE, 0x0001CD},
+{0x0001D0, 0x0001CF},
+{0x0001D2, 0x0001D1},
+{0x0001D4, 0x0001D3},
+{0x0001D6, 0x0001D5},
+{0x0001D8, 0x0001D7},
+{0x0001DA, 0x0001D9},
+{0x0001DC, 0x0001DB},
+{0x0001DD, 0x00018E},
+{0x0001DF, 0x0001DE},
+{0x0001E1, 0x0001E0},
+{0x0001E3, 0x0001E2},
+{0x0001E5, 0x0001E4},
+{0x0001E7, 0x0001E6},
+{0x0001E9, 0x0001E8},
+{0x0001EB, 0x0001EA},
+{0x0001ED, 0x0001EC},
+{0x0001EF, 0x0001EE},
+{0x0001F2, 0x0001F1},
+{0x0001F3, 0x0001F1},
+{0x0001F5, 0x0001F4},
+{0x0001F9, 0x0001F8},
+{0x0001FB, 0x0001FA},
+{0x0001FD, 0x0001FC},
+{0x0001FF, 0x0001FE},
+{0x000201, 0x000200},
+{0x000203, 0x000202},
+{0x000205, 0x000204},
+{0x000207, 0x000206},
+{0x000209, 0x000208},
+{0x00020B, 0x00020A},
+{0x00020D, 0x00020C},
+{0x00020F, 0x00020E},
+{0x000211, 0x000210},
+{0x000213, 0x000212},
+{0x000215, 0x000214},
+{0x000217, 0x000216},
+{0x000219, 0x000218},
+{0x00021B, 0x00021A},
+{0x00021D, 0x00021C},
+{0x00021F, 0x00021E},
+{0x000223, 0x000222},
+{0x000225, 0x000224},
+{0x000227, 0x000226},
+{0x000229, 0x000228},
+{0x00022B, 0x00022A},
+{0x00022D, 0x00022C},
+{0x00022F, 0x00022E},
+{0x000231, 0x000230},
+{0x000233, 0x000232},
+{0x00023C, 0x00023B},
+{0x00023F, 0x002C7E},
+{0x000240, 0x002C7F},
+{0x000242, 0x000241},
+{0x000247, 0x000246},
+{0x000249, 0x000248},
+{0x00024B, 0x00024A},
+{0x00024D, 0x00024C},
+{0x00024F, 0x00024E},
+{0x000250, 0x002C6F},
+{0x000251, 0x002C6D},
+{0x000252, 0x002C70},
+{0x000253, 0x000181},
+{0x000254, 0x000186},
+{0x000256, 0x000189},
+{0x000257, 0x00018A},
+{0x000259, 0x00018F},
+{0x00025B, 0x000190},
+{0x00025C, 0x00A7AB},
+{0x000260, 0x000193},
+{0x000261, 0x00A7AC},
+{0x000263, 0x000194},
+{0x000265, 0x00A78D},
+{0x000266, 0x00A7AA},
+{0x000268, 0x000197},
+{0x000269, 0x000196},
+{0x00026A, 0x00A7AE},
+{0x00026B, 0x002C62},
+{0x00026C, 0x00A7AD},
+{0x00026F, 0x00019C},
+{0x000271, 0x002C6E},
+{0x000272, 0x00019D},
+{0x000275, 0x00019F},
+{0x00027D, 0x002C64},
+{0x000280, 0x0001A6},
+{0x000282, 0x00A7C5},
+{0x000283, 0x0001A9},
+{0x000287, 0x00A7B1},
+{0x000288, 0x0001AE},
+{0x000289, 0x000244},
+{0x00028A, 0x0001B1},
+{0x00028B, 0x0001B2},
+{0x00028C, 0x000245},
+{0x000292, 0x0001B7},
+{0x00029D, 0x00A7B2},
+{0x00029E, 0x00A7B0},
+{0x000345, 0x000399},
+{0x000371, 0x000370},
+{0x000373, 0x000372},
+{0x000377, 0x000376},
+{0x00037B, 0x0003FD},
+{0x00037C, 0x0003FE},
+{0x00037D, 0x0003FF},
+{0x0003AC, 0x000386},
+{0x0003AD, 0x000388},
+{0x0003AE, 0x000389},
+{0x0003AF, 0x00038A},
+{0x0003B1, 0x000391},
+{0x0003B2, 0x000392},
+{0x0003B3, 0x000393},
+{0x0003B4, 0x000394},
+{0x0003B5, 0x000395},
+{0x0003B6, 0x000396},
+{0x0003B7, 0x000397},
+{0x0003B8, 0x000398},
+{0x0003B9, 0x000399},
+{0x0003BA, 0x00039A},
+{0x0003BB, 0x00039B},
+{0x0003BC, 0x00039C},
+{0x0003BD, 0x00039D},
+{0x0003BE, 0x00039E},
+{0x0003BF, 0x00039F},
+{0x0003C0, 0x0003A0},
+{0x0003C1, 0x0003A1},
+{0x0003C2, 0x0003A3},
+{0x0003C3, 0x0003A3},
+{0x0003C4, 0x0003A4},
+{0x0003C5, 0x0003A5},
+{0x0003C6, 0x0003A6},
+{0x0003C7, 0x0003A7},
+{0x0003C8, 0x0003A8},
+{0x0003C9, 0x0003A9},
+{0x0003CA, 0x0003AA},
+{0x0003CB, 0x0003AB},
+{0x0003CC, 0x00038C},
+{0x0003CD, 0x00038E},
+{0x0003CE, 0x00038F},
+{0x0003D0, 0x000392},
+{0x0003D1, 0x000398},
+{0x0003D5, 0x0003A6},
+{0x0003D6, 0x0003A0},
+{0x0003D7, 0x0003CF},
+{0x0003D9, 0x0003D8},
+{0x0003DB, 0x0003DA},
+{0x0003DD, 0x0003DC},
+{0x0003DF, 0x0003DE},
+{0x0003E1, 0x0003E0},
+{0x0003E3, 0x0003E2},
+{0x0003E5, 0x0003E4},
+{0x0003E7, 0x0003E6},
+{0x0003E9, 0x0003E8},
+{0x0003EB, 0x0003EA},
+{0x0003ED, 0x0003EC},
+{0x0003EF, 0x0003EE},
+{0x0003F0, 0x00039A},
+{0x0003F1, 0x0003A1},
+{0x0003F2, 0x0003F9},
+{0x0003F3, 0x00037F},
+{0x0003F5, 0x000395},
+{0x0003F8, 0x0003F7},
+{0x0003FB, 0x0003FA},
+{0x000430, 0x000410},
+{0x000431, 0x000411},
+{0x000432, 0x000412},
+{0x000433, 0x000413},
+{0x000434, 0x000414},
+{0x000435, 0x000415},
+{0x000436, 0x000416},
+{0x000437, 0x000417},
+{0x000438, 0x000418},
+{0x000439, 0x000419},
+{0x00043A, 0x00041A},
+{0x00043B, 0x00041B},
+{0x00043C, 0x00041C},
+{0x00043D, 0x00041D},
+{0x00043E, 0x00041E},
+{0x00043F, 0x00041F},
+{0x000440, 0x000420},
+{0x000441, 0x000421},
+{0x000442, 0x000422},
+{0x000443, 0x000423},
+{0x000444, 0x000424},
+{0x000445, 0x000425},
+{0x000446, 0x000426},
+{0x000447, 0x000427},
+{0x000448, 0x000428},
+{0x000449, 0x000429},
+{0x00044A, 0x00042A},
+{0x00044B, 0x00042B},
+{0x00044C, 0x00042C},
+{0x00044D, 0x00042D},
+{0x00044E, 0x00042E},
+{0x00044F, 0x00042F},
+{0x000450, 0x000400},
+{0x000451, 0x000401},
+{0x000452, 0x000402},
+{0x000453, 0x000403},
+{0x000454, 0x000404},
+{0x000455, 0x000405},
+{0x000456, 0x000406},
+{0x000457, 0x000407},
+{0x000458, 0x000408},
+{0x000459, 0x000409},
+{0x00045A, 0x00040A},
+{0x00045B, 0x00040B},
+{0x00045C, 0x00040C},
+{0x00045D, 0x00040D},
+{0x00045E, 0x00040E},
+{0x00045F, 0x00040F},
+{0x000461, 0x000460},
+{0x000463, 0x000462},
+{0x000465, 0x000464},
+{0x000467, 0x000466},
+{0x000469, 0x000468},
+{0x00046B, 0x00046A},
+{0x00046D, 0x00046C},
+{0x00046F, 0x00046E},
+{0x000471, 0x000470},
+{0x000473, 0x000472},
+{0x000475, 0x000474},
+{0x000477, 0x000476},
+{0x000479, 0x000478},
+{0x00047B, 0x00047A},
+{0x00047D, 0x00047C},
+{0x00047F, 0x00047E},
+{0x000481, 0x000480},
+{0x00048B, 0x00048A},
+{0x00048D, 0x00048C},
+{0x00048F, 0x00048E},
+{0x000491, 0x000490},
+{0x000493, 0x000492},
+{0x000495, 0x000494},
+{0x000497, 0x000496},
+{0x000499, 0x000498},
+{0x00049B, 0x00049A},
+{0x00049D, 0x00049C},
+{0x00049F, 0x00049E},
+{0x0004A1, 0x0004A0},
+{0x0004A3, 0x0004A2},
+{0x0004A5, 0x0004A4},
+{0x0004A7, 0x0004A6},
+{0x0004A9, 0x0004A8},
+{0x0004AB, 0x0004AA},
+{0x0004AD, 0x0004AC},
+{0x0004AF, 0x0004AE},
+{0x0004B1, 0x0004B0},
+{0x0004B3, 0x0004B2},
+{0x0004B5, 0x0004B4},
+{0x0004B7, 0x0004B6},
+{0x0004B9, 0x0004B8},
+{0x0004BB, 0x0004BA},
+{0x0004BD, 0x0004BC},
+{0x0004BF, 0x0004BE},
+{0x0004C2, 0x0004C1},
+{0x0004C4, 0x0004C3},
+{0x0004C6, 0x0004C5},
+{0x0004C8, 0x0004C7},
+{0x0004CA, 0x0004C9},
+{0x0004CC, 0x0004CB},
+{0x0004CE, 0x0004CD},
+{0x0004CF, 0x0004C0},
+{0x0004D1, 0x0004D0},
+{0x0004D3, 0x0004D2},
+{0x0004D5, 0x0004D4},
+{0x0004D7, 0x0004D6},
+{0x0004D9, 0x0004D8},
+{0x0004DB, 0x0004DA},
+{0x0004DD, 0x0004DC},
+{0x0004DF, 0x0004DE},
+{0x0004E1, 0x0004E0},
+{0x0004E3, 0x0004E2},
+{0x0004E5, 0x0004E4},
+{0x0004E7, 0x0004E6},
+{0x0004E9, 0x0004E8},
+{0x0004EB, 0x0004EA},
+{0x0004ED, 0x0004EC},
+{0x0004EF, 0x0004EE},
+{0x0004F1, 0x0004F0},
+{0x0004F3, 0x0004F2},
+{0x0004F5, 0x0004F4},
+{0x0004F7, 0x0004F6},
+{0x0004F9, 0x0004F8},
+{0x0004FB, 0x0004FA},
+{0x0004FD, 0x0004FC},
+{0x0004FF, 0x0004FE},
+{0x000501, 0x000500},
+{0x000503, 0x000502},
+{0x000505, 0x000504},
+{0x000507, 0x000506},
+{0x000509, 0x000508},
+{0x00050B, 0x00050A},
+{0x00050D, 0x00050C},
+{0x00050F, 0x00050E},
+{0x000511, 0x000510},
+{0x000513, 0x000512},
+{0x000515, 0x000514},
+{0x000517, 0x000516},
+{0x000519, 0x000518},
+{0x00051B, 0x00051A},
+{0x00051D, 0x00051C},
+{0x00051F, 0x00051E},
+{0x000521, 0x000520},
+{0x000523, 0x000522},
+{0x000525, 0x000524},
+{0x000527, 0x000526},
+{0x000529, 0x000528},
+{0x00052B, 0x00052A},
+{0x00052D, 0x00052C},
+{0x00052F, 0x00052E},
+{0x000561, 0x000531},
+{0x000562, 0x000532},
+{0x000563, 0x000533},
+{0x000564, 0x000534},
+{0x000565, 0x000535},
+{0x000566, 0x000536},
+{0x000567, 0x000537},
+{0x000568, 0x000538},
+{0x000569, 0x000539},
+{0x00056A, 0x00053A},
+{0x00056B, 0x00053B},
+{0x00056C, 0x00053C},
+{0x00056D, 0x00053D},
+{0x00056E, 0x00053E},
+{0x00056F, 0x00053F},
+{0x000570, 0x000540},
+{0x000571, 0x000541},
+{0x000572, 0x000542},
+{0x000573, 0x000543},
+{0x000574, 0x000544},
+{0x000575, 0x000545},
+{0x000576, 0x000546},
+{0x000577, 0x000547},
+{0x000578, 0x000548},
+{0x000579, 0x000549},
+{0x00057A, 0x00054A},
+{0x00057B, 0x00054B},
+{0x00057C, 0x00054C},
+{0x00057D, 0x00054D},
+{0x00057E, 0x00054E},
+{0x00057F, 0x00054F},
+{0x000580, 0x000550},
+{0x000581, 0x000551},
+{0x000582, 0x000552},
+{0x000583, 0x000553},
+{0x000584, 0x000554},
+{0x000585, 0x000555},
+{0x000586, 0x000556},
+{0x0010D0, 0x001C90},
+{0x0010D1, 0x001C91},
+{0x0010D2, 0x001C92},
+{0x0010D3, 0x001C93},
+{0x0010D4, 0x001C94},
+{0x0010D5, 0x001C95},
+{0x0010D6, 0x001C96},
+{0x0010D7, 0x001C97},
+{0x0010D8, 0x001C98},
+{0x0010D9, 0x001C99},
+{0x0010DA, 0x001C9A},
+{0x0010DB, 0x001C9B},
+{0x0010DC, 0x001C9C},
+{0x0010DD, 0x001C9D},
+{0x0010DE, 0x001C9E},
+{0x0010DF, 0x001C9F},
+{0x0010E0, 0x001CA0},
+{0x0010E1, 0x001CA1},
+{0x0010E2, 0x001CA2},
+{0x0010E3, 0x001CA3},
+{0x0010E4, 0x001CA4},
+{0x0010E5, 0x001CA5},
+{0x0010E6, 0x001CA6},
+{0x0010E7, 0x001CA7},
+{0x0010E8, 0x001CA8},
+{0x0010E9, 0x001CA9},
+{0x0010EA, 0x001CAA},
+{0x0010EB, 0x001CAB},
+{0x0010EC, 0x001CAC},
+{0x0010ED, 0x001CAD},
+{0x0010EE, 0x001CAE},
+{0x0010EF, 0x001CAF},
+{0x0010F0, 0x001CB0},
+{0x0010F1, 0x001CB1},
+{0x0010F2, 0x001CB2},
+{0x0010F3, 0x001CB3},
+{0x0010F4, 0x001CB4},
+{0x0010F5, 0x001CB5},
+{0x0010F6, 0x001CB6},
+{0x0010F7, 0x001CB7},
+{0x0010F8, 0x001CB8},
+{0x0010F9, 0x001CB9},
+{0x0010FA, 0x001CBA},
+{0x0010FD, 0x001CBD},
+{0x0010FE, 0x001CBE},
+{0x0010FF, 0x001CBF},
+{0x0013F8, 0x0013F0},
+{0x0013F9, 0x0013F1},
+{0x0013FA, 0x0013F2},
+{0x0013FB, 0x0013F3},
+{0x0013FC, 0x0013F4},
+{0x0013FD, 0x0013F5},
+{0x001C80, 0x000412},
+{0x001C81, 0x000414},
+{0x001C82, 0x00041E},
+{0x001C83, 0x000421},
+{0x001C84, 0x000422},
+{0x001C85, 0x000422},
+{0x001C86, 0x00042A},
+{0x001C87, 0x000462},
+{0x001C88, 0x00A64A},
+{0x001D79, 0x00A77D},
+{0x001D7D, 0x002C63},
+{0x001D8E, 0x00A7C6},
+{0x001E01, 0x001E00},
+{0x001E03, 0x001E02},
+{0x001E05, 0x001E04},
+{0x001E07, 0x001E06},
+{0x001E09, 0x001E08},
+{0x001E0B, 0x001E0A},
+{0x001E0D, 0x001E0C},
+{0x001E0F, 0x001E0E},
+{0x001E11, 0x001E10},
+{0x001E13, 0x001E12},
+{0x001E15, 0x001E14},
+{0x001E17, 0x001E16},
+{0x001E19, 0x001E18},
+{0x001E1B, 0x001E1A},
+{0x001E1D, 0x001E1C},
+{0x001E1F, 0x001E1E},
+{0x001E21, 0x001E20},
+{0x001E23, 0x001E22},
+{0x001E25, 0x001E24},
+{0x001E27, 0x001E26},
+{0x001E29, 0x001E28},
+{0x001E2B, 0x001E2A},
+{0x001E2D, 0x001E2C},
+{0x001E2F, 0x001E2E},
+{0x001E31, 0x001E30},
+{0x001E33, 0x001E32},
+{0x001E35, 0x001E34},
+{0x001E37, 0x001E36},
+{0x001E39, 0x001E38},
+{0x001E3B, 0x001E3A},
+{0x001E3D, 0x001E3C},
+{0x001E3F, 0x001E3E},
+{0x001E41, 0x001E40},
+{0x001E43, 0x001E42},
+{0x001E45, 0x001E44},
+{0x001E47, 0x001E46},
+{0x001E49, 0x001E48},
+{0x001E4B, 0x001E4A},
+{0x001E4D, 0x001E4C},
+{0x001E4F, 0x001E4E},
+{0x001E51, 0x001E50},
+{0x001E53, 0x001E52},
+{0x001E55, 0x001E54},
+{0x001E57, 0x001E56},
+{0x001E59, 0x001E58},
+{0x001E5B, 0x001E5A},
+{0x001E5D, 0x001E5C},
+{0x001E5F, 0x001E5E},
+{0x001E61, 0x001E60},
+{0x001E63, 0x001E62},
+{0x001E65, 0x001E64},
+{0x001E67, 0x001E66},
+{0x001E69, 0x001E68},
+{0x001E6B, 0x001E6A},
+{0x001E6D, 0x001E6C},
+{0x001E6F, 0x001E6E},
+{0x001E71, 0x001E70},
+{0x001E73, 0x001E72},
+{0x001E75, 0x001E74},
+{0x001E77, 0x001E76},
+{0x001E79, 0x001E78},
+{0x001E7B, 0x001E7A},
+{0x001E7D, 0x001E7C},
+{0x001E7F, 0x001E7E},
+{0x001E81, 0x001E80},
+{0x001E83, 0x001E82},
+{0x001E85, 0x001E84},
+{0x001E87, 0x001E86},
+{0x001E89, 0x001E88},
+{0x001E8B, 0x001E8A},
+{0x001E8D, 0x001E8C},
+{0x001E8F, 0x001E8E},
+{0x001E91, 0x001E90},
+{0x001E93, 0x001E92},
+{0x001E95, 0x001E94},
+{0x001E9B, 0x001E60},
+{0x001EA1, 0x001EA0},
+{0x001EA3, 0x001EA2},
+{0x001EA5, 0x001EA4},
+{0x001EA7, 0x001EA6},
+{0x001EA9, 0x001EA8},
+{0x001EAB, 0x001EAA},
+{0x001EAD, 0x001EAC},
+{0x001EAF, 0x001EAE},
+{0x001EB1, 0x001EB0},
+{0x001EB3, 0x001EB2},
+{0x001EB5, 0x001EB4},
+{0x001EB7, 0x001EB6},
+{0x001EB9, 0x001EB8},
+{0x001EBB, 0x001EBA},
+{0x001EBD, 0x001EBC},
+{0x001EBF, 0x001EBE},
+{0x001EC1, 0x001EC0},
+{0x001EC3, 0x001EC2},
+{0x001EC5, 0x001EC4},
+{0x001EC7, 0x001EC6},
+{0x001EC9, 0x001EC8},
+{0x001ECB, 0x001ECA},
+{0x001ECD, 0x001ECC},
+{0x001ECF, 0x001ECE},
+{0x001ED1, 0x001ED0},
+{0x001ED3, 0x001ED2},
+{0x001ED5, 0x001ED4},
+{0x001ED7, 0x001ED6},
+{0x001ED9, 0x001ED8},
+{0x001EDB, 0x001EDA},
+{0x001EDD, 0x001EDC},
+{0x001EDF, 0x001EDE},
+{0x001EE1, 0x001EE0},
+{0x001EE3, 0x001EE2},
+{0x001EE5, 0x001EE4},
+{0x001EE7, 0x001EE6},
+{0x001EE9, 0x001EE8},
+{0x001EEB, 0x001EEA},
+{0x001EED, 0x001EEC},
+{0x001EEF, 0x001EEE},
+{0x001EF1, 0x001EF0},
+{0x001EF3, 0x001EF2},
+{0x001EF5, 0x001EF4},
+{0x001EF7, 0x001EF6},
+{0x001EF9, 0x001EF8},
+{0x001EFB, 0x001EFA},
+{0x001EFD, 0x001EFC},
+{0x001EFF, 0x001EFE},
+{0x001F00, 0x001F08},
+{0x001F01, 0x001F09},
+{0x001F02, 0x001F0A},
+{0x001F03, 0x001F0B},
+{0x001F04, 0x001F0C},
+{0x001F05, 0x001F0D},
+{0x001F06, 0x001F0E},
+{0x001F07, 0x001F0F},
+{0x001F10, 0x001F18},
+{0x001F11, 0x001F19},
+{0x001F12, 0x001F1A},
+{0x001F13, 0x001F1B},
+{0x001F14, 0x001F1C},
+{0x001F15, 0x001F1D},
+{0x001F20, 0x001F28},
+{0x001F21, 0x001F29},
+{0x001F22, 0x001F2A},
+{0x001F23, 0x001F2B},
+{0x001F24, 0x001F2C},
+{0x001F25, 0x001F2D},
+{0x001F26, 0x001F2E},
+{0x001F27, 0x001F2F},
+{0x001F30, 0x001F38},
+{0x001F31, 0x001F39},
+{0x001F32, 0x001F3A},
+{0x001F33, 0x001F3B},
+{0x001F34, 0x001F3C},
+{0x001F35, 0x001F3D},
+{0x001F36, 0x001F3E},
+{0x001F37, 0x001F3F},
+{0x001F40, 0x001F48},
+{0x001F41, 0x001F49},
+{0x001F42, 0x001F4A},
+{0x001F43, 0x001F4B},
+{0x001F44, 0x001F4C},
+{0x001F45, 0x001F4D},
+{0x001F51, 0x001F59},
+{0x001F53, 0x001F5B},
+{0x001F55, 0x001F5D},
+{0x001F57, 0x001F5F},
+{0x001F60, 0x001F68},
+{0x001F61, 0x001F69},
+{0x001F62, 0x001F6A},
+{0x001F63, 0x001F6B},
+{0x001F64, 0x001F6C},
+{0x001F65, 0x001F6D},
+{0x001F66, 0x001F6E},
+{0x001F67, 0x001F6F},
+{0x001F70, 0x001FBA},
+{0x001F71, 0x001FBB},
+{0x001F72, 0x001FC8},
+{0x001F73, 0x001FC9},
+{0x001F74, 0x001FCA},
+{0x001F75, 0x001FCB},
+{0x001F76, 0x001FDA},
+{0x001F77, 0x001FDB},
+{0x001F78, 0x001FF8},
+{0x001F79, 0x001FF9},
+{0x001F7A, 0x001FEA},
+{0x001F7B, 0x001FEB},
+{0x001F7C, 0x001FFA},
+{0x001F7D, 0x001FFB},
+{0x001F80, 0x001F88},
+{0x001F81, 0x001F89},
+{0x001F82, 0x001F8A},
+{0x001F83, 0x001F8B},
+{0x001F84, 0x001F8C},
+{0x001F85, 0x001F8D},
+{0x001F86, 0x001F8E},
+{0x001F87, 0x001F8F},
+{0x001F90, 0x001F98},
+{0x001F91, 0x001F99},
+{0x001F92, 0x001F9A},
+{0x001F93, 0x001F9B},
+{0x001F94, 0x001F9C},
+{0x001F95, 0x001F9D},
+{0x001F96, 0x001F9E},
+{0x001F97, 0x001F9F},
+{0x001FA0, 0x001FA8},
+{0x001FA1, 0x001FA9},
+{0x001FA2, 0x001FAA},
+{0x001FA3, 0x001FAB},
+{0x001FA4, 0x001FAC},
+{0x001FA5, 0x001FAD},
+{0x001FA6, 0x001FAE},
+{0x001FA7, 0x001FAF},
+{0x001FB0, 0x001FB8},
+{0x001FB1, 0x001FB9},
+{0x001FB3, 0x001FBC},
+{0x001FBE, 0x000399},
+{0x001FC3, 0x001FCC},
+{0x001FD0, 0x001FD8},
+{0x001FD1, 0x001FD9},
+{0x001FE0, 0x001FE8},
+{0x001FE1, 0x001FE9},
+{0x001FE5, 0x001FEC},
+{0x001FF3, 0x001FFC},
+{0x00214E, 0x002132},
+{0x002170, 0x002160},
+{0x002171, 0x002161},
+{0x002172, 0x002162},
+{0x002173, 0x002163},
+{0x002174, 0x002164},
+{0x002175, 0x002165},
+{0x002176, 0x002166},
+{0x002177, 0x002167},
+{0x002178, 0x002168},
+{0x002179, 0x002169},
+{0x00217A, 0x00216A},
+{0x00217B, 0x00216B},
+{0x00217C, 0x00216C},
+{0x00217D, 0x00216D},
+{0x00217E, 0x00216E},
+{0x00217F, 0x00216F},
+{0x002184, 0x002183},
+{0x0024D0, 0x0024B6},
+{0x0024D1, 0x0024B7},
+{0x0024D2, 0x0024B8},
+{0x0024D3, 0x0024B9},
+{0x0024D4, 0x0024BA},
+{0x0024D5, 0x0024BB},
+{0x0024D6, 0x0024BC},
+{0x0024D7, 0x0024BD},
+{0x0024D8, 0x0024BE},
+{0x0024D9, 0x0024BF},
+{0x0024DA, 0x0024C0},
+{0x0024DB, 0x0024C1},
+{0x0024DC, 0x0024C2},
+{0x0024DD, 0x0024C3},
+{0x0024DE, 0x0024C4},
+{0x0024DF, 0x0024C5},
+{0x0024E0, 0x0024C6},
+{0x0024E1, 0x0024C7},
+{0x0024E2, 0x0024C8},
+{0x0024E3, 0x0024C9},
+{0x0024E4, 0x0024CA},
+{0x0024E5, 0x0024CB},
+{0x0024E6, 0x0024CC},
+{0x0024E7, 0x0024CD},
+{0x0024E8, 0x0024CE},
+{0x0024E9, 0x0024CF},
+{0x002C30, 0x002C00},
+{0x002C31, 0x002C01},
+{0x002C32, 0x002C02},
+{0x002C33, 0x002C03},
+{0x002C34, 0x002C04},
+{0x002C35, 0x002C05},
+{0x002C36, 0x002C06},
+{0x002C37, 0x002C07},
+{0x002C38, 0x002C08},
+{0x002C39, 0x002C09},
+{0x002C3A, 0x002C0A},
+{0x002C3B, 0x002C0B},
+{0x002C3C, 0x002C0C},
+{0x002C3D, 0x002C0D},
+{0x002C3E, 0x002C0E},
+{0x002C3F, 0x002C0F},
+{0x002C40, 0x002C10},
+{0x002C41, 0x002C11},
+{0x002C42, 0x002C12},
+{0x002C43, 0x002C13},
+{0x002C44, 0x002C14},
+{0x002C45, 0x002C15},
+{0x002C46, 0x002C16},
+{0x002C47, 0x002C17},
+{0x002C48, 0x002C18},
+{0x002C49, 0x002C19},
+{0x002C4A, 0x002C1A},
+{0x002C4B, 0x002C1B},
+{0x002C4C, 0x002C1C},
+{0x002C4D, 0x002C1D},
+{0x002C4E, 0x002C1E},
+{0x002C4F, 0x002C1F},
+{0x002C50, 0x002C20},
+{0x002C51, 0x002C21},
+{0x002C52, 0x002C22},
+{0x002C53, 0x002C23},
+{0x002C54, 0x002C24},
+{0x002C55, 0x002C25},
+{0x002C56, 0x002C26},
+{0x002C57, 0x002C27},
+{0x002C58, 0x002C28},
+{0x002C59, 0x002C29},
+{0x002C5A, 0x002C2A},
+{0x002C5B, 0x002C2B},
+{0x002C5C, 0x002C2C},
+{0x002C5D, 0x002C2D},
+{0x002C5E, 0x002C2E},
+{0x002C5F, 0x002C2F},
+{0x002C61, 0x002C60},
+{0x002C65, 0x00023A},
+{0x002C66, 0x00023E},
+{0x002C68, 0x002C67},
+{0x002C6A, 0x002C69},
+{0x002C6C, 0x002C6B},
+{0x002C73, 0x002C72},
+{0x002C76, 0x002C75},
+{0x002C81, 0x002C80},
+{0x002C83, 0x002C82},
+{0x002C85, 0x002C84},
+{0x002C87, 0x002C86},
+{0x002C89, 0x002C88},
+{0x002C8B, 0x002C8A},
+{0x002C8D, 0x002C8C},
+{0x002C8F, 0x002C8E},
+{0x002C91, 0x002C90},
+{0x002C93, 0x002C92},
+{0x002C95, 0x002C94},
+{0x002C97, 0x002C96},
+{0x002C99, 0x002C98},
+{0x002C9B, 0x002C9A},
+{0x002C9D, 0x002C9C},
+{0x002C9F, 0x002C9E},
+{0x002CA1, 0x002CA0},
+{0x002CA3, 0x002CA2},
+{0x002CA5, 0x002CA4},
+{0x002CA7, 0x002CA6},
+{0x002CA9, 0x002CA8},
+{0x002CAB, 0x002CAA},
+{0x002CAD, 0x002CAC},
+{0x002CAF, 0x002CAE},
+{0x002CB1, 0x002CB0},
+{0x002CB3, 0x002CB2},
+{0x002CB5, 0x002CB4},
+{0x002CB7, 0x002CB6},
+{0x002CB9, 0x002CB8},
+{0x002CBB, 0x002CBA},
+{0x002CBD, 0x002CBC},
+{0x002CBF, 0x002CBE},
+{0x002CC1, 0x002CC0},
+{0x002CC3, 0x002CC2},
+{0x002CC5, 0x002CC4},
+{0x002CC7, 0x002CC6},
+{0x002CC9, 0x002CC8},
+{0x002CCB, 0x002CCA},
+{0x002CCD, 0x002CCC},
+{0x002CCF, 0x002CCE},
+{0x002CD1, 0x002CD0},
+{0x002CD3, 0x002CD2},
+{0x002CD5, 0x002CD4},
+{0x002CD7, 0x002CD6},
+{0x002CD9, 0x002CD8},
+{0x002CDB, 0x002CDA},
+{0x002CDD, 0x002CDC},
+{0x002CDF, 0x002CDE},
+{0x002CE1, 0x002CE0},
+{0x002CE3, 0x002CE2},
+{0x002CEC, 0x002CEB},
+{0x002CEE, 0x002CED},
+{0x002CF3, 0x002CF2},
+{0x002D00, 0x0010A0},
+{0x002D01, 0x0010A1},
+{0x002D02, 0x0010A2},
+{0x002D03, 0x0010A3},
+{0x002D04, 0x0010A4},
+{0x002D05, 0x0010A5},
+{0x002D06, 0x0010A6},
+{0x002D07, 0x0010A7},
+{0x002D08, 0x0010A8},
+{0x002D09, 0x0010A9},
+{0x002D0A, 0x0010AA},
+{0x002D0B, 0x0010AB},
+{0x002D0C, 0x0010AC},
+{0x002D0D, 0x0010AD},
+{0x002D0E, 0x0010AE},
+{0x002D0F, 0x0010AF},
+{0x002D10, 0x0010B0},
+{0x002D11, 0x0010B1},
+{0x002D12, 0x0010B2},
+{0x002D13, 0x0010B3},
+{0x002D14, 0x0010B4},
+{0x002D15, 0x0010B5},
+{0x002D16, 0x0010B6},
+{0x002D17, 0x0010B7},
+{0x002D18, 0x0010B8},
+{0x002D19, 0x0010B9},
+{0x002D1A, 0x0010BA},
+{0x002D1B, 0x0010BB},
+{0x002D1C, 0x0010BC},
+{0x002D1D, 0x0010BD},
+{0x002D1E, 0x0010BE},
+{0x002D1F, 0x0010BF},
+{0x002D20, 0x0010C0},
+{0x002D21, 0x0010C1},
+{0x002D22, 0x0010C2},
+{0x002D23, 0x0010C3},
+{0x002D24, 0x0010C4},
+{0x002D25, 0x0010C5},
+{0x002D27, 0x0010C7},
+{0x002D2D, 0x0010CD},
+{0x00A641, 0x00A640},
+{0x00A643, 0x00A642},
+{0x00A645, 0x00A644},
+{0x00A647, 0x00A646},
+{0x00A649, 0x00A648},
+{0x00A64B, 0x00A64A},
+{0x00A64D, 0x00A64C},
+{0x00A64F, 0x00A64E},
+{0x00A651, 0x00A650},
+{0x00A653, 0x00A652},
+{0x00A655, 0x00A654},
+{0x00A657, 0x00A656},
+{0x00A659, 0x00A658},
+{0x00A65B, 0x00A65A},
+{0x00A65D, 0x00A65C},
+{0x00A65F, 0x00A65E},
+{0x00A661, 0x00A660},
+{0x00A663, 0x00A662},
+{0x00A665, 0x00A664},
+{0x00A667, 0x00A666},
+{0x00A669, 0x00A668},
+{0x00A66B, 0x00A66A},
+{0x00A66D, 0x00A66C},
+{0x00A681, 0x00A680},
+{0x00A683, 0x00A682},
+{0x00A685, 0x00A684},
+{0x00A687, 0x00A686},
+{0x00A689, 0x00A688},
+{0x00A68B, 0x00A68A},
+{0x00A68D, 0x00A68C},
+{0x00A68F, 0x00A68E},
+{0x00A691, 0x00A690},
+{0x00A693, 0x00A692},
+{0x00A695, 0x00A694},
+{0x00A697, 0x00A696},
+{0x00A699, 0x00A698},
+{0x00A69B, 0x00A69A},
+{0x00A723, 0x00A722},
+{0x00A725, 0x00A724},
+{0x00A727, 0x00A726},
+{0x00A729, 0x00A728},
+{0x00A72B, 0x00A72A},
+{0x00A72D, 0x00A72C},
+{0x00A72F, 0x00A72E},
+{0x00A733, 0x00A732},
+{0x00A735, 0x00A734},
+{0x00A737, 0x00A736},
+{0x00A739, 0x00A738},
+{0x00A73B, 0x00A73A},
+{0x00A73D, 0x00A73C},
+{0x00A73F, 0x00A73E},
+{0x00A741, 0x00A740},
+{0x00A743, 0x00A742},
+{0x00A745, 0x00A744},
+{0x00A747, 0x00A746},
+{0x00A749, 0x00A748},
+{0x00A74B, 0x00A74A},
+{0x00A74D, 0x00A74C},
+{0x00A74F, 0x00A74E},
+{0x00A751, 0x00A750},
+{0x00A753, 0x00A752},
+{0x00A755, 0x00A754},
+{0x00A757, 0x00A756},
+{0x00A759, 0x00A758},
+{0x00A75B, 0x00A75A},
+{0x00A75D, 0x00A75C},
+{0x00A75F, 0x00A75E},
+{0x00A761, 0x00A760},
+{0x00A763, 0x00A762},
+{0x00A765, 0x00A764},
+{0x00A767, 0x00A766},
+{0x00A769, 0x00A768},
+{0x00A76B, 0x00A76A},
+{0x00A76D, 0x00A76C},
+{0x00A76F, 0x00A76E},
+{0x00A77A, 0x00A779},
+{0x00A77C, 0x00A77B},
+{0x00A77F, 0x00A77E},
+{0x00A781, 0x00A780},
+{0x00A783, 0x00A782},
+{0x00A785, 0x00A784},
+{0x00A787, 0x00A786},
+{0x00A78C, 0x00A78B},
+{0x00A791, 0x00A790},
+{0x00A793, 0x00A792},
+{0x00A794, 0x00A7C4},
+{0x00A797, 0x00A796},
+{0x00A799, 0x00A798},
+{0x00A79B, 0x00A79A},
+{0x00A79D, 0x00A79C},
+{0x00A79F, 0x00A79E},
+{0x00A7A1, 0x00A7A0},
+{0x00A7A3, 0x00A7A2},
+{0x00A7A5, 0x00A7A4},
+{0x00A7A7, 0x00A7A6},
+{0x00A7A9, 0x00A7A8},
+{0x00A7B5, 0x00A7B4},
+{0x00A7B7, 0x00A7B6},
+{0x00A7B9, 0x00A7B8},
+{0x00A7BB, 0x00A7BA},
+{0x00A7BD, 0x00A7BC},
+{0x00A7BF, 0x00A7BE},
+{0x00A7C1, 0x00A7C0},
+{0x00A7C3, 0x00A7C2},
+{0x00A7C8, 0x00A7C7},
+{0x00A7CA, 0x00A7C9},
+{0x00A7D1, 0x00A7D0},
+{0x00A7D7, 0x00A7D6},
+{0x00A7D9, 0x00A7D8},
+{0x00A7F6, 0x00A7F5},
+{0x00AB53, 0x00A7B3},
+{0x00AB70, 0x0013A0},
+{0x00AB71, 0x0013A1},
+{0x00AB72, 0x0013A2},
+{0x00AB73, 0x0013A3},
+{0x00AB74, 0x0013A4},
+{0x00AB75, 0x0013A5},
+{0x00AB76, 0x0013A6},
+{0x00AB77, 0x0013A7},
+{0x00AB78, 0x0013A8},
+{0x00AB79, 0x0013A9},
+{0x00AB7A, 0x0013AA},
+{0x00AB7B, 0x0013AB},
+{0x00AB7C, 0x0013AC},
+{0x00AB7D, 0x0013AD},
+{0x00AB7E, 0x0013AE},
+{0x00AB7F, 0x0013AF},
+{0x00AB80, 0x0013B0},
+{0x00AB81, 0x0013B1},
+{0x00AB82, 0x0013B2},
+{0x00AB83, 0x0013B3},
+{0x00AB84, 0x0013B4},
+{0x00AB85, 0x0013B5},
+{0x00AB86, 0x0013B6},
+{0x00AB87, 0x0013B7},
+{0x00AB88, 0x0013B8},
+{0x00AB89, 0x0013B9},
+{0x00AB8A, 0x0013BA},
+{0x00AB8B, 0x0013BB},
+{0x00AB8C, 0x0013BC},
+{0x00AB8D, 0x0013BD},
+{0x00AB8E, 0x0013BE},
+{0x00AB8F, 0x0013BF},
+{0x00AB90, 0x0013C0},
+{0x00AB91, 0x0013C1},
+{0x00AB92, 0x0013C2},
+{0x00AB93, 0x0013C3},
+{0x00AB94, 0x0013C4},
+{0x00AB95, 0x0013C5},
+{0x00AB96, 0x0013C6},
+{0x00AB97, 0x0013C7},
+{0x00AB98, 0x0013C8},
+{0x00AB99, 0x0013C9},
+{0x00AB9A, 0x0013CA},
+{0x00AB9B, 0x0013CB},
+{0x00AB9C, 0x0013CC},
+{0x00AB9D, 0x0013CD},
+{0x00AB9E, 0x0013CE},
+{0x00AB9F, 0x0013CF},
+{0x00ABA0, 0x0013D0},
+{0x00ABA1, 0x0013D1},
+{0x00ABA2, 0x0013D2},
+{0x00ABA3, 0x0013D3},
+{0x00ABA4, 0x0013D4},
+{0x00ABA5, 0x0013D5},
+{0x00ABA6, 0x0013D6},
+{0x00ABA7, 0x0013D7},
+{0x00ABA8, 0x0013D8},
+{0x00ABA9, 0x0013D9},
+{0x00ABAA, 0x0013DA},
+{0x00ABAB, 0x0013DB},
+{0x00ABAC, 0x0013DC},
+{0x00ABAD, 0x0013DD},
+{0x00ABAE, 0x0013DE},
+{0x00ABAF, 0x0013DF},
+{0x00ABB0, 0x0013E0},
+{0x00ABB1, 0x0013E1},
+{0x00ABB2, 0x0013E2},
+{0x00ABB3, 0x0013E3},
+{0x00ABB4, 0x0013E4},
+{0x00ABB5, 0x0013E5},
+{0x00ABB6, 0x0013E6},
+{0x00ABB7, 0x0013E7},
+{0x00ABB8, 0x0013E8},
+{0x00ABB9, 0x0013E9},
+{0x00ABBA, 0x0013EA},
+{0x00ABBB, 0x0013EB},
+{0x00ABBC, 0x0013EC},
+{0x00ABBD, 0x0013ED},
+{0x00ABBE, 0x0013EE},
+{0x00ABBF, 0x0013EF},
+{0x00FF41, 0x00FF21},
+{0x00FF42, 0x00FF22},
+{0x00FF43, 0x00FF23},
+{0x00FF44, 0x00FF24},
+{0x00FF45, 0x00FF25},
+{0x00FF46, 0x00FF26},
+{0x00FF47, 0x00FF27},
+{0x00FF48, 0x00FF28},
+{0x00FF49, 0x00FF29},
+{0x00FF4A, 0x00FF2A},
+{0x00FF4B, 0x00FF2B},
+{0x00FF4C, 0x00FF2C},
+{0x00FF4D, 0x00FF2D},
+{0x00FF4E, 0x00FF2E},
+{0x00FF4F, 0x00FF2F},
+{0x00FF50, 0x00FF30},
+{0x00FF51, 0x00FF31},
+{0x00FF52, 0x00FF32},
+{0x00FF53, 0x00FF33},
+{0x00FF54, 0x00FF34},
+{0x00FF55, 0x00FF35},
+{0x00FF56, 0x00FF36},
+{0x00FF57, 0x00FF37},
+{0x00FF58, 0x00FF38},
+{0x00FF59, 0x00FF39},
+{0x00FF5A, 0x00FF3A},
+{0x010428, 0x010400},
+{0x010429, 0x010401},
+{0x01042A, 0x010402},
+{0x01042B, 0x010403},
+{0x01042C, 0x010404},
+{0x01042D, 0x010405},
+{0x01042E, 0x010406},
+{0x01042F, 0x010407},
+{0x010430, 0x010408},
+{0x010431, 0x010409},
+{0x010432, 0x01040A},
+{0x010433, 0x01040B},
+{0x010434, 0x01040C},
+{0x010435, 0x01040D},
+{0x010436, 0x01040E},
+{0x010437, 0x01040F},
+{0x010438, 0x010410},
+{0x010439, 0x010411},
+{0x01043A, 0x010412},
+{0x01043B, 0x010413},
+{0x01043C, 0x010414},
+{0x01043D, 0x010415},
+{0x01043E, 0x010416},
+{0x01043F, 0x010417},
+{0x010440, 0x010418},
+{0x010441, 0x010419},
+{0x010442, 0x01041A},
+{0x010443, 0x01041B},
+{0x010444, 0x01041C},
+{0x010445, 0x01041D},
+{0x010446, 0x01041E},
+{0x010447, 0x01041F},
+{0x010448, 0x010420},
+{0x010449, 0x010421},
+{0x01044A, 0x010422},
+{0x01044B, 0x010423},
+{0x01044C, 0x010424},
+{0x01044D, 0x010425},
+{0x01044E, 0x010426},
+{0x01044F, 0x010427},
+{0x0104D8, 0x0104B0},
+{0x0104D9, 0x0104B1},
+{0x0104DA, 0x0104B2},
+{0x0104DB, 0x0104B3},
+{0x0104DC, 0x0104B4},
+{0x0104DD, 0x0104B5},
+{0x0104DE, 0x0104B6},
+{0x0104DF, 0x0104B7},
+{0x0104E0, 0x0104B8},
+{0x0104E1, 0x0104B9},
+{0x0104E2, 0x0104BA},
+{0x0104E3, 0x0104BB},
+{0x0104E4, 0x0104BC},
+{0x0104E5, 0x0104BD},
+{0x0104E6, 0x0104BE},
+{0x0104E7, 0x0104BF},
+{0x0104E8, 0x0104C0},
+{0x0104E9, 0x0104C1},
+{0x0104EA, 0x0104C2},
+{0x0104EB, 0x0104C3},
+{0x0104EC, 0x0104C4},
+{0x0104ED, 0x0104C5},
+{0x0104EE, 0x0104C6},
+{0x0104EF, 0x0104C7},
+{0x0104F0, 0x0104C8},
+{0x0104F1, 0x0104C9},
+{0x0104F2, 0x0104CA},
+{0x0104F3, 0x0104CB},
+{0x0104F4, 0x0104CC},
+{0x0104F5, 0x0104CD},
+{0x0104F6, 0x0104CE},
+{0x0104F7, 0x0104CF},
+{0x0104F8, 0x0104D0},
+{0x0104F9, 0x0104D1},
+{0x0104FA, 0x0104D2},
+{0x0104FB, 0x0104D3},
+{0x010597, 0x010570},
+{0x010598, 0x010571},
+{0x010599, 0x010572},
+{0x01059A, 0x010573},
+{0x01059B, 0x010574},
+{0x01059C, 0x010575},
+{0x01059D, 0x010576},
+{0x01059E, 0x010577},
+{0x01059F, 0x010578},
+{0x0105A0, 0x010579},
+{0x0105A1, 0x01057A},
+{0x0105A3, 0x01057C},
+{0x0105A4, 0x01057D},
+{0x0105A5, 0x01057E},
+{0x0105A6, 0x01057F},
+{0x0105A7, 0x010580},
+{0x0105A8, 0x010581},
+{0x0105A9, 0x010582},
+{0x0105AA, 0x010583},
+{0x0105AB, 0x010584},
+{0x0105AC, 0x010585},
+{0x0105AD, 0x010586},
+{0x0105AE, 0x010587},
+{0x0105AF, 0x010588},
+{0x0105B0, 0x010589},
+{0x0105B1, 0x01058A},
+{0x0105B3, 0x01058C},
+{0x0105B4, 0x01058D},
+{0x0105B5, 0x01058E},
+{0x0105B6, 0x01058F},
+{0x0105B7, 0x010590},
+{0x0105B8, 0x010591},
+{0x0105B9, 0x010592},
+{0x0105BB, 0x010594},
+{0x0105BC, 0x010595},
+{0x010CC0, 0x010C80},
+{0x010CC1, 0x010C81},
+{0x010CC2, 0x010C82},
+{0x010CC3, 0x010C83},
+{0x010CC4, 0x010C84},
+{0x010CC5, 0x010C85},
+{0x010CC6, 0x010C86},
+{0x010CC7, 0x010C87},
+{0x010CC8, 0x010C88},
+{0x010CC9, 0x010C89},
+{0x010CCA, 0x010C8A},
+{0x010CCB, 0x010C8B},
+{0x010CCC, 0x010C8C},
+{0x010CCD, 0x010C8D},
+{0x010CCE, 0x010C8E},
+{0x010CCF, 0x010C8F},
+{0x010CD0, 0x010C90},
+{0x010CD1, 0x010C91},
+{0x010CD2, 0x010C92},
+{0x010CD3, 0x010C93},
+{0x010CD4, 0x010C94},
+{0x010CD5, 0x010C95},
+{0x010CD6, 0x010C96},
+{0x010CD7, 0x010C97},
+{0x010CD8, 0x010C98},
+{0x010CD9, 0x010C99},
+{0x010CDA, 0x010C9A},
+{0x010CDB, 0x010C9B},
+{0x010CDC, 0x010C9C},
+{0x010CDD, 0x010C9D},
+{0x010CDE, 0x010C9E},
+{0x010CDF, 0x010C9F},
+{0x010CE0, 0x010CA0},
+{0x010CE1, 0x010CA1},
+{0x010CE2, 0x010CA2},
+{0x010CE3, 0x010CA3},
+{0x010CE4, 0x010CA4},
+{0x010CE5, 0x010CA5},
+{0x010CE6, 0x010CA6},
+{0x010CE7, 0x010CA7},
+{0x010CE8, 0x010CA8},
+{0x010CE9, 0x010CA9},
+{0x010CEA, 0x010CAA},
+{0x010CEB, 0x010CAB},
+{0x010CEC, 0x010CAC},
+{0x010CED, 0x010CAD},
+{0x010CEE, 0x010CAE},
+{0x010CEF, 0x010CAF},
+{0x010CF0, 0x010CB0},
+{0x010CF1, 0x010CB1},
+{0x010CF2, 0x010CB2},
+{0x0118C0, 0x0118A0},
+{0x0118C1, 0x0118A1},
+{0x0118C2, 0x0118A2},
+{0x0118C3, 0x0118A3},
+{0x0118C4, 0x0118A4},
+{0x0118C5, 0x0118A5},
+{0x0118C6, 0x0118A6},
+{0x0118C7, 0x0118A7},
+{0x0118C8, 0x0118A8},
+{0x0118C9, 0x0118A9},
+{0x0118CA, 0x0118AA},
+{0x0118CB, 0x0118AB},
+{0x0118CC, 0x0118AC},
+{0x0118CD, 0x0118AD},
+{0x0118CE, 0x0118AE},
+{0x0118CF, 0x0118AF},
+{0x0118D0, 0x0118B0},
+{0x0118D1, 0x0118B1},
+{0x0118D2, 0x0118B2},
+{0x0118D3, 0x0118B3},
+{0x0118D4, 0x0118B4},
+{0x0118D5, 0x0118B5},
+{0x0118D6, 0x0118B6},
+{0x0118D7, 0x0118B7},
+{0x0118D8, 0x0118B8},
+{0x0118D9, 0x0118B9},
+{0x0118DA, 0x0118BA},
+{0x0118DB, 0x0118BB},
+{0x0118DC, 0x0118BC},
+{0x0118DD, 0x0118BD},
+{0x0118DE, 0x0118BE},
+{0x0118DF, 0x0118BF},
+{0x016E60, 0x016E40},
+{0x016E61, 0x016E41},
+{0x016E62, 0x016E42},
+{0x016E63, 0x016E43},
+{0x016E64, 0x016E44},
+{0x016E65, 0x016E45},
+{0x016E66, 0x016E46},
+{0x016E67, 0x016E47},
+{0x016E68, 0x016E48},
+{0x016E69, 0x016E49},
+{0x016E6A, 0x016E4A},
+{0x016E6B, 0x016E4B},
+{0x016E6C, 0x016E4C},
+{0x016E6D, 0x016E4D},
+{0x016E6E, 0x016E4E},
+{0x016E6F, 0x016E4F},
+{0x016E70, 0x016E50},
+{0x016E71, 0x016E51},
+{0x016E72, 0x016E52},
+{0x016E73, 0x016E53},
+{0x016E74, 0x016E54},
+{0x016E75, 0x016E55},
+{0x016E76, 0x016E56},
+{0x016E77, 0x016E57},
+{0x016E78, 0x016E58},
+{0x016E79, 0x016E59},
+{0x016E7A, 0x016E5A},
+{0x016E7B, 0x016E5B},
+{0x016E7C, 0x016E5C},
+{0x016E7D, 0x016E5D},
+{0x016E7E, 0x016E5E},
+{0x016E7F, 0x016E5F},
+{0x01E922, 0x01E900},
+{0x01E923, 0x01E901},
+{0x01E924, 0x01E902},
+{0x01E925, 0x01E903},
+{0x01E926, 0x01E904},
+{0x01E927, 0x01E905},
+{0x01E928, 0x01E906},
+{0x01E929, 0x01E907},
+{0x01E92A, 0x01E908},
+{0x01E92B, 0x01E909},
+{0x01E92C, 0x01E90A},
+{0x01E92D, 0x01E90B},
+{0x01E92E, 0x01E90C},
+{0x01E92F, 0x01E90D},
+{0x01E930, 0x01E90E},
+{0x01E931, 0x01E90F},
+{0x01E932, 0x01E910},
+{0x01E933, 0x01E911},
+{0x01E934, 0x01E912},
+{0x01E935, 0x01E913},
+{0x01E936, 0x01E914},
+{0x01E937, 0x01E915},
+{0x01E938, 0x01E916},
+{0x01E939, 0x01E917},
+{0x01E93A, 0x01E918},
+{0x01E93B, 0x01E919},
+{0x01E93C, 0x01E91A},
+{0x01E93D, 0x01E91B},
+{0x01E93E, 0x01E91C},
+{0x01E93F, 0x01E91D},
+{0x01E940, 0x01E91E},
+{0x01E941, 0x01E91F},
+{0x01E942, 0x01E920},
+{0x01E943, 0x01E921},
+};
+
+const std::vector<range_nfd> unicode_ranges_nfd = { // start, last, nfd
+{0x000000, 0x000000, 0x000000},
+{0x0000C0, 0x0000C5, 0x000041},
+{0x0000C7, 0x0000C7, 0x000043},
+{0x0000C8, 0x0000CB, 0x000045},
+{0x0000CC, 0x0000CF, 0x000049},
+{0x0000D1, 0x0000D1, 0x00004E},
+{0x0000D2, 0x0000D6, 0x00004F},
+{0x0000D9, 0x0000DC, 0x000055},
+{0x0000DD, 0x0000DD, 0x000059},
+{0x0000E0, 0x0000E5, 0x000061},
+{0x0000E7, 0x0000E7, 0x000063},
+{0x0000E8, 0x0000EB, 0x000065},
+{0x0000EC, 0x0000EF, 0x000069},
+{0x0000F1, 0x0000F1, 0x00006E},
+{0x0000F2, 0x0000F6, 0x00006F},
+{0x0000F9, 0x0000FC, 0x000075},
+{0x0000FD, 0x0000FD, 0x000079},
+{0x0000FF, 0x0000FF, 0x000079},
+{0x000100, 0x000100, 0x000041},
+{0x000101, 0x000101, 0x000061},
+{0x000102, 0x000102, 0x000041},
+{0x000103, 0x000103, 0x000061},
+{0x000104, 0x000104, 0x000041},
+{0x000105, 0x000105, 0x000061},
+{0x000106, 0x000106, 0x000043},
+{0x000107, 0x000107, 0x000063},
+{0x000108, 0x000108, 0x000043},
+{0x000109, 0x000109, 0x000063},
+{0x00010A, 0x00010A, 0x000043},
+{0x00010B, 0x00010B, 0x000063},
+{0x00010C, 0x00010C, 0x000043},
+{0x00010D, 0x00010D, 0x000063},
+{0x00010E, 0x00010E, 0x000044},
+{0x00010F, 0x00010F, 0x000064},
+{0x000112, 0x000112, 0x000045},
+{0x000113, 0x000113, 0x000065},
+{0x000114, 0x000114, 0x000045},
+{0x000115, 0x000115, 0x000065},
+{0x000116, 0x000116, 0x000045},
+{0x000117, 0x000117, 0x000065},
+{0x000118, 0x000118, 0x000045},
+{0x000119, 0x000119, 0x000065},
+{0x00011A, 0x00011A, 0x000045},
+{0x00011B, 0x00011B, 0x000065},
+{0x00011C, 0x00011C, 0x000047},
+{0x00011D, 0x00011D, 0x000067},
+{0x00011E, 0x00011E, 0x000047},
+{0x00011F, 0x00011F, 0x000067},
+{0x000120, 0x000120, 0x000047},
+{0x000121, 0x000121, 0x000067},
+{0x000122, 0x000122, 0x000047},
+{0x000123, 0x000123, 0x000067},
+{0x000124, 0x000124, 0x000048},
+{0x000125, 0x000125, 0x000068},
+{0x000128, 0x000128, 0x000049},
+{0x000129, 0x000129, 0x000069},
+{0x00012A, 0x00012A, 0x000049},
+{0x00012B, 0x00012B, 0x000069},
+{0x00012C, 0x00012C, 0x000049},
+{0x00012D, 0x00012D, 0x000069},
+{0x00012E, 0x00012E, 0x000049},
+{0x00012F, 0x00012F, 0x000069},
+{0x000130, 0x000130, 0x000049},
+{0x000134, 0x000134, 0x00004A},
+{0x000135, 0x000135, 0x00006A},
+{0x000136, 0x000136, 0x00004B},
+{0x000137, 0x000137, 0x00006B},
+{0x000139, 0x000139, 0x00004C},
+{0x00013A, 0x00013A, 0x00006C},
+{0x00013B, 0x00013B, 0x00004C},
+{0x00013C, 0x00013C, 0x00006C},
+{0x00013D, 0x00013D, 0x00004C},
+{0x00013E, 0x00013E, 0x00006C},
+{0x000143, 0x000143, 0x00004E},
+{0x000144, 0x000144, 0x00006E},
+{0x000145, 0x000145, 0x00004E},
+{0x000146, 0x000146, 0x00006E},
+{0x000147, 0x000147, 0x00004E},
+{0x000148, 0x000148, 0x00006E},
+{0x00014C, 0x00014C, 0x00004F},
+{0x00014D, 0x00014D, 0x00006F},
+{0x00014E, 0x00014E, 0x00004F},
+{0x00014F, 0x00014F, 0x00006F},
+{0x000150, 0x000150, 0x00004F},
+{0x000151, 0x000151, 0x00006F},
+{0x000154, 0x000154, 0x000052},
+{0x000155, 0x000155, 0x000072},
+{0x000156, 0x000156, 0x000052},
+{0x000157, 0x000157, 0x000072},
+{0x000158, 0x000158, 0x000052},
+{0x000159, 0x000159, 0x000072},
+{0x00015A, 0x00015A, 0x000053},
+{0x00015B, 0x00015B, 0x000073},
+{0x00015C, 0x00015C, 0x000053},
+{0x00015D, 0x00015D, 0x000073},
+{0x00015E, 0x00015E, 0x000053},
+{0x00015F, 0x00015F, 0x000073},
+{0x000160, 0x000160, 0x000053},
+{0x000161, 0x000161, 0x000073},
+{0x000162, 0x000162, 0x000054},
+{0x000163, 0x000163, 0x000074},
+{0x000164, 0x000164, 0x000054},
+{0x000165, 0x000165, 0x000074},
+{0x000168, 0x000168, 0x000055},
+{0x000169, 0x000169, 0x000075},
+{0x00016A, 0x00016A, 0x000055},
+{0x00016B, 0x00016B, 0x000075},
+{0x00016C, 0x00016C, 0x000055},
+{0x00016D, 0x00016D, 0x000075},
+{0x00016E, 0x00016E, 0x000055},
+{0x00016F, 0x00016F, 0x000075},
+{0x000170, 0x000170, 0x000055},
+{0x000171, 0x000171, 0x000075},
+{0x000172, 0x000172, 0x000055},
+{0x000173, 0x000173, 0x000075},
+{0x000174, 0x000174, 0x000057},
+{0x000175, 0x000175, 0x000077},
+{0x000176, 0x000176, 0x000059},
+{0x000177, 0x000177, 0x000079},
+{0x000178, 0x000178, 0x000059},
+{0x000179, 0x000179, 0x00005A},
+{0x00017A, 0x00017A, 0x00007A},
+{0x00017B, 0x00017B, 0x00005A},
+{0x00017C, 0x00017C, 0x00007A},
+{0x00017D, 0x00017D, 0x00005A},
+{0x00017E, 0x00017E, 0x00007A},
+{0x0001A0, 0x0001A0, 0x00004F},
+{0x0001A1, 0x0001A1, 0x00006F},
+{0x0001AF, 0x0001AF, 0x000055},
+{0x0001B0, 0x0001B0, 0x000075},
+{0x0001CD, 0x0001CD, 0x000041},
+{0x0001CE, 0x0001CE, 0x000061},
+{0x0001CF, 0x0001CF, 0x000049},
+{0x0001D0, 0x0001D0, 0x000069},
+{0x0001D1, 0x0001D1, 0x00004F},
+{0x0001D2, 0x0001D2, 0x00006F},
+{0x0001D3, 0x0001D3, 0x000055},
+{0x0001D4, 0x0001D4, 0x000075},
+{0x0001D5, 0x0001D5, 0x000055},
+{0x0001D6, 0x0001D6, 0x000075},
+{0x0001D7, 0x0001D7, 0x000055},
+{0x0001D8, 0x0001D8, 0x000075},
+{0x0001D9, 0x0001D9, 0x000055},
+{0x0001DA, 0x0001DA, 0x000075},
+{0x0001DB, 0x0001DB, 0x000055},
+{0x0001DC, 0x0001DC, 0x000075},
+{0x0001DE, 0x0001DE, 0x000041},
+{0x0001DF, 0x0001DF, 0x000061},
+{0x0001E0, 0x0001E0, 0x000041},
+{0x0001E1, 0x0001E1, 0x000061},
+{0x0001E2, 0x0001E2, 0x0000C6},
+{0x0001E3, 0x0001E3, 0x0000E6},
+{0x0001E6, 0x0001E6, 0x000047},
+{0x0001E7, 0x0001E7, 0x000067},
+{0x0001E8, 0x0001E8, 0x00004B},
+{0x0001E9, 0x0001E9, 0x00006B},
+{0x0001EA, 0x0001EA, 0x00004F},
+{0x0001EB, 0x0001EB, 0x00006F},
+{0x0001EC, 0x0001EC, 0x00004F},
+{0x0001ED, 0x0001ED, 0x00006F},
+{0x0001EE, 0x0001EE, 0x0001B7},
+{0x0001EF, 0x0001EF, 0x000292},
+{0x0001F0, 0x0001F0, 0x00006A},
+{0x0001F4, 0x0001F4, 0x000047},
+{0x0001F5, 0x0001F5, 0x000067},
+{0x0001F8, 0x0001F8, 0x00004E},
+{0x0001F9, 0x0001F9, 0x00006E},
+{0x0001FA, 0x0001FA, 0x000041},
+{0x0001FB, 0x0001FB, 0x000061},
+{0x0001FC, 0x0001FC, 0x0000C6},
+{0x0001FD, 0x0001FD, 0x0000E6},
+{0x0001FE, 0x0001FE, 0x0000D8},
+{0x0001FF, 0x0001FF, 0x0000F8},
+{0x000200, 0x000200, 0x000041},
+{0x000201, 0x000201, 0x000061},
+{0x000202, 0x000202, 0x000041},
+{0x000203, 0x000203, 0x000061},
+{0x000204, 0x000204, 0x000045},
+{0x000205, 0x000205, 0x000065},
+{0x000206, 0x000206, 0x000045},
+{0x000207, 0x000207, 0x000065},
+{0x000208, 0x000208, 0x000049},
+{0x000209, 0x000209, 0x000069},
+{0x00020A, 0x00020A, 0x000049},
+{0x00020B, 0x00020B, 0x000069},
+{0x00020C, 0x00020C, 0x00004F},
+{0x00020D, 0x00020D, 0x00006F},
+{0x00020E, 0x00020E, 0x00004F},
+{0x00020F, 0x00020F, 0x00006F},
+{0x000210, 0x000210, 0x000052},
+{0x000211, 0x000211, 0x000072},
+{0x000212, 0x000212, 0x000052},
+{0x000213, 0x000213, 0x000072},
+{0x000214, 0x000214, 0x000055},
+{0x000215, 0x000215, 0x000075},
+{0x000216, 0x000216, 0x000055},
+{0x000217, 0x000217, 0x000075},
+{0x000218, 0x000218, 0x000053},
+{0x000219, 0x000219, 0x000073},
+{0x00021A, 0x00021A, 0x000054},
+{0x00021B, 0x00021B, 0x000074},
+{0x00021E, 0x00021E, 0x000048},
+{0x00021F, 0x00021F, 0x000068},
+{0x000226, 0x000226, 0x000041},
+{0x000227, 0x000227, 0x000061},
+{0x000228, 0x000228, 0x000045},
+{0x000229, 0x000229, 0x000065},
+{0x00022A, 0x00022A, 0x00004F},
+{0x00022B, 0x00022B, 0x00006F},
+{0x00022C, 0x00022C, 0x00004F},
+{0x00022D, 0x00022D, 0x00006F},
+{0x00022E, 0x00022E, 0x00004F},
+{0x00022F, 0x00022F, 0x00006F},
+{0x000230, 0x000230, 0x00004F},
+{0x000231, 0x000231, 0x00006F},
+{0x000232, 0x000232, 0x000059},
+{0x000233, 0x000233, 0x000079},
+{0x000340, 0x000340, 0x000300},
+{0x000341, 0x000341, 0x000301},
+{0x000343, 0x000343, 0x000313},
+{0x000344, 0x000344, 0x000308},
+{0x000374, 0x000374, 0x0002B9},
+{0x00037E, 0x00037E, 0x00003B},
+{0x000385, 0x000385, 0x0000A8},
+{0x000386, 0x000386, 0x000391},
+{0x000387, 0x000387, 0x0000B7},
+{0x000388, 0x000388, 0x000395},
+{0x000389, 0x000389, 0x000397},
+{0x00038A, 0x00038A, 0x000399},
+{0x00038C, 0x00038C, 0x00039F},
+{0x00038E, 0x00038E, 0x0003A5},
+{0x00038F, 0x00038F, 0x0003A9},
+{0x000390, 0x000390, 0x0003B9},
+{0x0003AA, 0x0003AA, 0x000399},
+{0x0003AB, 0x0003AB, 0x0003A5},
+{0x0003AC, 0x0003AC, 0x0003B1},
+{0x0003AD, 0x0003AD, 0x0003B5},
+{0x0003AE, 0x0003AE, 0x0003B7},
+{0x0003AF, 0x0003AF, 0x0003B9},
+{0x0003B0, 0x0003B0, 0x0003C5},
+{0x0003CA, 0x0003CA, 0x0003B9},
+{0x0003CB, 0x0003CB, 0x0003C5},
+{0x0003CC, 0x0003CC, 0x0003BF},
+{0x0003CD, 0x0003CD, 0x0003C5},
+{0x0003CE, 0x0003CE, 0x0003C9},
+{0x0003D3, 0x0003D4, 0x0003D2},
+{0x000400, 0x000401, 0x000415},
+{0x000403, 0x000403, 0x000413},
+{0x000407, 0x000407, 0x000406},
+{0x00040C, 0x00040C, 0x00041A},
+{0x00040D, 0x00040D, 0x000418},
+{0x00040E, 0x00040E, 0x000423},
+{0x000419, 0x000419, 0x000418},
+{0x000439, 0x000439, 0x000438},
+{0x000450, 0x000451, 0x000435},
+{0x000453, 0x000453, 0x000433},
+{0x000457, 0x000457, 0x000456},
+{0x00045C, 0x00045C, 0x00043A},
+{0x00045D, 0x00045D, 0x000438},
+{0x00045E, 0x00045E, 0x000443},
+{0x000476, 0x000476, 0x000474},
+{0x000477, 0x000477, 0x000475},
+{0x0004C1, 0x0004C1, 0x000416},
+{0x0004C2, 0x0004C2, 0x000436},
+{0x0004D0, 0x0004D0, 0x000410},
+{0x0004D1, 0x0004D1, 0x000430},
+{0x0004D2, 0x0004D2, 0x000410},
+{0x0004D3, 0x0004D3, 0x000430},
+{0x0004D6, 0x0004D6, 0x000415},
+{0x0004D7, 0x0004D7, 0x000435},
+{0x0004DA, 0x0004DA, 0x0004D8},
+{0x0004DB, 0x0004DB, 0x0004D9},
+{0x0004DC, 0x0004DC, 0x000416},
+{0x0004DD, 0x0004DD, 0x000436},
+{0x0004DE, 0x0004DE, 0x000417},
+{0x0004DF, 0x0004DF, 0x000437},
+{0x0004E2, 0x0004E2, 0x000418},
+{0x0004E3, 0x0004E3, 0x000438},
+{0x0004E4, 0x0004E4, 0x000418},
+{0x0004E5, 0x0004E5, 0x000438},
+{0x0004E6, 0x0004E6, 0x00041E},
+{0x0004E7, 0x0004E7, 0x00043E},
+{0x0004EA, 0x0004EA, 0x0004E8},
+{0x0004EB, 0x0004EB, 0x0004E9},
+{0x0004EC, 0x0004EC, 0x00042D},
+{0x0004ED, 0x0004ED, 0x00044D},
+{0x0004EE, 0x0004EE, 0x000423},
+{0x0004EF, 0x0004EF, 0x000443},
+{0x0004F0, 0x0004F0, 0x000423},
+{0x0004F1, 0x0004F1, 0x000443},
+{0x0004F2, 0x0004F2, 0x000423},
+{0x0004F3, 0x0004F3, 0x000443},
+{0x0004F4, 0x0004F4, 0x000427},
+{0x0004F5, 0x0004F5, 0x000447},
+{0x0004F8, 0x0004F8, 0x00042B},
+{0x0004F9, 0x0004F9, 0x00044B},
+{0x000622, 0x000623, 0x000627},
+{0x000624, 0x000624, 0x000648},
+{0x000625, 0x000625, 0x000627},
+{0x000626, 0x000626, 0x00064A},
+{0x0006C0, 0x0006C0, 0x0006D5},
+{0x0006C2, 0x0006C2, 0x0006C1},
+{0x0006D3, 0x0006D3, 0x0006D2},
+{0x000929, 0x000929, 0x000928},
+{0x000931, 0x000931, 0x000930},
+{0x000934, 0x000934, 0x000933},
+{0x000958, 0x000958, 0x000915},
+{0x000959, 0x000959, 0x000916},
+{0x00095A, 0x00095A, 0x000917},
+{0x00095B, 0x00095B, 0x00091C},
+{0x00095C, 0x00095C, 0x000921},
+{0x00095D, 0x00095D, 0x000922},
+{0x00095E, 0x00095E, 0x00092B},
+{0x00095F, 0x00095F, 0x00092F},
+{0x0009CB, 0x0009CC, 0x0009C7},
+{0x0009DC, 0x0009DC, 0x0009A1},
+{0x0009DD, 0x0009DD, 0x0009A2},
+{0x0009DF, 0x0009DF, 0x0009AF},
+{0x000A33, 0x000A33, 0x000A32},
+{0x000A36, 0x000A36, 0x000A38},
+{0x000A59, 0x000A59, 0x000A16},
+{0x000A5A, 0x000A5A, 0x000A17},
+{0x000A5B, 0x000A5B, 0x000A1C},
+{0x000A5E, 0x000A5E, 0x000A2B},
+{0x000B48, 0x000B48, 0x000B47},
+{0x000B4B, 0x000B4C, 0x000B47},
+{0x000B5C, 0x000B5C, 0x000B21},
+{0x000B5D, 0x000B5D, 0x000B22},
+{0x000B94, 0x000B94, 0x000B92},
+{0x000BCA, 0x000BCA, 0x000BC6},
+{0x000BCB, 0x000BCB, 0x000BC7},
+{0x000BCC, 0x000BCC, 0x000BC6},
+{0x000C48, 0x000C48, 0x000C46},
+{0x000CC0, 0x000CC0, 0x000CBF},
+{0x000CC7, 0x000CC8, 0x000CC6},
+{0x000CCA, 0x000CCB, 0x000CC6},
+{0x000D4A, 0x000D4A, 0x000D46},
+{0x000D4B, 0x000D4B, 0x000D47},
+{0x000D4C, 0x000D4C, 0x000D46},
+{0x000DDA, 0x000DDA, 0x000DD9},
+{0x000DDC, 0x000DDE, 0x000DD9},
+{0x000F43, 0x000F43, 0x000F42},
+{0x000F4D, 0x000F4D, 0x000F4C},
+{0x000F52, 0x000F52, 0x000F51},
+{0x000F57, 0x000F57, 0x000F56},
+{0x000F5C, 0x000F5C, 0x000F5B},
+{0x000F69, 0x000F69, 0x000F40},
+{0x000F73, 0x000F73, 0x000F71},
+{0x000F75, 0x000F75, 0x000F71},
+{0x000F76, 0x000F76, 0x000FB2},
+{0x000F78, 0x000F78, 0x000FB3},
+{0x000F81, 0x000F81, 0x000F71},
+{0x000F93, 0x000F93, 0x000F92},
+{0x000F9D, 0x000F9D, 0x000F9C},
+{0x000FA2, 0x000FA2, 0x000FA1},
+{0x000FA7, 0x000FA7, 0x000FA6},
+{0x000FAC, 0x000FAC, 0x000FAB},
+{0x000FB9, 0x000FB9, 0x000F90},
+{0x001026, 0x001026, 0x001025},
+{0x001B06, 0x001B06, 0x001B05},
+{0x001B08, 0x001B08, 0x001B07},
+{0x001B0A, 0x001B0A, 0x001B09},
+{0x001B0C, 0x001B0C, 0x001B0B},
+{0x001B0E, 0x001B0E, 0x001B0D},
+{0x001B12, 0x001B12, 0x001B11},
+{0x001B3B, 0x001B3B, 0x001B3A},
+{0x001B3D, 0x001B3D, 0x001B3C},
+{0x001B40, 0x001B40, 0x001B3E},
+{0x001B41, 0x001B41, 0x001B3F},
+{0x001B43, 0x001B43, 0x001B42},
+{0x001E00, 0x001E00, 0x000041},
+{0x001E01, 0x001E01, 0x000061},
+{0x001E02, 0x001E02, 0x000042},
+{0x001E03, 0x001E03, 0x000062},
+{0x001E04, 0x001E04, 0x000042},
+{0x001E05, 0x001E05, 0x000062},
+{0x001E06, 0x001E06, 0x000042},
+{0x001E07, 0x001E07, 0x000062},
+{0x001E08, 0x001E08, 0x000043},
+{0x001E09, 0x001E09, 0x000063},
+{0x001E0A, 0x001E0A, 0x000044},
+{0x001E0B, 0x001E0B, 0x000064},
+{0x001E0C, 0x001E0C, 0x000044},
+{0x001E0D, 0x001E0D, 0x000064},
+{0x001E0E, 0x001E0E, 0x000044},
+{0x001E0F, 0x001E0F, 0x000064},
+{0x001E10, 0x001E10, 0x000044},
+{0x001E11, 0x001E11, 0x000064},
+{0x001E12, 0x001E12, 0x000044},
+{0x001E13, 0x001E13, 0x000064},
+{0x001E14, 0x001E14, 0x000045},
+{0x001E15, 0x001E15, 0x000065},
+{0x001E16, 0x001E16, 0x000045},
+{0x001E17, 0x001E17, 0x000065},
+{0x001E18, 0x001E18, 0x000045},
+{0x001E19, 0x001E19, 0x000065},
+{0x001E1A, 0x001E1A, 0x000045},
+{0x001E1B, 0x001E1B, 0x000065},
+{0x001E1C, 0x001E1C, 0x000045},
+{0x001E1D, 0x001E1D, 0x000065},
+{0x001E1E, 0x001E1E, 0x000046},
+{0x001E1F, 0x001E1F, 0x000066},
+{0x001E20, 0x001E20, 0x000047},
+{0x001E21, 0x001E21, 0x000067},
+{0x001E22, 0x001E22, 0x000048},
+{0x001E23, 0x001E23, 0x000068},
+{0x001E24, 0x001E24, 0x000048},
+{0x001E25, 0x001E25, 0x000068},
+{0x001E26, 0x001E26, 0x000048},
+{0x001E27, 0x001E27, 0x000068},
+{0x001E28, 0x001E28, 0x000048},
+{0x001E29, 0x001E29, 0x000068},
+{0x001E2A, 0x001E2A, 0x000048},
+{0x001E2B, 0x001E2B, 0x000068},
+{0x001E2C, 0x001E2C, 0x000049},
+{0x001E2D, 0x001E2D, 0x000069},
+{0x001E2E, 0x001E2E, 0x000049},
+{0x001E2F, 0x001E2F, 0x000069},
+{0x001E30, 0x001E30, 0x00004B},
+{0x001E31, 0x001E31, 0x00006B},
+{0x001E32, 0x001E32, 0x00004B},
+{0x001E33, 0x001E33, 0x00006B},
+{0x001E34, 0x001E34, 0x00004B},
+{0x001E35, 0x001E35, 0x00006B},
+{0x001E36, 0x001E36, 0x00004C},
+{0x001E37, 0x001E37, 0x00006C},
+{0x001E38, 0x001E38, 0x00004C},
+{0x001E39, 0x001E39, 0x00006C},
+{0x001E3A, 0x001E3A, 0x00004C},
+{0x001E3B, 0x001E3B, 0x00006C},
+{0x001E3C, 0x001E3C, 0x00004C},
+{0x001E3D, 0x001E3D, 0x00006C},
+{0x001E3E, 0x001E3E, 0x00004D},
+{0x001E3F, 0x001E3F, 0x00006D},
+{0x001E40, 0x001E40, 0x00004D},
+{0x001E41, 0x001E41, 0x00006D},
+{0x001E42, 0x001E42, 0x00004D},
+{0x001E43, 0x001E43, 0x00006D},
+{0x001E44, 0x001E44, 0x00004E},
+{0x001E45, 0x001E45, 0x00006E},
+{0x001E46, 0x001E46, 0x00004E},
+{0x001E47, 0x001E47, 0x00006E},
+{0x001E48, 0x001E48, 0x00004E},
+{0x001E49, 0x001E49, 0x00006E},
+{0x001E4A, 0x001E4A, 0x00004E},
+{0x001E4B, 0x001E4B, 0x00006E},
+{0x001E4C, 0x001E4C, 0x00004F},
+{0x001E4D, 0x001E4D, 0x00006F},
+{0x001E4E, 0x001E4E, 0x00004F},
+{0x001E4F, 0x001E4F, 0x00006F},
+{0x001E50, 0x001E50, 0x00004F},
+{0x001E51, 0x001E51, 0x00006F},
+{0x001E52, 0x001E52, 0x00004F},
+{0x001E53, 0x001E53, 0x00006F},
+{0x001E54, 0x001E54, 0x000050},
+{0x001E55, 0x001E55, 0x000070},
+{0x001E56, 0x001E56, 0x000050},
+{0x001E57, 0x001E57, 0x000070},
+{0x001E58, 0x001E58, 0x000052},
+{0x001E59, 0x001E59, 0x000072},
+{0x001E5A, 0x001E5A, 0x000052},
+{0x001E5B, 0x001E5B, 0x000072},
+{0x001E5C, 0x001E5C, 0x000052},
+{0x001E5D, 0x001E5D, 0x000072},
+{0x001E5E, 0x001E5E, 0x000052},
+{0x001E5F, 0x001E5F, 0x000072},
+{0x001E60, 0x001E60, 0x000053},
+{0x001E61, 0x001E61, 0x000073},
+{0x001E62, 0x001E62, 0x000053},
+{0x001E63, 0x001E63, 0x000073},
+{0x001E64, 0x001E64, 0x000053},
+{0x001E65, 0x001E65, 0x000073},
+{0x001E66, 0x001E66, 0x000053},
+{0x001E67, 0x001E67, 0x000073},
+{0x001E68, 0x001E68, 0x000053},
+{0x001E69, 0x001E69, 0x000073},
+{0x001E6A, 0x001E6A, 0x000054},
+{0x001E6B, 0x001E6B, 0x000074},
+{0x001E6C, 0x001E6C, 0x000054},
+{0x001E6D, 0x001E6D, 0x000074},
+{0x001E6E, 0x001E6E, 0x000054},
+{0x001E6F, 0x001E6F, 0x000074},
+{0x001E70, 0x001E70, 0x000054},
+{0x001E71, 0x001E71, 0x000074},
+{0x001E72, 0x001E72, 0x000055},
+{0x001E73, 0x001E73, 0x000075},
+{0x001E74, 0x001E74, 0x000055},
+{0x001E75, 0x001E75, 0x000075},
+{0x001E76, 0x001E76, 0x000055},
+{0x001E77, 0x001E77, 0x000075},
+{0x001E78, 0x001E78, 0x000055},
+{0x001E79, 0x001E79, 0x000075},
+{0x001E7A, 0x001E7A, 0x000055},
+{0x001E7B, 0x001E7B, 0x000075},
+{0x001E7C, 0x001E7C, 0x000056},
+{0x001E7D, 0x001E7D, 0x000076},
+{0x001E7E, 0x001E7E, 0x000056},
+{0x001E7F, 0x001E7F, 0x000076},
+{0x001E80, 0x001E80, 0x000057},
+{0x001E81, 0x001E81, 0x000077},
+{0x001E82, 0x001E82, 0x000057},
+{0x001E83, 0x001E83, 0x000077},
+{0x001E84, 0x001E84, 0x000057},
+{0x001E85, 0x001E85, 0x000077},
+{0x001E86, 0x001E86, 0x000057},
+{0x001E87, 0x001E87, 0x000077},
+{0x001E88, 0x001E88, 0x000057},
+{0x001E89, 0x001E89, 0x000077},
+{0x001E8A, 0x001E8A, 0x000058},
+{0x001E8B, 0x001E8B, 0x000078},
+{0x001E8C, 0x001E8C, 0x000058},
+{0x001E8D, 0x001E8D, 0x000078},
+{0x001E8E, 0x001E8E, 0x000059},
+{0x001E8F, 0x001E8F, 0x000079},
+{0x001E90, 0x001E90, 0x00005A},
+{0x001E91, 0x001E91, 0x00007A},
+{0x001E92, 0x001E92, 0x00005A},
+{0x001E93, 0x001E93, 0x00007A},
+{0x001E94, 0x001E94, 0x00005A},
+{0x001E95, 0x001E95, 0x00007A},
+{0x001E96, 0x001E96, 0x000068},
+{0x001E97, 0x001E97, 0x000074},
+{0x001E98, 0x001E98, 0x000077},
+{0x001E99, 0x001E99, 0x000079},
+{0x001E9B, 0x001E9B, 0x00017F},
+{0x001EA0, 0x001EA0, 0x000041},
+{0x001EA1, 0x001EA1, 0x000061},
+{0x001EA2, 0x001EA2, 0x000041},
+{0x001EA3, 0x001EA3, 0x000061},
+{0x001EA4, 0x001EA4, 0x000041},
+{0x001EA5, 0x001EA5, 0x000061},
+{0x001EA6, 0x001EA6, 0x000041},
+{0x001EA7, 0x001EA7, 0x000061},
+{0x001EA8, 0x001EA8, 0x000041},
+{0x001EA9, 0x001EA9, 0x000061},
+{0x001EAA, 0x001EAA, 0x000041},
+{0x001EAB, 0x001EAB, 0x000061},
+{0x001EAC, 0x001EAC, 0x000041},
+{0x001EAD, 0x001EAD, 0x000061},
+{0x001EAE, 0x001EAE, 0x000041},
+{0x001EAF, 0x001EAF, 0x000061},
+{0x001EB0, 0x001EB0, 0x000041},
+{0x001EB1, 0x001EB1, 0x000061},
+{0x001EB2, 0x001EB2, 0x000041},
+{0x001EB3, 0x001EB3, 0x000061},
+{0x001EB4, 0x001EB4, 0x000041},
+{0x001EB5, 0x001EB5, 0x000061},
+{0x001EB6, 0x001EB6, 0x000041},
+{0x001EB7, 0x001EB7, 0x000061},
+{0x001EB8, 0x001EB8, 0x000045},
+{0x001EB9, 0x001EB9, 0x000065},
+{0x001EBA, 0x001EBA, 0x000045},
+{0x001EBB, 0x001EBB, 0x000065},
+{0x001EBC, 0x001EBC, 0x000045},
+{0x001EBD, 0x001EBD, 0x000065},
+{0x001EBE, 0x001EBE, 0x000045},
+{0x001EBF, 0x001EBF, 0x000065},
+{0x001EC0, 0x001EC0, 0x000045},
+{0x001EC1, 0x001EC1, 0x000065},
+{0x001EC2, 0x001EC2, 0x000045},
+{0x001EC3, 0x001EC3, 0x000065},
+{0x001EC4, 0x001EC4, 0x000045},
+{0x001EC5, 0x001EC5, 0x000065},
+{0x001EC6, 0x001EC6, 0x000045},
+{0x001EC7, 0x001EC7, 0x000065},
+{0x001EC8, 0x001EC8, 0x000049},
+{0x001EC9, 0x001EC9, 0x000069},
+{0x001ECA, 0x001ECA, 0x000049},
+{0x001ECB, 0x001ECB, 0x000069},
+{0x001ECC, 0x001ECC, 0x00004F},
+{0x001ECD, 0x001ECD, 0x00006F},
+{0x001ECE, 0x001ECE, 0x00004F},
+{0x001ECF, 0x001ECF, 0x00006F},
+{0x001ED0, 0x001ED0, 0x00004F},
+{0x001ED1, 0x001ED1, 0x00006F},
+{0x001ED2, 0x001ED2, 0x00004F},
+{0x001ED3, 0x001ED3, 0x00006F},
+{0x001ED4, 0x001ED4, 0x00004F},
+{0x001ED5, 0x001ED5, 0x00006F},
+{0x001ED6, 0x001ED6, 0x00004F},
+{0x001ED7, 0x001ED7, 0x00006F},
+{0x001ED8, 0x001ED8, 0x00004F},
+{0x001ED9, 0x001ED9, 0x00006F},
+{0x001EDA, 0x001EDA, 0x00004F},
+{0x001EDB, 0x001EDB, 0x00006F},
+{0x001EDC, 0x001EDC, 0x00004F},
+{0x001EDD, 0x001EDD, 0x00006F},
+{0x001EDE, 0x001EDE, 0x00004F},
+{0x001EDF, 0x001EDF, 0x00006F},
+{0x001EE0, 0x001EE0, 0x00004F},
+{0x001EE1, 0x001EE1, 0x00006F},
+{0x001EE2, 0x001EE2, 0x00004F},
+{0x001EE3, 0x001EE3, 0x00006F},
+{0x001EE4, 0x001EE4, 0x000055},
+{0x001EE5, 0x001EE5, 0x000075},
+{0x001EE6, 0x001EE6, 0x000055},
+{0x001EE7, 0x001EE7, 0x000075},
+{0x001EE8, 0x001EE8, 0x000055},
+{0x001EE9, 0x001EE9, 0x000075},
+{0x001EEA, 0x001EEA, 0x000055},
+{0x001EEB, 0x001EEB, 0x000075},
+{0x001EEC, 0x001EEC, 0x000055},
+{0x001EED, 0x001EED, 0x000075},
+{0x001EEE, 0x001EEE, 0x000055},
+{0x001EEF, 0x001EEF, 0x000075},
+{0x001EF0, 0x001EF0, 0x000055},
+{0x001EF1, 0x001EF1, 0x000075},
+{0x001EF2, 0x001EF2, 0x000059},
+{0x001EF3, 0x001EF3, 0x000079},
+{0x001EF4, 0x001EF4, 0x000059},
+{0x001EF5, 0x001EF5, 0x000079},
+{0x001EF6, 0x001EF6, 0x000059},
+{0x001EF7, 0x001EF7, 0x000079},
+{0x001EF8, 0x001EF8, 0x000059},
+{0x001EF9, 0x001EF9, 0x000079},
+{0x001F00, 0x001F07, 0x0003B1},
+{0x001F08, 0x001F0F, 0x000391},
+{0x001F10, 0x001F15, 0x0003B5},
+{0x001F18, 0x001F1D, 0x000395},
+{0x001F20, 0x001F27, 0x0003B7},
+{0x001F28, 0x001F2F, 0x000397},
+{0x001F30, 0x001F37, 0x0003B9},
+{0x001F38, 0x001F3F, 0x000399},
+{0x001F40, 0x001F45, 0x0003BF},
+{0x001F48, 0x001F4D, 0x00039F},
+{0x001F50, 0x001F57, 0x0003C5},
+{0x001F59, 0x001F59, 0x0003A5},
+{0x001F5B, 0x001F5B, 0x0003A5},
+{0x001F5D, 0x001F5D, 0x0003A5},
+{0x001F5F, 0x001F5F, 0x0003A5},
+{0x001F60, 0x001F67, 0x0003C9},
+{0x001F68, 0x001F6F, 0x0003A9},
+{0x001F70, 0x001F71, 0x0003B1},
+{0x001F72, 0x001F73, 0x0003B5},
+{0x001F74, 0x001F75, 0x0003B7},
+{0x001F76, 0x001F77, 0x0003B9},
+{0x001F78, 0x001F79, 0x0003BF},
+{0x001F7A, 0x001F7B, 0x0003C5},
+{0x001F7C, 0x001F7D, 0x0003C9},
+{0x001F80, 0x001F87, 0x0003B1},
+{0x001F88, 0x001F8F, 0x000391},
+{0x001F90, 0x001F97, 0x0003B7},
+{0x001F98, 0x001F9F, 0x000397},
+{0x001FA0, 0x001FA7, 0x0003C9},
+{0x001FA8, 0x001FAF, 0x0003A9},
+{0x001FB0, 0x001FB4, 0x0003B1},
+{0x001FB6, 0x001FB7, 0x0003B1},
+{0x001FB8, 0x001FBC, 0x000391},
+{0x001FBE, 0x001FBE, 0x0003B9},
+{0x001FC1, 0x001FC1, 0x0000A8},
+{0x001FC2, 0x001FC4, 0x0003B7},
+{0x001FC6, 0x001FC7, 0x0003B7},
+{0x001FC8, 0x001FC9, 0x000395},
+{0x001FCA, 0x001FCC, 0x000397},
+{0x001FCD, 0x001FCF, 0x001FBF},
+{0x001FD0, 0x001FD3, 0x0003B9},
+{0x001FD6, 0x001FD7, 0x0003B9},
+{0x001FD8, 0x001FDB, 0x000399},
+{0x001FDD, 0x001FDF, 0x001FFE},
+{0x001FE0, 0x001FE3, 0x0003C5},
+{0x001FE4, 0x001FE5, 0x0003C1},
+{0x001FE6, 0x001FE7, 0x0003C5},
+{0x001FE8, 0x001FEB, 0x0003A5},
+{0x001FEC, 0x001FEC, 0x0003A1},
+{0x001FED, 0x001FEE, 0x0000A8},
+{0x001FEF, 0x001FEF, 0x000060},
+{0x001FF2, 0x001FF4, 0x0003C9},
+{0x001FF6, 0x001FF7, 0x0003C9},
+{0x001FF8, 0x001FF9, 0x00039F},
+{0x001FFA, 0x001FFC, 0x0003A9},
+{0x001FFD, 0x001FFD, 0x0000B4},
+{0x002000, 0x002000, 0x002002},
+{0x002001, 0x002001, 0x002003},
+{0x002126, 0x002126, 0x0003A9},
+{0x00212A, 0x00212A, 0x00004B},
+{0x00212B, 0x00212B, 0x000041},
+{0x00219A, 0x00219A, 0x002190},
+{0x00219B, 0x00219B, 0x002192},
+{0x0021AE, 0x0021AE, 0x002194},
+{0x0021CD, 0x0021CD, 0x0021D0},
+{0x0021CE, 0x0021CE, 0x0021D4},
+{0x0021CF, 0x0021CF, 0x0021D2},
+{0x002204, 0x002204, 0x002203},
+{0x002209, 0x002209, 0x002208},
+{0x00220C, 0x00220C, 0x00220B},
+{0x002224, 0x002224, 0x002223},
+{0x002226, 0x002226, 0x002225},
+{0x002241, 0x002241, 0x00223C},
+{0x002244, 0x002244, 0x002243},
+{0x002247, 0x002247, 0x002245},
+{0x002249, 0x002249, 0x002248},
+{0x002260, 0x002260, 0x00003D},
+{0x002262, 0x002262, 0x002261},
+{0x00226D, 0x00226D, 0x00224D},
+{0x00226E, 0x00226E, 0x00003C},
+{0x00226F, 0x00226F, 0x00003E},
+{0x002270, 0x002270, 0x002264},
+{0x002271, 0x002271, 0x002265},
+{0x002274, 0x002274, 0x002272},
+{0x002275, 0x002275, 0x002273},
+{0x002278, 0x002278, 0x002276},
+{0x002279, 0x002279, 0x002277},
+{0x002280, 0x002280, 0x00227A},
+{0x002281, 0x002281, 0x00227B},
+{0x002284, 0x002284, 0x002282},
+{0x002285, 0x002285, 0x002283},
+{0x002288, 0x002288, 0x002286},
+{0x002289, 0x002289, 0x002287},
+{0x0022AC, 0x0022AC, 0x0022A2},
+{0x0022AD, 0x0022AD, 0x0022A8},
+{0x0022AE, 0x0022AE, 0x0022A9},
+{0x0022AF, 0x0022AF, 0x0022AB},
+{0x0022E0, 0x0022E0, 0x00227C},
+{0x0022E1, 0x0022E1, 0x00227D},
+{0x0022E2, 0x0022E2, 0x002291},
+{0x0022E3, 0x0022E3, 0x002292},
+{0x0022EA, 0x0022EA, 0x0022B2},
+{0x0022EB, 0x0022EB, 0x0022B3},
+{0x0022EC, 0x0022EC, 0x0022B4},
+{0x0022ED, 0x0022ED, 0x0022B5},
+{0x002329, 0x002329, 0x003008},
+{0x00232A, 0x00232A, 0x003009},
+{0x002ADC, 0x002ADC, 0x002ADD},
+{0x00304C, 0x00304C, 0x00304B},
+{0x00304E, 0x00304E, 0x00304D},
+{0x003050, 0x003050, 0x00304F},
+{0x003052, 0x003052, 0x003051},
+{0x003054, 0x003054, 0x003053},
+{0x003056, 0x003056, 0x003055},
+{0x003058, 0x003058, 0x003057},
+{0x00305A, 0x00305A, 0x003059},
+{0x00305C, 0x00305C, 0x00305B},
+{0x00305E, 0x00305E, 0x00305D},
+{0x003060, 0x003060, 0x00305F},
+{0x003062, 0x003062, 0x003061},
+{0x003065, 0x003065, 0x003064},
+{0x003067, 0x003067, 0x003066},
+{0x003069, 0x003069, 0x003068},
+{0x003070, 0x003071, 0x00306F},
+{0x003073, 0x003074, 0x003072},
+{0x003076, 0x003077, 0x003075},
+{0x003079, 0x00307A, 0x003078},
+{0x00307C, 0x00307D, 0x00307B},
+{0x003094, 0x003094, 0x003046},
+{0x00309E, 0x00309E, 0x00309D},
+{0x0030AC, 0x0030AC, 0x0030AB},
+{0x0030AE, 0x0030AE, 0x0030AD},
+{0x0030B0, 0x0030B0, 0x0030AF},
+{0x0030B2, 0x0030B2, 0x0030B1},
+{0x0030B4, 0x0030B4, 0x0030B3},
+{0x0030B6, 0x0030B6, 0x0030B5},
+{0x0030B8, 0x0030B8, 0x0030B7},
+{0x0030BA, 0x0030BA, 0x0030B9},
+{0x0030BC, 0x0030BC, 0x0030BB},
+{0x0030BE, 0x0030BE, 0x0030BD},
+{0x0030C0, 0x0030C0, 0x0030BF},
+{0x0030C2, 0x0030C2, 0x0030C1},
+{0x0030C5, 0x0030C5, 0x0030C4},
+{0x0030C7, 0x0030C7, 0x0030C6},
+{0x0030C9, 0x0030C9, 0x0030C8},
+{0x0030D0, 0x0030D1, 0x0030CF},
+{0x0030D3, 0x0030D4, 0x0030D2},
+{0x0030D6, 0x0030D7, 0x0030D5},
+{0x0030D9, 0x0030DA, 0x0030D8},
+{0x0030DC, 0x0030DD, 0x0030DB},
+{0x0030F4, 0x0030F4, 0x0030A6},
+{0x0030F7, 0x0030F7, 0x0030EF},
+{0x0030F8, 0x0030F8, 0x0030F0},
+{0x0030F9, 0x0030F9, 0x0030F1},
+{0x0030FA, 0x0030FA, 0x0030F2},
+{0x0030FE, 0x0030FE, 0x0030FD},
+{0x00AC00, 0x00AE4B, 0x001100},
+{0x00AE4C, 0x00B097, 0x001101},
+{0x00B098, 0x00B2E3, 0x001102},
+{0x00B2E4, 0x00B52F, 0x001103},
+{0x00B530, 0x00B77B, 0x001104},
+{0x00B77C, 0x00B9C7, 0x001105},
+{0x00B9C8, 0x00BC13, 0x001106},
+{0x00BC14, 0x00BE5F, 0x001107},
+{0x00BE60, 0x00C0AB, 0x001108},
+{0x00C0AC, 0x00C2F7, 0x001109},
+{0x00C2F8, 0x00C543, 0x00110A},
+{0x00C544, 0x00C78F, 0x00110B},
+{0x00C790, 0x00C9DB, 0x00110C},
+{0x00C9DC, 0x00CC27, 0x00110D},
+{0x00CC28, 0x00CE73, 0x00110E},
+{0x00CE74, 0x00D0BF, 0x00110F},
+{0x00D0C0, 0x00D30B, 0x001110},
+{0x00D30C, 0x00D557, 0x001111},
+{0x00D558, 0x00D7A3, 0x001112},
+{0x00F900, 0x00F900, 0x008C48},
+{0x00F901, 0x00F901, 0x0066F4},
+{0x00F902, 0x00F902, 0x008ECA},
+{0x00F903, 0x00F903, 0x008CC8},
+{0x00F904, 0x00F904, 0x006ED1},
+{0x00F905, 0x00F905, 0x004E32},
+{0x00F906, 0x00F906, 0x0053E5},
+{0x00F907, 0x00F908, 0x009F9C},
+{0x00F909, 0x00F909, 0x005951},
+{0x00F90A, 0x00F90A, 0x0091D1},
+{0x00F90B, 0x00F90B, 0x005587},
+{0x00F90C, 0x00F90C, 0x005948},
+{0x00F90D, 0x00F90D, 0x0061F6},
+{0x00F90E, 0x00F90E, 0x007669},
+{0x00F90F, 0x00F90F, 0x007F85},
+{0x00F910, 0x00F910, 0x00863F},
+{0x00F911, 0x00F911, 0x0087BA},
+{0x00F912, 0x00F912, 0x0088F8},
+{0x00F913, 0x00F913, 0x00908F},
+{0x00F914, 0x00F914, 0x006A02},
+{0x00F915, 0x00F915, 0x006D1B},
+{0x00F916, 0x00F916, 0x0070D9},
+{0x00F917, 0x00F917, 0x0073DE},
+{0x00F918, 0x00F918, 0x00843D},
+{0x00F919, 0x00F919, 0x00916A},
+{0x00F91A, 0x00F91A, 0x0099F1},
+{0x00F91B, 0x00F91B, 0x004E82},
+{0x00F91C, 0x00F91C, 0x005375},
+{0x00F91D, 0x00F91D, 0x006B04},
+{0x00F91E, 0x00F91E, 0x00721B},
+{0x00F91F, 0x00F91F, 0x00862D},
+{0x00F920, 0x00F920, 0x009E1E},
+{0x00F921, 0x00F921, 0x005D50},
+{0x00F922, 0x00F922, 0x006FEB},
+{0x00F923, 0x00F923, 0x0085CD},
+{0x00F924, 0x00F924, 0x008964},
+{0x00F925, 0x00F925, 0x0062C9},
+{0x00F926, 0x00F926, 0x0081D8},
+{0x00F927, 0x00F927, 0x00881F},
+{0x00F928, 0x00F928, 0x005ECA},
+{0x00F929, 0x00F929, 0x006717},
+{0x00F92A, 0x00F92A, 0x006D6A},
+{0x00F92B, 0x00F92B, 0x0072FC},
+{0x00F92C, 0x00F92C, 0x0090CE},
+{0x00F92D, 0x00F92D, 0x004F86},
+{0x00F92E, 0x00F92E, 0x0051B7},
+{0x00F92F, 0x00F92F, 0x0052DE},
+{0x00F930, 0x00F930, 0x0064C4},
+{0x00F931, 0x00F931, 0x006AD3},
+{0x00F932, 0x00F932, 0x007210},
+{0x00F933, 0x00F933, 0x0076E7},
+{0x00F934, 0x00F934, 0x008001},
+{0x00F935, 0x00F935, 0x008606},
+{0x00F936, 0x00F936, 0x00865C},
+{0x00F937, 0x00F937, 0x008DEF},
+{0x00F938, 0x00F938, 0x009732},
+{0x00F939, 0x00F939, 0x009B6F},
+{0x00F93A, 0x00F93A, 0x009DFA},
+{0x00F93B, 0x00F93B, 0x00788C},
+{0x00F93C, 0x00F93C, 0x00797F},
+{0x00F93D, 0x00F93D, 0x007DA0},
+{0x00F93E, 0x00F93E, 0x0083C9},
+{0x00F93F, 0x00F93F, 0x009304},
+{0x00F940, 0x00F940, 0x009E7F},
+{0x00F941, 0x00F941, 0x008AD6},
+{0x00F942, 0x00F942, 0x0058DF},
+{0x00F943, 0x00F943, 0x005F04},
+{0x00F944, 0x00F944, 0x007C60},
+{0x00F945, 0x00F945, 0x00807E},
+{0x00F946, 0x00F946, 0x007262},
+{0x00F947, 0x00F947, 0x0078CA},
+{0x00F948, 0x00F948, 0x008CC2},
+{0x00F949, 0x00F949, 0x0096F7},
+{0x00F94A, 0x00F94A, 0x0058D8},
+{0x00F94B, 0x00F94B, 0x005C62},
+{0x00F94C, 0x00F94C, 0x006A13},
+{0x00F94D, 0x00F94D, 0x006DDA},
+{0x00F94E, 0x00F94E, 0x006F0F},
+{0x00F94F, 0x00F94F, 0x007D2F},
+{0x00F950, 0x00F950, 0x007E37},
+{0x00F951, 0x00F951, 0x00964B},
+{0x00F952, 0x00F952, 0x0052D2},
+{0x00F953, 0x00F953, 0x00808B},
+{0x00F954, 0x00F954, 0x0051DC},
+{0x00F955, 0x00F955, 0x0051CC},
+{0x00F956, 0x00F956, 0x007A1C},
+{0x00F957, 0x00F957, 0x007DBE},
+{0x00F958, 0x00F958, 0x0083F1},
+{0x00F959, 0x00F959, 0x009675},
+{0x00F95A, 0x00F95A, 0x008B80},
+{0x00F95B, 0x00F95B, 0x0062CF},
+{0x00F95C, 0x00F95C, 0x006A02},
+{0x00F95D, 0x00F95D, 0x008AFE},
+{0x00F95E, 0x00F95E, 0x004E39},
+{0x00F95F, 0x00F95F, 0x005BE7},
+{0x00F960, 0x00F960, 0x006012},
+{0x00F961, 0x00F961, 0x007387},
+{0x00F962, 0x00F962, 0x007570},
+{0x00F963, 0x00F963, 0x005317},
+{0x00F964, 0x00F964, 0x0078FB},
+{0x00F965, 0x00F965, 0x004FBF},
+{0x00F966, 0x00F966, 0x005FA9},
+{0x00F967, 0x00F967, 0x004E0D},
+{0x00F968, 0x00F968, 0x006CCC},
+{0x00F969, 0x00F969, 0x006578},
+{0x00F96A, 0x00F96A, 0x007D22},
+{0x00F96B, 0x00F96B, 0x0053C3},
+{0x00F96C, 0x00F96C, 0x00585E},
+{0x00F96D, 0x00F96D, 0x007701},
+{0x00F96E, 0x00F96E, 0x008449},
+{0x00F96F, 0x00F96F, 0x008AAA},
+{0x00F970, 0x00F970, 0x006BBA},
+{0x00F971, 0x00F971, 0x008FB0},
+{0x00F972, 0x00F972, 0x006C88},
+{0x00F973, 0x00F973, 0x0062FE},
+{0x00F974, 0x00F974, 0x0082E5},
+{0x00F975, 0x00F975, 0x0063A0},
+{0x00F976, 0x00F976, 0x007565},
+{0x00F977, 0x00F977, 0x004EAE},
+{0x00F978, 0x00F978, 0x005169},
+{0x00F979, 0x00F979, 0x0051C9},
+{0x00F97A, 0x00F97A, 0x006881},
+{0x00F97B, 0x00F97B, 0x007CE7},
+{0x00F97C, 0x00F97C, 0x00826F},
+{0x00F97D, 0x00F97D, 0x008AD2},
+{0x00F97E, 0x00F97E, 0x0091CF},
+{0x00F97F, 0x00F97F, 0x0052F5},
+{0x00F980, 0x00F980, 0x005442},
+{0x00F981, 0x00F981, 0x005973},
+{0x00F982, 0x00F982, 0x005EEC},
+{0x00F983, 0x00F983, 0x0065C5},
+{0x00F984, 0x00F984, 0x006FFE},
+{0x00F985, 0x00F985, 0x00792A},
+{0x00F986, 0x00F986, 0x0095AD},
+{0x00F987, 0x00F987, 0x009A6A},
+{0x00F988, 0x00F988, 0x009E97},
+{0x00F989, 0x00F989, 0x009ECE},
+{0x00F98A, 0x00F98A, 0x00529B},
+{0x00F98B, 0x00F98B, 0x0066C6},
+{0x00F98C, 0x00F98C, 0x006B77},
+{0x00F98D, 0x00F98D, 0x008F62},
+{0x00F98E, 0x00F98E, 0x005E74},
+{0x00F98F, 0x00F98F, 0x006190},
+{0x00F990, 0x00F990, 0x006200},
+{0x00F991, 0x00F991, 0x00649A},
+{0x00F992, 0x00F992, 0x006F23},
+{0x00F993, 0x00F993, 0x007149},
+{0x00F994, 0x00F994, 0x007489},
+{0x00F995, 0x00F995, 0x0079CA},
+{0x00F996, 0x00F996, 0x007DF4},
+{0x00F997, 0x00F997, 0x00806F},
+{0x00F998, 0x00F998, 0x008F26},
+{0x00F999, 0x00F999, 0x0084EE},
+{0x00F99A, 0x00F99A, 0x009023},
+{0x00F99B, 0x00F99B, 0x00934A},
+{0x00F99C, 0x00F99C, 0x005217},
+{0x00F99D, 0x00F99D, 0x0052A3},
+{0x00F99E, 0x00F99E, 0x0054BD},
+{0x00F99F, 0x00F99F, 0x0070C8},
+{0x00F9A0, 0x00F9A0, 0x0088C2},
+{0x00F9A1, 0x00F9A1, 0x008AAA},
+{0x00F9A2, 0x00F9A2, 0x005EC9},
+{0x00F9A3, 0x00F9A3, 0x005FF5},
+{0x00F9A4, 0x00F9A4, 0x00637B},
+{0x00F9A5, 0x00F9A5, 0x006BAE},
+{0x00F9A6, 0x00F9A6, 0x007C3E},
+{0x00F9A7, 0x00F9A7, 0x007375},
+{0x00F9A8, 0x00F9A8, 0x004EE4},
+{0x00F9A9, 0x00F9A9, 0x0056F9},
+{0x00F9AA, 0x00F9AA, 0x005BE7},
+{0x00F9AB, 0x00F9AB, 0x005DBA},
+{0x00F9AC, 0x00F9AC, 0x00601C},
+{0x00F9AD, 0x00F9AD, 0x0073B2},
+{0x00F9AE, 0x00F9AE, 0x007469},
+{0x00F9AF, 0x00F9AF, 0x007F9A},
+{0x00F9B0, 0x00F9B0, 0x008046},
+{0x00F9B1, 0x00F9B1, 0x009234},
+{0x00F9B2, 0x00F9B2, 0x0096F6},
+{0x00F9B3, 0x00F9B3, 0x009748},
+{0x00F9B4, 0x00F9B4, 0x009818},
+{0x00F9B5, 0x00F9B5, 0x004F8B},
+{0x00F9B6, 0x00F9B6, 0x0079AE},
+{0x00F9B7, 0x00F9B7, 0x0091B4},
+{0x00F9B8, 0x00F9B8, 0x0096B8},
+{0x00F9B9, 0x00F9B9, 0x0060E1},
+{0x00F9BA, 0x00F9BA, 0x004E86},
+{0x00F9BB, 0x00F9BB, 0x0050DA},
+{0x00F9BC, 0x00F9BC, 0x005BEE},
+{0x00F9BD, 0x00F9BD, 0x005C3F},
+{0x00F9BE, 0x00F9BE, 0x006599},
+{0x00F9BF, 0x00F9BF, 0x006A02},
+{0x00F9C0, 0x00F9C0, 0x0071CE},
+{0x00F9C1, 0x00F9C1, 0x007642},
+{0x00F9C2, 0x00F9C2, 0x0084FC},
+{0x00F9C3, 0x00F9C3, 0x00907C},
+{0x00F9C4, 0x00F9C4, 0x009F8D},
+{0x00F9C5, 0x00F9C5, 0x006688},
+{0x00F9C6, 0x00F9C6, 0x00962E},
+{0x00F9C7, 0x00F9C7, 0x005289},
+{0x00F9C8, 0x00F9C8, 0x00677B},
+{0x00F9C9, 0x00F9C9, 0x0067F3},
+{0x00F9CA, 0x00F9CA, 0x006D41},
+{0x00F9CB, 0x00F9CB, 0x006E9C},
+{0x00F9CC, 0x00F9CC, 0x007409},
+{0x00F9CD, 0x00F9CD, 0x007559},
+{0x00F9CE, 0x00F9CE, 0x00786B},
+{0x00F9CF, 0x00F9CF, 0x007D10},
+{0x00F9D0, 0x00F9D0, 0x00985E},
+{0x00F9D1, 0x00F9D1, 0x00516D},
+{0x00F9D2, 0x00F9D2, 0x00622E},
+{0x00F9D3, 0x00F9D3, 0x009678},
+{0x00F9D4, 0x00F9D4, 0x00502B},
+{0x00F9D5, 0x00F9D5, 0x005D19},
+{0x00F9D6, 0x00F9D6, 0x006DEA},
+{0x00F9D7, 0x00F9D7, 0x008F2A},
+{0x00F9D8, 0x00F9D8, 0x005F8B},
+{0x00F9D9, 0x00F9D9, 0x006144},
+{0x00F9DA, 0x00F9DA, 0x006817},
+{0x00F9DB, 0x00F9DB, 0x007387},
+{0x00F9DC, 0x00F9DC, 0x009686},
+{0x00F9DD, 0x00F9DD, 0x005229},
+{0x00F9DE, 0x00F9DE, 0x00540F},
+{0x00F9DF, 0x00F9DF, 0x005C65},
+{0x00F9E0, 0x00F9E0, 0x006613},
+{0x00F9E1, 0x00F9E1, 0x00674E},
+{0x00F9E2, 0x00F9E2, 0x0068A8},
+{0x00F9E3, 0x00F9E3, 0x006CE5},
+{0x00F9E4, 0x00F9E4, 0x007406},
+{0x00F9E5, 0x00F9E5, 0x0075E2},
+{0x00F9E6, 0x00F9E6, 0x007F79},
+{0x00F9E7, 0x00F9E7, 0x0088CF},
+{0x00F9E8, 0x00F9E8, 0x0088E1},
+{0x00F9E9, 0x00F9E9, 0x0091CC},
+{0x00F9EA, 0x00F9EA, 0x0096E2},
+{0x00F9EB, 0x00F9EB, 0x00533F},
+{0x00F9EC, 0x00F9EC, 0x006EBA},
+{0x00F9ED, 0x00F9ED, 0x00541D},
+{0x00F9EE, 0x00F9EE, 0x0071D0},
+{0x00F9EF, 0x00F9EF, 0x007498},
+{0x00F9F0, 0x00F9F0, 0x0085FA},
+{0x00F9F1, 0x00F9F1, 0x0096A3},
+{0x00F9F2, 0x00F9F2, 0x009C57},
+{0x00F9F3, 0x00F9F3, 0x009E9F},
+{0x00F9F4, 0x00F9F4, 0x006797},
+{0x00F9F5, 0x00F9F5, 0x006DCB},
+{0x00F9F6, 0x00F9F6, 0x0081E8},
+{0x00F9F7, 0x00F9F7, 0x007ACB},
+{0x00F9F8, 0x00F9F8, 0x007B20},
+{0x00F9F9, 0x00F9F9, 0x007C92},
+{0x00F9FA, 0x00F9FA, 0x0072C0},
+{0x00F9FB, 0x00F9FB, 0x007099},
+{0x00F9FC, 0x00F9FC, 0x008B58},
+{0x00F9FD, 0x00F9FD, 0x004EC0},
+{0x00F9FE, 0x00F9FE, 0x008336},
+{0x00F9FF, 0x00F9FF, 0x00523A},
+{0x00FA00, 0x00FA00, 0x005207},
+{0x00FA01, 0x00FA01, 0x005EA6},
+{0x00FA02, 0x00FA02, 0x0062D3},
+{0x00FA03, 0x00FA03, 0x007CD6},
+{0x00FA04, 0x00FA04, 0x005B85},
+{0x00FA05, 0x00FA05, 0x006D1E},
+{0x00FA06, 0x00FA06, 0x0066B4},
+{0x00FA07, 0x00FA07, 0x008F3B},
+{0x00FA08, 0x00FA08, 0x00884C},
+{0x00FA09, 0x00FA09, 0x00964D},
+{0x00FA0A, 0x00FA0A, 0x00898B},
+{0x00FA0B, 0x00FA0B, 0x005ED3},
+{0x00FA0C, 0x00FA0C, 0x005140},
+{0x00FA0D, 0x00FA0D, 0x0055C0},
+{0x00FA10, 0x00FA10, 0x00585A},
+{0x00FA12, 0x00FA12, 0x006674},
+{0x00FA15, 0x00FA15, 0x0051DE},
+{0x00FA16, 0x00FA16, 0x00732A},
+{0x00FA17, 0x00FA17, 0x0076CA},
+{0x00FA18, 0x00FA18, 0x00793C},
+{0x00FA19, 0x00FA19, 0x00795E},
+{0x00FA1A, 0x00FA1A, 0x007965},
+{0x00FA1B, 0x00FA1B, 0x00798F},
+{0x00FA1C, 0x00FA1C, 0x009756},
+{0x00FA1D, 0x00FA1D, 0x007CBE},
+{0x00FA1E, 0x00FA1E, 0x007FBD},
+{0x00FA20, 0x00FA20, 0x008612},
+{0x00FA22, 0x00FA22, 0x008AF8},
+{0x00FA25, 0x00FA25, 0x009038},
+{0x00FA26, 0x00FA26, 0x0090FD},
+{0x00FA2A, 0x00FA2A, 0x0098EF},
+{0x00FA2B, 0x00FA2B, 0x0098FC},
+{0x00FA2C, 0x00FA2C, 0x009928},
+{0x00FA2D, 0x00FA2D, 0x009DB4},
+{0x00FA2E, 0x00FA2E, 0x0090DE},
+{0x00FA2F, 0x00FA2F, 0x0096B7},
+{0x00FA30, 0x00FA30, 0x004FAE},
+{0x00FA31, 0x00FA31, 0x0050E7},
+{0x00FA32, 0x00FA32, 0x00514D},
+{0x00FA33, 0x00FA33, 0x0052C9},
+{0x00FA34, 0x00FA34, 0x0052E4},
+{0x00FA35, 0x00FA35, 0x005351},
+{0x00FA36, 0x00FA36, 0x00559D},
+{0x00FA37, 0x00FA37, 0x005606},
+{0x00FA38, 0x00FA38, 0x005668},
+{0x00FA39, 0x00FA39, 0x005840},
+{0x00FA3A, 0x00FA3A, 0x0058A8},
+{0x00FA3B, 0x00FA3B, 0x005C64},
+{0x00FA3C, 0x00FA3C, 0x005C6E},
+{0x00FA3D, 0x00FA3D, 0x006094},
+{0x00FA3E, 0x00FA3E, 0x006168},
+{0x00FA3F, 0x00FA3F, 0x00618E},
+{0x00FA40, 0x00FA40, 0x0061F2},
+{0x00FA41, 0x00FA41, 0x00654F},
+{0x00FA42, 0x00FA42, 0x0065E2},
+{0x00FA43, 0x00FA43, 0x006691},
+{0x00FA44, 0x00FA44, 0x006885},
+{0x00FA45, 0x00FA45, 0x006D77},
+{0x00FA46, 0x00FA46, 0x006E1A},
+{0x00FA47, 0x00FA47, 0x006F22},
+{0x00FA48, 0x00FA48, 0x00716E},
+{0x00FA49, 0x00FA49, 0x00722B},
+{0x00FA4A, 0x00FA4A, 0x007422},
+{0x00FA4B, 0x00FA4B, 0x007891},
+{0x00FA4C, 0x00FA4C, 0x00793E},
+{0x00FA4D, 0x00FA4D, 0x007949},
+{0x00FA4E, 0x00FA4E, 0x007948},
+{0x00FA4F, 0x00FA4F, 0x007950},
+{0x00FA50, 0x00FA50, 0x007956},
+{0x00FA51, 0x00FA51, 0x00795D},
+{0x00FA52, 0x00FA52, 0x00798D},
+{0x00FA53, 0x00FA53, 0x00798E},
+{0x00FA54, 0x00FA54, 0x007A40},
+{0x00FA55, 0x00FA55, 0x007A81},
+{0x00FA56, 0x00FA56, 0x007BC0},
+{0x00FA57, 0x00FA57, 0x007DF4},
+{0x00FA58, 0x00FA58, 0x007E09},
+{0x00FA59, 0x00FA59, 0x007E41},
+{0x00FA5A, 0x00FA5A, 0x007F72},
+{0x00FA5B, 0x00FA5B, 0x008005},
+{0x00FA5C, 0x00FA5C, 0x0081ED},
+{0x00FA5D, 0x00FA5E, 0x008279},
+{0x00FA5F, 0x00FA5F, 0x008457},
+{0x00FA60, 0x00FA60, 0x008910},
+{0x00FA61, 0x00FA61, 0x008996},
+{0x00FA62, 0x00FA62, 0x008B01},
+{0x00FA63, 0x00FA63, 0x008B39},
+{0x00FA64, 0x00FA64, 0x008CD3},
+{0x00FA65, 0x00FA65, 0x008D08},
+{0x00FA66, 0x00FA66, 0x008FB6},
+{0x00FA67, 0x00FA67, 0x009038},
+{0x00FA68, 0x00FA68, 0x0096E3},
+{0x00FA69, 0x00FA69, 0x0097FF},
+{0x00FA6A, 0x00FA6A, 0x00983B},
+{0x00FA6B, 0x00FA6B, 0x006075},
+{0x00FA6C, 0x00FA6C, 0x0242EE},
+{0x00FA6D, 0x00FA6D, 0x008218},
+{0x00FA70, 0x00FA70, 0x004E26},
+{0x00FA71, 0x00FA71, 0x0051B5},
+{0x00FA72, 0x00FA72, 0x005168},
+{0x00FA73, 0x00FA73, 0x004F80},
+{0x00FA74, 0x00FA74, 0x005145},
+{0x00FA75, 0x00FA75, 0x005180},
+{0x00FA76, 0x00FA76, 0x0052C7},
+{0x00FA77, 0x00FA77, 0x0052FA},
+{0x00FA78, 0x00FA78, 0x00559D},
+{0x00FA79, 0x00FA79, 0x005555},
+{0x00FA7A, 0x00FA7A, 0x005599},
+{0x00FA7B, 0x00FA7B, 0x0055E2},
+{0x00FA7C, 0x00FA7C, 0x00585A},
+{0x00FA7D, 0x00FA7D, 0x0058B3},
+{0x00FA7E, 0x00FA7E, 0x005944},
+{0x00FA7F, 0x00FA7F, 0x005954},
+{0x00FA80, 0x00FA80, 0x005A62},
+{0x00FA81, 0x00FA81, 0x005B28},
+{0x00FA82, 0x00FA82, 0x005ED2},
+{0x00FA83, 0x00FA83, 0x005ED9},
+{0x00FA84, 0x00FA84, 0x005F69},
+{0x00FA85, 0x00FA85, 0x005FAD},
+{0x00FA86, 0x00FA86, 0x0060D8},
+{0x00FA87, 0x00FA87, 0x00614E},
+{0x00FA88, 0x00FA88, 0x006108},
+{0x00FA89, 0x00FA89, 0x00618E},
+{0x00FA8A, 0x00FA8A, 0x006160},
+{0x00FA8B, 0x00FA8B, 0x0061F2},
+{0x00FA8C, 0x00FA8C, 0x006234},
+{0x00FA8D, 0x00FA8D, 0x0063C4},
+{0x00FA8E, 0x00FA8E, 0x00641C},
+{0x00FA8F, 0x00FA8F, 0x006452},
+{0x00FA90, 0x00FA90, 0x006556},
+{0x00FA91, 0x00FA91, 0x006674},
+{0x00FA92, 0x00FA92, 0x006717},
+{0x00FA93, 0x00FA93, 0x00671B},
+{0x00FA94, 0x00FA94, 0x006756},
+{0x00FA95, 0x00FA95, 0x006B79},
+{0x00FA96, 0x00FA96, 0x006BBA},
+{0x00FA97, 0x00FA97, 0x006D41},
+{0x00FA98, 0x00FA98, 0x006EDB},
+{0x00FA99, 0x00FA99, 0x006ECB},
+{0x00FA9A, 0x00FA9A, 0x006F22},
+{0x00FA9B, 0x00FA9B, 0x00701E},
+{0x00FA9C, 0x00FA9C, 0x00716E},
+{0x00FA9D, 0x00FA9D, 0x0077A7},
+{0x00FA9E, 0x00FA9E, 0x007235},
+{0x00FA9F, 0x00FA9F, 0x0072AF},
+{0x00FAA0, 0x00FAA0, 0x00732A},
+{0x00FAA1, 0x00FAA1, 0x007471},
+{0x00FAA2, 0x00FAA2, 0x007506},
+{0x00FAA3, 0x00FAA3, 0x00753B},
+{0x00FAA4, 0x00FAA4, 0x00761D},
+{0x00FAA5, 0x00FAA5, 0x00761F},
+{0x00FAA6, 0x00FAA6, 0x0076CA},
+{0x00FAA7, 0x00FAA7, 0x0076DB},
+{0x00FAA8, 0x00FAA8, 0x0076F4},
+{0x00FAA9, 0x00FAA9, 0x00774A},
+{0x00FAAA, 0x00FAAA, 0x007740},
+{0x00FAAB, 0x00FAAB, 0x0078CC},
+{0x00FAAC, 0x00FAAC, 0x007AB1},
+{0x00FAAD, 0x00FAAD, 0x007BC0},
+{0x00FAAE, 0x00FAAE, 0x007C7B},
+{0x00FAAF, 0x00FAAF, 0x007D5B},
+{0x00FAB0, 0x00FAB0, 0x007DF4},
+{0x00FAB1, 0x00FAB1, 0x007F3E},
+{0x00FAB2, 0x00FAB2, 0x008005},
+{0x00FAB3, 0x00FAB3, 0x008352},
+{0x00FAB4, 0x00FAB4, 0x0083EF},
+{0x00FAB5, 0x00FAB5, 0x008779},
+{0x00FAB6, 0x00FAB6, 0x008941},
+{0x00FAB7, 0x00FAB7, 0x008986},
+{0x00FAB8, 0x00FAB8, 0x008996},
+{0x00FAB9, 0x00FAB9, 0x008ABF},
+{0x00FABA, 0x00FABA, 0x008AF8},
+{0x00FABB, 0x00FABB, 0x008ACB},
+{0x00FABC, 0x00FABC, 0x008B01},
+{0x00FABD, 0x00FABD, 0x008AFE},
+{0x00FABE, 0x00FABE, 0x008AED},
+{0x00FABF, 0x00FABF, 0x008B39},
+{0x00FAC0, 0x00FAC0, 0x008B8A},
+{0x00FAC1, 0x00FAC1, 0x008D08},
+{0x00FAC2, 0x00FAC2, 0x008F38},
+{0x00FAC3, 0x00FAC3, 0x009072},
+{0x00FAC4, 0x00FAC4, 0x009199},
+{0x00FAC5, 0x00FAC5, 0x009276},
+{0x00FAC6, 0x00FAC6, 0x00967C},
+{0x00FAC7, 0x00FAC7, 0x0096E3},
+{0x00FAC8, 0x00FAC8, 0x009756},
+{0x00FAC9, 0x00FAC9, 0x0097DB},
+{0x00FACA, 0x00FACA, 0x0097FF},
+{0x00FACB, 0x00FACB, 0x00980B},
+{0x00FACC, 0x00FACC, 0x00983B},
+{0x00FACD, 0x00FACD, 0x009B12},
+{0x00FACE, 0x00FACE, 0x009F9C},
+{0x00FACF, 0x00FACF, 0x02284A},
+{0x00FAD0, 0x00FAD0, 0x022844},
+{0x00FAD1, 0x00FAD1, 0x0233D5},
+{0x00FAD2, 0x00FAD2, 0x003B9D},
+{0x00FAD3, 0x00FAD3, 0x004018},
+{0x00FAD4, 0x00FAD4, 0x004039},
+{0x00FAD5, 0x00FAD5, 0x025249},
+{0x00FAD6, 0x00FAD6, 0x025CD0},
+{0x00FAD7, 0x00FAD7, 0x027ED3},
+{0x00FAD8, 0x00FAD8, 0x009F43},
+{0x00FAD9, 0x00FAD9, 0x009F8E},
+{0x00FB1D, 0x00FB1D, 0x0005D9},
+{0x00FB1F, 0x00FB1F, 0x0005F2},
+{0x00FB2A, 0x00FB2D, 0x0005E9},
+{0x00FB2E, 0x00FB30, 0x0005D0},
+{0x00FB31, 0x00FB31, 0x0005D1},
+{0x00FB32, 0x00FB32, 0x0005D2},
+{0x00FB33, 0x00FB33, 0x0005D3},
+{0x00FB34, 0x00FB34, 0x0005D4},
+{0x00FB35, 0x00FB35, 0x0005D5},
+{0x00FB36, 0x00FB36, 0x0005D6},
+{0x00FB38, 0x00FB38, 0x0005D8},
+{0x00FB39, 0x00FB39, 0x0005D9},
+{0x00FB3A, 0x00FB3A, 0x0005DA},
+{0x00FB3B, 0x00FB3B, 0x0005DB},
+{0x00FB3C, 0x00FB3C, 0x0005DC},
+{0x00FB3E, 0x00FB3E, 0x0005DE},
+{0x00FB40, 0x00FB40, 0x0005E0},
+{0x00FB41, 0x00FB41, 0x0005E1},
+{0x00FB43, 0x00FB43, 0x0005E3},
+{0x00FB44, 0x00FB44, 0x0005E4},
+{0x00FB46, 0x00FB46, 0x0005E6},
+{0x00FB47, 0x00FB47, 0x0005E7},
+{0x00FB48, 0x00FB48, 0x0005E8},
+{0x00FB49, 0x00FB49, 0x0005E9},
+{0x00FB4A, 0x00FB4A, 0x0005EA},
+{0x00FB4B, 0x00FB4B, 0x0005D5},
+{0x00FB4C, 0x00FB4C, 0x0005D1},
+{0x00FB4D, 0x00FB4D, 0x0005DB},
+{0x00FB4E, 0x00FB4E, 0x0005E4},
+{0x01109A, 0x01109A, 0x011099},
+{0x01109C, 0x01109C, 0x01109B},
+{0x0110AB, 0x0110AB, 0x0110A5},
+{0x01112E, 0x01112E, 0x011131},
+{0x01112F, 0x01112F, 0x011132},
+{0x01134B, 0x01134C, 0x011347},
+{0x0114BB, 0x0114BC, 0x0114B9},
+{0x0114BE, 0x0114BE, 0x0114B9},
+{0x0115BA, 0x0115BA, 0x0115B8},
+{0x0115BB, 0x0115BB, 0x0115B9},
+{0x011938, 0x011938, 0x011935},
+{0x01D15E, 0x01D15E, 0x01D157},
+{0x01D15F, 0x01D164, 0x01D158},
+{0x01D1BB, 0x01D1BB, 0x01D1B9},
+{0x01D1BC, 0x01D1BC, 0x01D1BA},
+{0x01D1BD, 0x01D1BD, 0x01D1B9},
+{0x01D1BE, 0x01D1BE, 0x01D1BA},
+{0x01D1BF, 0x01D1BF, 0x01D1B9},
+{0x01D1C0, 0x01D1C0, 0x01D1BA},
+{0x02F800, 0x02F800, 0x004E3D},
+{0x02F801, 0x02F801, 0x004E38},
+{0x02F802, 0x02F802, 0x004E41},
+{0x02F803, 0x02F803, 0x020122},
+{0x02F804, 0x02F804, 0x004F60},
+{0x02F805, 0x02F805, 0x004FAE},
+{0x02F806, 0x02F806, 0x004FBB},
+{0x02F807, 0x02F807, 0x005002},
+{0x02F808, 0x02F808, 0x00507A},
+{0x02F809, 0x02F809, 0x005099},
+{0x02F80A, 0x02F80A, 0x0050E7},
+{0x02F80B, 0x02F80B, 0x0050CF},
+{0x02F80C, 0x02F80C, 0x00349E},
+{0x02F80D, 0x02F80D, 0x02063A},
+{0x02F80E, 0x02F80E, 0x00514D},
+{0x02F80F, 0x02F80F, 0x005154},
+{0x02F810, 0x02F810, 0x005164},
+{0x02F811, 0x02F811, 0x005177},
+{0x02F812, 0x02F812, 0x02051C},
+{0x02F813, 0x02F813, 0x0034B9},
+{0x02F814, 0x02F814, 0x005167},
+{0x02F815, 0x02F815, 0x00518D},
+{0x02F816, 0x02F816, 0x02054B},
+{0x02F817, 0x02F817, 0x005197},
+{0x02F818, 0x02F818, 0x0051A4},
+{0x02F819, 0x02F819, 0x004ECC},
+{0x02F81A, 0x02F81A, 0x0051AC},
+{0x02F81B, 0x02F81B, 0x0051B5},
+{0x02F81C, 0x02F81C, 0x0291DF},
+{0x02F81D, 0x02F81D, 0x0051F5},
+{0x02F81E, 0x02F81E, 0x005203},
+{0x02F81F, 0x02F81F, 0x0034DF},
+{0x02F820, 0x02F820, 0x00523B},
+{0x02F821, 0x02F821, 0x005246},
+{0x02F822, 0x02F822, 0x005272},
+{0x02F823, 0x02F823, 0x005277},
+{0x02F824, 0x02F824, 0x003515},
+{0x02F825, 0x02F825, 0x0052C7},
+{0x02F826, 0x02F826, 0x0052C9},
+{0x02F827, 0x02F827, 0x0052E4},
+{0x02F828, 0x02F828, 0x0052FA},
+{0x02F829, 0x02F829, 0x005305},
+{0x02F82A, 0x02F82A, 0x005306},
+{0x02F82B, 0x02F82B, 0x005317},
+{0x02F82C, 0x02F82C, 0x005349},
+{0x02F82D, 0x02F82D, 0x005351},
+{0x02F82E, 0x02F82E, 0x00535A},
+{0x02F82F, 0x02F82F, 0x005373},
+{0x02F830, 0x02F830, 0x00537D},
+{0x02F831, 0x02F833, 0x00537F},
+{0x02F834, 0x02F834, 0x020A2C},
+{0x02F835, 0x02F835, 0x007070},
+{0x02F836, 0x02F836, 0x0053CA},
+{0x02F837, 0x02F837, 0x0053DF},
+{0x02F838, 0x02F838, 0x020B63},
+{0x02F839, 0x02F839, 0x0053EB},
+{0x02F83A, 0x02F83A, 0x0053F1},
+{0x02F83B, 0x02F83B, 0x005406},
+{0x02F83C, 0x02F83C, 0x00549E},
+{0x02F83D, 0x02F83D, 0x005438},
+{0x02F83E, 0x02F83E, 0x005448},
+{0x02F83F, 0x02F83F, 0x005468},
+{0x02F840, 0x02F840, 0x0054A2},
+{0x02F841, 0x02F841, 0x0054F6},
+{0x02F842, 0x02F842, 0x005510},
+{0x02F843, 0x02F843, 0x005553},
+{0x02F844, 0x02F844, 0x005563},
+{0x02F845, 0x02F846, 0x005584},
+{0x02F847, 0x02F847, 0x005599},
+{0x02F848, 0x02F848, 0x0055AB},
+{0x02F849, 0x02F849, 0x0055B3},
+{0x02F84A, 0x02F84A, 0x0055C2},
+{0x02F84B, 0x02F84B, 0x005716},
+{0x02F84C, 0x02F84C, 0x005606},
+{0x02F84D, 0x02F84D, 0x005717},
+{0x02F84E, 0x02F84E, 0x005651},
+{0x02F84F, 0x02F84F, 0x005674},
+{0x02F850, 0x02F850, 0x005207},
+{0x02F851, 0x02F851, 0x0058EE},
+{0x02F852, 0x02F852, 0x0057CE},
+{0x02F853, 0x02F853, 0x0057F4},
+{0x02F854, 0x02F854, 0x00580D},
+{0x02F855, 0x02F855, 0x00578B},
+{0x02F856, 0x02F856, 0x005832},
+{0x02F857, 0x02F857, 0x005831},
+{0x02F858, 0x02F858, 0x0058AC},
+{0x02F859, 0x02F859, 0x0214E4},
+{0x02F85A, 0x02F85A, 0x0058F2},
+{0x02F85B, 0x02F85B, 0x0058F7},
+{0x02F85C, 0x02F85C, 0x005906},
+{0x02F85D, 0x02F85D, 0x00591A},
+{0x02F85E, 0x02F85E, 0x005922},
+{0x02F85F, 0x02F85F, 0x005962},
+{0x02F860, 0x02F860, 0x0216A8},
+{0x02F861, 0x02F861, 0x0216EA},
+{0x02F862, 0x02F862, 0x0059EC},
+{0x02F863, 0x02F863, 0x005A1B},
+{0x02F864, 0x02F864, 0x005A27},
+{0x02F865, 0x02F865, 0x0059D8},
+{0x02F866, 0x02F866, 0x005A66},
+{0x02F867, 0x02F867, 0x0036EE},
+{0x02F868, 0x02F868, 0x0036FC},
+{0x02F869, 0x02F869, 0x005B08},
+{0x02F86A, 0x02F86B, 0x005B3E},
+{0x02F86C, 0x02F86C, 0x0219C8},
+{0x02F86D, 0x02F86D, 0x005BC3},
+{0x02F86E, 0x02F86E, 0x005BD8},
+{0x02F86F, 0x02F86F, 0x005BE7},
+{0x02F870, 0x02F870, 0x005BF3},
+{0x02F871, 0x02F871, 0x021B18},
+{0x02F872, 0x02F872, 0x005BFF},
+{0x02F873, 0x02F873, 0x005C06},
+{0x02F874, 0x02F874, 0x005F53},
+{0x02F875, 0x02F875, 0x005C22},
+{0x02F876, 0x02F876, 0x003781},
+{0x02F877, 0x02F877, 0x005C60},
+{0x02F878, 0x02F878, 0x005C6E},
+{0x02F879, 0x02F879, 0x005CC0},
+{0x02F87A, 0x02F87A, 0x005C8D},
+{0x02F87B, 0x02F87B, 0x021DE4},
+{0x02F87C, 0x02F87C, 0x005D43},
+{0x02F87D, 0x02F87D, 0x021DE6},
+{0x02F87E, 0x02F87E, 0x005D6E},
+{0x02F87F, 0x02F87F, 0x005D6B},
+{0x02F880, 0x02F880, 0x005D7C},
+{0x02F881, 0x02F881, 0x005DE1},
+{0x02F882, 0x02F882, 0x005DE2},
+{0x02F883, 0x02F883, 0x00382F},
+{0x02F884, 0x02F884, 0x005DFD},
+{0x02F885, 0x02F885, 0x005E28},
+{0x02F886, 0x02F886, 0x005E3D},
+{0x02F887, 0x02F887, 0x005E69},
+{0x02F888, 0x02F888, 0x003862},
+{0x02F889, 0x02F889, 0x022183},
+{0x02F88A, 0x02F88A, 0x00387C},
+{0x02F88B, 0x02F88B, 0x005EB0},
+{0x02F88C, 0x02F88C, 0x005EB3},
+{0x02F88D, 0x02F88D, 0x005EB6},
+{0x02F88E, 0x02F88E, 0x005ECA},
+{0x02F88F, 0x02F88F, 0x02A392},
+{0x02F890, 0x02F890, 0x005EFE},
+{0x02F891, 0x02F892, 0x022331},
+{0x02F893, 0x02F893, 0x008201},
+{0x02F894, 0x02F895, 0x005F22},
+{0x02F896, 0x02F896, 0x0038C7},
+{0x02F897, 0x02F897, 0x0232B8},
+{0x02F898, 0x02F898, 0x0261DA},
+{0x02F899, 0x02F899, 0x005F62},
+{0x02F89A, 0x02F89A, 0x005F6B},
+{0x02F89B, 0x02F89B, 0x0038E3},
+{0x02F89C, 0x02F89C, 0x005F9A},
+{0x02F89D, 0x02F89D, 0x005FCD},
+{0x02F89E, 0x02F89E, 0x005FD7},
+{0x02F89F, 0x02F89F, 0x005FF9},
+{0x02F8A0, 0x02F8A0, 0x006081},
+{0x02F8A1, 0x02F8A1, 0x00393A},
+{0x02F8A2, 0x02F8A2, 0x00391C},
+{0x02F8A3, 0x02F8A3, 0x006094},
+{0x02F8A4, 0x02F8A4, 0x0226D4},
+{0x02F8A5, 0x02F8A5, 0x0060C7},
+{0x02F8A6, 0x02F8A6, 0x006148},
+{0x02F8A7, 0x02F8A7, 0x00614C},
+{0x02F8A8, 0x02F8A8, 0x00614E},
+{0x02F8A9, 0x02F8A9, 0x00614C},
+{0x02F8AA, 0x02F8AA, 0x00617A},
+{0x02F8AB, 0x02F8AB, 0x00618E},
+{0x02F8AC, 0x02F8AC, 0x0061B2},
+{0x02F8AD, 0x02F8AD, 0x0061A4},
+{0x02F8AE, 0x02F8AE, 0x0061AF},
+{0x02F8AF, 0x02F8AF, 0x0061DE},
+{0x02F8B0, 0x02F8B0, 0x0061F2},
+{0x02F8B1, 0x02F8B1, 0x0061F6},
+{0x02F8B2, 0x02F8B2, 0x006210},
+{0x02F8B3, 0x02F8B3, 0x00621B},
+{0x02F8B4, 0x02F8B4, 0x00625D},
+{0x02F8B5, 0x02F8B5, 0x0062B1},
+{0x02F8B6, 0x02F8B6, 0x0062D4},
+{0x02F8B7, 0x02F8B7, 0x006350},
+{0x02F8B8, 0x02F8B8, 0x022B0C},
+{0x02F8B9, 0x02F8B9, 0x00633D},
+{0x02F8BA, 0x02F8BA, 0x0062FC},
+{0x02F8BB, 0x02F8BB, 0x006368},
+{0x02F8BC, 0x02F8BC, 0x006383},
+{0x02F8BD, 0x02F8BD, 0x0063E4},
+{0x02F8BE, 0x02F8BE, 0x022BF1},
+{0x02F8BF, 0x02F8BF, 0x006422},
+{0x02F8C0, 0x02F8C0, 0x0063C5},
+{0x02F8C1, 0x02F8C1, 0x0063A9},
+{0x02F8C2, 0x02F8C2, 0x003A2E},
+{0x02F8C3, 0x02F8C3, 0x006469},
+{0x02F8C4, 0x02F8C4, 0x00647E},
+{0x02F8C5, 0x02F8C5, 0x00649D},
+{0x02F8C6, 0x02F8C6, 0x006477},
+{0x02F8C7, 0x02F8C7, 0x003A6C},
+{0x02F8C8, 0x02F8C8, 0x00654F},
+{0x02F8C9, 0x02F8C9, 0x00656C},
+{0x02F8CA, 0x02F8CA, 0x02300A},
+{0x02F8CB, 0x02F8CB, 0x0065E3},
+{0x02F8CC, 0x02F8CC, 0x0066F8},
+{0x02F8CD, 0x02F8CD, 0x006649},
+{0x02F8CE, 0x02F8CE, 0x003B19},
+{0x02F8CF, 0x02F8CF, 0x006691},
+{0x02F8D0, 0x02F8D0, 0x003B08},
+{0x02F8D1, 0x02F8D1, 0x003AE4},
+{0x02F8D2, 0x02F8D2, 0x005192},
+{0x02F8D3, 0x02F8D3, 0x005195},
+{0x02F8D4, 0x02F8D4, 0x006700},
+{0x02F8D5, 0x02F8D5, 0x00669C},
+{0x02F8D6, 0x02F8D6, 0x0080AD},
+{0x02F8D7, 0x02F8D7, 0x0043D9},
+{0x02F8D8, 0x02F8D8, 0x006717},
+{0x02F8D9, 0x02F8D9, 0x00671B},
+{0x02F8DA, 0x02F8DA, 0x006721},
+{0x02F8DB, 0x02F8DB, 0x00675E},
+{0x02F8DC, 0x02F8DC, 0x006753},
+{0x02F8DD, 0x02F8DD, 0x0233C3},
+{0x02F8DE, 0x02F8DE, 0x003B49},
+{0x02F8DF, 0x02F8DF, 0x0067FA},
+{0x02F8E0, 0x02F8E0, 0x006785},
+{0x02F8E1, 0x02F8E1, 0x006852},
+{0x02F8E2, 0x02F8E2, 0x006885},
+{0x02F8E3, 0x02F8E3, 0x02346D},
+{0x02F8E4, 0x02F8E4, 0x00688E},
+{0x02F8E5, 0x02F8E5, 0x00681F},
+{0x02F8E6, 0x02F8E6, 0x006914},
+{0x02F8E7, 0x02F8E7, 0x003B9D},
+{0x02F8E8, 0x02F8E8, 0x006942},
+{0x02F8E9, 0x02F8E9, 0x0069A3},
+{0x02F8EA, 0x02F8EA, 0x0069EA},
+{0x02F8EB, 0x02F8EB, 0x006AA8},
+{0x02F8EC, 0x02F8EC, 0x0236A3},
+{0x02F8ED, 0x02F8ED, 0x006ADB},
+{0x02F8EE, 0x02F8EE, 0x003C18},
+{0x02F8EF, 0x02F8EF, 0x006B21},
+{0x02F8F0, 0x02F8F0, 0x0238A7},
+{0x02F8F1, 0x02F8F1, 0x006B54},
+{0x02F8F2, 0x02F8F2, 0x003C4E},
+{0x02F8F3, 0x02F8F3, 0x006B72},
+{0x02F8F4, 0x02F8F4, 0x006B9F},
+{0x02F8F5, 0x02F8F5, 0x006BBA},
+{0x02F8F6, 0x02F8F6, 0x006BBB},
+{0x02F8F7, 0x02F8F7, 0x023A8D},
+{0x02F8F8, 0x02F8F8, 0x021D0B},
+{0x02F8F9, 0x02F8F9, 0x023AFA},
+{0x02F8FA, 0x02F8FA, 0x006C4E},
+{0x02F8FB, 0x02F8FB, 0x023CBC},
+{0x02F8FC, 0x02F8FC, 0x006CBF},
+{0x02F8FD, 0x02F8FD, 0x006CCD},
+{0x02F8FE, 0x02F8FE, 0x006C67},
+{0x02F8FF, 0x02F8FF, 0x006D16},
+{0x02F900, 0x02F900, 0x006D3E},
+{0x02F901, 0x02F901, 0x006D77},
+{0x02F902, 0x02F902, 0x006D41},
+{0x02F903, 0x02F903, 0x006D69},
+{0x02F904, 0x02F904, 0x006D78},
+{0x02F905, 0x02F905, 0x006D85},
+{0x02F906, 0x02F906, 0x023D1E},
+{0x02F907, 0x02F907, 0x006D34},
+{0x02F908, 0x02F908, 0x006E2F},
+{0x02F909, 0x02F909, 0x006E6E},
+{0x02F90A, 0x02F90A, 0x003D33},
+{0x02F90B, 0x02F90B, 0x006ECB},
+{0x02F90C, 0x02F90C, 0x006EC7},
+{0x02F90D, 0x02F90D, 0x023ED1},
+{0x02F90E, 0x02F90E, 0x006DF9},
+{0x02F90F, 0x02F90F, 0x006F6E},
+{0x02F910, 0x02F910, 0x023F5E},
+{0x02F911, 0x02F911, 0x023F8E},
+{0x02F912, 0x02F912, 0x006FC6},
+{0x02F913, 0x02F913, 0x007039},
+{0x02F914, 0x02F914, 0x00701E},
+{0x02F915, 0x02F915, 0x00701B},
+{0x02F916, 0x02F916, 0x003D96},
+{0x02F917, 0x02F917, 0x00704A},
+{0x02F918, 0x02F918, 0x00707D},
+{0x02F919, 0x02F919, 0x007077},
+{0x02F91A, 0x02F91A, 0x0070AD},
+{0x02F91B, 0x02F91B, 0x020525},
+{0x02F91C, 0x02F91C, 0x007145},
+{0x02F91D, 0x02F91D, 0x024263},
+{0x02F91E, 0x02F91E, 0x00719C},
+{0x02F91F, 0x02F91F, 0x0243AB},
+{0x02F920, 0x02F920, 0x007228},
+{0x02F921, 0x02F921, 0x007235},
+{0x02F922, 0x02F922, 0x007250},
+{0x02F923, 0x02F923, 0x024608},
+{0x02F924, 0x02F924, 0x007280},
+{0x02F925, 0x02F925, 0x007295},
+{0x02F926, 0x02F926, 0x024735},
+{0x02F927, 0x02F927, 0x024814},
+{0x02F928, 0x02F928, 0x00737A},
+{0x02F929, 0x02F929, 0x00738B},
+{0x02F92A, 0x02F92A, 0x003EAC},
+{0x02F92B, 0x02F92B, 0x0073A5},
+{0x02F92C, 0x02F92D, 0x003EB8},
+{0x02F92E, 0x02F92E, 0x007447},
+{0x02F92F, 0x02F92F, 0x00745C},
+{0x02F930, 0x02F930, 0x007471},
+{0x02F931, 0x02F931, 0x007485},
+{0x02F932, 0x02F932, 0x0074CA},
+{0x02F933, 0x02F933, 0x003F1B},
+{0x02F934, 0x02F934, 0x007524},
+{0x02F935, 0x02F935, 0x024C36},
+{0x02F936, 0x02F936, 0x00753E},
+{0x02F937, 0x02F937, 0x024C92},
+{0x02F938, 0x02F938, 0x007570},
+{0x02F939, 0x02F939, 0x02219F},
+{0x02F93A, 0x02F93A, 0x007610},
+{0x02F93B, 0x02F93B, 0x024FA1},
+{0x02F93C, 0x02F93C, 0x024FB8},
+{0x02F93D, 0x02F93D, 0x025044},
+{0x02F93E, 0x02F93E, 0x003FFC},
+{0x02F93F, 0x02F93F, 0x004008},
+{0x02F940, 0x02F940, 0x0076F4},
+{0x02F941, 0x02F941, 0x0250F3},
+{0x02F942, 0x02F942, 0x0250F2},
+{0x02F943, 0x02F943, 0x025119},
+{0x02F944, 0x02F944, 0x025133},
+{0x02F945, 0x02F945, 0x00771E},
+{0x02F946, 0x02F947, 0x00771F},
+{0x02F948, 0x02F948, 0x00774A},
+{0x02F949, 0x02F949, 0x004039},
+{0x02F94A, 0x02F94A, 0x00778B},
+{0x02F94B, 0x02F94B, 0x004046},
+{0x02F94C, 0x02F94C, 0x004096},
+{0x02F94D, 0x02F94D, 0x02541D},
+{0x02F94E, 0x02F94E, 0x00784E},
+{0x02F94F, 0x02F94F, 0x00788C},
+{0x02F950, 0x02F950, 0x0078CC},
+{0x02F951, 0x02F951, 0x0040E3},
+{0x02F952, 0x02F952, 0x025626},
+{0x02F953, 0x02F953, 0x007956},
+{0x02F954, 0x02F954, 0x02569A},
+{0x02F955, 0x02F955, 0x0256C5},
+{0x02F956, 0x02F956, 0x00798F},
+{0x02F957, 0x02F957, 0x0079EB},
+{0x02F958, 0x02F958, 0x00412F},
+{0x02F959, 0x02F959, 0x007A40},
+{0x02F95A, 0x02F95A, 0x007A4A},
+{0x02F95B, 0x02F95B, 0x007A4F},
+{0x02F95C, 0x02F95C, 0x02597C},
+{0x02F95D, 0x02F95E, 0x025AA7},
+{0x02F95F, 0x02F95F, 0x007AEE},
+{0x02F960, 0x02F960, 0x004202},
+{0x02F961, 0x02F961, 0x025BAB},
+{0x02F962, 0x02F962, 0x007BC6},
+{0x02F963, 0x02F963, 0x007BC9},
+{0x02F964, 0x02F964, 0x004227},
+{0x02F965, 0x02F965, 0x025C80},
+{0x02F966, 0x02F966, 0x007CD2},
+{0x02F967, 0x02F967, 0x0042A0},
+{0x02F968, 0x02F968, 0x007CE8},
+{0x02F969, 0x02F969, 0x007CE3},
+{0x02F96A, 0x02F96A, 0x007D00},
+{0x02F96B, 0x02F96B, 0x025F86},
+{0x02F96C, 0x02F96C, 0x007D63},
+{0x02F96D, 0x02F96D, 0x004301},
+{0x02F96E, 0x02F96E, 0x007DC7},
+{0x02F96F, 0x02F96F, 0x007E02},
+{0x02F970, 0x02F970, 0x007E45},
+{0x02F971, 0x02F971, 0x004334},
+{0x02F972, 0x02F972, 0x026228},
+{0x02F973, 0x02F973, 0x026247},
+{0x02F974, 0x02F974, 0x004359},
+{0x02F975, 0x02F975, 0x0262D9},
+{0x02F976, 0x02F976, 0x007F7A},
+{0x02F977, 0x02F977, 0x02633E},
+{0x02F978, 0x02F978, 0x007F95},
+{0x02F979, 0x02F979, 0x007FFA},
+{0x02F97A, 0x02F97A, 0x008005},
+{0x02F97B, 0x02F97B, 0x0264DA},
+{0x02F97C, 0x02F97C, 0x026523},
+{0x02F97D, 0x02F97D, 0x008060},
+{0x02F97E, 0x02F97E, 0x0265A8},
+{0x02F97F, 0x02F97F, 0x008070},
+{0x02F980, 0x02F980, 0x02335F},
+{0x02F981, 0x02F981, 0x0043D5},
+{0x02F982, 0x02F982, 0x0080B2},
+{0x02F983, 0x02F983, 0x008103},
+{0x02F984, 0x02F984, 0x00440B},
+{0x02F985, 0x02F985, 0x00813E},
+{0x02F986, 0x02F986, 0x005AB5},
+{0x02F987, 0x02F987, 0x0267A7},
+{0x02F988, 0x02F988, 0x0267B5},
+{0x02F989, 0x02F989, 0x023393},
+{0x02F98A, 0x02F98A, 0x02339C},
+{0x02F98B, 0x02F98B, 0x008201},
+{0x02F98C, 0x02F98C, 0x008204},
+{0x02F98D, 0x02F98D, 0x008F9E},
+{0x02F98E, 0x02F98E, 0x00446B},
+{0x02F98F, 0x02F98F, 0x008291},
+{0x02F990, 0x02F990, 0x00828B},
+{0x02F991, 0x02F991, 0x00829D},
+{0x02F992, 0x02F992, 0x0052B3},
+{0x02F993, 0x02F993, 0x0082B1},
+{0x02F994, 0x02F994, 0x0082B3},
+{0x02F995, 0x02F995, 0x0082BD},
+{0x02F996, 0x02F996, 0x0082E6},
+{0x02F997, 0x02F997, 0x026B3C},
+{0x02F998, 0x02F998, 0x0082E5},
+{0x02F999, 0x02F999, 0x00831D},
+{0x02F99A, 0x02F99A, 0x008363},
+{0x02F99B, 0x02F99B, 0x0083AD},
+{0x02F99C, 0x02F99C, 0x008323},
+{0x02F99D, 0x02F99D, 0x0083BD},
+{0x02F99E, 0x02F99E, 0x0083E7},
+{0x02F99F, 0x02F99F, 0x008457},
+{0x02F9A0, 0x02F9A0, 0x008353},
+{0x02F9A1, 0x02F9A1, 0x0083CA},
+{0x02F9A2, 0x02F9A2, 0x0083CC},
+{0x02F9A3, 0x02F9A3, 0x0083DC},
+{0x02F9A4, 0x02F9A4, 0x026C36},
+{0x02F9A5, 0x02F9A5, 0x026D6B},
+{0x02F9A6, 0x02F9A6, 0x026CD5},
+{0x02F9A7, 0x02F9A7, 0x00452B},
+{0x02F9A8, 0x02F9A8, 0x0084F1},
+{0x02F9A9, 0x02F9A9, 0x0084F3},
+{0x02F9AA, 0x02F9AA, 0x008516},
+{0x02F9AB, 0x02F9AB, 0x0273CA},
+{0x02F9AC, 0x02F9AC, 0x008564},
+{0x02F9AD, 0x02F9AD, 0x026F2C},
+{0x02F9AE, 0x02F9AE, 0x00455D},
+{0x02F9AF, 0x02F9AF, 0x004561},
+{0x02F9B0, 0x02F9B0, 0x026FB1},
+{0x02F9B1, 0x02F9B1, 0x0270D2},
+{0x02F9B2, 0x02F9B2, 0x00456B},
+{0x02F9B3, 0x02F9B3, 0x008650},
+{0x02F9B4, 0x02F9B4, 0x00865C},
+{0x02F9B5, 0x02F9B5, 0x008667},
+{0x02F9B6, 0x02F9B6, 0x008669},
+{0x02F9B7, 0x02F9B7, 0x0086A9},
+{0x02F9B8, 0x02F9B8, 0x008688},
+{0x02F9B9, 0x02F9B9, 0x00870E},
+{0x02F9BA, 0x02F9BA, 0x0086E2},
+{0x02F9BB, 0x02F9BB, 0x008779},
+{0x02F9BC, 0x02F9BC, 0x008728},
+{0x02F9BD, 0x02F9BD, 0x00876B},
+{0x02F9BE, 0x02F9BE, 0x008786},
+{0x02F9BF, 0x02F9BF, 0x0045D7},
+{0x02F9C0, 0x02F9C0, 0x0087E1},
+{0x02F9C1, 0x02F9C1, 0x008801},
+{0x02F9C2, 0x02F9C2, 0x0045F9},
+{0x02F9C3, 0x02F9C3, 0x008860},
+{0x02F9C4, 0x02F9C4, 0x008863},
+{0x02F9C5, 0x02F9C5, 0x027667},
+{0x02F9C6, 0x02F9C6, 0x0088D7},
+{0x02F9C7, 0x02F9C7, 0x0088DE},
+{0x02F9C8, 0x02F9C8, 0x004635},
+{0x02F9C9, 0x02F9C9, 0x0088FA},
+{0x02F9CA, 0x02F9CA, 0x0034BB},
+{0x02F9CB, 0x02F9CB, 0x0278AE},
+{0x02F9CC, 0x02F9CC, 0x027966},
+{0x02F9CD, 0x02F9CD, 0x0046BE},
+{0x02F9CE, 0x02F9CE, 0x0046C7},
+{0x02F9CF, 0x02F9CF, 0x008AA0},
+{0x02F9D0, 0x02F9D0, 0x008AED},
+{0x02F9D1, 0x02F9D1, 0x008B8A},
+{0x02F9D2, 0x02F9D2, 0x008C55},
+{0x02F9D3, 0x02F9D3, 0x027CA8},
+{0x02F9D4, 0x02F9D4, 0x008CAB},
+{0x02F9D5, 0x02F9D5, 0x008CC1},
+{0x02F9D6, 0x02F9D6, 0x008D1B},
+{0x02F9D7, 0x02F9D7, 0x008D77},
+{0x02F9D8, 0x02F9D8, 0x027F2F},
+{0x02F9D9, 0x02F9D9, 0x020804},
+{0x02F9DA, 0x02F9DA, 0x008DCB},
+{0x02F9DB, 0x02F9DB, 0x008DBC},
+{0x02F9DC, 0x02F9DC, 0x008DF0},
+{0x02F9DD, 0x02F9DD, 0x0208DE},
+{0x02F9DE, 0x02F9DE, 0x008ED4},
+{0x02F9DF, 0x02F9DF, 0x008F38},
+{0x02F9E0, 0x02F9E0, 0x0285D2},
+{0x02F9E1, 0x02F9E1, 0x0285ED},
+{0x02F9E2, 0x02F9E2, 0x009094},
+{0x02F9E3, 0x02F9E3, 0x0090F1},
+{0x02F9E4, 0x02F9E4, 0x009111},
+{0x02F9E5, 0x02F9E5, 0x02872E},
+{0x02F9E6, 0x02F9E6, 0x00911B},
+{0x02F9E7, 0x02F9E7, 0x009238},
+{0x02F9E8, 0x02F9E8, 0x0092D7},
+{0x02F9E9, 0x02F9E9, 0x0092D8},
+{0x02F9EA, 0x02F9EA, 0x00927C},
+{0x02F9EB, 0x02F9EB, 0x0093F9},
+{0x02F9EC, 0x02F9EC, 0x009415},
+{0x02F9ED, 0x02F9ED, 0x028BFA},
+{0x02F9EE, 0x02F9EE, 0x00958B},
+{0x02F9EF, 0x02F9EF, 0x004995},
+{0x02F9F0, 0x02F9F0, 0x0095B7},
+{0x02F9F1, 0x02F9F1, 0x028D77},
+{0x02F9F2, 0x02F9F2, 0x0049E6},
+{0x02F9F3, 0x02F9F3, 0x0096C3},
+{0x02F9F4, 0x02F9F4, 0x005DB2},
+{0x02F9F5, 0x02F9F5, 0x009723},
+{0x02F9F6, 0x02F9F6, 0x029145},
+{0x02F9F7, 0x02F9F7, 0x02921A},
+{0x02F9F8, 0x02F9F8, 0x004A6E},
+{0x02F9F9, 0x02F9F9, 0x004A76},
+{0x02F9FA, 0x02F9FA, 0x0097E0},
+{0x02F9FB, 0x02F9FB, 0x02940A},
+{0x02F9FC, 0x02F9FC, 0x004AB2},
+{0x02F9FD, 0x02F9FD, 0x029496},
+{0x02F9FE, 0x02F9FF, 0x00980B},
+{0x02FA00, 0x02FA00, 0x009829},
+{0x02FA01, 0x02FA01, 0x0295B6},
+{0x02FA02, 0x02FA02, 0x0098E2},
+{0x02FA03, 0x02FA03, 0x004B33},
+{0x02FA04, 0x02FA04, 0x009929},
+{0x02FA05, 0x02FA05, 0x0099A7},
+{0x02FA06, 0x02FA06, 0x0099C2},
+{0x02FA07, 0x02FA07, 0x0099FE},
+{0x02FA08, 0x02FA08, 0x004BCE},
+{0x02FA09, 0x02FA09, 0x029B30},
+{0x02FA0A, 0x02FA0A, 0x009B12},
+{0x02FA0B, 0x02FA0B, 0x009C40},
+{0x02FA0C, 0x02FA0C, 0x009CFD},
+{0x02FA0D, 0x02FA0D, 0x004CCE},
+{0x02FA0E, 0x02FA0E, 0x004CED},
+{0x02FA0F, 0x02FA0F, 0x009D67},
+{0x02FA10, 0x02FA10, 0x02A0CE},
+{0x02FA11, 0x02FA11, 0x004CF8},
+{0x02FA12, 0x02FA12, 0x02A105},
+{0x02FA13, 0x02FA13, 0x02A20E},
+{0x02FA14, 0x02FA14, 0x02A291},
+{0x02FA15, 0x02FA15, 0x009EBB},
+{0x02FA16, 0x02FA16, 0x004D56},
+{0x02FA17, 0x02FA17, 0x009EF9},
+{0x02FA18, 0x02FA18, 0x009EFE},
+{0x02FA19, 0x02FA19, 0x009F05},
+{0x02FA1A, 0x02FA1A, 0x009F0F},
+{0x02FA1B, 0x02FA1B, 0x009F16},
+{0x02FA1C, 0x02FA1C, 0x009F3B},
+{0x02FA1D, 0x02FA1D, 0x02A600},
+};
diff --git a/llama/unicode-data.h b/llama/unicode-data.h
new file mode 100644
index 00000000..bb31731d
--- /dev/null
+++ b/llama/unicode-data.h
@@ -0,0 +1,46 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <vector>
+#include <unordered_map>
+#include <unordered_set>
+
+struct range_nfd {
+ uint32_t first;
+ uint32_t last;
+ uint32_t nfd;
+};
+
+static const uint32_t MAX_CODEPOINTS = 0x110000;
+
+extern const std::vector<std::pair<uint32_t, uint16_t>> unicode_ranges_flags;
+extern const std::unordered_set<uint32_t> unicode_set_whitespace;
+extern const std::unordered_map<uint32_t, uint32_t> unicode_map_lowercase;
+extern const std::unordered_map<uint32_t, uint32_t> unicode_map_uppercase;
+extern const std::vector<range_nfd> unicode_ranges_nfd;
diff --git a/llama/unicode.cpp b/llama/unicode.cpp
new file mode 100644
index 00000000..d3d8b05f
--- /dev/null
+++ b/llama/unicode.cpp
@@ -0,0 +1,844 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if defined(_MSC_VER)
+#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING
+#endif
+
+#include "unicode.h"
+#include "unicode-data.h"
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <map>
+#include <regex>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+#include <locale>
+#include <codecvt>
+
+size_t unicode_len_utf8(char src) {
+ const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };
+ uint8_t highbits = static_cast<uint8_t>(src) >> 4;
+ return lookup[highbits];
+}
+
+static std::string unicode_cpts_to_utf8(const std::vector<uint32_t> & cps) {
+ std::string result;
+ for (size_t i = 0; i < cps.size(); ++i) {
+ result.append(unicode_cpt_to_utf8(cps[i]));
+ }
+ return result;
+}
+
+uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
+ assert(offset < utf8.size());
+ if (!(utf8[offset + 0] & 0x80)) {
+ auto result = utf8[offset + 0];
+ offset += 1;
+ return result;
+ }
+ if (!(utf8[offset + 0] & 0x40)) {
+ throw std::invalid_argument("invalid character");
+ }
+ if (!(utf8[offset + 0] & 0x20)) {
+ if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80)) {
+ throw std::invalid_argument("invalid character");
+ }
+ auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f);
+ offset += 2;
+ return result;
+ }
+ if (!(utf8[offset + 0] & 0x10)) {
+ if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80)) {
+ throw std::invalid_argument("invalid character");
+ }
+ auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f);
+ offset += 3;
+ return result;
+ }
+ if (!(utf8[offset + 0] & 0x08)) {
+ if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) {
+ throw std::invalid_argument("invalid character");
+ }
+ auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f);
+ offset += 4;
+ return result;
+ }
+ throw std::invalid_argument("failed to convert utf8 to codepoint");
+}
+
+//static std::vector<uint16_t> unicode_cpt_to_utf16(uint32_t cp) {
+// std::vector<uint16_t> result;
+// if (/* 0x0000 <= cp && */ cp <= 0xffff) {
+// result.emplace_back(cp);
+// return result;
+// }
+// if (0x10000 <= cp && cp <= 0x10ffff) {
+// result.emplace_back(0xd800 | ((cp - 0x10000) >> 10));
+// result.emplace_back(0xdc00 | ((cp - 0x10000) & 0x03ff));
+// return result;
+// }
+// throw std::invalid_argument("failed to convert codepoint to utf16");
+//}
+
+//static std::vector<uint16_t> unicode_cpts_to_utf16(const std::vector<uint32_t> & cps) {
+// std::vector<uint16_t> result;
+// for (size_t i = 0; i < cps.size(); ++i) {
+// auto temp = unicode_cpt_to_utf16(cps[i]);
+// result.insert(result.end(), temp.begin(), temp.end());
+// }
+// return result;
+//}
+
+//static uint32_t unicode_cpt_from_utf16(const std::vector<uint16_t> & utf16, size_t & offset) {
+// assert(offset < utf16.size());
+// if (((utf16[0] >> 10) << 10) != 0xd800) {
+// auto result = utf16[offset + 0];
+// offset += 1;
+// return result;
+// }
+//
+// if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) {
+// throw std::invalid_argument("invalid character");
+// }
+//
+// auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff));
+// offset += 2;
+// return result;
+//}
+
+//static std::vector<uint32_t> unicode_cpts_from_utf16(const std::vector<uint16_t> & utf16) {
+// std::vector<uint32_t> result;
+// size_t offset = 0;
+// while (offset < utf16.size()) {
+// result.push_back(unicode_cpt_from_utf16(utf16, offset));
+// }
+// return result;
+//}
+
+static std::vector<codepoint_flags> unicode_cpt_flags_array() {
+ std::vector<codepoint_flags> cpt_flags(MAX_CODEPOINTS, codepoint_flags::UNDEFINED);
+
+ assert (unicode_ranges_flags.front().first == 0);
+ assert (unicode_ranges_flags.back().first == MAX_CODEPOINTS);
+ for (size_t i = 1; i < unicode_ranges_flags.size(); ++i) {
+ const auto range_ini = unicode_ranges_flags[i-1]; // codepoint_ini, flags
+ const auto range_end = unicode_ranges_flags[i]; // codepoint_end, flags
+ for (uint32_t cpt = range_ini.first; cpt < range_end.first; ++cpt) {
+ cpt_flags[cpt] = range_ini.second;
+ }
+ }
+
+ for (auto cpt : unicode_set_whitespace) {
+ cpt_flags[cpt].is_whitespace = true;
+ }
+
+ for (auto p : unicode_map_lowercase) {
+ cpt_flags[p.second].is_lowercase = true;
+ }
+
+ for (auto p : unicode_map_uppercase) {
+ cpt_flags[p.second].is_uppercase = true;
+ }
+
+ for (auto &range : unicode_ranges_nfd) { // start, last, nfd
+ cpt_flags[range.nfd].is_nfd = true;
+ }
+
+ return cpt_flags;
+}
+
+static std::unordered_map<uint8_t, std::string> unicode_byte_to_utf8_map() {
+ std::unordered_map<uint8_t, std::string> map;
+ for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~'
+ assert(0 <= ch && ch < 256);
+ map[ch] = unicode_cpt_to_utf8(ch);
+ }
+ for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬'
+ assert(0 <= ch && ch < 256);
+ map[ch] = unicode_cpt_to_utf8(ch);
+ }
+ for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ'
+ assert(0 <= ch && ch < 256);
+ map[ch] = unicode_cpt_to_utf8(ch);
+ }
+ auto n = 0;
+ for (int ch = 0; ch < 256; ++ch) {
+ if (map.find(ch) == map.end()) {
+ map[ch] = unicode_cpt_to_utf8(256 + n);
+ ++n;
+ }
+ }
+ return map;
+}
+
+static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {
+ std::unordered_map<std::string, uint8_t> map;
+ for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~'
+ assert(0 <= ch && ch < 256);
+ map[unicode_cpt_to_utf8(ch)] = ch;
+ }
+ for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬'
+ assert(0 <= ch && ch < 256);
+ map[unicode_cpt_to_utf8(ch)] = ch;
+ }
+ for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ'
+ assert(0 <= ch && ch < 256);
+ map[unicode_cpt_to_utf8(ch)] = ch;
+ }
+ auto n = 0;
+ for (int ch = 0; ch < 256; ++ch) {
+ if (map.find(unicode_cpt_to_utf8(ch)) == map.end()) {
+ map[unicode_cpt_to_utf8(256 + n)] = ch;
+ ++n;
+ }
+ }
+ return map;
+}
+
+static inline std::wstring unicode_wstring_from_utf8(const std::string & s) {
+ std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
+ return conv.from_bytes(s);
+}
+
+static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) {
+ std::vector<std::string> bpe_encoded_words;
+ for (const auto & word : bpe_words) {
+ std::string text_utf;
+ auto utf_word = unicode_cpts_from_utf8(word);
+ for (size_t i = 0; i < utf_word.size(); ++i) {
+ text_utf += unicode_cpt_to_utf8(utf_word[i]);
+ }
+
+ std::string encoded_token;
+ for (char & c : text_utf) {
+ encoded_token += unicode_byte_to_utf8(c);
+ }
+ bpe_encoded_words.emplace_back(encoded_token);
+ }
+ return bpe_encoded_words;
+}
+
+// GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+
+static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & text, const std::vector<size_t> & offsets) {
+ std::vector<size_t> bpe_offsets; // store the offset of each word
+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
+
+ const auto cpts = unicode_cpts_from_utf8(text);
+
+ size_t start = 0;
+ for (auto offset : offsets) {
+ const size_t offset_ini = start;
+ const size_t offset_end = start + offset;
+ assert(offset_end <= cpts.size());
+ start = offset_end;
+
+ static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF;
+ auto _get_cpt = [&] (const size_t pos) -> uint32_t {
+ return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;
+ };
+
+ auto _get_flags = [&] (const size_t pos) -> codepoint_flags {
+ return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : codepoint_flags{};
+ };
+
+ size_t _prev_end = offset_ini;
+ auto _add_token = [&] (const size_t end) -> size_t {
+ assert(_prev_end <= end && end <= offset_end);
+ size_t len = end - _prev_end;
+ if (len > 0) {
+ bpe_offsets.push_back(len);
+ }
+ _prev_end = end;
+ //if (len > 0) {
+ // std::string s = "";
+ // for(size_t p = end-len; p < end; p++)
+ // s += unicode_cpt_to_utf8(cpts[p]);
+ // printf(">>> '%s'\n", s.c_str());
+ //}
+ return len;
+ };
+
+ for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {
+ const uint32_t cpt = _get_cpt(pos);
+ const auto flags = _get_flags(pos);
+
+ // regex: 's|'t|'re|'ve|'m|'ll|'d
+ if (cpt == '\'' && pos+1 < offset_end) {
+ uint32_t cpt_next = _get_cpt(pos+1);
+ if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {
+ pos += _add_token(pos+2);
+ continue;
+ }
+ if (pos+2 < offset_end) {
+ uint32_t cpt_next_next = _get_cpt(pos+2);
+ if ((cpt_next == 'r' && cpt_next_next == 'e') ||
+ (cpt_next == 'v' && cpt_next_next == 'e') ||
+ (cpt_next == 'l' && cpt_next_next == 'l')) {
+ pos += _add_token(pos+3);
+ continue;
+ }
+ }
+ }
+
+ auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags);
+ // regex: <space>?\p{L}+
+ if (flags2.is_letter) {
+ pos += (cpt == ' ');
+ while (flags2.is_letter) {
+ flags2 = _get_flags(++pos);
+ }
+ _add_token(pos);
+ continue;
+ }
+ // regex: <space>?\p{N}+
+ if (flags2.is_number) {
+ pos += (cpt == ' ');
+ while (flags2.is_number) {
+ flags2 = _get_flags(++pos);
+ }
+ _add_token(pos);
+ continue;
+ }
+ // regex: <space>?[^\s\p{L}\p{N}]+
+ if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) {
+ pos += (cpt == ' ');
+ while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) {
+ flags2 = _get_flags(++pos);
+ }
+ _add_token(pos);
+ continue;
+ }
+
+ size_t num_whitespaces = 0;
+ while (_get_flags(pos+num_whitespaces).is_whitespace) {
+ num_whitespaces++;
+ }
+
+ // regex: \s+(?!\S)
+ if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) {
+ pos += num_whitespaces - 1;
+ _add_token(pos);
+ continue;
+ }
+
+ // regex: \s+
+ if (num_whitespaces > 0) {
+ pos += num_whitespaces;
+ _add_token(pos);
+ continue;
+ }
+
+ // no matches
+ _add_token(++pos);
+ }
+ }
+
+ return bpe_offsets;
+}
+
+// LLAMA3 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
+static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string & text, const std::vector<size_t> & offsets) {
+ std::vector<size_t> bpe_offsets; // store the offset of each word
+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
+
+ const auto cpts = unicode_cpts_from_utf8(text);
+
+ size_t start = 0;
+ for (auto offset : offsets) {
+ const size_t offset_ini = start;
+ const size_t offset_end = start + offset;
+ assert(offset_end <= cpts.size());
+ start = offset_end;
+
+ static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF;
+ auto _get_cpt = [&] (const size_t pos) -> uint32_t {
+ return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;
+ };
+
+ auto _get_flags = [&] (const size_t pos) -> codepoint_flags {
+ return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : codepoint_flags{};
+ };
+
+ size_t _prev_end = offset_ini;
+ auto _add_token = [&] (const size_t end) -> size_t {
+ assert(_prev_end <= end && end <= offset_end);
+ size_t len = end - _prev_end;
+ if (len > 0) {
+ bpe_offsets.push_back(len);
+ }
+ _prev_end = end;
+ //if (len > 0) {
+ // std::string s = "";
+ // for(size_t p = end-len; p < end; p++)
+ // s += unicode_cpt_to_utf8(cpts[p]);
+ // printf(">>> '%s'\n", s.c_str());
+ //}
+ return len;
+ };
+
+ for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {
+ const uint32_t cpt = _get_cpt(pos);
+ const auto flags = _get_flags(pos);
+
+ // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive
+ if (cpt == '\'' && pos+1 < offset_end) {
+ uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1));
+ if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {
+ pos += _add_token(pos+2);
+ continue;
+ }
+ if (pos+2 < offset_end) {
+ uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2));
+ if ((cpt_next == 'r' && cpt_next_next == 'e') ||
+ (cpt_next == 'v' && cpt_next_next == 'e') ||
+ (cpt_next == 'l' && cpt_next_next == 'l')) {
+ pos += _add_token(pos+3);
+ continue;
+ }
+ }
+ }
+
+ // regex: [^\r\n\p{L}\p{N}]?\p{L}+
+ if (!(cpt == '\r' || cpt == '\n' || flags.is_number)) {
+ if (flags.is_letter || _get_flags(pos+1).is_letter) { // one or more letters
+ pos++;
+ while (_get_flags(pos).is_letter) {
+ pos++;
+ }
+ _add_token(pos);
+ continue;
+ }
+ }
+
+ // regex: \p{N}{1,3}
+ if (flags.is_number) {
+ size_t ini = pos;
+ while (_get_flags(pos).is_number) {
+ if (++pos - ini >= 3 ) {
+ _add_token(pos);
+ ini = pos;
+ }
+ }
+ _add_token(pos);
+ continue;
+ }
+
+ // regex: <space>?[^\s\p{L}\p{N}]+[\r\n]*
+ auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags);
+ if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags.as_uint()) {
+ pos += (cpt == ' ');
+ while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) {
+ flags2 = _get_flags(++pos);
+ }
+ uint32_t cpt2 = _get_cpt(pos);
+ while (cpt2 == '\r' || cpt2 == '\n') {
+ cpt2 = _get_cpt(++pos);
+ }
+ _add_token(pos);
+ continue;
+ }
+
+ size_t num_whitespaces = 0;
+ size_t last_end_r_or_n = 0;
+ while (_get_flags(pos+num_whitespaces).is_whitespace) {
+ uint32_t cpt2 = _get_cpt(pos+num_whitespaces);
+ if (cpt2 == '\r' || cpt2 == '\n') {
+ last_end_r_or_n = pos + num_whitespaces + 1;
+ }
+ num_whitespaces++;
+ }
+
+ // regex: \s*[\r\n]+
+ if (last_end_r_or_n > 0) {
+ pos = last_end_r_or_n;
+ _add_token(pos);
+ continue;
+ }
+
+ // regex: \s+(?!\S)
+ if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) {
+ pos += num_whitespaces - 1;
+ _add_token(pos);
+ continue;
+ }
+
+ // regex: \s+
+ if (num_whitespaces > 0) {
+ pos += num_whitespaces;
+ _add_token(pos);
+ continue;
+ }
+
+ // no matches
+ _add_token(++pos);
+ }
+ }
+
+ return bpe_offsets;
+}
+
+// use std::wregex to split the text
+static std::vector<size_t> unicode_regex_split_stl(const std::wstring & wtext, const std::wstring & regex_expr, const std::vector<size_t> & offsets) {
+ std::wregex expr(regex_expr);
+ std::vector<size_t> bpe_offsets; // store the offset of each word
+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
+ size_t start = 0;
+ for (auto offset : offsets) {
+ std::wcregex_iterator it(wtext.data() + start, wtext.data() + start + offset, expr);
+ std::wcregex_iterator end;
+
+ int64_t start_idx = 0;
+ while (it != end) {
+ std::wcmatch match = *it;
+ if (match.position() > start_idx) {
+ bpe_offsets.emplace_back(match.position() - start_idx);
+ }
+ bpe_offsets.emplace_back(match.length());
+ start_idx = match.position() + match.length();
+ ++it;
+ }
+
+ if (start_idx < (int64_t) offset) {
+ bpe_offsets.emplace_back(offset - start_idx);
+ }
+ start += offset;
+ }
+
+ return bpe_offsets;
+}
+
+// use std::regex to split the text
+static std::vector<size_t> unicode_regex_split_stl(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {
+ std::regex expr(regex_expr);
+ std::vector<size_t> bpe_offsets; // store the offset of each word
+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
+ size_t start = 0;
+ for (auto offset : offsets) {
+ std::cregex_iterator it(text.data() + start, text.data() + start + offset, expr);
+ std::cregex_iterator end;
+
+ int64_t start_idx = 0;
+ while (it != end) {
+ std::cmatch match = *it;
+ if (match.position() > start_idx) {
+ bpe_offsets.emplace_back(match.position() - start_idx);
+ }
+ bpe_offsets.emplace_back(match.length());
+ start_idx = match.position() + match.length();
+ ++it;
+ }
+
+ if (start_idx < (int64_t) offset) {
+ bpe_offsets.emplace_back(offset - start_idx);
+ }
+ start += offset;
+ }
+
+ return bpe_offsets;
+}
+
+static std::vector<size_t> unicode_regex_split_custom(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {
+ std::vector<size_t> bpe_offsets;
+
+ if (regex_expr == "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)") {
+ bpe_offsets = unicode_regex_split_custom_gpt2(text, offsets);
+ } else if (
+ regex_expr == "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" ||
+ regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") {
+
+ bpe_offsets = unicode_regex_split_custom_llama3(text, offsets);
+ }
+
+ return bpe_offsets;
+}
+
+//
+// interface
+//
+
+std::string unicode_cpt_to_utf8(uint32_t cp) {
+ std::string result;
+
+ if (/* 0x00 <= cp && */ cp <= 0x7f) {
+ result.push_back(cp);
+ return result;
+ }
+ if (0x80 <= cp && cp <= 0x7ff) {
+ result.push_back(0xc0 | ((cp >> 6) & 0x1f));
+ result.push_back(0x80 | (cp & 0x3f));
+ return result;
+ }
+ if (0x800 <= cp && cp <= 0xffff) {
+ result.push_back(0xe0 | ((cp >> 12) & 0x0f));
+ result.push_back(0x80 | ((cp >> 6) & 0x3f));
+ result.push_back(0x80 | (cp & 0x3f));
+ return result;
+ }
+ if (0x10000 <= cp && cp <= 0x10ffff) {
+ result.push_back(0xf0 | ((cp >> 18) & 0x07));
+ result.push_back(0x80 | ((cp >> 12) & 0x3f));
+ result.push_back(0x80 | ((cp >> 6) & 0x3f));
+ result.push_back(0x80 | (cp & 0x3f));
+ return result;
+ }
+
+ throw std::invalid_argument("invalid codepoint");
+}
+
+std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts) {
+ auto comp = [] (const uint32_t cpt, const range_nfd & range) {
+ return cpt < range.first;
+ };
+ std::vector<uint32_t> result(cpts.size());
+ for (size_t i = 0; i < cpts.size(); ++i) {
+ const uint32_t cpt = cpts[i];
+ auto it = std::upper_bound(unicode_ranges_nfd.cbegin(), unicode_ranges_nfd.cend(), cpt, comp) - 1;
+ result[i] = (it->first <= cpt && cpt <= it->last) ? it->nfd : cpt;
+ }
+ return result;
+}
+
+std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8) {
+ std::vector<uint32_t> result;
+ result.reserve(utf8.size());
+ size_t offset = 0;
+ while (offset < utf8.size()) {
+ result.push_back(unicode_cpt_from_utf8(utf8, offset));
+ }
+ return result;
+}
+
+codepoint_flags unicode_cpt_flags(const uint32_t cp) {
+ static const codepoint_flags undef(codepoint_flags::UNDEFINED);
+ static const auto cpt_flags = unicode_cpt_flags_array();
+ return cp < cpt_flags.size() ? cpt_flags[cp] : undef;
+}
+
+codepoint_flags unicode_cpt_flags(const std::string & utf8) {
+ static const codepoint_flags undef(codepoint_flags::UNDEFINED);
+ if (utf8.empty()) {
+ return undef; // undefined
+ }
+ size_t offset = 0;
+ return unicode_cpt_flags(unicode_cpt_from_utf8(utf8, offset));
+}
+
+std::string unicode_byte_to_utf8(uint8_t byte) {
+ static std::unordered_map<uint8_t, std::string> map = unicode_byte_to_utf8_map();
+ return map.at(byte);
+}
+
+uint8_t unicode_utf8_to_byte(const std::string & utf8) {
+ static std::unordered_map<std::string, uint8_t> map = unicode_utf8_to_byte_map();
+ return map.at(utf8);
+}
+
+uint32_t unicode_tolower(uint32_t cp) {
+ auto it = unicode_map_lowercase.find(cp);
+ return it == unicode_map_lowercase.end() ? cp : it->second;
+}
+
+std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs) {
+ // unicode categories
+ static const std::map<std::string, int> k_ucat_enum = {
+ { "\\p{N}", codepoint_flags::NUMBER },
+ { "\\p{L}", codepoint_flags::LETTER },
+ { "\\p{P}", codepoint_flags::PUNCTUATION },
+ };
+
+ static const std::map<int, int> k_ucat_cpt = {
+ { codepoint_flags::NUMBER, 0xD1 },
+ { codepoint_flags::LETTER, 0xD2 },
+ { codepoint_flags::PUNCTUATION, 0xD3 },
+ };
+
+ static const std::map<int, std::string> k_ucat_map = {
+ { codepoint_flags::NUMBER, "\x30-\x39" }, // 0-9
+ { codepoint_flags::LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z
+ { codepoint_flags::PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\}
+ };
+
+ // compute collapsed codepoints only if needed by at least one regex
+ bool need_collapse = false;
+ for (auto & regex_expr : regex_exprs) {
+ // search for unicode categories
+ for (const auto & ucat : k_ucat_enum) {
+ if (std::string::npos != regex_expr.find(ucat.first)) {
+ need_collapse = true;
+ break;
+ }
+ }
+ }
+
+ const auto cpts = unicode_cpts_from_utf8(text);
+
+ // generate a "collapsed" representation of the text, where all codepoints are replaced by a single byte
+ // ref: https://github.com/ggerganov/llama.cpp/pull/6920#issuecomment-2081479935
+ std::string text_collapsed;
+ if (need_collapse) {
+ // collapse all unicode categories
+ text_collapsed.resize(cpts.size());
+
+ for (size_t i = 0; i < cpts.size(); ++i) {
+ // keep single-byte codepoints as is
+ if (cpts[i] < 128) {
+ text_collapsed[i] = cpts[i];
+ continue;
+ }
+
+ const auto flags = unicode_cpt_flags(cpts[i]);
+
+ if (flags.is_whitespace) {
+ //NOTE: C++ std::regex \s does not mach 0x85, Rust and Python regex does.
+ //text_collapsed[i] = (char) 0x85; // <Next Line> as whitespace fallback
+ text_collapsed[i] = (char) 0x0B; // <vertical tab> as whitespace fallback
+ } else if (k_ucat_cpt.find(flags.category_flag()) != k_ucat_cpt.end()) {
+ text_collapsed[i] = k_ucat_cpt.at(flags.category_flag());
+ } else {
+ text_collapsed[i] = (char) 0xD0; // fallback
+ }
+ }
+ }
+
+ std::vector<size_t> bpe_offsets = { cpts.size() };
+
+ for (auto & regex_expr : regex_exprs) {
+ // first, see if we have an efficient custom regex implementation
+ auto tmp = unicode_regex_split_custom(text, regex_expr, bpe_offsets);
+
+ if (!tmp.empty()) {
+ bpe_offsets = std::move(tmp);
+ continue;
+ }
+
+ // fallback to general-purpose std::regex / std::wregex
+ try {
+ // if a unicode category is used in the regex, we use the collapsed text and replace the unicode category
+ // with the corresponding collapsed representation
+ bool use_collapsed = false;
+ for (auto & ucat : k_ucat_enum) {
+ if (std::string::npos != regex_expr.find(ucat.first)) {
+ use_collapsed = true;
+ break;
+ }
+ }
+
+ if (use_collapsed) {
+ // sanity-check that the original regex does not contain any non-ASCII characters
+ const auto cpts_regex = unicode_cpts_from_utf8(regex_expr);
+ for (size_t i = 0; i < cpts_regex.size(); ++i) {
+ if (cpts_regex[i] >= 128) {
+ throw std::runtime_error("Regex includes both unicode categories and non-ASCII characters - not supported");
+ }
+ }
+
+ // generate a collapsed representation of the regex
+ std::string regex_expr_collapsed;
+
+ // track if we are inside [], because nested [] are not allowed
+ bool inside = false;
+ for (size_t i = 0; i < regex_expr.size(); ++i) {
+ if (regex_expr[i] == '[' && (i == 0 || regex_expr[i - 1] != '\\')) {
+ regex_expr_collapsed += '[';
+ inside = true;
+ continue;
+ }
+
+ if (inside && regex_expr[i] == ']' && regex_expr[i - 1] != '\\') {
+ regex_expr_collapsed += ']';
+ inside = false;
+ continue;
+ }
+
+ if (regex_expr[i + 0] == '\\' && i + 4 < regex_expr.size() &&
+ regex_expr[i + 1] == 'p' &&
+ regex_expr[i + 2] == '{' &&
+ regex_expr[i + 4] == '}') {
+ const std::string pat = regex_expr.substr(i, 5);
+ if (k_ucat_enum.find(pat) != k_ucat_enum.end()) {
+ if (!inside) {
+ regex_expr_collapsed += '[';
+ }
+ regex_expr_collapsed += k_ucat_cpt.at(k_ucat_enum.at(pat));
+ regex_expr_collapsed += k_ucat_map.at(k_ucat_enum.at(pat));
+ if (!inside) {
+ regex_expr_collapsed += ']';
+ }
+ i += 4;
+ continue;
+ }
+ }
+
+ regex_expr_collapsed += regex_expr[i];
+ }
+
+ //printf("text_collapsed: %s\n", text_collapsed.c_str());
+ //printf("regex_expr_collapsed: %s\n", regex_expr_collapsed.c_str());
+ bpe_offsets = unicode_regex_split_stl(text_collapsed, regex_expr_collapsed, bpe_offsets);
+ } else {
+ // no unicode category used, we can use std::wregex directly
+ const std::wstring wregex_expr = unicode_wstring_from_utf8(regex_expr);
+
+ // std::wregex \s does not mach non-ASCII whitespaces, using 0x0B as fallback
+ std::wstring wtext(cpts.begin(), cpts.end());
+ for (size_t i = 0; i < wtext.size(); ++i) {
+ if (wtext[i] > 0x7F && unicode_cpt_flags(wtext[i]).is_whitespace) {
+ wtext[i] = 0x0B;
+ }
+ }
+
+ //printf("text: %s\n", text.c_str());
+ //printf("regex_expr: %s\n", regex_expr.c_str());
+ bpe_offsets = unicode_regex_split_stl(wtext, wregex_expr, bpe_offsets);
+ }
+ } catch (std::regex_error & e) {
+ fprintf(stderr, "Failed to process regex: '%s'\n", regex_expr.c_str());
+ fprintf(stderr, "Regex error: %s\n", e.what());
+ throw std::runtime_error("Failed to process regex");
+ }
+ }
+
+ std::vector<std::string> bpe_words;
+ bpe_words.reserve(bpe_offsets.size()); // reserve memory for the approximate size
+
+ size_t start = 0;
+ for (size_t & offset : bpe_offsets) {
+ bpe_words.emplace_back();
+ for (size_t i = start; i < start + offset; ++i) {
+ bpe_words.back() += unicode_cpt_to_utf8(cpts[i]);
+ }
+ start += offset;
+ }
+
+ return unicode_byte_encoding_process(bpe_words);
+}
diff --git a/llama/unicode.h b/llama/unicode.h
new file mode 100644
index 00000000..34f2bf7a
--- /dev/null
+++ b/llama/unicode.h
@@ -0,0 +1,93 @@
+/**
+ * llama.cpp - commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177 - do not edit this file
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+// TODO: prefix all symbols with "llama_"
+
+struct codepoint_flags {
+ enum {
+ UNDEFINED = 0x0001,
+ NUMBER = 0x0002, // regex: \p{N}
+ LETTER = 0x0004, // regex: \p{L}
+ SEPARATOR = 0x0008, // regex: \p{Z}
+ ACCENT_MARK = 0x0010, // regex: \p{M}
+ PUNCTUATION = 0x0020, // regex: \p{P}
+ SYMBOL = 0x0040, // regex: \p{S}
+ CONTROL = 0x0080, // regex: \p{C}
+ MASK_CATEGORIES = 0x00FF,
+ };
+
+ // codepoint type
+ uint16_t is_undefined : 1;
+ uint16_t is_number : 1; // regex: \p{N}
+ uint16_t is_letter : 1; // regex: \p{L}
+ uint16_t is_separator : 1; // regex: \p{Z}
+ uint16_t is_accent_mark : 1; // regex: \p{M}
+ uint16_t is_punctuation : 1; // regex: \p{P}
+ uint16_t is_symbol : 1; // regex: \p{S}
+ uint16_t is_control : 1; // regex: \p{C}
+ // helper flags
+ uint16_t is_whitespace : 1; // regex: \s
+ uint16_t is_lowercase : 1;
+ uint16_t is_uppercase : 1;
+ uint16_t is_nfd : 1;
+
+ // decode from uint16
+ inline codepoint_flags(const uint16_t flags=0) {
+ *reinterpret_cast<uint16_t*>(this) = flags;
+ }
+
+ inline uint16_t as_uint() const {
+ return *reinterpret_cast<const uint16_t*>(this);
+ }
+
+ inline uint16_t category_flag() const {
+ return this->as_uint() & MASK_CATEGORIES;
+ }
+};
+
+size_t unicode_len_utf8(char src);
+
+std::string unicode_cpt_to_utf8(uint32_t cp);
+uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset);
+std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8);
+
+std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts);
+
+codepoint_flags unicode_cpt_flags(const uint32_t cp);
+codepoint_flags unicode_cpt_flags(const std::string & utf8);
+
+std::string unicode_byte_to_utf8(uint8_t byte);
+uint8_t unicode_utf8_to_byte(const std::string & utf8);
+
+uint32_t unicode_tolower(uint32_t cp);
+
+std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs);
diff --git a/llm/dynamic_shim.c b/llm/dynamic_shim.c
deleted file mode 100644
index 8b5d67c9..00000000
--- a/llm/dynamic_shim.c
+++ /dev/null
@@ -1,136 +0,0 @@
-#include "dynamic_shim.h"
-
-#include <stdio.h>
-#include <string.h>
-
-#ifdef __linux__
-#include <dlfcn.h>
-#define LOAD_LIBRARY(lib, flags) dlopen(lib, flags | RTLD_DEEPBIND)
-#define LOAD_SYMBOL(handle, sym) dlsym(handle, sym)
-#define LOAD_ERR() dlerror()
-#define UNLOAD_LIBRARY(handle) dlclose(handle)
-#elif _WIN32
-#include <windows.h>
-#define LOAD_LIBRARY(lib, flags) LoadLibrary(lib)
-#define LOAD_SYMBOL(handle, sym) GetProcAddress(handle, sym)
-#define UNLOAD_LIBRARY(handle) FreeLibrary(handle)
-// TODO - refactor this with proper error message handling on windows
-inline static char *LOAD_ERR() {
- static char errbuf[8];
- snprintf(errbuf, 8, "0x%lx", GetLastError());
- return errbuf;
-}
-#else
-#include <dlfcn.h>
-#define LOAD_LIBRARY(lib, flags) dlopen(lib, flags)
-#define LOAD_SYMBOL(handle, sym) dlsym(handle, sym)
-#define LOAD_ERR() dlerror()
-#define UNLOAD_LIBRARY(handle) dlclose(handle)
-#endif
-
-void dynamic_shim_init(const char *libPath, struct dynamic_llama_server *s,
- ext_server_resp_t *err) {
- int i = 0;
- struct lookup {
- char *s;
- void **p;
- } l[] = {
- {"llama_server_init", (void *)&s->llama_server_init},
- {"llama_server_start", (void *)&s->llama_server_start},
- {"llama_server_stop", (void *)&s->llama_server_stop},
- {"llama_server_completion", (void *)&s->llama_server_completion},
- {"llama_server_completion_next_result",
- (void *)&s->llama_server_completion_next_result},
- {"llama_server_completion_cancel",
- (void *)&s->llama_server_completion_cancel},
- {"llama_server_release_task_result",
- (void *)&s->llama_server_release_task_result},
- {"llama_server_tokenize", (void *)&s->llama_server_tokenize},
- {"llama_server_detokenize", (void *)&s->llama_server_detokenize},
- {"llama_server_embedding", (void *)&s->llama_server_embedding},
- {"llama_server_release_json_resp",
- (void *)&s->llama_server_release_json_resp},
- {"", NULL},
- };
-
- printf("Lazy loading %s library\n", libPath);
- s->handle = LOAD_LIBRARY(libPath, RTLD_NOW);
- if (!s->handle) {
- err->id = -1;
- snprintf(err->msg, err->msg_len,
- "Unable to load dynamic server library: %s", LOAD_ERR());
- return;
- }
-
- for (i = 0; l[i].p != NULL; i++) {
- *l[i].p = LOAD_SYMBOL(s->handle, l[i].s);
- if (!l[i].p) {
- UNLOAD_LIBRARY(s->handle);
- err->id = -1;
- snprintf(err->msg, err->msg_len, "symbol lookup for %s failed: %s",
- l[i].s, LOAD_ERR());
- return;
- }
- }
-}
-
-inline void dynamic_shim_llama_server_init(struct dynamic_llama_server s,
- ext_server_params_t *sparams,
- ext_server_resp_t *err) {
- s.llama_server_init(sparams, err);
-}
-
-inline void dynamic_shim_llama_server_start(struct dynamic_llama_server s) {
- s.llama_server_start();
-}
-
-inline void dynamic_shim_llama_server_stop(struct dynamic_llama_server s) {
- s.llama_server_stop();
-}
-
-inline void dynamic_shim_llama_server_completion(struct dynamic_llama_server s,
- const char *json_req,
- ext_server_resp_t *resp) {
- s.llama_server_completion(json_req, resp);
-}
-
-inline void dynamic_shim_llama_server_completion_next_result(
- struct dynamic_llama_server s, const int task_id,
- ext_server_task_result_t *result) {
- s.llama_server_completion_next_result(task_id, result);
-}
-
-inline void dynamic_shim_llama_server_completion_cancel(
- struct dynamic_llama_server s, const int task_id, ext_server_resp_t *err) {
- s.llama_server_completion_cancel(task_id, err);
-}
-inline void dynamic_shim_llama_server_release_task_result(
- struct dynamic_llama_server s, ext_server_task_result_t *result) {
- s.llama_server_release_task_result(result);
-}
-
-inline void dynamic_shim_llama_server_tokenize(struct dynamic_llama_server s,
- const char *json_req,
- char **json_resp,
- ext_server_resp_t *err) {
- s.llama_server_tokenize(json_req, json_resp, err);
-}
-
-inline void dynamic_shim_llama_server_detokenize(struct dynamic_llama_server s,
- const char *json_req,
- char **json_resp,
- ext_server_resp_t *err) {
- s.llama_server_detokenize(json_req, json_resp, err);
-}
-
-inline void dynamic_shim_llama_server_embedding(struct dynamic_llama_server s,
- const char *json_req,
- char **json_resp,
- ext_server_resp_t *err) {
- s.llama_server_embedding(json_req, json_resp, err);
-}
-
-inline void dynamic_shim_llama_server_release_json_resp(
- struct dynamic_llama_server s, char **json_resp) {
- s.llama_server_release_json_resp(json_resp);
-}
diff --git a/llm/dynamic_shim.h b/llm/dynamic_shim.h
deleted file mode 100644
index 5e4e78b7..00000000
--- a/llm/dynamic_shim.h
+++ /dev/null
@@ -1,74 +0,0 @@
-#include <stdlib.h>
-
-#include "server.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-struct dynamic_llama_server {
- void *handle;
- void (*llama_server_init)(ext_server_params_t *sparams,
- ext_server_resp_t *err);
- void (*llama_server_start)();
- void (*llama_server_stop)();
- void (*llama_server_completion)(const char *json_req,
- ext_server_resp_t *resp);
- void (*llama_server_completion_next_result)(const int task_id,
- ext_server_task_result_t *result);
- void (*llama_server_completion_cancel)(const int task_id,
- ext_server_resp_t *err);
- void (*llama_server_release_task_result)(ext_server_task_result_t *result);
- void (*llama_server_tokenize)(const char *json_req, char **json_resp,
- ext_server_resp_t *err);
- void (*llama_server_detokenize)(const char *json_req, char **json_resp,
- ext_server_resp_t *err);
- void (*llama_server_embedding)(const char *json_req, char **json_resp,
- ext_server_resp_t *err);
- void (*llama_server_release_json_resp)(char **json_resp);
-};
-
-void dynamic_shim_init(const char *libPath, struct dynamic_llama_server *s,
- ext_server_resp_t *err);
-
-// No good way to call C function pointers from Go so inline the indirection
-void dynamic_shim_llama_server_init(struct dynamic_llama_server s,
- ext_server_params_t *sparams,
- ext_server_resp_t *err);
-
-void dynamic_shim_llama_server_start(struct dynamic_llama_server s);
-
-void dynamic_shim_llama_server_stop(struct dynamic_llama_server s);
-
-void dynamic_shim_llama_server_completion(struct dynamic_llama_server s,
- const char *json_req,
- ext_server_resp_t *resp);
-
-void dynamic_shim_llama_server_completion_next_result(
- struct dynamic_llama_server s, const int task_id,
- ext_server_task_result_t *result);
-
-void dynamic_shim_llama_server_completion_cancel(struct dynamic_llama_server s,
- const int task_id,
- ext_server_resp_t *err);
-
-void dynamic_shim_llama_server_release_task_result(
- struct dynamic_llama_server s, ext_server_task_result_t *result);
-
-void dynamic_shim_llama_server_tokenize(struct dynamic_llama_server s,
- const char *json_req, char **json_resp,
- ext_server_resp_t *err);
-
-void dynamic_shim_llama_server_detokenize(struct dynamic_llama_server s,
- const char *json_req,
- char **json_resp,
- ext_server_resp_t *err);
-
-void dynamic_shim_llama_server_embedding(struct dynamic_llama_server s,
- const char *json_req, char **json_resp,
- ext_server_resp_t *err);
-void dynamic_shim_llama_server_release_json_resp(struct dynamic_llama_server s,
- char **json_resp);
-
-#ifdef __cplusplus
-}
-#endif
\ No newline at end of file
diff --git a/llm/ext_server.go b/llm/ext_server.go
deleted file mode 100644
index 0f7d441a..00000000
--- a/llm/ext_server.go
+++ /dev/null
@@ -1,424 +0,0 @@
-package llm
-
-/*
-#cgo CFLAGS: -I${SRCDIR}/llama.cpp/gguf -I${SRCDIR}/llama.cpp/gguf/common -I${SRCDIR}/llama.cpp/gguf/examples/server
-#cgo CFLAGS: -DNDEBUG -DLLAMA_SERVER_LIBRARY=1 -D_XOPEN_SOURCE=600 -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64
-#cgo CFLAGS: -Wmissing-noreturn -Wall -Wextra -Wcast-qual -Wno-unused-function -Wno-array-bounds
-#cgo CPPFLAGS: -Ofast -Wall -Wextra -Wno-unused-function -Wno-unused-variable -Wno-deprecated-declarations -Wno-unused-but-set-variable
-#cgo darwin CFLAGS: -D_DARWIN_C_SOURCE
-#cgo darwin CPPFLAGS: -DGGML_USE_ACCELERATE
-#cgo darwin CPPFLAGS: -DGGML_USE_METAL -DGGML_METAL_NDEBUG
-#cgo darwin LDFLAGS: -lc++ -framework Accelerate
-#cgo darwin LDFLAGS: -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders
-#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/common/libcommon.a
-#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/examples/server/libext_server.a
-#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/libllama.a
-#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/libggml_static.a
-#cgo linux CFLAGS: -D_GNU_SOURCE
-#cgo linux windows CFLAGS: -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_MMV_Y=1 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_USE_CUBLAS
-#cgo linux LDFLAGS: -L/usr/local/cuda/targets/x86_64-linux/lib -L/usr/local/cuda/lib64 -L/usr/local/cuda/targets/x86_64-linux/lib/stubs
-#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/examples/server/libext_server.a
-#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/common/libcommon.a
-#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/libllama.a
-#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/libggml_static.a
-#cgo linux LDFLAGS: -lrt -lpthread -ldl -lstdc++ -lm
-#cgo windows LDFLAGS: -L${SRCDIR}/llama.cpp/gguf/build/wincpu/dist/lib
-#cgo windows LDFLAGS: -lcpu_server -lpthread
-
-#include <stdlib.h>
-#include "server.h"
-
-*/
-import "C"
-import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "log"
- "os"
- "strings"
- "sync"
- "time"
- "unsafe"
-
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/gpu"
-)
-
-func newExtServerResp(len C.size_t) C.ext_server_resp_t {
- var resp C.ext_server_resp_t
- resp.msg_len = len
- bytes := make([]byte, len)
- resp.msg = (*C.char)(C.CBytes(bytes))
- return resp
-}
-
-func freeExtServerResp(resp C.ext_server_resp_t) {
- if resp.msg_len == 0 {
- return
- }
- C.free(unsafe.Pointer(resp.msg))
-}
-
-func extServerResponseToErr(resp C.ext_server_resp_t) error {
- return fmt.Errorf(C.GoString(resp.msg))
-}
-
-type extServer interface {
- LLM
- llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t)
- llama_server_start()
- llama_server_stop()
- llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t)
- llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t)
- llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t)
- llama_server_release_task_result(result *C.ext_server_task_result_t)
- llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
- llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
- llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
- llama_server_release_json_resp(json_resp **C.char)
-}
-
-type llamaExtServer struct {
- api.Options
-}
-
-// Note: current implementation does not support concurrent instantiations
-var mutex sync.Mutex
-
-func (llm *llamaExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) {
- C.llama_server_init(sparams, err)
-}
-func (llm *llamaExtServer) llama_server_start() {
- C.llama_server_start()
-}
-func (llm *llamaExtServer) llama_server_stop() {
- C.llama_server_stop()
-}
-
-func (llm *llamaExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) {
- C.llama_server_completion(json_req, resp)
-}
-func (llm *llamaExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) {
- C.llama_server_completion_next_result(task_id, resp)
-}
-func (llm *llamaExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) {
- C.llama_server_completion_cancel(task_id, err)
-}
-func (llm *llamaExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) {
- C.llama_server_release_task_result(result)
-}
-
-func (llm *llamaExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
- C.llama_server_tokenize(json_req, json_resp, err)
-}
-func (llm *llamaExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
- C.llama_server_detokenize(json_req, json_resp, err)
-}
-func (llm *llamaExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
- C.llama_server_embedding(json_req, json_resp, err)
-}
-func (llm *llamaExtServer) llama_server_release_json_resp(json_resp **C.char) {
- C.llama_server_release_json_resp(json_resp)
-}
-
-func newDefaultExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
- server := &llamaExtServer{opts}
- return newExtServer(server, model, adapters, projectors, numLayers, opts)
-}
-
-func newExtServer(server extServer, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
- if !mutex.TryLock() {
- log.Printf("concurrent llm servers not yet supported, waiting for prior server to complete")
- mutex.Lock()
- }
- fileInfo, err := os.Stat(model)
- if err != nil {
- return nil, err
- }
- var sparams C.ext_server_params_t
- sparams.model = C.CString(model)
- defer C.free(unsafe.Pointer(sparams.model))
-
- numGPU := gpu.NumGPU(numLayers, fileInfo.Size(), opts)
-
- sparams.embedding = true
- sparams.n_ctx = C.uint(opts.NumCtx)
- sparams.n_batch = C.uint(opts.NumBatch)
- sparams.n_gpu_layers = C.int(numGPU)
- sparams.main_gpu = C.int(opts.MainGPU)
- sparams.n_parallel = 1 // TODO - wire up concurrency
-
- // Always use the value encoded in the model
- sparams.rope_freq_base = 0.0
- sparams.rope_freq_scale = 0.0
- sparams.memory_f16 = C.bool(opts.F16KV)
- sparams.use_mlock = C.bool(opts.UseMLock)
- sparams.use_mmap = C.bool(opts.UseMMap)
- sparams.numa = C.bool(opts.UseNUMA)
-
- sparams.lora_adapters = nil
- for i := 0; i < len(adapters); i++ {
- la := (*C.ext_server_lora_adapter_t)(C.malloc(C.sizeof_ext_server_lora_adapter_t))
- defer C.free(unsafe.Pointer(la))
- la.adapter = C.CString(adapters[i])
- defer C.free(unsafe.Pointer(la.adapter))
- la.scale = C.float(1.0) // TODO expose scale/weights up through ollama UX
- la.next = nil
- if i == 0 {
- sparams.lora_adapters = la
- } else {
- tmp := sparams.lora_adapters
- for ; tmp.next != nil; tmp = tmp.next {
- }
- tmp.next = la
- }
- }
-
- if len(projectors) > 0 {
- // TODO: applying multiple projectors is not supported by the llama.cpp server yet
- sparams.mmproj = C.CString(projectors[0])
- defer C.free(unsafe.Pointer(sparams.mmproj))
- } else {
- sparams.mmproj = nil
- }
-
- sparams.n_threads = C.uint(opts.NumThread)
-
- log.Printf("Initializing internal llama server")
- resp := newExtServerResp(128)
- defer freeExtServerResp(resp)
- server.llama_server_init(&sparams, &resp)
- if resp.id < 0 {
- return nil, extServerResponseToErr(resp)
- }
-
- log.Printf("Starting internal llama main loop")
- server.llama_server_start()
- return server, nil
-}
-
-func (llm *llamaExtServer) Predict(ctx context.Context, pred PredictOpts, fn func(PredictResult)) error {
- return predict(llm, llm.Options, ctx, pred, fn)
-}
-
-func predict(llm extServer, opts api.Options, ctx context.Context, predict PredictOpts, fn func(PredictResult)) error {
- resp := newExtServerResp(128)
- defer freeExtServerResp(resp)
- var imageData []ImageData
- if len(predict.Images) > 0 {
- for cnt, i := range predict.Images {
- imageData = append(imageData, ImageData{Data: i, ID: cnt})
- }
- }
- log.Printf("loaded %d images", len(imageData))
-
- request := map[string]any{
- "prompt": predict.Prompt,
- "stream": true,
- "n_predict": opts.NumPredict,
- "n_keep": opts.NumKeep,
- "temperature": opts.Temperature,
- "top_k": opts.TopK,
- "top_p": opts.TopP,
- "tfs_z": opts.TFSZ,
- "typical_p": opts.TypicalP,
- "repeat_last_n": opts.RepeatLastN,
- "repeat_penalty": opts.RepeatPenalty,
- "presence_penalty": opts.PresencePenalty,
- "frequency_penalty": opts.FrequencyPenalty,
- "mirostat": opts.Mirostat,
- "mirostat_tau": opts.MirostatTau,
- "mirostat_eta": opts.MirostatEta,
- "penalize_nl": opts.PenalizeNewline,
- "seed": opts.Seed,
- "stop": opts.Stop,
- "image_data": imageData,
- "cache_prompt": true,
- }
-
- if predict.Format == "json" {
- request["grammar"] = jsonGrammar
- }
-
- retryDelay := 100 * time.Microsecond
- for retries := 0; retries < maxRetries; retries++ {
- if retries > 0 {
- time.Sleep(retryDelay) // wait before retrying
- retryDelay *= 2 // exponential backoff
- }
-
- // Handling JSON marshaling with special characters unescaped.
- buffer := &bytes.Buffer{}
- enc := json.NewEncoder(buffer)
- enc.SetEscapeHTML(false)
-
- if err := enc.Encode(request); err != nil {
- return fmt.Errorf("failed to marshal data: %w", err)
- }
-
- req := C.CString(buffer.String())
- defer C.free(unsafe.Pointer(req))
-
- llm.llama_server_completion(req, &resp)
- if resp.id < 0 {
- return extServerResponseToErr(resp)
- }
-
- retryNeeded := false
- out:
- for {
- select {
- case <-ctx.Done():
- // This handles the request cancellation
- llm.llama_server_completion_cancel(resp.id, &resp)
- if resp.id < 0 {
- return extServerResponseToErr(resp)
- } else {
- return nil
- }
- default:
- var result C.ext_server_task_result_t
- llm.llama_server_completion_next_result(resp.id, &result)
- json_resp := C.GoString(result.json_resp)
- llm.llama_server_release_task_result(&result)
-
- var p prediction
- if err := json.Unmarshal([]byte(json_resp), &p); err != nil {
- llm.llama_server_completion_cancel(resp.id, &resp)
- if resp.id < 0 {
- return fmt.Errorf("error unmarshaling llm prediction response: %w and cancel %s", err, C.GoString(resp.msg))
- } else {
- return fmt.Errorf("error unmarshaling llm prediction response: %w", err)
- }
- }
-
- if bool(result.error) && strings.Contains(json_resp, "slot unavailable") {
- retryNeeded = true
- // task will already be canceled
- break out
- }
-
- if p.Content != "" {
- fn(PredictResult{
- Content: p.Content,
- })
- }
-
- if p.Stop {
- fn(PredictResult{
- Done: true,
- PromptEvalCount: p.Timings.PromptN,
- PromptEvalDuration: parseDurationMs(p.Timings.PromptMS),
- EvalCount: p.Timings.PredictedN,
- EvalDuration: parseDurationMs(p.Timings.PredictedMS),
- })
- return nil
- }
- }
- }
- if !retryNeeded {
- return nil // success
- }
- }
-
- // should never reach here ideally
- return fmt.Errorf("max retries exceeded")
-}
-func (llm *llamaExtServer) Encode(ctx context.Context, prompt string) ([]int, error) {
- return encode(llm, ctx, prompt)
-}
-
-func encode(llm extServer, ctx context.Context, prompt string) ([]int, error) {
- data, err := json.Marshal(TokenizeRequest{Content: prompt})
- if err != nil {
- return nil, fmt.Errorf("marshaling encode data: %w", err)
- }
- req := C.CString(string(data))
- defer C.free(unsafe.Pointer(req))
- var json_resp *C.char
- resp := newExtServerResp(128)
- defer freeExtServerResp(resp)
- llm.llama_server_tokenize(req, &json_resp, &resp)
- if resp.id < 0 {
- return nil, extServerResponseToErr(resp)
- }
- defer llm.llama_server_release_json_resp(&json_resp)
-
- var encoded TokenizeResponse
- if err2 := json.Unmarshal([]byte(C.GoString(json_resp)), &encoded); err2 != nil {
- return nil, fmt.Errorf("unmarshal encode response: %w", err2)
- }
-
- return encoded.Tokens, err
-}
-
-func (llm *llamaExtServer) Decode(ctx context.Context, tokens []int) (string, error) {
- return decode(llm, ctx, tokens)
-}
-
-func decode(llm extServer, ctx context.Context, tokens []int) (string, error) {
- if len(tokens) == 0 {
- return "", nil
- }
- data, err := json.Marshal(DetokenizeRequest{Tokens: tokens})
- if err != nil {
- return "", fmt.Errorf("marshaling decode data: %w", err)
- }
-
- req := C.CString(string(data))
- defer C.free(unsafe.Pointer(req))
- var json_resp *C.char
- resp := newExtServerResp(128)
- defer freeExtServerResp(resp)
- llm.llama_server_detokenize(req, &json_resp, &resp)
- if resp.id < 0 {
- return "", extServerResponseToErr(resp)
- }
- defer llm.llama_server_release_json_resp(&json_resp)
-
- var decoded DetokenizeResponse
- if err2 := json.Unmarshal([]byte(C.GoString(json_resp)), &decoded); err2 != nil {
- return "", fmt.Errorf("unmarshal encode response: %w", err2)
- }
-
- return decoded.Content, err
-}
-
-func (llm *llamaExtServer) Embedding(ctx context.Context, input string) ([]float64, error) {
- return embedding(llm, ctx, input)
-}
-func embedding(llm extServer, ctx context.Context, input string) ([]float64, error) {
- data, err := json.Marshal(TokenizeRequest{Content: input})
- if err != nil {
- return nil, fmt.Errorf("error marshaling embed data: %w", err)
- }
-
- req := C.CString(string(data))
- defer C.free(unsafe.Pointer(req))
- var json_resp *C.char
- resp := newExtServerResp(128)
- defer freeExtServerResp(resp)
- llm.llama_server_embedding(req, &json_resp, &resp)
- if resp.id < 0 {
- return nil, extServerResponseToErr(resp)
- }
- defer llm.llama_server_release_json_resp(&json_resp)
-
- var embedding EmbeddingResponse
- if err := json.Unmarshal([]byte(C.GoString(json_resp)), &embedding); err != nil {
- return nil, fmt.Errorf("unmarshal tokenize response: %w", err)
- }
-
- return embedding.Embedding, nil
-}
-
-func (llm *llamaExtServer) Close() {
- close(llm)
-}
-
-func close(llm extServer) {
- llm.llama_server_stop()
- mutex.Unlock()
-}
diff --git a/llm/ext_server/CMakeLists.txt b/llm/ext_server/CMakeLists.txt
new file mode 100644
index 00000000..51730245
--- /dev/null
+++ b/llm/ext_server/CMakeLists.txt
@@ -0,0 +1,15 @@
+set(TARGET ollama_llama_server)
+option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON)
+set(LLAMA_SERVER_LDFLAGS $ENV{LLAMA_SERVER_LDFLAGS})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+add_executable(${TARGET} server.cpp utils.hpp httplib.h)
+install(TARGETS ${TARGET} RUNTIME)
+target_compile_definitions(${TARGET} PRIVATE
+ SERVER_VERBOSE=$<BOOL:${LLAMA_SERVER_VERBOSE}>
+)
+target_link_libraries(${TARGET} PRIVATE ggml llama common llava ${CMAKE_THREAD_LIBS_INIT} ${LLAMA_SERVER_LDFLAGS})
+if (WIN32)
+ TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ws2_32)
+ target_link_options(${TARGET} PRIVATE -municode -Wl,/subsystem:console)
+endif()
+target_compile_features(${TARGET} PRIVATE cxx_std_11)
\ No newline at end of file
diff --git a/llm/ext_server/httplib.h b/llm/ext_server/httplib.h
new file mode 100644
index 00000000..28746000
--- /dev/null
+++ b/llm/ext_server/httplib.h
@@ -0,0 +1,8794 @@
+//
+// httplib.h
+//
+// Copyright (c) 2023 Yuji Hirose. All rights reserved.
+// MIT License
+//
+
+#ifndef CPPHTTPLIB_HTTPLIB_H
+#define CPPHTTPLIB_HTTPLIB_H
+
+#define CPPHTTPLIB_VERSION "0.12.2"
+
+/*
+ * Configuration
+ */
+
+#ifndef CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND
+#define CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND 5
+#endif
+
+#ifndef CPPHTTPLIB_KEEPALIVE_MAX_COUNT
+#define CPPHTTPLIB_KEEPALIVE_MAX_COUNT 5
+#endif
+
+#ifndef CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND
+#define CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND 300
+#endif
+
+#ifndef CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND
+#define CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND 0
+#endif
+
+#ifndef CPPHTTPLIB_READ_TIMEOUT_SECOND
+#define CPPHTTPLIB_READ_TIMEOUT_SECOND 5
+#endif
+
+#ifndef CPPHTTPLIB_READ_TIMEOUT_USECOND
+#define CPPHTTPLIB_READ_TIMEOUT_USECOND 0
+#endif
+
+#ifndef CPPHTTPLIB_WRITE_TIMEOUT_SECOND
+#define CPPHTTPLIB_WRITE_TIMEOUT_SECOND 5
+#endif
+
+#ifndef CPPHTTPLIB_WRITE_TIMEOUT_USECOND
+#define CPPHTTPLIB_WRITE_TIMEOUT_USECOND 0
+#endif
+
+#ifndef CPPHTTPLIB_IDLE_INTERVAL_SECOND
+#define CPPHTTPLIB_IDLE_INTERVAL_SECOND 0
+#endif
+
+#ifndef CPPHTTPLIB_IDLE_INTERVAL_USECOND
+#ifdef _WIN32
+#define CPPHTTPLIB_IDLE_INTERVAL_USECOND 10000
+#else
+#define CPPHTTPLIB_IDLE_INTERVAL_USECOND 0
+#endif
+#endif
+
+#ifndef CPPHTTPLIB_REQUEST_URI_MAX_LENGTH
+#define CPPHTTPLIB_REQUEST_URI_MAX_LENGTH 8192
+#endif
+
+#ifndef CPPHTTPLIB_HEADER_MAX_LENGTH
+#define CPPHTTPLIB_HEADER_MAX_LENGTH 8192
+#endif
+
+#ifndef CPPHTTPLIB_REDIRECT_MAX_COUNT
+#define CPPHTTPLIB_REDIRECT_MAX_COUNT 20
+#endif
+
+#ifndef CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT
+#define CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT 1024
+#endif
+
+#ifndef CPPHTTPLIB_PAYLOAD_MAX_LENGTH
+#define CPPHTTPLIB_PAYLOAD_MAX_LENGTH ((std::numeric_limits<size_t>::max)())
+#endif
+
+#ifndef CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH
+#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 8192
+#endif
+
+#ifndef CPPHTTPLIB_TCP_NODELAY
+#define CPPHTTPLIB_TCP_NODELAY false
+#endif
+
+#ifndef CPPHTTPLIB_RECV_BUFSIZ
+#define CPPHTTPLIB_RECV_BUFSIZ size_t(4096u)
+#endif
+
+#ifndef CPPHTTPLIB_COMPRESSION_BUFSIZ
+#define CPPHTTPLIB_COMPRESSION_BUFSIZ size_t(16384u)
+#endif
+
+#ifndef CPPHTTPLIB_THREAD_POOL_COUNT
+#define CPPHTTPLIB_THREAD_POOL_COUNT \
+ ((std::max)(8u, std::thread::hardware_concurrency() > 0 \
+ ? std::thread::hardware_concurrency() - 1 \
+ : 0))
+#endif
+
+#ifndef CPPHTTPLIB_RECV_FLAGS
+#define CPPHTTPLIB_RECV_FLAGS 0
+#endif
+
+#ifndef CPPHTTPLIB_SEND_FLAGS
+#define CPPHTTPLIB_SEND_FLAGS 0
+#endif
+
+#ifndef CPPHTTPLIB_LISTEN_BACKLOG
+#define CPPHTTPLIB_LISTEN_BACKLOG 5
+#endif
+
+/*
+ * Headers
+ */
+
+#ifdef _WIN32
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif //_CRT_SECURE_NO_WARNINGS
+
+#ifndef _CRT_NONSTDC_NO_DEPRECATE
+#define _CRT_NONSTDC_NO_DEPRECATE
+#endif //_CRT_NONSTDC_NO_DEPRECATE
+
+#if defined(_MSC_VER)
+#if _MSC_VER < 1900
+#error Sorry, Visual Studio versions prior to 2015 are not supported
+#endif
+
+#pragma comment(lib, "ws2_32.lib")
+
+#ifdef _WIN64
+using ssize_t = __int64;
+#else
+using ssize_t = long;
+#endif
+#endif // _MSC_VER
+
+#ifndef S_ISREG
+#define S_ISREG(m) (((m)&S_IFREG) == S_IFREG)
+#endif // S_ISREG
+
+#ifndef S_ISDIR
+#define S_ISDIR(m) (((m)&S_IFDIR) == S_IFDIR)
+#endif // S_ISDIR
+
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif // NOMINMAX
+
+#include <io.h>
+#include <winsock2.h>
+#include <ws2tcpip.h>
+
+#ifndef WSA_FLAG_NO_HANDLE_INHERIT
+#define WSA_FLAG_NO_HANDLE_INHERIT 0x80
+#endif
+
+#ifndef strcasecmp
+#define strcasecmp _stricmp
+#endif // strcasecmp
+
+using socket_t = SOCKET;
+#ifdef CPPHTTPLIB_USE_POLL
+#define poll(fds, nfds, timeout) WSAPoll(fds, nfds, timeout)
+#endif
+
+#else // not _WIN32
+
+#include <arpa/inet.h>
+#ifndef _AIX
+#include <ifaddrs.h>
+#endif
+#include <net/if.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#ifdef __linux__
+#include <resolv.h>
+#endif
+#include <netinet/tcp.h>
+#ifdef CPPHTTPLIB_USE_POLL
+#include <poll.h>
+#endif
+#include <csignal>
+#include <pthread.h>
+#include <sys/select.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+using socket_t = int;
+#ifndef INVALID_SOCKET
+#define INVALID_SOCKET (-1)
+#endif
+#endif //_WIN32
+
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <cassert>
+#include <cctype>
+#include <climits>
+#include <condition_variable>
+#include <cstring>
+#include <errno.h>
+#include <fcntl.h>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <list>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <random>
+#include <regex>
+#include <set>
+#include <sstream>
+#include <string>
+#include <sys/stat.h>
+#include <thread>
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+#ifdef _WIN32
+#include <wincrypt.h>
+
+// these are defined in wincrypt.h and it breaks compilation if BoringSSL is
+// used
+#undef X509_NAME
+#undef X509_CERT_PAIR
+#undef X509_EXTENSIONS
+#undef PKCS7_SIGNER_INFO
+
+#ifdef _MSC_VER
+#pragma comment(lib, "crypt32.lib")
+#pragma comment(lib, "cryptui.lib")
+#endif
+#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__)
+#include <TargetConditionals.h>
+#if TARGET_OS_OSX
+#include <CoreFoundation/CoreFoundation.h>
+#include <Security/Security.h>
+#endif // TARGET_OS_OSX
+#endif // _WIN32
+
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/ssl.h>
+#include <openssl/x509v3.h>
+
+#if defined(_WIN32) && defined(OPENSSL_USE_APPLINK)
+#include <openssl/applink.c>
+#endif
+
+#include <iostream>
+#include <sstream>
+
+#if OPENSSL_VERSION_NUMBER < 0x1010100fL
+#error Sorry, OpenSSL versions prior to 1.1.1 are not supported
+#elif OPENSSL_VERSION_NUMBER < 0x30000000L
+#define SSL_get1_peer_certificate SSL_get_peer_certificate
+#endif
+
+#endif
+
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+#include <zlib.h>
+#endif
+
+#ifdef CPPHTTPLIB_BROTLI_SUPPORT
+#include <brotli/decode.h>
+#include <brotli/encode.h>
+#endif
+
+/*
+ * Declaration
+ */
+namespace httplib {
+
+namespace detail {
+
+/*
+ * Backport std::make_unique from C++14.
+ *
+ * NOTE: This code came up with the following stackoverflow post:
+ * https://stackoverflow.com/questions/10149840/c-arrays-and-make-unique
+ *
+ */
+
+template <class T, class... Args>
+typename std::enable_if<!std::is_array<T>::value, std::unique_ptr<T>>::type
+make_unique(Args &&...args) {
+ return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
+}
+
+template <class T>
+typename std::enable_if<std::is_array<T>::value, std::unique_ptr<T>>::type
+make_unique(std::size_t n) {
+ typedef typename std::remove_extent<T>::type RT;
+ return std::unique_ptr<T>(new RT[n]);
+}
+
+struct ci {
+ bool operator()(const std::string &s1, const std::string &s2) const {
+ return std::lexicographical_compare(s1.begin(), s1.end(), s2.begin(),
+ s2.end(),
+ [](unsigned char c1, unsigned char c2) {
+ return ::tolower(c1) < ::tolower(c2);
+ });
+ }
+};
+
+// This is based on
+// "http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4189".
+
+struct scope_exit {
+ explicit scope_exit(std::function<void(void)> &&f)
+ : exit_function(std::move(f)), execute_on_destruction{true} {}
+
+ scope_exit(scope_exit &&rhs)
+ : exit_function(std::move(rhs.exit_function)),
+ execute_on_destruction{rhs.execute_on_destruction} {
+ rhs.release();
+ }
+
+ ~scope_exit() {
+ if (execute_on_destruction) { this->exit_function(); }
+ }
+
+ void release() { this->execute_on_destruction = false; }
+
+private:
+ scope_exit(const scope_exit &) = delete;
+ void operator=(const scope_exit &) = delete;
+ scope_exit &operator=(scope_exit &&) = delete;
+
+ std::function<void(void)> exit_function;
+ bool execute_on_destruction;
+};
+
+} // namespace detail
+
+using Headers = std::multimap<std::string, std::string, detail::ci>;
+
+using Params = std::multimap<std::string, std::string>;
+using Match = std::smatch;
+
+using Progress = std::function<bool(uint64_t current, uint64_t total)>;
+
+struct Response;
+using ResponseHandler = std::function<bool(const Response &response)>;
+
+struct MultipartFormData {
+ std::string name;
+ std::string content;
+ std::string filename;
+ std::string content_type;
+};
+using MultipartFormDataItems = std::vector<MultipartFormData>;
+using MultipartFormDataMap = std::multimap<std::string, MultipartFormData>;
+
+class DataSink {
+public:
+ DataSink() : os(&sb_), sb_(*this) {}
+
+ DataSink(const DataSink &) = delete;
+ DataSink &operator=(const DataSink &) = delete;
+ DataSink(DataSink &&) = delete;
+ DataSink &operator=(DataSink &&) = delete;
+
+ std::function<bool(const char *data, size_t data_len)> write;
+ std::function<void()> done;
+ std::function<void(const Headers &trailer)> done_with_trailer;
+ std::ostream os;
+
+private:
+ class data_sink_streambuf : public std::streambuf {
+ public:
+ explicit data_sink_streambuf(DataSink &sink) : sink_(sink) {}
+
+ protected:
+ std::streamsize xsputn(const char *s, std::streamsize n) {
+ sink_.write(s, static_cast<size_t>(n));
+ return n;
+ }
+
+ private:
+ DataSink &sink_;
+ };
+
+ data_sink_streambuf sb_;
+};
+
+using ContentProvider =
+ std::function<bool(size_t offset, size_t length, DataSink &sink)>;
+
+using ContentProviderWithoutLength =
+ std::function<bool(size_t offset, DataSink &sink)>;
+
+using ContentProviderResourceReleaser = std::function<void(bool success)>;
+
+struct MultipartFormDataProvider {
+ std::string name;
+ ContentProviderWithoutLength provider;
+ std::string filename;
+ std::string content_type;
+};
+using MultipartFormDataProviderItems = std::vector<MultipartFormDataProvider>;
+
+using ContentReceiverWithProgress =
+ std::function<bool(const char *data, size_t data_length, uint64_t offset,
+ uint64_t total_length)>;
+
+using ContentReceiver =
+ std::function<bool(const char *data, size_t data_length)>;
+
+using MultipartContentHeader =
+ std::function<bool(const MultipartFormData &file)>;
+
+class ContentReader {
+public:
+ using Reader = std::function<bool(ContentReceiver receiver)>;
+ using MultipartReader = std::function<bool(MultipartContentHeader header,
+ ContentReceiver receiver)>;
+
+ ContentReader(Reader reader, MultipartReader multipart_reader)
+ : reader_(std::move(reader)),
+ multipart_reader_(std::move(multipart_reader)) {}
+
+ bool operator()(MultipartContentHeader header,
+ ContentReceiver receiver) const {
+ return multipart_reader_(std::move(header), std::move(receiver));
+ }
+
+ bool operator()(ContentReceiver receiver) const {
+ return reader_(std::move(receiver));
+ }
+
+ Reader reader_;
+ MultipartReader multipart_reader_;
+};
+
+using Range = std::pair<ssize_t, ssize_t>;
+using Ranges = std::vector<Range>;
+
+struct Request {
+ std::string method;
+ std::string path;
+ Headers headers;
+ std::string body;
+
+ std::string remote_addr;
+ int remote_port = -1;
+ std::string local_addr;
+ int local_port = -1;
+
+ // for server
+ std::string version;
+ std::string target;
+ Params params;
+ MultipartFormDataMap files;
+ Ranges ranges;
+ Match matches;
+
+ // for client
+ ResponseHandler response_handler;
+ ContentReceiverWithProgress content_receiver;
+ Progress progress;
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ const SSL *ssl = nullptr;
+#endif
+
+ bool has_header(const std::string &key) const;
+ std::string get_header_value(const std::string &key, size_t id = 0) const;
+ template <typename T>
+ T get_header_value(const std::string &key, size_t id = 0) const;
+ size_t get_header_value_count(const std::string &key) const;
+ void set_header(const std::string &key, const std::string &val);
+
+ bool has_param(const std::string &key) const;
+ std::string get_param_value(const std::string &key, size_t id = 0) const;
+ size_t get_param_value_count(const std::string &key) const;
+
+ bool is_multipart_form_data() const;
+
+ bool has_file(const std::string &key) const;
+ MultipartFormData get_file_value(const std::string &key) const;
+ std::vector<MultipartFormData> get_file_values(const std::string &key) const;
+
+ // private members...
+ size_t redirect_count_ = CPPHTTPLIB_REDIRECT_MAX_COUNT;
+ size_t content_length_ = 0;
+ ContentProvider content_provider_;
+ bool is_chunked_content_provider_ = false;
+ size_t authorization_count_ = 0;
+};
+
+struct Response {
+ std::string version;
+ int status = -1;
+ std::string reason;
+ Headers headers;
+ std::string body;
+ std::string location; // Redirect location
+
+ bool has_header(const std::string &key) const;
+ std::string get_header_value(const std::string &key, size_t id = 0) const;
+ template <typename T>
+ T get_header_value(const std::string &key, size_t id = 0) const;
+ size_t get_header_value_count(const std::string &key) const;
+ void set_header(const std::string &key, const std::string &val);
+
+ void set_redirect(const std::string &url, int status = 302);
+ void set_content(const char *s, size_t n, const std::string &content_type);
+ void set_content(const std::string &s, const std::string &content_type);
+
+ void set_content_provider(
+ size_t length, const std::string &content_type, ContentProvider provider,
+ ContentProviderResourceReleaser resource_releaser = nullptr);
+
+ void set_content_provider(
+ const std::string &content_type, ContentProviderWithoutLength provider,
+ ContentProviderResourceReleaser resource_releaser = nullptr);
+
+ void set_chunked_content_provider(
+ const std::string &content_type, ContentProviderWithoutLength provider,
+ ContentProviderResourceReleaser resource_releaser = nullptr);
+
+ Response() = default;
+ Response(const Response &) = default;
+ Response &operator=(const Response &) = default;
+ Response(Response &&) = default;
+ Response &operator=(Response &&) = default;
+ ~Response() {
+ if (content_provider_resource_releaser_) {
+ content_provider_resource_releaser_(content_provider_success_);
+ }
+ }
+
+ // private members...
+ size_t content_length_ = 0;
+ ContentProvider content_provider_;
+ ContentProviderResourceReleaser content_provider_resource_releaser_;
+ bool is_chunked_content_provider_ = false;
+ bool content_provider_success_ = false;
+};
+
+class Stream {
+public:
+ virtual ~Stream() = default;
+
+ virtual bool is_readable() const = 0;
+ virtual bool is_writable() const = 0;
+
+ virtual ssize_t read(char *ptr, size_t size) = 0;
+ virtual ssize_t write(const char *ptr, size_t size) = 0;
+ virtual void get_remote_ip_and_port(std::string &ip, int &port) const = 0;
+ virtual void get_local_ip_and_port(std::string &ip, int &port) const = 0;
+ virtual socket_t socket() const = 0;
+
+ template <typename... Args>
+ ssize_t write_format(const char *fmt, const Args &...args);
+ ssize_t write(const char *ptr);
+ ssize_t write(const std::string &s);
+};
+
+class TaskQueue {
+public:
+ TaskQueue() = default;
+ virtual ~TaskQueue() = default;
+
+ virtual void enqueue(std::function<void()> fn) = 0;
+ virtual void shutdown() = 0;
+
+ virtual void on_idle() {}
+};
+
+class ThreadPool : public TaskQueue {
+public:
+ explicit ThreadPool(size_t n) : shutdown_(false) {
+ while (n) {
+ threads_.emplace_back(worker(*this));
+ n--;
+ }
+ }
+
+ ThreadPool(const ThreadPool &) = delete;
+ ~ThreadPool() override = default;
+
+ void enqueue(std::function<void()> fn) override {
+ {
+ std::unique_lock<std::mutex> lock(mutex_);
+ jobs_.push_back(std::move(fn));
+ }
+
+ cond_.notify_one();
+ }
+
+ void shutdown() override {
+ // Stop all worker threads...
+ {
+ std::unique_lock<std::mutex> lock(mutex_);
+ shutdown_ = true;
+ }
+
+ cond_.notify_all();
+
+ // Join...
+ for (auto &t : threads_) {
+ t.join();
+ }
+ }
+
+private:
+ struct worker {
+ explicit worker(ThreadPool &pool) : pool_(pool) {}
+
+ void operator()() {
+ for (;;) {
+ std::function<void()> fn;
+ {
+ std::unique_lock<std::mutex> lock(pool_.mutex_);
+
+ pool_.cond_.wait(
+ lock, [&] { return !pool_.jobs_.empty() || pool_.shutdown_; });
+
+ if (pool_.shutdown_ && pool_.jobs_.empty()) { break; }
+
+ fn = std::move(pool_.jobs_.front());
+ pool_.jobs_.pop_front();
+ }
+
+ assert(true == static_cast<bool>(fn));
+ fn();
+ }
+ }
+
+ ThreadPool &pool_;
+ };
+ friend struct worker;
+
+ std::vector<std::thread> threads_;
+ std::list<std::function<void()>> jobs_;
+
+ bool shutdown_;
+
+ std::condition_variable cond_;
+ std::mutex mutex_;
+};
+
+using Logger = std::function<void(const Request &, const Response &)>;
+
+using SocketOptions = std::function<void(socket_t sock)>;
+
+void default_socket_options(socket_t sock);
+
+class Server {
+public:
+ using Handler = std::function<void(const Request &, Response &)>;
+
+ using ExceptionHandler =
+ std::function<void(const Request &, Response &, std::exception_ptr ep)>;
+
+ enum class HandlerResponse {
+ Handled,
+ Unhandled,
+ };
+ using HandlerWithResponse =
+ std::function<HandlerResponse(const Request &, Response &)>;
+
+ using HandlerWithContentReader = std::function<void(
+ const Request &, Response &, const ContentReader &content_reader)>;
+
+ using Expect100ContinueHandler =
+ std::function<int(const Request &, Response &)>;
+
+ Server();
+
+ virtual ~Server();
+
+ virtual bool is_valid() const;
+
+ Server &Get(const std::string &pattern, Handler handler);
+ Server &Post(const std::string &pattern, Handler handler);
+ Server &Post(const std::string &pattern, HandlerWithContentReader handler);
+ Server &Put(const std::string &pattern, Handler handler);
+ Server &Put(const std::string &pattern, HandlerWithContentReader handler);
+ Server &Patch(const std::string &pattern, Handler handler);
+ Server &Patch(const std::string &pattern, HandlerWithContentReader handler);
+ Server &Delete(const std::string &pattern, Handler handler);
+ Server &Delete(const std::string &pattern, HandlerWithContentReader handler);
+ Server &Options(const std::string &pattern, Handler handler);
+
+ bool set_base_dir(const std::string &dir,
+ const std::string &mount_point = std::string());
+ bool set_mount_point(const std::string &mount_point, const std::string &dir,
+ Headers headers = Headers());
+ bool remove_mount_point(const std::string &mount_point);
+ Server &set_file_extension_and_mimetype_mapping(const std::string &ext,
+ const std::string &mime);
+ Server &set_file_request_handler(Handler handler);
+
+ Server &set_error_handler(HandlerWithResponse handler);
+ Server &set_error_handler(Handler handler);
+ Server &set_exception_handler(ExceptionHandler handler);
+ Server &set_pre_routing_handler(HandlerWithResponse handler);
+ Server &set_post_routing_handler(Handler handler);
+
+ Server &set_expect_100_continue_handler(Expect100ContinueHandler handler);
+ Server &set_logger(Logger logger);
+
+ Server &set_address_family(int family);
+ Server &set_tcp_nodelay(bool on);
+ Server &set_socket_options(SocketOptions socket_options);
+
+ Server &set_default_headers(Headers headers);
+
+ Server &set_keep_alive_max_count(size_t count);
+ Server &set_keep_alive_timeout(time_t sec);
+
+ Server &set_read_timeout(time_t sec, time_t usec = 0);
+ template <class Rep, class Period>
+ Server &set_read_timeout(const std::chrono::duration<Rep, Period> &duration);
+
+ Server &set_write_timeout(time_t sec, time_t usec = 0);
+ template <class Rep, class Period>
+ Server &set_write_timeout(const std::chrono::duration<Rep, Period> &duration);
+
+ Server &set_idle_interval(time_t sec, time_t usec = 0);
+ template <class Rep, class Period>
+ Server &set_idle_interval(const std::chrono::duration<Rep, Period> &duration);
+
+ Server &set_payload_max_length(size_t length);
+
+ bool bind_to_port(const std::string &host, int port, int socket_flags = 0);
+ int bind_to_any_port(const std::string &host, int socket_flags = 0);
+ bool listen_after_bind();
+
+ bool listen(const std::string &host, int port, int socket_flags = 0);
+
+ bool is_running() const;
+ void wait_until_ready() const;
+ void stop();
+
+ std::function<TaskQueue *(void)> new_task_queue;
+
+protected:
+ bool process_request(Stream &strm, bool close_connection,
+ bool &connection_closed,
+ const std::function<void(Request &)> &setup_request);
+
+ std::atomic<socket_t> svr_sock_{INVALID_SOCKET};
+ size_t keep_alive_max_count_ = CPPHTTPLIB_KEEPALIVE_MAX_COUNT;
+ time_t keep_alive_timeout_sec_ = CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND;
+ time_t read_timeout_sec_ = CPPHTTPLIB_READ_TIMEOUT_SECOND;
+ time_t read_timeout_usec_ = CPPHTTPLIB_READ_TIMEOUT_USECOND;
+ time_t write_timeout_sec_ = CPPHTTPLIB_WRITE_TIMEOUT_SECOND;
+ time_t write_timeout_usec_ = CPPHTTPLIB_WRITE_TIMEOUT_USECOND;
+ time_t idle_interval_sec_ = CPPHTTPLIB_IDLE_INTERVAL_SECOND;
+ time_t idle_interval_usec_ = CPPHTTPLIB_IDLE_INTERVAL_USECOND;
+ size_t payload_max_length_ = CPPHTTPLIB_PAYLOAD_MAX_LENGTH;
+
+private:
+ using Handlers = std::vector<std::pair<std::regex, Handler>>;
+ using HandlersForContentReader =
+ std::vector<std::pair<std::regex, HandlerWithContentReader>>;
+
+ socket_t create_server_socket(const std::string &host, int port,
+ int socket_flags,
+ SocketOptions socket_options) const;
+ int bind_internal(const std::string &host, int port, int socket_flags);
+ bool listen_internal();
+
+ bool routing(Request &req, Response &res, Stream &strm);
+ bool handle_file_request(const Request &req, Response &res,
+ bool head = false);
+ bool dispatch_request(Request &req, Response &res, const Handlers &handlers);
+ bool
+ dispatch_request_for_content_reader(Request &req, Response &res,
+ ContentReader content_reader,
+ const HandlersForContentReader &handlers);
+
+ bool parse_request_line(const char *s, Request &req);
+ void apply_ranges(const Request &req, Response &res,
+ std::string &content_type, std::string &boundary);
+ bool write_response(Stream &strm, bool close_connection, const Request &req,
+ Response &res);
+ bool write_response_with_content(Stream &strm, bool close_connection,
+ const Request &req, Response &res);
+ bool write_response_core(Stream &strm, bool close_connection,
+ const Request &req, Response &res,
+ bool need_apply_ranges);
+ bool write_content_with_provider(Stream &strm, const Request &req,
+ Response &res, const std::string &boundary,
+ const std::string &content_type);
+ bool read_content(Stream &strm, Request &req, Response &res);
+ bool
+ read_content_with_content_receiver(Stream &strm, Request &req, Response &res,
+ ContentReceiver receiver,
+ MultipartContentHeader multipart_header,
+ ContentReceiver multipart_receiver);
+ bool read_content_core(Stream &strm, Request &req, Response &res,
+ ContentReceiver receiver,
+ MultipartContentHeader multipart_header,
+ ContentReceiver multipart_receiver);
+
+ virtual bool process_and_close_socket(socket_t sock);
+
+ struct MountPointEntry {
+ std::string mount_point;
+ std::string base_dir;
+ Headers headers;
+ };
+ std::vector<MountPointEntry> base_dirs_;
+
+ std::atomic<bool> is_running_{false};
+ std::atomic<bool> done_{false};
+ std::map<std::string, std::string> file_extension_and_mimetype_map_;
+ Handler file_request_handler_;
+ Handlers get_handlers_;
+ Handlers post_handlers_;
+ HandlersForContentReader post_handlers_for_content_reader_;
+ Handlers put_handlers_;
+ HandlersForContentReader put_handlers_for_content_reader_;
+ Handlers patch_handlers_;
+ HandlersForContentReader patch_handlers_for_content_reader_;
+ Handlers delete_handlers_;
+ HandlersForContentReader delete_handlers_for_content_reader_;
+ Handlers options_handlers_;
+ HandlerWithResponse error_handler_;
+ ExceptionHandler exception_handler_;
+ HandlerWithResponse pre_routing_handler_;
+ Handler post_routing_handler_;
+ Logger logger_;
+ Expect100ContinueHandler expect_100_continue_handler_;
+
+ int address_family_ = AF_UNSPEC;
+ bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY;
+ SocketOptions socket_options_ = default_socket_options;
+
+ Headers default_headers_;
+};
+
+enum class Error {
+ Success = 0,
+ Unknown,
+ Connection,
+ BindIPAddress,
+ Read,
+ Write,
+ ExceedRedirectCount,
+ Canceled,
+ SSLConnection,
+ SSLLoadingCerts,
+ SSLServerVerification,
+ UnsupportedMultipartBoundaryChars,
+ Compression,
+ ConnectionTimeout,
+
+ // For internal use only
+ SSLPeerCouldBeClosed_,
+};
+
+std::string to_string(const Error error);
+
+std::ostream &operator<<(std::ostream &os, const Error &obj);
+
+class Result {
+public:
+ Result(std::unique_ptr<Response> &&res, Error err,
+ Headers &&request_headers = Headers{})
+ : res_(std::move(res)), err_(err),
+ request_headers_(std::move(request_headers)) {}
+ // Response
+ operator bool() const { return res_ != nullptr; }
+ bool operator==(std::nullptr_t) const { return res_ == nullptr; }
+ bool operator!=(std::nullptr_t) const { return res_ != nullptr; }
+ const Response &value() const { return *res_; }
+ Response &value() { return *res_; }
+ const Response &operator*() const { return *res_; }
+ Response &operator*() { return *res_; }
+ const Response *operator->() const { return res_.get(); }
+ Response *operator->() { return res_.get(); }
+
+ // Error
+ Error error() const { return err_; }
+
+ // Request Headers
+ bool has_request_header(const std::string &key) const;
+ std::string get_request_header_value(const std::string &key,
+ size_t id = 0) const;
+ template <typename T>
+ T get_request_header_value(const std::string &key, size_t id = 0) const;
+ size_t get_request_header_value_count(const std::string &key) const;
+
+private:
+ std::unique_ptr<Response> res_;
+ Error err_;
+ Headers request_headers_;
+};
+
+class ClientImpl {
+public:
+ explicit ClientImpl(const std::string &host);
+
+ explicit ClientImpl(const std::string &host, int port);
+
+ explicit ClientImpl(const std::string &host, int port,
+ const std::string &client_cert_path,
+ const std::string &client_key_path);
+
+ virtual ~ClientImpl();
+
+ virtual bool is_valid() const;
+
+ Result Get(const std::string &path);
+ Result Get(const std::string &path, const Headers &headers);
+ Result Get(const std::string &path, Progress progress);
+ Result Get(const std::string &path, const Headers &headers,
+ Progress progress);
+ Result Get(const std::string &path, ContentReceiver content_receiver);
+ Result Get(const std::string &path, const Headers &headers,
+ ContentReceiver content_receiver);
+ Result Get(const std::string &path, ContentReceiver content_receiver,
+ Progress progress);
+ Result Get(const std::string &path, const Headers &headers,
+ ContentReceiver content_receiver, Progress progress);
+ Result Get(const std::string &path, ResponseHandler response_handler,
+ ContentReceiver content_receiver);
+ Result Get(const std::string &path, const Headers &headers,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver);
+ Result Get(const std::string &path, ResponseHandler response_handler,
+ ContentReceiver content_receiver, Progress progress);
+ Result Get(const std::string &path, const Headers &headers,
+ ResponseHandler response_handler, ContentReceiver content_receiver,
+ Progress progress);
+
+ Result Get(const std::string &path, const Params ¶ms,
+ const Headers &headers, Progress progress = nullptr);
+ Result Get(const std::string &path, const Params ¶ms,
+ const Headers &headers, ContentReceiver content_receiver,
+ Progress progress = nullptr);
+ Result Get(const std::string &path, const Params ¶ms,
+ const Headers &headers, ResponseHandler response_handler,
+ ContentReceiver content_receiver, Progress progress = nullptr);
+
+ Result Head(const std::string &path);
+ Result Head(const std::string &path, const Headers &headers);
+
+ Result Post(const std::string &path);
+ Result Post(const std::string &path, const Headers &headers);
+ Result Post(const std::string &path, const char *body, size_t content_length,
+ const std::string &content_type);
+ Result Post(const std::string &path, const Headers &headers, const char *body,
+ size_t content_length, const std::string &content_type);
+ Result Post(const std::string &path, const std::string &body,
+ const std::string &content_type);
+ Result Post(const std::string &path, const Headers &headers,
+ const std::string &body, const std::string &content_type);
+ Result Post(const std::string &path, size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type);
+ Result Post(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+ Result Post(const std::string &path, const Headers &headers,
+ size_t content_length, ContentProvider content_provider,
+ const std::string &content_type);
+ Result Post(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+ Result Post(const std::string &path, const Params ¶ms);
+ Result Post(const std::string &path, const Headers &headers,
+ const Params ¶ms);
+ Result Post(const std::string &path, const MultipartFormDataItems &items);
+ Result Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items);
+ Result Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items, const std::string &boundary);
+ Result Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items);
+
+ Result Put(const std::string &path);
+ Result Put(const std::string &path, const char *body, size_t content_length,
+ const std::string &content_type);
+ Result Put(const std::string &path, const Headers &headers, const char *body,
+ size_t content_length, const std::string &content_type);
+ Result Put(const std::string &path, const std::string &body,
+ const std::string &content_type);
+ Result Put(const std::string &path, const Headers &headers,
+ const std::string &body, const std::string &content_type);
+ Result Put(const std::string &path, size_t content_length,
+ ContentProvider content_provider, const std::string &content_type);
+ Result Put(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+ Result Put(const std::string &path, const Headers &headers,
+ size_t content_length, ContentProvider content_provider,
+ const std::string &content_type);
+ Result Put(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+ Result Put(const std::string &path, const Params ¶ms);
+ Result Put(const std::string &path, const Headers &headers,
+ const Params ¶ms);
+ Result Put(const std::string &path, const MultipartFormDataItems &items);
+ Result Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items);
+ Result Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items, const std::string &boundary);
+ Result Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items);
+
+ Result Patch(const std::string &path);
+ Result Patch(const std::string &path, const char *body, size_t content_length,
+ const std::string &content_type);
+ Result Patch(const std::string &path, const Headers &headers,
+ const char *body, size_t content_length,
+ const std::string &content_type);
+ Result Patch(const std::string &path, const std::string &body,
+ const std::string &content_type);
+ Result Patch(const std::string &path, const Headers &headers,
+ const std::string &body, const std::string &content_type);
+ Result Patch(const std::string &path, size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type);
+ Result Patch(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+ Result Patch(const std::string &path, const Headers &headers,
+ size_t content_length, ContentProvider content_provider,
+ const std::string &content_type);
+ Result Patch(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+
+ Result Delete(const std::string &path);
+ Result Delete(const std::string &path, const Headers &headers);
+ Result Delete(const std::string &path, const char *body,
+ size_t content_length, const std::string &content_type);
+ Result Delete(const std::string &path, const Headers &headers,
+ const char *body, size_t content_length,
+ const std::string &content_type);
+ Result Delete(const std::string &path, const std::string &body,
+ const std::string &content_type);
+ Result Delete(const std::string &path, const Headers &headers,
+ const std::string &body, const std::string &content_type);
+
+ Result Options(const std::string &path);
+ Result Options(const std::string &path, const Headers &headers);
+
+ bool send(Request &req, Response &res, Error &error);
+ Result send(const Request &req);
+
+ size_t is_socket_open() const;
+
+ socket_t socket() const;
+
+ void stop();
+
+ void set_hostname_addr_map(std::map<std::string, std::string> addr_map);
+
+ void set_default_headers(Headers headers);
+
+ void set_address_family(int family);
+ void set_tcp_nodelay(bool on);
+ void set_socket_options(SocketOptions socket_options);
+
+ void set_connection_timeout(time_t sec, time_t usec = 0);
+ template <class Rep, class Period>
+ void
+ set_connection_timeout(const std::chrono::duration<Rep, Period> &duration);
+
+ void set_read_timeout(time_t sec, time_t usec = 0);
+ template <class Rep, class Period>
+ void set_read_timeout(const std::chrono::duration<Rep, Period> &duration);
+
+ void set_write_timeout(time_t sec, time_t usec = 0);
+ template <class Rep, class Period>
+ void set_write_timeout(const std::chrono::duration<Rep, Period> &duration);
+
+ void set_basic_auth(const std::string &username, const std::string &password);
+ void set_bearer_token_auth(const std::string &token);
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ void set_digest_auth(const std::string &username,
+ const std::string &password);
+#endif
+
+ void set_keep_alive(bool on);
+ void set_follow_location(bool on);
+
+ void set_url_encode(bool on);
+
+ void set_compress(bool on);
+
+ void set_decompress(bool on);
+
+ void set_interface(const std::string &intf);
+
+ void set_proxy(const std::string &host, int port);
+ void set_proxy_basic_auth(const std::string &username,
+ const std::string &password);
+ void set_proxy_bearer_token_auth(const std::string &token);
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ void set_proxy_digest_auth(const std::string &username,
+ const std::string &password);
+#endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ void set_ca_cert_path(const std::string &ca_cert_file_path,
+ const std::string &ca_cert_dir_path = std::string());
+ void set_ca_cert_store(X509_STORE *ca_cert_store);
+#endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ void enable_server_certificate_verification(bool enabled);
+#endif
+
+ void set_logger(Logger logger);
+
+protected:
+ struct Socket {
+ socket_t sock = INVALID_SOCKET;
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ SSL *ssl = nullptr;
+#endif
+
+ bool is_open() const { return sock != INVALID_SOCKET; }
+ };
+
+ virtual bool create_and_connect_socket(Socket &socket, Error &error);
+
+ // All of:
+ // shutdown_ssl
+ // shutdown_socket
+ // close_socket
+ // should ONLY be called when socket_mutex_ is locked.
+ // Also, shutdown_ssl and close_socket should also NOT be called concurrently
+ // with a DIFFERENT thread sending requests using that socket.
+ virtual void shutdown_ssl(Socket &socket, bool shutdown_gracefully);
+ void shutdown_socket(Socket &socket);
+ void close_socket(Socket &socket);
+
+ bool process_request(Stream &strm, Request &req, Response &res,
+ bool close_connection, Error &error);
+
+ bool write_content_with_provider(Stream &strm, const Request &req,
+ Error &error);
+
+ void copy_settings(const ClientImpl &rhs);
+
+ // Socket endpoint information
+ const std::string host_;
+ const int port_;
+ const std::string host_and_port_;
+
+ // Current open socket
+ Socket socket_;
+ mutable std::mutex socket_mutex_;
+ std::recursive_mutex request_mutex_;
+
+ // These are all protected under socket_mutex
+ size_t socket_requests_in_flight_ = 0;
+ std::thread::id socket_requests_are_from_thread_ = std::thread::id();
+ bool socket_should_be_closed_when_request_is_done_ = false;
+
+ // Hostname-IP map
+ std::map<std::string, std::string> addr_map_;
+
+ // Default headers
+ Headers default_headers_;
+
+ // Settings
+ std::string client_cert_path_;
+ std::string client_key_path_;
+
+ time_t connection_timeout_sec_ = CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND;
+ time_t connection_timeout_usec_ = CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND;
+ time_t read_timeout_sec_ = CPPHTTPLIB_READ_TIMEOUT_SECOND;
+ time_t read_timeout_usec_ = CPPHTTPLIB_READ_TIMEOUT_USECOND;
+ time_t write_timeout_sec_ = CPPHTTPLIB_WRITE_TIMEOUT_SECOND;
+ time_t write_timeout_usec_ = CPPHTTPLIB_WRITE_TIMEOUT_USECOND;
+
+ std::string basic_auth_username_;
+ std::string basic_auth_password_;
+ std::string bearer_token_auth_token_;
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ std::string digest_auth_username_;
+ std::string digest_auth_password_;
+#endif
+
+ bool keep_alive_ = false;
+ bool follow_location_ = false;
+
+ bool url_encode_ = true;
+
+ int address_family_ = AF_UNSPEC;
+ bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY;
+ SocketOptions socket_options_ = nullptr;
+
+ bool compress_ = false;
+ bool decompress_ = true;
+
+ std::string interface_;
+
+ std::string proxy_host_;
+ int proxy_port_ = -1;
+
+ std::string proxy_basic_auth_username_;
+ std::string proxy_basic_auth_password_;
+ std::string proxy_bearer_token_auth_token_;
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ std::string proxy_digest_auth_username_;
+ std::string proxy_digest_auth_password_;
+#endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ std::string ca_cert_file_path_;
+ std::string ca_cert_dir_path_;
+
+ X509_STORE *ca_cert_store_ = nullptr;
+#endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ bool server_certificate_verification_ = true;
+#endif
+
+ Logger logger_;
+
+private:
+ bool send_(Request &req, Response &res, Error &error);
+ Result send_(Request &&req);
+
+ socket_t create_client_socket(Error &error) const;
+ bool read_response_line(Stream &strm, const Request &req, Response &res);
+ bool write_request(Stream &strm, Request &req, bool close_connection,
+ Error &error);
+ bool redirect(Request &req, Response &res, Error &error);
+ bool handle_request(Stream &strm, Request &req, Response &res,
+ bool close_connection, Error &error);
+ std::unique_ptr<Response> send_with_content_provider(
+ Request &req, const char *body, size_t content_length,
+ ContentProvider content_provider,
+ ContentProviderWithoutLength content_provider_without_length,
+ const std::string &content_type, Error &error);
+ Result send_with_content_provider(
+ const std::string &method, const std::string &path,
+ const Headers &headers, const char *body, size_t content_length,
+ ContentProvider content_provider,
+ ContentProviderWithoutLength content_provider_without_length,
+ const std::string &content_type);
+ ContentProviderWithoutLength get_multipart_content_provider(
+ const std::string &boundary, const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items);
+
+ std::string adjust_host_string(const std::string &host) const;
+
+ virtual bool process_socket(const Socket &socket,
+ std::function<bool(Stream &strm)> callback);
+ virtual bool is_ssl() const;
+};
+
+class Client {
+public:
+ // Universal interface
+ explicit Client(const std::string &scheme_host_port);
+
+ explicit Client(const std::string &scheme_host_port,
+ const std::string &client_cert_path,
+ const std::string &client_key_path);
+
+ // HTTP only interface
+ explicit Client(const std::string &host, int port);
+
+ explicit Client(const std::string &host, int port,
+ const std::string &client_cert_path,
+ const std::string &client_key_path);
+
+ Client(Client &&) = default;
+
+ ~Client();
+
+ bool is_valid() const;
+
+ Result Get(const std::string &path);
+ Result Get(const std::string &path, const Headers &headers);
+ Result Get(const std::string &path, Progress progress);
+ Result Get(const std::string &path, const Headers &headers,
+ Progress progress);
+ Result Get(const std::string &path, ContentReceiver content_receiver);
+ Result Get(const std::string &path, const Headers &headers,
+ ContentReceiver content_receiver);
+ Result Get(const std::string &path, ContentReceiver content_receiver,
+ Progress progress);
+ Result Get(const std::string &path, const Headers &headers,
+ ContentReceiver content_receiver, Progress progress);
+ Result Get(const std::string &path, ResponseHandler response_handler,
+ ContentReceiver content_receiver);
+ Result Get(const std::string &path, const Headers &headers,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver);
+ Result Get(const std::string &path, const Headers &headers,
+ ResponseHandler response_handler, ContentReceiver content_receiver,
+ Progress progress);
+ Result Get(const std::string &path, ResponseHandler response_handler,
+ ContentReceiver content_receiver, Progress progress);
+
+ Result Get(const std::string &path, const Params ¶ms,
+ const Headers &headers, Progress progress = nullptr);
+ Result Get(const std::string &path, const Params ¶ms,
+ const Headers &headers, ContentReceiver content_receiver,
+ Progress progress = nullptr);
+ Result Get(const std::string &path, const Params ¶ms,
+ const Headers &headers, ResponseHandler response_handler,
+ ContentReceiver content_receiver, Progress progress = nullptr);
+
+ Result Head(const std::string &path);
+ Result Head(const std::string &path, const Headers &headers);
+
+ Result Post(const std::string &path);
+ Result Post(const std::string &path, const Headers &headers);
+ Result Post(const std::string &path, const char *body, size_t content_length,
+ const std::string &content_type);
+ Result Post(const std::string &path, const Headers &headers, const char *body,
+ size_t content_length, const std::string &content_type);
+ Result Post(const std::string &path, const std::string &body,
+ const std::string &content_type);
+ Result Post(const std::string &path, const Headers &headers,
+ const std::string &body, const std::string &content_type);
+ Result Post(const std::string &path, size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type);
+ Result Post(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+ Result Post(const std::string &path, const Headers &headers,
+ size_t content_length, ContentProvider content_provider,
+ const std::string &content_type);
+ Result Post(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+ Result Post(const std::string &path, const Params ¶ms);
+ Result Post(const std::string &path, const Headers &headers,
+ const Params ¶ms);
+ Result Post(const std::string &path, const MultipartFormDataItems &items);
+ Result Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items);
+ Result Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items, const std::string &boundary);
+ Result Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items);
+
+ Result Put(const std::string &path);
+ Result Put(const std::string &path, const char *body, size_t content_length,
+ const std::string &content_type);
+ Result Put(const std::string &path, const Headers &headers, const char *body,
+ size_t content_length, const std::string &content_type);
+ Result Put(const std::string &path, const std::string &body,
+ const std::string &content_type);
+ Result Put(const std::string &path, const Headers &headers,
+ const std::string &body, const std::string &content_type);
+ Result Put(const std::string &path, size_t content_length,
+ ContentProvider content_provider, const std::string &content_type);
+ Result Put(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+ Result Put(const std::string &path, const Headers &headers,
+ size_t content_length, ContentProvider content_provider,
+ const std::string &content_type);
+ Result Put(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+ Result Put(const std::string &path, const Params ¶ms);
+ Result Put(const std::string &path, const Headers &headers,
+ const Params ¶ms);
+ Result Put(const std::string &path, const MultipartFormDataItems &items);
+ Result Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items);
+ Result Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items, const std::string &boundary);
+ Result Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items);
+
+ Result Patch(const std::string &path);
+ Result Patch(const std::string &path, const char *body, size_t content_length,
+ const std::string &content_type);
+ Result Patch(const std::string &path, const Headers &headers,
+ const char *body, size_t content_length,
+ const std::string &content_type);
+ Result Patch(const std::string &path, const std::string &body,
+ const std::string &content_type);
+ Result Patch(const std::string &path, const Headers &headers,
+ const std::string &body, const std::string &content_type);
+ Result Patch(const std::string &path, size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type);
+ Result Patch(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+ Result Patch(const std::string &path, const Headers &headers,
+ size_t content_length, ContentProvider content_provider,
+ const std::string &content_type);
+ Result Patch(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type);
+
+ Result Delete(const std::string &path);
+ Result Delete(const std::string &path, const Headers &headers);
+ Result Delete(const std::string &path, const char *body,
+ size_t content_length, const std::string &content_type);
+ Result Delete(const std::string &path, const Headers &headers,
+ const char *body, size_t content_length,
+ const std::string &content_type);
+ Result Delete(const std::string &path, const std::string &body,
+ const std::string &content_type);
+ Result Delete(const std::string &path, const Headers &headers,
+ const std::string &body, const std::string &content_type);
+
+ Result Options(const std::string &path);
+ Result Options(const std::string &path, const Headers &headers);
+
+ bool send(Request &req, Response &res, Error &error);
+ Result send(const Request &req);
+
+ size_t is_socket_open() const;
+
+ socket_t socket() const;
+
+ void stop();
+
+ void set_hostname_addr_map(std::map<std::string, std::string> addr_map);
+
+ void set_default_headers(Headers headers);
+
+ void set_address_family(int family);
+ void set_tcp_nodelay(bool on);
+ void set_socket_options(SocketOptions socket_options);
+
+ void set_connection_timeout(time_t sec, time_t usec = 0);
+ template <class Rep, class Period>
+ void
+ set_connection_timeout(const std::chrono::duration<Rep, Period> &duration);
+
+ void set_read_timeout(time_t sec, time_t usec = 0);
+ template <class Rep, class Period>
+ void set_read_timeout(const std::chrono::duration<Rep, Period> &duration);
+
+ void set_write_timeout(time_t sec, time_t usec = 0);
+ template <class Rep, class Period>
+ void set_write_timeout(const std::chrono::duration<Rep, Period> &duration);
+
+ void set_basic_auth(const std::string &username, const std::string &password);
+ void set_bearer_token_auth(const std::string &token);
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ void set_digest_auth(const std::string &username,
+ const std::string &password);
+#endif
+
+ void set_keep_alive(bool on);
+ void set_follow_location(bool on);
+
+ void set_url_encode(bool on);
+
+ void set_compress(bool on);
+
+ void set_decompress(bool on);
+
+ void set_interface(const std::string &intf);
+
+ void set_proxy(const std::string &host, int port);
+ void set_proxy_basic_auth(const std::string &username,
+ const std::string &password);
+ void set_proxy_bearer_token_auth(const std::string &token);
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ void set_proxy_digest_auth(const std::string &username,
+ const std::string &password);
+#endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ void enable_server_certificate_verification(bool enabled);
+#endif
+
+ void set_logger(Logger logger);
+
+ // SSL
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ void set_ca_cert_path(const std::string &ca_cert_file_path,
+ const std::string &ca_cert_dir_path = std::string());
+
+ void set_ca_cert_store(X509_STORE *ca_cert_store);
+
+ long get_openssl_verify_result() const;
+
+ SSL_CTX *ssl_context() const;
+#endif
+
+private:
+ std::unique_ptr<ClientImpl> cli_;
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ bool is_ssl_ = false;
+#endif
+};
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+class SSLServer : public Server {
+public:
+ SSLServer(const char *cert_path, const char *private_key_path,
+ const char *client_ca_cert_file_path = nullptr,
+ const char *client_ca_cert_dir_path = nullptr,
+ const char *private_key_password = nullptr);
+
+ SSLServer(X509 *cert, EVP_PKEY *private_key,
+ X509_STORE *client_ca_cert_store = nullptr);
+
+ SSLServer(
+ const std::function<bool(SSL_CTX &ssl_ctx)> &setup_ssl_ctx_callback);
+
+ ~SSLServer() override;
+
+ bool is_valid() const override;
+
+ SSL_CTX *ssl_context() const;
+
+private:
+ bool process_and_close_socket(socket_t sock) override;
+
+ SSL_CTX *ctx_;
+ std::mutex ctx_mutex_;
+};
+
+class SSLClient : public ClientImpl {
+public:
+ explicit SSLClient(const std::string &host);
+
+ explicit SSLClient(const std::string &host, int port);
+
+ explicit SSLClient(const std::string &host, int port,
+ const std::string &client_cert_path,
+ const std::string &client_key_path);
+
+ explicit SSLClient(const std::string &host, int port, X509 *client_cert,
+ EVP_PKEY *client_key);
+
+ ~SSLClient() override;
+
+ bool is_valid() const override;
+
+ void set_ca_cert_store(X509_STORE *ca_cert_store);
+
+ long get_openssl_verify_result() const;
+
+ SSL_CTX *ssl_context() const;
+
+private:
+ bool create_and_connect_socket(Socket &socket, Error &error) override;
+ void shutdown_ssl(Socket &socket, bool shutdown_gracefully) override;
+ void shutdown_ssl_impl(Socket &socket, bool shutdown_socket);
+
+ bool process_socket(const Socket &socket,
+ std::function<bool(Stream &strm)> callback) override;
+ bool is_ssl() const override;
+
+ bool connect_with_proxy(Socket &sock, Response &res, bool &success,
+ Error &error);
+ bool initialize_ssl(Socket &socket, Error &error);
+
+ bool load_certs();
+
+ bool verify_host(X509 *server_cert) const;
+ bool verify_host_with_subject_alt_name(X509 *server_cert) const;
+ bool verify_host_with_common_name(X509 *server_cert) const;
+ bool check_host_name(const char *pattern, size_t pattern_len) const;
+
+ SSL_CTX *ctx_;
+ std::mutex ctx_mutex_;
+ std::once_flag initialize_cert_;
+
+ std::vector<std::string> host_components_;
+
+ long verify_result_ = 0;
+
+ friend class ClientImpl;
+};
+#endif
+
+/*
+ * Implementation of template methods.
+ */
+
+namespace detail {
+
+template <typename T, typename U>
+inline void duration_to_sec_and_usec(const T &duration, U callback) {
+ auto sec = std::chrono::duration_cast<std::chrono::seconds>(duration).count();
+ auto usec = std::chrono::duration_cast<std::chrono::microseconds>(
+ duration - std::chrono::seconds(sec))
+ .count();
+ callback(static_cast<time_t>(sec), static_cast<time_t>(usec));
+}
+
+template <typename T>
+inline T get_header_value(const Headers & /*headers*/,
+ const std::string & /*key*/, size_t /*id*/ = 0,
+ uint64_t /*def*/ = 0) {}
+
+template <>
+inline uint64_t get_header_value<uint64_t>(const Headers &headers,
+ const std::string &key, size_t id,
+ uint64_t def) {
+ auto rng = headers.equal_range(key);
+ auto it = rng.first;
+ std::advance(it, static_cast<ssize_t>(id));
+ if (it != rng.second) {
+ return std::strtoull(it->second.data(), nullptr, 10);
+ }
+ return def;
+}
+
+} // namespace detail
+
+template <typename T>
+inline T Request::get_header_value(const std::string &key, size_t id) const {
+ return detail::get_header_value<T>(headers, key, id, 0);
+}
+
+template <typename T>
+inline T Response::get_header_value(const std::string &key, size_t id) const {
+ return detail::get_header_value<T>(headers, key, id, 0);
+}
+
+template <typename... Args>
+inline ssize_t Stream::write_format(const char *fmt, const Args &...args) {
+ const auto bufsiz = 2048;
+ std::array<char, bufsiz> buf{};
+
+ auto sn = snprintf(buf.data(), buf.size() - 1, fmt, args...);
+ if (sn <= 0) { return sn; }
+
+ auto n = static_cast<size_t>(sn);
+
+ if (n >= buf.size() - 1) {
+ std::vector<char> glowable_buf(buf.size());
+
+ while (n >= glowable_buf.size() - 1) {
+ glowable_buf.resize(glowable_buf.size() * 2);
+ n = static_cast<size_t>(
+ snprintf(&glowable_buf[0], glowable_buf.size() - 1, fmt, args...));
+ }
+ return write(&glowable_buf[0], n);
+ } else {
+ return write(buf.data(), n);
+ }
+}
+
+inline void default_socket_options(socket_t sock) {
+ int yes = 1;
+#ifdef _WIN32
+ setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char *>(&yes),
+ sizeof(yes));
+ setsockopt(sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE,
+ reinterpret_cast<char *>(&yes), sizeof(yes));
+#else
+#ifdef SO_REUSEPORT
+ setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, reinterpret_cast<void *>(&yes),
+ sizeof(yes));
+#else
+ setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<void *>(&yes),
+ sizeof(yes));
+#endif
+#endif
+}
+
+template <class Rep, class Period>
+inline Server &
+Server::set_read_timeout(const std::chrono::duration<Rep, Period> &duration) {
+ detail::duration_to_sec_and_usec(
+ duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); });
+ return *this;
+}
+
+template <class Rep, class Period>
+inline Server &
+Server::set_write_timeout(const std::chrono::duration<Rep, Period> &duration) {
+ detail::duration_to_sec_and_usec(
+ duration, [&](time_t sec, time_t usec) { set_write_timeout(sec, usec); });
+ return *this;
+}
+
+template <class Rep, class Period>
+inline Server &
+Server::set_idle_interval(const std::chrono::duration<Rep, Period> &duration) {
+ detail::duration_to_sec_and_usec(
+ duration, [&](time_t sec, time_t usec) { set_idle_interval(sec, usec); });
+ return *this;
+}
+
+inline std::string to_string(const Error error) {
+ switch (error) {
+ case Error::Success: return "Success (no error)";
+ case Error::Connection: return "Could not establish connection";
+ case Error::BindIPAddress: return "Failed to bind IP address";
+ case Error::Read: return "Failed to read connection";
+ case Error::Write: return "Failed to write connection";
+ case Error::ExceedRedirectCount: return "Maximum redirect count exceeded";
+ case Error::Canceled: return "Connection handling canceled";
+ case Error::SSLConnection: return "SSL connection failed";
+ case Error::SSLLoadingCerts: return "SSL certificate loading failed";
+ case Error::SSLServerVerification: return "SSL server verification failed";
+ case Error::UnsupportedMultipartBoundaryChars:
+ return "Unsupported HTTP multipart boundary characters";
+ case Error::Compression: return "Compression failed";
+ case Error::ConnectionTimeout: return "Connection timed out";
+ case Error::Unknown: return "Unknown";
+ default: break;
+ }
+
+ return "Invalid";
+}
+
+inline std::ostream &operator<<(std::ostream &os, const Error &obj) {
+ os << to_string(obj);
+ os << " (" << static_cast<std::underlying_type<Error>::type>(obj) << ')';
+ return os;
+}
+
+template <typename T>
+inline T Result::get_request_header_value(const std::string &key,
+ size_t id) const {
+ return detail::get_header_value<T>(request_headers_, key, id, 0);
+}
+
+template <class Rep, class Period>
+inline void ClientImpl::set_connection_timeout(
+ const std::chrono::duration<Rep, Period> &duration) {
+ detail::duration_to_sec_and_usec(duration, [&](time_t sec, time_t usec) {
+ set_connection_timeout(sec, usec);
+ });
+}
+
+template <class Rep, class Period>
+inline void ClientImpl::set_read_timeout(
+ const std::chrono::duration<Rep, Period> &duration) {
+ detail::duration_to_sec_and_usec(
+ duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); });
+}
+
+template <class Rep, class Period>
+inline void ClientImpl::set_write_timeout(
+ const std::chrono::duration<Rep, Period> &duration) {
+ detail::duration_to_sec_and_usec(
+ duration, [&](time_t sec, time_t usec) { set_write_timeout(sec, usec); });
+}
+
+template <class Rep, class Period>
+inline void Client::set_connection_timeout(
+ const std::chrono::duration<Rep, Period> &duration) {
+ cli_->set_connection_timeout(duration);
+}
+
+template <class Rep, class Period>
+inline void
+Client::set_read_timeout(const std::chrono::duration<Rep, Period> &duration) {
+ cli_->set_read_timeout(duration);
+}
+
+template <class Rep, class Period>
+inline void
+Client::set_write_timeout(const std::chrono::duration<Rep, Period> &duration) {
+ cli_->set_write_timeout(duration);
+}
+
+/*
+ * Forward declarations and types that will be part of the .h file if split into
+ * .h + .cc.
+ */
+
+std::string hosted_at(const std::string &hostname);
+
+void hosted_at(const std::string &hostname, std::vector<std::string> &addrs);
+
+std::string append_query_params(const std::string &path, const Params ¶ms);
+
+std::pair<std::string, std::string> make_range_header(Ranges ranges);
+
+std::pair<std::string, std::string>
+make_basic_authentication_header(const std::string &username,
+ const std::string &password,
+ bool is_proxy = false);
+
+namespace detail {
+
+std::string encode_query_param(const std::string &value);
+
+std::string decode_url(const std::string &s, bool convert_plus_to_space);
+
+void read_file(const std::string &path, std::string &out);
+
+std::string trim_copy(const std::string &s);
+
+void split(const char *b, const char *e, char d,
+ std::function<void(const char *, const char *)> fn);
+
+bool process_client_socket(socket_t sock, time_t read_timeout_sec,
+ time_t read_timeout_usec, time_t write_timeout_sec,
+ time_t write_timeout_usec,
+ std::function<bool(Stream &)> callback);
+
+socket_t create_client_socket(
+ const std::string &host, const std::string &ip, int port,
+ int address_family, bool tcp_nodelay, SocketOptions socket_options,
+ time_t connection_timeout_sec, time_t connection_timeout_usec,
+ time_t read_timeout_sec, time_t read_timeout_usec, time_t write_timeout_sec,
+ time_t write_timeout_usec, const std::string &intf, Error &error);
+
+const char *get_header_value(const Headers &headers, const std::string &key,
+ size_t id = 0, const char *def = nullptr);
+
+std::string params_to_query_str(const Params ¶ms);
+
+void parse_query_text(const std::string &s, Params ¶ms);
+
+bool parse_multipart_boundary(const std::string &content_type,
+ std::string &boundary);
+
+bool parse_range_header(const std::string &s, Ranges &ranges);
+
+int close_socket(socket_t sock);
+
+ssize_t send_socket(socket_t sock, const void *ptr, size_t size, int flags);
+
+ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags);
+
+enum class EncodingType { None = 0, Gzip, Brotli };
+
+EncodingType encoding_type(const Request &req, const Response &res);
+
+class BufferStream : public Stream {
+public:
+ BufferStream() = default;
+ ~BufferStream() override = default;
+
+ bool is_readable() const override;
+ bool is_writable() const override;
+ ssize_t read(char *ptr, size_t size) override;
+ ssize_t write(const char *ptr, size_t size) override;
+ void get_remote_ip_and_port(std::string &ip, int &port) const override;
+ void get_local_ip_and_port(std::string &ip, int &port) const override;
+ socket_t socket() const override;
+
+ const std::string &get_buffer() const;
+
+private:
+ std::string buffer;
+ size_t position = 0;
+};
+
+class compressor {
+public:
+ virtual ~compressor() = default;
+
+ typedef std::function<bool(const char *data, size_t data_len)> Callback;
+ virtual bool compress(const char *data, size_t data_length, bool last,
+ Callback callback) = 0;
+};
+
+class decompressor {
+public:
+ virtual ~decompressor() = default;
+
+ virtual bool is_valid() const = 0;
+
+ typedef std::function<bool(const char *data, size_t data_len)> Callback;
+ virtual bool decompress(const char *data, size_t data_length,
+ Callback callback) = 0;
+};
+
+class nocompressor : public compressor {
+public:
+ virtual ~nocompressor() = default;
+
+ bool compress(const char *data, size_t data_length, bool /*last*/,
+ Callback callback) override;
+};
+
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+class gzip_compressor : public compressor {
+public:
+ gzip_compressor();
+ ~gzip_compressor();
+
+ bool compress(const char *data, size_t data_length, bool last,
+ Callback callback) override;
+
+private:
+ bool is_valid_ = false;
+ z_stream strm_;
+};
+
+class gzip_decompressor : public decompressor {
+public:
+ gzip_decompressor();
+ ~gzip_decompressor();
+
+ bool is_valid() const override;
+
+ bool decompress(const char *data, size_t data_length,
+ Callback callback) override;
+
+private:
+ bool is_valid_ = false;
+ z_stream strm_;
+};
+#endif
+
+#ifdef CPPHTTPLIB_BROTLI_SUPPORT
+class brotli_compressor : public compressor {
+public:
+ brotli_compressor();
+ ~brotli_compressor();
+
+ bool compress(const char *data, size_t data_length, bool last,
+ Callback callback) override;
+
+private:
+ BrotliEncoderState *state_ = nullptr;
+};
+
+class brotli_decompressor : public decompressor {
+public:
+ brotli_decompressor();
+ ~brotli_decompressor();
+
+ bool is_valid() const override;
+
+ bool decompress(const char *data, size_t data_length,
+ Callback callback) override;
+
+private:
+ BrotliDecoderResult decoder_r;
+ BrotliDecoderState *decoder_s = nullptr;
+};
+#endif
+
+// NOTE: until the read size reaches `fixed_buffer_size`, use `fixed_buffer`
+// to store data. The call can set memory on stack for performance.
+class stream_line_reader {
+public:
+ stream_line_reader(Stream &strm, char *fixed_buffer,
+ size_t fixed_buffer_size);
+ const char *ptr() const;
+ size_t size() const;
+ bool end_with_crlf() const;
+ bool getline();
+
+private:
+ void append(char c);
+
+ Stream &strm_;
+ char *fixed_buffer_;
+ const size_t fixed_buffer_size_;
+ size_t fixed_buffer_used_size_ = 0;
+ std::string glowable_buffer_;
+};
+
+} // namespace detail
+
+// ----------------------------------------------------------------------------
+
+/*
+ * Implementation that will be part of the .cc file if split into .h + .cc.
+ */
+
+namespace detail {
+
+inline bool is_hex(char c, int &v) {
+ if (0x20 <= c && isdigit(c)) {
+ v = c - '0';
+ return true;
+ } else if ('A' <= c && c <= 'F') {
+ v = c - 'A' + 10;
+ return true;
+ } else if ('a' <= c && c <= 'f') {
+ v = c - 'a' + 10;
+ return true;
+ }
+ return false;
+}
+
+inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt,
+ int &val) {
+ if (i >= s.size()) { return false; }
+
+ val = 0;
+ for (; cnt; i++, cnt--) {
+ if (!s[i]) { return false; }
+ int v = 0;
+ if (is_hex(s[i], v)) {
+ val = val * 16 + v;
+ } else {
+ return false;
+ }
+ }
+ return true;
+}
+
+inline std::string from_i_to_hex(size_t n) {
+ const char *charset = "0123456789abcdef";
+ std::string ret;
+ do {
+ ret = charset[n & 15] + ret;
+ n >>= 4;
+ } while (n > 0);
+ return ret;
+}
+
+inline size_t to_utf8(int code, char *buff) {
+ if (code < 0x0080) {
+ buff[0] = (code & 0x7F);
+ return 1;
+ } else if (code < 0x0800) {
+ buff[0] = static_cast<char>(0xC0 | ((code >> 6) & 0x1F));
+ buff[1] = static_cast<char>(0x80 | (code & 0x3F));
+ return 2;
+ } else if (code < 0xD800) {
+ buff[0] = static_cast<char>(0xE0 | ((code >> 12) & 0xF));
+ buff[1] = static_cast<char>(0x80 | ((code >> 6) & 0x3F));
+ buff[2] = static_cast<char>(0x80 | (code & 0x3F));
+ return 3;
+ } else if (code < 0xE000) { // D800 - DFFF is invalid...
+ return 0;
+ } else if (code < 0x10000) {
+ buff[0] = static_cast<char>(0xE0 | ((code >> 12) & 0xF));
+ buff[1] = static_cast<char>(0x80 | ((code >> 6) & 0x3F));
+ buff[2] = static_cast<char>(0x80 | (code & 0x3F));
+ return 3;
+ } else if (code < 0x110000) {
+ buff[0] = static_cast<char>(0xF0 | ((code >> 18) & 0x7));
+ buff[1] = static_cast<char>(0x80 | ((code >> 12) & 0x3F));
+ buff[2] = static_cast<char>(0x80 | ((code >> 6) & 0x3F));
+ buff[3] = static_cast<char>(0x80 | (code & 0x3F));
+ return 4;
+ }
+
+ // NOTREACHED
+ return 0;
+}
+
+// NOTE: This code came up with the following stackoverflow post:
+// https://stackoverflow.com/questions/180947/base64-decode-snippet-in-c
+inline std::string base64_encode(const std::string &in) {
+ static const auto lookup =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+ std::string out;
+ out.reserve(in.size());
+
+ int val = 0;
+ int valb = -6;
+
+ for (auto c : in) {
+ val = (val << 8) + static_cast<uint8_t>(c);
+ valb += 8;
+ while (valb >= 0) {
+ out.push_back(lookup[(val >> valb) & 0x3F]);
+ valb -= 6;
+ }
+ }
+
+ if (valb > -6) { out.push_back(lookup[((val << 8) >> (valb + 8)) & 0x3F]); }
+
+ while (out.size() % 4) {
+ out.push_back('=');
+ }
+
+ return out;
+}
+
+inline bool is_file(const std::string &path) {
+#ifdef _WIN32
+ return _access_s(path.c_str(), 0) == 0;
+#else
+ struct stat st;
+ return stat(path.c_str(), &st) >= 0 && S_ISREG(st.st_mode);
+#endif
+}
+
+inline bool is_dir(const std::string &path) {
+ struct stat st;
+ return stat(path.c_str(), &st) >= 0 && S_ISDIR(st.st_mode);
+}
+
+inline bool is_valid_path(const std::string &path) {
+ size_t level = 0;
+ size_t i = 0;
+
+ // Skip slash
+ while (i < path.size() && path[i] == '/') {
+ i++;
+ }
+
+ while (i < path.size()) {
+ // Read component
+ auto beg = i;
+ while (i < path.size() && path[i] != '/') {
+ i++;
+ }
+
+ auto len = i - beg;
+ assert(len > 0);
+
+ if (!path.compare(beg, len, ".")) {
+ ;
+ } else if (!path.compare(beg, len, "..")) {
+ if (level == 0) { return false; }
+ level--;
+ } else {
+ level++;
+ }
+
+ // Skip slash
+ while (i < path.size() && path[i] == '/') {
+ i++;
+ }
+ }
+
+ return true;
+}
+
+inline std::string encode_query_param(const std::string &value) {
+ std::ostringstream escaped;
+ escaped.fill('0');
+ escaped << std::hex;
+
+ for (auto c : value) {
+ if (std::isalnum(static_cast<uint8_t>(c)) || c == '-' || c == '_' ||
+ c == '.' || c == '!' || c == '~' || c == '*' || c == '\'' || c == '(' ||
+ c == ')') {
+ escaped << c;
+ } else {
+ escaped << std::uppercase;
+ escaped << '%' << std::setw(2)
+ << static_cast<int>(static_cast<unsigned char>(c));
+ escaped << std::nouppercase;
+ }
+ }
+
+ return escaped.str();
+}
+
+inline std::string encode_url(const std::string &s) {
+ std::string result;
+ result.reserve(s.size());
+
+ for (size_t i = 0; s[i]; i++) {
+ switch (s[i]) {
+ case ' ': result += "%20"; break;
+ case '+': result += "%2B"; break;
+ case '\r': result += "%0D"; break;
+ case '\n': result += "%0A"; break;
+ case '\'': result += "%27"; break;
+ case ',': result += "%2C"; break;
+ // case ':': result += "%3A"; break; // ok? probably...
+ case ';': result += "%3B"; break;
+ default:
+ auto c = static_cast<uint8_t>(s[i]);
+ if (c >= 0x80) {
+ result += '%';
+ char hex[4];
+ auto len = snprintf(hex, sizeof(hex) - 1, "%02X", c);
+ assert(len == 2);
+ result.append(hex, static_cast<size_t>(len));
+ } else {
+ result += s[i];
+ }
+ break;
+ }
+ }
+
+ return result;
+}
+
+inline std::string decode_url(const std::string &s,
+ bool convert_plus_to_space) {
+ std::string result;
+
+ for (size_t i = 0; i < s.size(); i++) {
+ if (s[i] == '%' && i + 1 < s.size()) {
+ if (s[i + 1] == 'u') {
+ int val = 0;
+ if (from_hex_to_i(s, i + 2, 4, val)) {
+ // 4 digits Unicode codes
+ char buff[4];
+ size_t len = to_utf8(val, buff);
+ if (len > 0) { result.append(buff, len); }
+ i += 5; // 'u0000'
+ } else {
+ result += s[i];
+ }
+ } else {
+ int val = 0;
+ if (from_hex_to_i(s, i + 1, 2, val)) {
+ // 2 digits hex codes
+ result += static_cast<char>(val);
+ i += 2; // '00'
+ } else {
+ result += s[i];
+ }
+ }
+ } else if (convert_plus_to_space && s[i] == '+') {
+ result += ' ';
+ } else {
+ result += s[i];
+ }
+ }
+
+ return result;
+}
+
+inline void read_file(const std::string &path, std::string &out) {
+ std::ifstream fs(path, std::ios_base::binary);
+ fs.seekg(0, std::ios_base::end);
+ auto size = fs.tellg();
+ fs.seekg(0);
+ out.resize(static_cast<size_t>(size));
+ fs.read(&out[0], static_cast<std::streamsize>(size));
+}
+
+inline std::string file_extension(const std::string &path) {
+ std::smatch m;
+ static auto re = std::regex("\\.([a-zA-Z0-9]+)$");
+ if (std::regex_search(path, m, re)) { return m[1].str(); }
+ return std::string();
+}
+
+inline bool is_space_or_tab(char c) { return c == ' ' || c == '\t'; }
+
+inline std::pair<size_t, size_t> trim(const char *b, const char *e, size_t left,
+ size_t right) {
+ while (b + left < e && is_space_or_tab(b[left])) {
+ left++;
+ }
+ while (right > 0 && is_space_or_tab(b[right - 1])) {
+ right--;
+ }
+ return std::make_pair(left, right);
+}
+
+inline std::string trim_copy(const std::string &s) {
+ auto r = trim(s.data(), s.data() + s.size(), 0, s.size());
+ return s.substr(r.first, r.second - r.first);
+}
+
+inline void split(const char *b, const char *e, char d,
+ std::function<void(const char *, const char *)> fn) {
+ size_t i = 0;
+ size_t beg = 0;
+
+ while (e ? (b + i < e) : (b[i] != '\0')) {
+ if (b[i] == d) {
+ auto r = trim(b, e, beg, i);
+ if (r.first < r.second) { fn(&b[r.first], &b[r.second]); }
+ beg = i + 1;
+ }
+ i++;
+ }
+
+ if (i) {
+ auto r = trim(b, e, beg, i);
+ if (r.first < r.second) { fn(&b[r.first], &b[r.second]); }
+ }
+}
+
+inline stream_line_reader::stream_line_reader(Stream &strm, char *fixed_buffer,
+ size_t fixed_buffer_size)
+ : strm_(strm), fixed_buffer_(fixed_buffer),
+ fixed_buffer_size_(fixed_buffer_size) {}
+
+inline const char *stream_line_reader::ptr() const {
+ if (glowable_buffer_.empty()) {
+ return fixed_buffer_;
+ } else {
+ return glowable_buffer_.data();
+ }
+}
+
+inline size_t stream_line_reader::size() const {
+ if (glowable_buffer_.empty()) {
+ return fixed_buffer_used_size_;
+ } else {
+ return glowable_buffer_.size();
+ }
+}
+
+inline bool stream_line_reader::end_with_crlf() const {
+ auto end = ptr() + size();
+ return size() >= 2 && end[-2] == '\r' && end[-1] == '\n';
+}
+
+inline bool stream_line_reader::getline() {
+ fixed_buffer_used_size_ = 0;
+ glowable_buffer_.clear();
+
+ for (size_t i = 0;; i++) {
+ char byte;
+ auto n = strm_.read(&byte, 1);
+
+ if (n < 0) {
+ return false;
+ } else if (n == 0) {
+ if (i == 0) {
+ return false;
+ } else {
+ break;
+ }
+ }
+
+ append(byte);
+
+ if (byte == '\n') { break; }
+ }
+
+ return true;
+}
+
+inline void stream_line_reader::append(char c) {
+ if (fixed_buffer_used_size_ < fixed_buffer_size_ - 1) {
+ fixed_buffer_[fixed_buffer_used_size_++] = c;
+ fixed_buffer_[fixed_buffer_used_size_] = '\0';
+ } else {
+ if (glowable_buffer_.empty()) {
+ assert(fixed_buffer_[fixed_buffer_used_size_] == '\0');
+ glowable_buffer_.assign(fixed_buffer_, fixed_buffer_used_size_);
+ }
+ glowable_buffer_ += c;
+ }
+}
+
+inline int close_socket(socket_t sock) {
+#ifdef _WIN32
+ return closesocket(sock);
+#else
+ return close(sock);
+#endif
+}
+
+template <typename T> inline ssize_t handle_EINTR(T fn) {
+ ssize_t res = false;
+ while (true) {
+ res = fn();
+ if (res < 0 && errno == EINTR) { continue; }
+ break;
+ }
+ return res;
+}
+
+inline ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags) {
+ return handle_EINTR([&]() {
+ return recv(sock,
+#ifdef _WIN32
+ static_cast<char *>(ptr), static_cast<int>(size),
+#else
+ ptr, size,
+#endif
+ flags);
+ });
+}
+
+inline ssize_t send_socket(socket_t sock, const void *ptr, size_t size,
+ int flags) {
+ return handle_EINTR([&]() {
+ return send(sock,
+#ifdef _WIN32
+ static_cast<const char *>(ptr), static_cast<int>(size),
+#else
+ ptr, size,
+#endif
+ flags);
+ });
+}
+
+inline ssize_t select_read(socket_t sock, time_t sec, time_t usec) {
+#ifdef CPPHTTPLIB_USE_POLL
+ struct pollfd pfd_read;
+ pfd_read.fd = sock;
+ pfd_read.events = POLLIN;
+
+ auto timeout = static_cast<int>(sec * 1000 + usec / 1000);
+
+ return handle_EINTR([&]() { return poll(&pfd_read, 1, timeout); });
+#else
+#ifndef _WIN32
+ if (sock >= FD_SETSIZE) { return 1; }
+#endif
+
+ fd_set fds;
+ FD_ZERO(&fds);
+ FD_SET(sock, &fds);
+
+ timeval tv;
+ tv.tv_sec = static_cast<long>(sec);
+ tv.tv_usec = static_cast<decltype(tv.tv_usec)>(usec);
+
+ return handle_EINTR([&]() {
+ return select(static_cast<int>(sock + 1), &fds, nullptr, nullptr, &tv);
+ });
+#endif
+}
+
+inline ssize_t select_write(socket_t sock, time_t sec, time_t usec) {
+#ifdef CPPHTTPLIB_USE_POLL
+ struct pollfd pfd_read;
+ pfd_read.fd = sock;
+ pfd_read.events = POLLOUT;
+
+ auto timeout = static_cast<int>(sec * 1000 + usec / 1000);
+
+ return handle_EINTR([&]() { return poll(&pfd_read, 1, timeout); });
+#else
+#ifndef _WIN32
+ if (sock >= FD_SETSIZE) { return 1; }
+#endif
+
+ fd_set fds;
+ FD_ZERO(&fds);
+ FD_SET(sock, &fds);
+
+ timeval tv;
+ tv.tv_sec = static_cast<long>(sec);
+ tv.tv_usec = static_cast<decltype(tv.tv_usec)>(usec);
+
+ return handle_EINTR([&]() {
+ return select(static_cast<int>(sock + 1), nullptr, &fds, nullptr, &tv);
+ });
+#endif
+}
+
+inline Error wait_until_socket_is_ready(socket_t sock, time_t sec,
+ time_t usec) {
+#ifdef CPPHTTPLIB_USE_POLL
+ struct pollfd pfd_read;
+ pfd_read.fd = sock;
+ pfd_read.events = POLLIN | POLLOUT;
+
+ auto timeout = static_cast<int>(sec * 1000 + usec / 1000);
+
+ auto poll_res = handle_EINTR([&]() { return poll(&pfd_read, 1, timeout); });
+
+ if (poll_res == 0) { return Error::ConnectionTimeout; }
+
+ if (poll_res > 0 && pfd_read.revents & (POLLIN | POLLOUT)) {
+ int error = 0;
+ socklen_t len = sizeof(error);
+ auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR,
+ reinterpret_cast<char *>(&error), &len);
+ auto successful = res >= 0 && !error;
+ return successful ? Error::Success : Error::Connection;
+ }
+
+ return Error::Connection;
+#else
+#ifndef _WIN32
+ if (sock >= FD_SETSIZE) { return Error::Connection; }
+#endif
+
+ fd_set fdsr;
+ FD_ZERO(&fdsr);
+ FD_SET(sock, &fdsr);
+
+ auto fdsw = fdsr;
+ auto fdse = fdsr;
+
+ timeval tv;
+ tv.tv_sec = static_cast<long>(sec);
+ tv.tv_usec = static_cast<decltype(tv.tv_usec)>(usec);
+
+ auto ret = handle_EINTR([&]() {
+ return select(static_cast<int>(sock + 1), &fdsr, &fdsw, &fdse, &tv);
+ });
+
+ if (ret == 0) { return Error::ConnectionTimeout; }
+
+ if (ret > 0 && (FD_ISSET(sock, &fdsr) || FD_ISSET(sock, &fdsw))) {
+ int error = 0;
+ socklen_t len = sizeof(error);
+ auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR,
+ reinterpret_cast<char *>(&error), &len);
+ auto successful = res >= 0 && !error;
+ return successful ? Error::Success : Error::Connection;
+ }
+ return Error::Connection;
+#endif
+}
+
+inline bool is_socket_alive(socket_t sock) {
+ const auto val = detail::select_read(sock, 0, 0);
+ if (val == 0) {
+ return true;
+ } else if (val < 0 && errno == EBADF) {
+ return false;
+ }
+ char buf[1];
+ return detail::read_socket(sock, &buf[0], sizeof(buf), MSG_PEEK) > 0;
+}
+
+class SocketStream : public Stream {
+public:
+ SocketStream(socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec,
+ time_t write_timeout_sec, time_t write_timeout_usec);
+ ~SocketStream() override;
+
+ bool is_readable() const override;
+ bool is_writable() const override;
+ ssize_t read(char *ptr, size_t size) override;
+ ssize_t write(const char *ptr, size_t size) override;
+ void get_remote_ip_and_port(std::string &ip, int &port) const override;
+ void get_local_ip_and_port(std::string &ip, int &port) const override;
+ socket_t socket() const override;
+
+private:
+ socket_t sock_;
+ time_t read_timeout_sec_;
+ time_t read_timeout_usec_;
+ time_t write_timeout_sec_;
+ time_t write_timeout_usec_;
+
+ std::vector<char> read_buff_;
+ size_t read_buff_off_ = 0;
+ size_t read_buff_content_size_ = 0;
+
+ static const size_t read_buff_size_ = 1024 * 4;
+};
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+class SSLSocketStream : public Stream {
+public:
+ SSLSocketStream(socket_t sock, SSL *ssl, time_t read_timeout_sec,
+ time_t read_timeout_usec, time_t write_timeout_sec,
+ time_t write_timeout_usec);
+ ~SSLSocketStream() override;
+
+ bool is_readable() const override;
+ bool is_writable() const override;
+ ssize_t read(char *ptr, size_t size) override;
+ ssize_t write(const char *ptr, size_t size) override;
+ void get_remote_ip_and_port(std::string &ip, int &port) const override;
+ void get_local_ip_and_port(std::string &ip, int &port) const override;
+ socket_t socket() const override;
+
+private:
+ socket_t sock_;
+ SSL *ssl_;
+ time_t read_timeout_sec_;
+ time_t read_timeout_usec_;
+ time_t write_timeout_sec_;
+ time_t write_timeout_usec_;
+};
+#endif
+
+inline bool keep_alive(socket_t sock, time_t keep_alive_timeout_sec) {
+ using namespace std::chrono;
+ auto start = steady_clock::now();
+ while (true) {
+ auto val = select_read(sock, 0, 10000);
+ if (val < 0) {
+ return false;
+ } else if (val == 0) {
+ auto current = steady_clock::now();
+ auto duration = duration_cast<milliseconds>(current - start);
+ auto timeout = keep_alive_timeout_sec * 1000;
+ if (duration.count() > timeout) { return false; }
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ } else {
+ return true;
+ }
+ }
+}
+
+template <typename T>
+inline bool
+process_server_socket_core(const std::atomic<socket_t> &svr_sock, socket_t sock,
+ size_t keep_alive_max_count,
+ time_t keep_alive_timeout_sec, T callback) {
+ assert(keep_alive_max_count > 0);
+ auto ret = false;
+ auto count = keep_alive_max_count;
+ while (svr_sock != INVALID_SOCKET && count > 0 &&
+ keep_alive(sock, keep_alive_timeout_sec)) {
+ auto close_connection = count == 1;
+ auto connection_closed = false;
+ ret = callback(close_connection, connection_closed);
+ if (!ret || connection_closed) { break; }
+ count--;
+ }
+ return ret;
+}
+
+template <typename T>
+inline bool
+process_server_socket(const std::atomic<socket_t> &svr_sock, socket_t sock,
+ size_t keep_alive_max_count,
+ time_t keep_alive_timeout_sec, time_t read_timeout_sec,
+ time_t read_timeout_usec, time_t write_timeout_sec,
+ time_t write_timeout_usec, T callback) {
+ return process_server_socket_core(
+ svr_sock, sock, keep_alive_max_count, keep_alive_timeout_sec,
+ [&](bool close_connection, bool &connection_closed) {
+ SocketStream strm(sock, read_timeout_sec, read_timeout_usec,
+ write_timeout_sec, write_timeout_usec);
+ return callback(strm, close_connection, connection_closed);
+ });
+}
+
+inline bool process_client_socket(socket_t sock, time_t read_timeout_sec,
+ time_t read_timeout_usec,
+ time_t write_timeout_sec,
+ time_t write_timeout_usec,
+ std::function<bool(Stream &)> callback) {
+ SocketStream strm(sock, read_timeout_sec, read_timeout_usec,
+ write_timeout_sec, write_timeout_usec);
+ return callback(strm);
+}
+
+inline int shutdown_socket(socket_t sock) {
+#ifdef _WIN32
+ return shutdown(sock, SD_BOTH);
+#else
+ return shutdown(sock, SHUT_RDWR);
+#endif
+}
+
+template <typename BindOrConnect>
+socket_t create_socket(const std::string &host, const std::string &ip, int port,
+ int address_family, int socket_flags, bool tcp_nodelay,
+ SocketOptions socket_options,
+ BindOrConnect bind_or_connect) {
+ // Get address info
+ const char *node = nullptr;
+ struct addrinfo hints;
+ struct addrinfo *result;
+
+ memset(&hints, 0, sizeof(struct addrinfo));
+ hints.ai_socktype = SOCK_STREAM;
+ hints.ai_protocol = 0;
+
+ if (!ip.empty()) {
+ node = ip.c_str();
+ // Ask getaddrinfo to convert IP in c-string to address
+ hints.ai_family = AF_UNSPEC;
+ hints.ai_flags = AI_NUMERICHOST;
+ } else {
+ if (!host.empty()) { node = host.c_str(); }
+ hints.ai_family = address_family;
+ hints.ai_flags = socket_flags;
+ }
+
+#ifndef _WIN32
+ if (hints.ai_family == AF_UNIX) {
+ const auto addrlen = host.length();
+ if (addrlen > sizeof(sockaddr_un::sun_path)) return INVALID_SOCKET;
+
+ auto sock = socket(hints.ai_family, hints.ai_socktype, hints.ai_protocol);
+ if (sock != INVALID_SOCKET) {
+ sockaddr_un addr{};
+ addr.sun_family = AF_UNIX;
+ std::copy(host.begin(), host.end(), addr.sun_path);
+
+ hints.ai_addr = reinterpret_cast<sockaddr *>(&addr);
+ hints.ai_addrlen = static_cast<socklen_t>(
+ sizeof(addr) - sizeof(addr.sun_path) + addrlen);
+
+ fcntl(sock, F_SETFD, FD_CLOEXEC);
+ if (socket_options) { socket_options(sock); }
+
+ if (!bind_or_connect(sock, hints)) {
+ close_socket(sock);
+ sock = INVALID_SOCKET;
+ }
+ }
+ return sock;
+ }
+#endif
+
+ auto service = std::to_string(port);
+
+ if (getaddrinfo(node, service.c_str(), &hints, &result)) {
+#if defined __linux__ && !defined __ANDROID__
+ res_init();
+#endif
+ return INVALID_SOCKET;
+ }
+
+ for (auto rp = result; rp; rp = rp->ai_next) {
+ // Create a socket
+#ifdef _WIN32
+ auto sock =
+ WSASocketW(rp->ai_family, rp->ai_socktype, rp->ai_protocol, nullptr, 0,
+ WSA_FLAG_NO_HANDLE_INHERIT | WSA_FLAG_OVERLAPPED);
+ /**
+ * Since the WSA_FLAG_NO_HANDLE_INHERIT is only supported on Windows 7 SP1
+ * and above the socket creation fails on older Windows Systems.
+ *
+ * Let's try to create a socket the old way in this case.
+ *
+ * Reference:
+ * https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-wsasocketa
+ *
+ * WSA_FLAG_NO_HANDLE_INHERIT:
+ * This flag is supported on Windows 7 with SP1, Windows Server 2008 R2 with
+ * SP1, and later
+ *
+ */
+ if (sock == INVALID_SOCKET) {
+ sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
+ }
+#else
+ auto sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
+#endif
+ if (sock == INVALID_SOCKET) { continue; }
+
+#ifndef _WIN32
+ if (fcntl(sock, F_SETFD, FD_CLOEXEC) == -1) {
+ close_socket(sock);
+ continue;
+ }
+#endif
+
+ if (tcp_nodelay) {
+ int yes = 1;
+ setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<char *>(&yes),
+ sizeof(yes));
+ }
+
+ if (socket_options) { socket_options(sock); }
+
+ if (rp->ai_family == AF_INET6) {
+ int no = 0;
+ setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast<char *>(&no),
+ sizeof(no));
+ }
+
+ // bind or connect
+ if (bind_or_connect(sock, *rp)) {
+ freeaddrinfo(result);
+ return sock;
+ }
+
+ close_socket(sock);
+ }
+
+ freeaddrinfo(result);
+ return INVALID_SOCKET;
+}
+
+inline void set_nonblocking(socket_t sock, bool nonblocking) {
+#ifdef _WIN32
+ auto flags = nonblocking ? 1UL : 0UL;
+ ioctlsocket(sock, FIONBIO, &flags);
+#else
+ auto flags = fcntl(sock, F_GETFL, 0);
+ fcntl(sock, F_SETFL,
+ nonblocking ? (flags | O_NONBLOCK) : (flags & (~O_NONBLOCK)));
+#endif
+}
+
+inline bool is_connection_error() {
+#ifdef _WIN32
+ return WSAGetLastError() != WSAEWOULDBLOCK;
+#else
+ return errno != EINPROGRESS;
+#endif
+}
+
+inline bool bind_ip_address(socket_t sock, const std::string &host) {
+ struct addrinfo hints;
+ struct addrinfo *result;
+
+ memset(&hints, 0, sizeof(struct addrinfo));
+ hints.ai_family = AF_UNSPEC;
+ hints.ai_socktype = SOCK_STREAM;
+ hints.ai_protocol = 0;
+
+ if (getaddrinfo(host.c_str(), "0", &hints, &result)) { return false; }
+
+ auto ret = false;
+ for (auto rp = result; rp; rp = rp->ai_next) {
+ const auto &ai = *rp;
+ if (!::bind(sock, ai.ai_addr, static_cast<socklen_t>(ai.ai_addrlen))) {
+ ret = true;
+ break;
+ }
+ }
+
+ freeaddrinfo(result);
+ return ret;
+}
+
+#if !defined _WIN32 && !defined ANDROID && !defined _AIX
+#define USE_IF2IP
+#endif
+
+#ifdef USE_IF2IP
+inline std::string if2ip(int address_family, const std::string &ifn) {
+ struct ifaddrs *ifap;
+ getifaddrs(&ifap);
+ std::string addr_candidate;
+ for (auto ifa = ifap; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr && ifn == ifa->ifa_name &&
+ (AF_UNSPEC == address_family ||
+ ifa->ifa_addr->sa_family == address_family)) {
+ if (ifa->ifa_addr->sa_family == AF_INET) {
+ auto sa = reinterpret_cast<struct sockaddr_in *>(ifa->ifa_addr);
+ char buf[INET_ADDRSTRLEN];
+ if (inet_ntop(AF_INET, &sa->sin_addr, buf, INET_ADDRSTRLEN)) {
+ freeifaddrs(ifap);
+ return std::string(buf, INET_ADDRSTRLEN);
+ }
+ } else if (ifa->ifa_addr->sa_family == AF_INET6) {
+ auto sa = reinterpret_cast<struct sockaddr_in6 *>(ifa->ifa_addr);
+ if (!IN6_IS_ADDR_LINKLOCAL(&sa->sin6_addr)) {
+ char buf[INET6_ADDRSTRLEN] = {};
+ if (inet_ntop(AF_INET6, &sa->sin6_addr, buf, INET6_ADDRSTRLEN)) {
+ // equivalent to mac's IN6_IS_ADDR_UNIQUE_LOCAL
+ auto s6_addr_head = sa->sin6_addr.s6_addr[0];
+ if (s6_addr_head == 0xfc || s6_addr_head == 0xfd) {
+ addr_candidate = std::string(buf, INET6_ADDRSTRLEN);
+ } else {
+ freeifaddrs(ifap);
+ return std::string(buf, INET6_ADDRSTRLEN);
+ }
+ }
+ }
+ }
+ }
+ }
+ freeifaddrs(ifap);
+ return addr_candidate;
+}
+#endif
+
+inline socket_t create_client_socket(
+ const std::string &host, const std::string &ip, int port,
+ int address_family, bool tcp_nodelay, SocketOptions socket_options,
+ time_t connection_timeout_sec, time_t connection_timeout_usec,
+ time_t read_timeout_sec, time_t read_timeout_usec, time_t write_timeout_sec,
+ time_t write_timeout_usec, const std::string &intf, Error &error) {
+ auto sock = create_socket(
+ host, ip, port, address_family, 0, tcp_nodelay, std::move(socket_options),
+ [&](socket_t sock2, struct addrinfo &ai) -> bool {
+ if (!intf.empty()) {
+#ifdef USE_IF2IP
+ auto ip_from_if = if2ip(address_family, intf);
+ if (ip_from_if.empty()) { ip_from_if = intf; }
+ if (!bind_ip_address(sock2, ip_from_if.c_str())) {
+ error = Error::BindIPAddress;
+ return false;
+ }
+#endif
+ }
+
+ set_nonblocking(sock2, true);
+
+ auto ret =
+ ::connect(sock2, ai.ai_addr, static_cast<socklen_t>(ai.ai_addrlen));
+
+ if (ret < 0) {
+ if (is_connection_error()) {
+ error = Error::Connection;
+ return false;
+ }
+ error = wait_until_socket_is_ready(sock2, connection_timeout_sec,
+ connection_timeout_usec);
+ if (error != Error::Success) { return false; }
+ }
+
+ set_nonblocking(sock2, false);
+
+ {
+#ifdef _WIN32
+ auto timeout = static_cast<uint32_t>(read_timeout_sec * 1000 +
+ read_timeout_usec / 1000);
+ setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout,
+ sizeof(timeout));
+#else
+ timeval tv;
+ tv.tv_sec = static_cast<long>(read_timeout_sec);
+ tv.tv_usec = static_cast<decltype(tv.tv_usec)>(read_timeout_usec);
+ setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv));
+#endif
+ }
+ {
+
+#ifdef _WIN32
+ auto timeout = static_cast<uint32_t>(write_timeout_sec * 1000 +
+ write_timeout_usec / 1000);
+ setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout,
+ sizeof(timeout));
+#else
+ timeval tv;
+ tv.tv_sec = static_cast<long>(write_timeout_sec);
+ tv.tv_usec = static_cast<decltype(tv.tv_usec)>(write_timeout_usec);
+ setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv));
+#endif
+ }
+
+ error = Error::Success;
+ return true;
+ });
+
+ if (sock != INVALID_SOCKET) {
+ error = Error::Success;
+ } else {
+ if (error == Error::Success) { error = Error::Connection; }
+ }
+
+ return sock;
+}
+
+inline bool get_ip_and_port(const struct sockaddr_storage &addr,
+ socklen_t addr_len, std::string &ip, int &port) {
+ if (addr.ss_family == AF_INET) {
+ port = ntohs(reinterpret_cast<const struct sockaddr_in *>(&addr)->sin_port);
+ } else if (addr.ss_family == AF_INET6) {
+ port =
+ ntohs(reinterpret_cast<const struct sockaddr_in6 *>(&addr)->sin6_port);
+ } else {
+ return false;
+ }
+
+ std::array<char, NI_MAXHOST> ipstr{};
+ if (getnameinfo(reinterpret_cast<const struct sockaddr *>(&addr), addr_len,
+ ipstr.data(), static_cast<socklen_t>(ipstr.size()), nullptr,
+ 0, NI_NUMERICHOST)) {
+ return false;
+ }
+
+ ip = ipstr.data();
+ return true;
+}
+
+inline void get_local_ip_and_port(socket_t sock, std::string &ip, int &port) {
+ struct sockaddr_storage addr;
+ socklen_t addr_len = sizeof(addr);
+ if (!getsockname(sock, reinterpret_cast<struct sockaddr *>(&addr),
+ &addr_len)) {
+ get_ip_and_port(addr, addr_len, ip, port);
+ }
+}
+
+inline void get_remote_ip_and_port(socket_t sock, std::string &ip, int &port) {
+ struct sockaddr_storage addr;
+ socklen_t addr_len = sizeof(addr);
+
+ if (!getpeername(sock, reinterpret_cast<struct sockaddr *>(&addr),
+ &addr_len)) {
+#ifndef _WIN32
+ if (addr.ss_family == AF_UNIX) {
+#if defined(__linux__)
+ struct ucred ucred;
+ socklen_t len = sizeof(ucred);
+ if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &len) == 0) {
+ port = ucred.pid;
+ }
+#elif defined(SOL_LOCAL) && defined(SO_PEERPID) // __APPLE__
+ pid_t pid;
+ socklen_t len = sizeof(pid);
+ if (getsockopt(sock, SOL_LOCAL, SO_PEERPID, &pid, &len) == 0) {
+ port = pid;
+ }
+#endif
+ return;
+ }
+#endif
+ get_ip_and_port(addr, addr_len, ip, port);
+ }
+}
+
+inline constexpr unsigned int str2tag_core(const char *s, size_t l,
+ unsigned int h) {
+ return (l == 0)
+ ? h
+ : str2tag_core(
+ s + 1, l - 1,
+ // Unsets the 6 high bits of h, therefore no overflow happens
+ (((std::numeric_limits<unsigned int>::max)() >> 6) &
+ h * 33) ^
+ static_cast<unsigned char>(*s));
+}
+
+inline unsigned int str2tag(const std::string &s) {
+ return str2tag_core(s.data(), s.size(), 0);
+}
+
+namespace udl {
+
+inline constexpr unsigned int operator"" _t(const char *s, size_t l) {
+ return str2tag_core(s, l, 0);
+}
+
+} // namespace udl
+
+inline const char *
+find_content_type(const std::string &path,
+ const std::map<std::string, std::string> &user_data) {
+ auto ext = file_extension(path);
+
+ auto it = user_data.find(ext);
+ if (it != user_data.end()) { return it->second.c_str(); }
+
+ using udl::operator""_t;
+
+ switch (str2tag(ext)) {
+ default: return nullptr;
+ case "css"_t: return "text/css";
+ case "csv"_t: return "text/csv";
+ case "htm"_t:
+ case "html"_t: return "text/html";
+ case "js"_t:
+ case "mjs"_t: return "text/javascript";
+ case "txt"_t: return "text/plain";
+ case "vtt"_t: return "text/vtt";
+
+ case "apng"_t: return "image/apng";
+ case "avif"_t: return "image/avif";
+ case "bmp"_t: return "image/bmp";
+ case "gif"_t: return "image/gif";
+ case "png"_t: return "image/png";
+ case "svg"_t: return "image/svg+xml";
+ case "webp"_t: return "image/webp";
+ case "ico"_t: return "image/x-icon";
+ case "tif"_t: return "image/tiff";
+ case "tiff"_t: return "image/tiff";
+ case "jpg"_t:
+ case "jpeg"_t: return "image/jpeg";
+
+ case "mp4"_t: return "video/mp4";
+ case "mpeg"_t: return "video/mpeg";
+ case "webm"_t: return "video/webm";
+
+ case "mp3"_t: return "audio/mp3";
+ case "mpga"_t: return "audio/mpeg";
+ case "weba"_t: return "audio/webm";
+ case "wav"_t: return "audio/wave";
+
+ case "otf"_t: return "font/otf";
+ case "ttf"_t: return "font/ttf";
+ case "woff"_t: return "font/woff";
+ case "woff2"_t: return "font/woff2";
+
+ case "7z"_t: return "application/x-7z-compressed";
+ case "atom"_t: return "application/atom+xml";
+ case "pdf"_t: return "application/pdf";
+ case "json"_t: return "application/json";
+ case "rss"_t: return "application/rss+xml";
+ case "tar"_t: return "application/x-tar";
+ case "xht"_t:
+ case "xhtml"_t: return "application/xhtml+xml";
+ case "xslt"_t: return "application/xslt+xml";
+ case "xml"_t: return "application/xml";
+ case "gz"_t: return "application/gzip";
+ case "zip"_t: return "application/zip";
+ case "wasm"_t: return "application/wasm";
+ }
+}
+
+inline const char *status_message(int status) {
+ switch (status) {
+ case 100: return "Continue";
+ case 101: return "Switching Protocol";
+ case 102: return "Processing";
+ case 103: return "Early Hints";
+ case 200: return "OK";
+ case 201: return "Created";
+ case 202: return "Accepted";
+ case 203: return "Non-Authoritative Information";
+ case 204: return "No Content";
+ case 205: return "Reset Content";
+ case 206: return "Partial Content";
+ case 207: return "Multi-Status";
+ case 208: return "Already Reported";
+ case 226: return "IM Used";
+ case 300: return "Multiple Choice";
+ case 301: return "Moved Permanently";
+ case 302: return "Found";
+ case 303: return "See Other";
+ case 304: return "Not Modified";
+ case 305: return "Use Proxy";
+ case 306: return "unused";
+ case 307: return "Temporary Redirect";
+ case 308: return "Permanent Redirect";
+ case 400: return "Bad Request";
+ case 401: return "Unauthorized";
+ case 402: return "Payment Required";
+ case 403: return "Forbidden";
+ case 404: return "Not Found";
+ case 405: return "Method Not Allowed";
+ case 406: return "Not Acceptable";
+ case 407: return "Proxy Authentication Required";
+ case 408: return "Request Timeout";
+ case 409: return "Conflict";
+ case 410: return "Gone";
+ case 411: return "Length Required";
+ case 412: return "Precondition Failed";
+ case 413: return "Payload Too Large";
+ case 414: return "URI Too Long";
+ case 415: return "Unsupported Media Type";
+ case 416: return "Range Not Satisfiable";
+ case 417: return "Expectation Failed";
+ case 418: return "I'm a teapot";
+ case 421: return "Misdirected Request";
+ case 422: return "Unprocessable Entity";
+ case 423: return "Locked";
+ case 424: return "Failed Dependency";
+ case 425: return "Too Early";
+ case 426: return "Upgrade Required";
+ case 428: return "Precondition Required";
+ case 429: return "Too Many Requests";
+ case 431: return "Request Header Fields Too Large";
+ case 451: return "Unavailable For Legal Reasons";
+ case 501: return "Not Implemented";
+ case 502: return "Bad Gateway";
+ case 503: return "Service Unavailable";
+ case 504: return "Gateway Timeout";
+ case 505: return "HTTP Version Not Supported";
+ case 506: return "Variant Also Negotiates";
+ case 507: return "Insufficient Storage";
+ case 508: return "Loop Detected";
+ case 510: return "Not Extended";
+ case 511: return "Network Authentication Required";
+
+ default:
+ case 500: return "Internal Server Error";
+ }
+}
+
+inline bool can_compress_content_type(const std::string &content_type) {
+ using udl::operator""_t;
+
+ auto tag = str2tag(content_type);
+
+ switch (tag) {
+ case "image/svg+xml"_t:
+ case "application/javascript"_t:
+ case "application/json"_t:
+ case "application/xml"_t:
+ case "application/protobuf"_t:
+ case "application/xhtml+xml"_t: return true;
+
+ default:
+ return !content_type.rfind("text/", 0) && tag != "text/event-stream"_t;
+ }
+}
+
+inline EncodingType encoding_type(const Request &req, const Response &res) {
+ auto ret =
+ detail::can_compress_content_type(res.get_header_value("Content-Type"));
+ if (!ret) { return EncodingType::None; }
+
+ const auto &s = req.get_header_value("Accept-Encoding");
+ (void)(s);
+
+#ifdef CPPHTTPLIB_BROTLI_SUPPORT
+ // TODO: 'Accept-Encoding' has br, not br;q=0
+ ret = s.find("br") != std::string::npos;
+ if (ret) { return EncodingType::Brotli; }
+#endif
+
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+ // TODO: 'Accept-Encoding' has gzip, not gzip;q=0
+ ret = s.find("gzip") != std::string::npos;
+ if (ret) { return EncodingType::Gzip; }
+#endif
+
+ return EncodingType::None;
+}
+
+inline bool nocompressor::compress(const char *data, size_t data_length,
+ bool /*last*/, Callback callback) {
+ if (!data_length) { return true; }
+ return callback(data, data_length);
+}
+
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+inline gzip_compressor::gzip_compressor() {
+ std::memset(&strm_, 0, sizeof(strm_));
+ strm_.zalloc = Z_NULL;
+ strm_.zfree = Z_NULL;
+ strm_.opaque = Z_NULL;
+
+ is_valid_ = deflateInit2(&strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8,
+ Z_DEFAULT_STRATEGY) == Z_OK;
+}
+
+inline gzip_compressor::~gzip_compressor() { deflateEnd(&strm_); }
+
+inline bool gzip_compressor::compress(const char *data, size_t data_length,
+ bool last, Callback callback) {
+ assert(is_valid_);
+
+ do {
+ constexpr size_t max_avail_in =
+ (std::numeric_limits<decltype(strm_.avail_in)>::max)();
+
+ strm_.avail_in = static_cast<decltype(strm_.avail_in)>(
+ (std::min)(data_length, max_avail_in));
+ strm_.next_in = const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));
+
+ data_length -= strm_.avail_in;
+ data += strm_.avail_in;
+
+ auto flush = (last && data_length == 0) ? Z_FINISH : Z_NO_FLUSH;
+ int ret = Z_OK;
+
+ std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
+ do {
+ strm_.avail_out = static_cast<uInt>(buff.size());
+ strm_.next_out = reinterpret_cast<Bytef *>(buff.data());
+
+ ret = deflate(&strm_, flush);
+ if (ret == Z_STREAM_ERROR) { return false; }
+
+ if (!callback(buff.data(), buff.size() - strm_.avail_out)) {
+ return false;
+ }
+ } while (strm_.avail_out == 0);
+
+ assert((flush == Z_FINISH && ret == Z_STREAM_END) ||
+ (flush == Z_NO_FLUSH && ret == Z_OK));
+ assert(strm_.avail_in == 0);
+ } while (data_length > 0);
+
+ return true;
+}
+
+inline gzip_decompressor::gzip_decompressor() {
+ std::memset(&strm_, 0, sizeof(strm_));
+ strm_.zalloc = Z_NULL;
+ strm_.zfree = Z_NULL;
+ strm_.opaque = Z_NULL;
+
+ // 15 is the value of wbits, which should be at the maximum possible value
+ // to ensure that any gzip stream can be decoded. The offset of 32 specifies
+ // that the stream type should be automatically detected either gzip or
+ // deflate.
+ is_valid_ = inflateInit2(&strm_, 32 + 15) == Z_OK;
+}
+
+inline gzip_decompressor::~gzip_decompressor() { inflateEnd(&strm_); }
+
+inline bool gzip_decompressor::is_valid() const { return is_valid_; }
+
+inline bool gzip_decompressor::decompress(const char *data, size_t data_length,
+ Callback callback) {
+ assert(is_valid_);
+
+ int ret = Z_OK;
+
+ do {
+ constexpr size_t max_avail_in =
+ (std::numeric_limits<decltype(strm_.avail_in)>::max)();
+
+ strm_.avail_in = static_cast<decltype(strm_.avail_in)>(
+ (std::min)(data_length, max_avail_in));
+ strm_.next_in = const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));
+
+ data_length -= strm_.avail_in;
+ data += strm_.avail_in;
+
+ std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
+ while (strm_.avail_in > 0) {
+ strm_.avail_out = static_cast<uInt>(buff.size());
+ strm_.next_out = reinterpret_cast<Bytef *>(buff.data());
+
+ auto prev_avail_in = strm_.avail_in;
+
+ ret = inflate(&strm_, Z_NO_FLUSH);
+
+ if (prev_avail_in - strm_.avail_in == 0) { return false; }
+
+ assert(ret != Z_STREAM_ERROR);
+ switch (ret) {
+ case Z_NEED_DICT:
+ case Z_DATA_ERROR:
+ case Z_MEM_ERROR: inflateEnd(&strm_); return false;
+ }
+
+ if (!callback(buff.data(), buff.size() - strm_.avail_out)) {
+ return false;
+ }
+ }
+
+ if (ret != Z_OK && ret != Z_STREAM_END) return false;
+
+ } while (data_length > 0);
+
+ return true;
+}
+#endif
+
+#ifdef CPPHTTPLIB_BROTLI_SUPPORT
+inline brotli_compressor::brotli_compressor() {
+ state_ = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
+}
+
+inline brotli_compressor::~brotli_compressor() {
+ BrotliEncoderDestroyInstance(state_);
+}
+
+inline bool brotli_compressor::compress(const char *data, size_t data_length,
+ bool last, Callback callback) {
+ std::array<uint8_t, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
+
+ auto operation = last ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS;
+ auto available_in = data_length;
+ auto next_in = reinterpret_cast<const uint8_t *>(data);
+
+ for (;;) {
+ if (last) {
+ if (BrotliEncoderIsFinished(state_)) { break; }
+ } else {
+ if (!available_in) { break; }
+ }
+
+ auto available_out = buff.size();
+ auto next_out = buff.data();
+
+ if (!BrotliEncoderCompressStream(state_, operation, &available_in, &next_in,
+ &available_out, &next_out, nullptr)) {
+ return false;
+ }
+
+ auto output_bytes = buff.size() - available_out;
+ if (output_bytes) {
+ callback(reinterpret_cast<const char *>(buff.data()), output_bytes);
+ }
+ }
+
+ return true;
+}
+
+inline brotli_decompressor::brotli_decompressor() {
+ decoder_s = BrotliDecoderCreateInstance(0, 0, 0);
+ decoder_r = decoder_s ? BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT
+ : BROTLI_DECODER_RESULT_ERROR;
+}
+
+inline brotli_decompressor::~brotli_decompressor() {
+ if (decoder_s) { BrotliDecoderDestroyInstance(decoder_s); }
+}
+
+inline bool brotli_decompressor::is_valid() const { return decoder_s; }
+
+inline bool brotli_decompressor::decompress(const char *data,
+ size_t data_length,
+ Callback callback) {
+ if (decoder_r == BROTLI_DECODER_RESULT_SUCCESS ||
+ decoder_r == BROTLI_DECODER_RESULT_ERROR) {
+ return 0;
+ }
+
+ const uint8_t *next_in = (const uint8_t *)data;
+ size_t avail_in = data_length;
+ size_t total_out;
+
+ decoder_r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
+
+ std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
+ while (decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
+ char *next_out = buff.data();
+ size_t avail_out = buff.size();
+
+ decoder_r = BrotliDecoderDecompressStream(
+ decoder_s, &avail_in, &next_in, &avail_out,
+ reinterpret_cast<uint8_t **>(&next_out), &total_out);
+
+ if (decoder_r == BROTLI_DECODER_RESULT_ERROR) { return false; }
+
+ if (!callback(buff.data(), buff.size() - avail_out)) { return false; }
+ }
+
+ return decoder_r == BROTLI_DECODER_RESULT_SUCCESS ||
+ decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
+}
+#endif
+
+inline bool has_header(const Headers &headers, const std::string &key) {
+ return headers.find(key) != headers.end();
+}
+
+inline const char *get_header_value(const Headers &headers,
+ const std::string &key, size_t id,
+ const char *def) {
+ auto rng = headers.equal_range(key);
+ auto it = rng.first;
+ std::advance(it, static_cast<ssize_t>(id));
+ if (it != rng.second) { return it->second.c_str(); }
+ return def;
+}
+
+inline bool compare_case_ignore(const std::string &a, const std::string &b) {
+ if (a.size() != b.size()) { return false; }
+ for (size_t i = 0; i < b.size(); i++) {
+ if (::tolower(a[i]) != ::tolower(b[i])) { return false; }
+ }
+ return true;
+}
+
+template <typename T>
+inline bool parse_header(const char *beg, const char *end, T fn) {
+ // Skip trailing spaces and tabs.
+ while (beg < end && is_space_or_tab(end[-1])) {
+ end--;
+ }
+
+ auto p = beg;
+ while (p < end && *p != ':') {
+ p++;
+ }
+
+ if (p == end) { return false; }
+
+ auto key_end = p;
+
+ if (*p++ != ':') { return false; }
+
+ while (p < end && is_space_or_tab(*p)) {
+ p++;
+ }
+
+ if (p < end) {
+ auto key = std::string(beg, key_end);
+ auto val = compare_case_ignore(key, "Location")
+ ? std::string(p, end)
+ : decode_url(std::string(p, end), false);
+ fn(std::move(key), std::move(val));
+ return true;
+ }
+
+ return false;
+}
+
+inline bool read_headers(Stream &strm, Headers &headers) {
+ const auto bufsiz = 2048;
+ char buf[bufsiz];
+ stream_line_reader line_reader(strm, buf, bufsiz);
+
+ for (;;) {
+ if (!line_reader.getline()) { return false; }
+
+ // Check if the line ends with CRLF.
+ auto line_terminator_len = 2;
+ if (line_reader.end_with_crlf()) {
+ // Blank line indicates end of headers.
+ if (line_reader.size() == 2) { break; }
+#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR
+ } else {
+ // Blank line indicates end of headers.
+ if (line_reader.size() == 1) { break; }
+ line_terminator_len = 1;
+ }
+#else
+ } else {
+ continue; // Skip invalid line.
+ }
+#endif
+
+ if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; }
+
+ // Exclude line terminator
+ auto end = line_reader.ptr() + line_reader.size() - line_terminator_len;
+
+ parse_header(line_reader.ptr(), end,
+ [&](std::string &&key, std::string &&val) {
+ headers.emplace(std::move(key), std::move(val));
+ });
+ }
+
+ return true;
+}
+
+inline bool read_content_with_length(Stream &strm, uint64_t len,
+ Progress progress,
+ ContentReceiverWithProgress out) {
+ char buf[CPPHTTPLIB_RECV_BUFSIZ];
+
+ uint64_t r = 0;
+ while (r < len) {
+ auto read_len = static_cast<size_t>(len - r);
+ auto n = strm.read(buf, (std::min)(read_len, CPPHTTPLIB_RECV_BUFSIZ));
+ if (n <= 0) { return false; }
+
+ if (!out(buf, static_cast<size_t>(n), r, len)) { return false; }
+ r += static_cast<uint64_t>(n);
+
+ if (progress) {
+ if (!progress(r, len)) { return false; }
+ }
+ }
+
+ return true;
+}
+
+inline void skip_content_with_length(Stream &strm, uint64_t len) {
+ char buf[CPPHTTPLIB_RECV_BUFSIZ];
+ uint64_t r = 0;
+ while (r < len) {
+ auto read_len = static_cast<size_t>(len - r);
+ auto n = strm.read(buf, (std::min)(read_len, CPPHTTPLIB_RECV_BUFSIZ));
+ if (n <= 0) { return; }
+ r += static_cast<uint64_t>(n);
+ }
+}
+
+inline bool read_content_without_length(Stream &strm,
+ ContentReceiverWithProgress out) {
+ char buf[CPPHTTPLIB_RECV_BUFSIZ];
+ uint64_t r = 0;
+ for (;;) {
+ auto n = strm.read(buf, CPPHTTPLIB_RECV_BUFSIZ);
+ if (n < 0) {
+ return false;
+ } else if (n == 0) {
+ return true;
+ }
+
+ if (!out(buf, static_cast<size_t>(n), r, 0)) { return false; }
+ r += static_cast<uint64_t>(n);
+ }
+
+ return true;
+}
+
+template <typename T>
+inline bool read_content_chunked(Stream &strm, T &x,
+ ContentReceiverWithProgress out) {
+ const auto bufsiz = 16;
+ char buf[bufsiz];
+
+ stream_line_reader line_reader(strm, buf, bufsiz);
+
+ if (!line_reader.getline()) { return false; }
+
+ unsigned long chunk_len;
+ while (true) {
+ char *end_ptr;
+
+ chunk_len = std::strtoul(line_reader.ptr(), &end_ptr, 16);
+
+ if (end_ptr == line_reader.ptr()) { return false; }
+ if (chunk_len == ULONG_MAX) { return false; }
+
+ if (chunk_len == 0) { break; }
+
+ if (!read_content_with_length(strm, chunk_len, nullptr, out)) {
+ return false;
+ }
+
+ if (!line_reader.getline()) { return false; }
+
+ if (strcmp(line_reader.ptr(), "\r\n")) { return false; }
+
+ if (!line_reader.getline()) { return false; }
+ }
+
+ assert(chunk_len == 0);
+
+ // Trailer
+ if (!line_reader.getline()) { return false; }
+
+ while (strcmp(line_reader.ptr(), "\r\n")) {
+ if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; }
+
+ // Exclude line terminator
+ constexpr auto line_terminator_len = 2;
+ auto end = line_reader.ptr() + line_reader.size() - line_terminator_len;
+
+ parse_header(line_reader.ptr(), end,
+ [&](std::string &&key, std::string &&val) {
+ x.headers.emplace(std::move(key), std::move(val));
+ });
+
+ if (!line_reader.getline()) { return false; }
+ }
+
+ return true;
+}
+
+inline bool is_chunked_transfer_encoding(const Headers &headers) {
+ return !strcasecmp(get_header_value(headers, "Transfer-Encoding", 0, ""),
+ "chunked");
+}
+
+template <typename T, typename U>
+bool prepare_content_receiver(T &x, int &status,
+ ContentReceiverWithProgress receiver,
+ bool decompress, U callback) {
+ if (decompress) {
+ std::string encoding = x.get_header_value("Content-Encoding");
+ std::unique_ptr<decompressor> decompressor;
+
+ if (encoding == "gzip" || encoding == "deflate") {
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+ decompressor = detail::make_unique<gzip_decompressor>();
+#else
+ status = 415;
+ return false;
+#endif
+ } else if (encoding.find("br") != std::string::npos) {
+#ifdef CPPHTTPLIB_BROTLI_SUPPORT
+ decompressor = detail::make_unique<brotli_decompressor>();
+#else
+ status = 415;
+ return false;
+#endif
+ }
+
+ if (decompressor) {
+ if (decompressor->is_valid()) {
+ ContentReceiverWithProgress out = [&](const char *buf, size_t n,
+ uint64_t off, uint64_t len) {
+ return decompressor->decompress(buf, n,
+ [&](const char *buf2, size_t n2) {
+ return receiver(buf2, n2, off, len);
+ });
+ };
+ return callback(std::move(out));
+ } else {
+ status = 500;
+ return false;
+ }
+ }
+ }
+
+ ContentReceiverWithProgress out = [&](const char *buf, size_t n, uint64_t off,
+ uint64_t len) {
+ return receiver(buf, n, off, len);
+ };
+ return callback(std::move(out));
+}
+
+template <typename T>
+bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status,
+ Progress progress, ContentReceiverWithProgress receiver,
+ bool decompress) {
+ return prepare_content_receiver(
+ x, status, std::move(receiver), decompress,
+ [&](const ContentReceiverWithProgress &out) {
+ auto ret = true;
+ auto exceed_payload_max_length = false;
+
+ if (is_chunked_transfer_encoding(x.headers)) {
+ ret = read_content_chunked(strm, x, out);
+ } else if (!has_header(x.headers, "Content-Length")) {
+ ret = read_content_without_length(strm, out);
+ } else {
+ auto len = get_header_value<uint64_t>(x.headers, "Content-Length");
+ if (len > payload_max_length) {
+ exceed_payload_max_length = true;
+ skip_content_with_length(strm, len);
+ ret = false;
+ } else if (len > 0) {
+ ret = read_content_with_length(strm, len, std::move(progress), out);
+ }
+ }
+
+ if (!ret) { status = exceed_payload_max_length ? 413 : 400; }
+ return ret;
+ });
+} // namespace detail
+
+inline ssize_t write_headers(Stream &strm, const Headers &headers) {
+ ssize_t write_len = 0;
+ for (const auto &x : headers) {
+ auto len =
+ strm.write_format("%s: %s\r\n", x.first.c_str(), x.second.c_str());
+ if (len < 0) { return len; }
+ write_len += len;
+ }
+ auto len = strm.write("\r\n");
+ if (len < 0) { return len; }
+ write_len += len;
+ return write_len;
+}
+
+inline bool write_data(Stream &strm, const char *d, size_t l) {
+ size_t offset = 0;
+ while (offset < l) {
+ auto length = strm.write(d + offset, l - offset);
+ if (length < 0) { return false; }
+ offset += static_cast<size_t>(length);
+ }
+ return true;
+}
+
+template <typename T>
+inline bool write_content(Stream &strm, const ContentProvider &content_provider,
+ size_t offset, size_t length, T is_shutting_down,
+ Error &error) {
+ size_t end_offset = offset + length;
+ auto ok = true;
+ DataSink data_sink;
+
+ data_sink.write = [&](const char *d, size_t l) -> bool {
+ if (ok) {
+ if (strm.is_writable() && write_data(strm, d, l)) {
+ offset += l;
+ } else {
+ ok = false;
+ }
+ }
+ return ok;
+ };
+
+ while (offset < end_offset && !is_shutting_down()) {
+ if (!strm.is_writable()) {
+ error = Error::Write;
+ return false;
+ } else if (!content_provider(offset, end_offset - offset, data_sink)) {
+ error = Error::Canceled;
+ return false;
+ } else if (!ok) {
+ error = Error::Write;
+ return false;
+ }
+ }
+
+ error = Error::Success;
+ return true;
+}
+
+template <typename T>
+inline bool write_content(Stream &strm, const ContentProvider &content_provider,
+ size_t offset, size_t length,
+ const T &is_shutting_down) {
+ auto error = Error::Success;
+ return write_content(strm, content_provider, offset, length, is_shutting_down,
+ error);
+}
+
+template <typename T>
+inline bool
+write_content_without_length(Stream &strm,
+ const ContentProvider &content_provider,
+ const T &is_shutting_down) {
+ size_t offset = 0;
+ auto data_available = true;
+ auto ok = true;
+ DataSink data_sink;
+
+ data_sink.write = [&](const char *d, size_t l) -> bool {
+ if (ok) {
+ offset += l;
+ if (!strm.is_writable() || !write_data(strm, d, l)) { ok = false; }
+ }
+ return ok;
+ };
+
+ data_sink.done = [&](void) { data_available = false; };
+
+ while (data_available && !is_shutting_down()) {
+ if (!strm.is_writable()) {
+ return false;
+ } else if (!content_provider(offset, 0, data_sink)) {
+ return false;
+ } else if (!ok) {
+ return false;
+ }
+ }
+ return true;
+}
+
+template <typename T, typename U>
+inline bool
+write_content_chunked(Stream &strm, const ContentProvider &content_provider,
+ const T &is_shutting_down, U &compressor, Error &error) {
+ size_t offset = 0;
+ auto data_available = true;
+ auto ok = true;
+ DataSink data_sink;
+
+ data_sink.write = [&](const char *d, size_t l) -> bool {
+ if (ok) {
+ data_available = l > 0;
+ offset += l;
+
+ std::string payload;
+ if (compressor.compress(d, l, false,
+ [&](const char *data, size_t data_len) {
+ payload.append(data, data_len);
+ return true;
+ })) {
+ if (!payload.empty()) {
+ // Emit chunked response header and footer for each chunk
+ auto chunk =
+ from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n";
+ if (!strm.is_writable() ||
+ !write_data(strm, chunk.data(), chunk.size())) {
+ ok = false;
+ }
+ }
+ } else {
+ ok = false;
+ }
+ }
+ return ok;
+ };
+
+ auto done_with_trailer = [&](const Headers *trailer) {
+ if (!ok) { return; }
+
+ data_available = false;
+
+ std::string payload;
+ if (!compressor.compress(nullptr, 0, true,
+ [&](const char *data, size_t data_len) {
+ payload.append(data, data_len);
+ return true;
+ })) {
+ ok = false;
+ return;
+ }
+
+ if (!payload.empty()) {
+ // Emit chunked response header and footer for each chunk
+ auto chunk = from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n";
+ if (!strm.is_writable() ||
+ !write_data(strm, chunk.data(), chunk.size())) {
+ ok = false;
+ return;
+ }
+ }
+
+ static const std::string done_marker("0\r\n");
+ if (!write_data(strm, done_marker.data(), done_marker.size())) {
+ ok = false;
+ }
+
+ // Trailer
+ if (trailer) {
+ for (const auto &kv : *trailer) {
+ std::string field_line = kv.first + ": " + kv.second + "\r\n";
+ if (!write_data(strm, field_line.data(), field_line.size())) {
+ ok = false;
+ }
+ }
+ }
+
+ static const std::string crlf("\r\n");
+ if (!write_data(strm, crlf.data(), crlf.size())) { ok = false; }
+ };
+
+ data_sink.done = [&](void) { done_with_trailer(nullptr); };
+
+ data_sink.done_with_trailer = [&](const Headers &trailer) {
+ done_with_trailer(&trailer);
+ };
+
+ while (data_available && !is_shutting_down()) {
+ if (!strm.is_writable()) {
+ error = Error::Write;
+ return false;
+ } else if (!content_provider(offset, 0, data_sink)) {
+ error = Error::Canceled;
+ return false;
+ } else if (!ok) {
+ error = Error::Write;
+ return false;
+ }
+ }
+
+ error = Error::Success;
+ return true;
+}
+
+template <typename T, typename U>
+inline bool write_content_chunked(Stream &strm,
+ const ContentProvider &content_provider,
+ const T &is_shutting_down, U &compressor) {
+ auto error = Error::Success;
+ return write_content_chunked(strm, content_provider, is_shutting_down,
+ compressor, error);
+}
+
+template <typename T>
+inline bool redirect(T &cli, Request &req, Response &res,
+ const std::string &path, const std::string &location,
+ Error &error) {
+ Request new_req = req;
+ new_req.path = path;
+ new_req.redirect_count_ -= 1;
+
+ if (res.status == 303 && (req.method != "GET" && req.method != "HEAD")) {
+ new_req.method = "GET";
+ new_req.body.clear();
+ new_req.headers.clear();
+ }
+
+ Response new_res;
+
+ auto ret = cli.send(new_req, new_res, error);
+ if (ret) {
+ req = new_req;
+ res = new_res;
+ res.location = location;
+ }
+ return ret;
+}
+
+inline std::string params_to_query_str(const Params ¶ms) {
+ std::string query;
+
+ for (auto it = params.begin(); it != params.end(); ++it) {
+ if (it != params.begin()) { query += "&"; }
+ query += it->first;
+ query += "=";
+ query += encode_query_param(it->second);
+ }
+ return query;
+}
+
+inline void parse_query_text(const std::string &s, Params ¶ms) {
+ std::set<std::string> cache;
+ split(s.data(), s.data() + s.size(), '&', [&](const char *b, const char *e) {
+ std::string kv(b, e);
+ if (cache.find(kv) != cache.end()) { return; }
+ cache.insert(kv);
+
+ std::string key;
+ std::string val;
+ split(b, e, '=', [&](const char *b2, const char *e2) {
+ if (key.empty()) {
+ key.assign(b2, e2);
+ } else {
+ val.assign(b2, e2);
+ }
+ });
+
+ if (!key.empty()) {
+ params.emplace(decode_url(key, true), decode_url(val, true));
+ }
+ });
+}
+
+inline bool parse_multipart_boundary(const std::string &content_type,
+ std::string &boundary) {
+ auto boundary_keyword = "boundary=";
+ auto pos = content_type.find(boundary_keyword);
+ if (pos == std::string::npos) { return false; }
+ auto end = content_type.find(';', pos);
+ auto beg = pos + strlen(boundary_keyword);
+ boundary = content_type.substr(beg, end - beg);
+ if (boundary.length() >= 2 && boundary.front() == '"' &&
+ boundary.back() == '"') {
+ boundary = boundary.substr(1, boundary.size() - 2);
+ }
+ return !boundary.empty();
+}
+
+#ifdef CPPHTTPLIB_NO_EXCEPTIONS
+inline bool parse_range_header(const std::string &s, Ranges &ranges) {
+#else
+inline bool parse_range_header(const std::string &s, Ranges &ranges) try {
+#endif
+ static auto re_first_range = std::regex(R"(bytes=(\d*-\d*(?:,\s*\d*-\d*)*))");
+ std::smatch m;
+ if (std::regex_match(s, m, re_first_range)) {
+ auto pos = static_cast<size_t>(m.position(1));
+ auto len = static_cast<size_t>(m.length(1));
+ bool all_valid_ranges = true;
+ split(&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) {
+ if (!all_valid_ranges) return;
+ static auto re_another_range = std::regex(R"(\s*(\d*)-(\d*))");
+ std::cmatch cm;
+ if (std::regex_match(b, e, cm, re_another_range)) {
+ ssize_t first = -1;
+ if (!cm.str(1).empty()) {
+ first = static_cast<ssize_t>(std::stoll(cm.str(1)));
+ }
+
+ ssize_t last = -1;
+ if (!cm.str(2).empty()) {
+ last = static_cast<ssize_t>(std::stoll(cm.str(2)));
+ }
+
+ if (first != -1 && last != -1 && first > last) {
+ all_valid_ranges = false;
+ return;
+ }
+ ranges.emplace_back(std::make_pair(first, last));
+ }
+ });
+ return all_valid_ranges;
+ }
+ return false;
+#ifdef CPPHTTPLIB_NO_EXCEPTIONS
+}
+#else
+} catch (...) { return false; }
+#endif
+
+class MultipartFormDataParser {
+public:
+ MultipartFormDataParser() = default;
+
+ void set_boundary(std::string &&boundary) {
+ boundary_ = boundary;
+ dash_boundary_crlf_ = dash_ + boundary_ + crlf_;
+ crlf_dash_boundary_ = crlf_ + dash_ + boundary_;
+ }
+
+ bool is_valid() const { return is_valid_; }
+
+ bool parse(const char *buf, size_t n, const ContentReceiver &content_callback,
+ const MultipartContentHeader &header_callback) {
+
+ // TODO: support 'filename*'
+ static const std::regex re_content_disposition(
+ R"~(^Content-Disposition:\s*form-data;\s*name="(.*?)"(?:;\s*filename="(.*?)")?(?:;\s*filename\*=\S+)?\s*$)~",
+ std::regex_constants::icase);
+
+ buf_append(buf, n);
+
+ while (buf_size() > 0) {
+ switch (state_) {
+ case 0: { // Initial boundary
+ buf_erase(buf_find(dash_boundary_crlf_));
+ if (dash_boundary_crlf_.size() > buf_size()) { return true; }
+ if (!buf_start_with(dash_boundary_crlf_)) { return false; }
+ buf_erase(dash_boundary_crlf_.size());
+ state_ = 1;
+ break;
+ }
+ case 1: { // New entry
+ clear_file_info();
+ state_ = 2;
+ break;
+ }
+ case 2: { // Headers
+ auto pos = buf_find(crlf_);
+ if (pos > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; }
+ while (pos < buf_size()) {
+ // Empty line
+ if (pos == 0) {
+ if (!header_callback(file_)) {
+ is_valid_ = false;
+ return false;
+ }
+ buf_erase(crlf_.size());
+ state_ = 3;
+ break;
+ }
+
+ static const std::string header_name = "content-type:";
+ const auto header = buf_head(pos);
+ if (start_with_case_ignore(header, header_name)) {
+ file_.content_type = trim_copy(header.substr(header_name.size()));
+ } else {
+ std::smatch m;
+ if (std::regex_match(header, m, re_content_disposition)) {
+ file_.name = m[1];
+ file_.filename = m[2];
+ } else {
+ is_valid_ = false;
+ return false;
+ }
+ }
+ buf_erase(pos + crlf_.size());
+ pos = buf_find(crlf_);
+ }
+ if (state_ != 3) { return true; }
+ break;
+ }
+ case 3: { // Body
+ if (crlf_dash_boundary_.size() > buf_size()) { return true; }
+ auto pos = buf_find(crlf_dash_boundary_);
+ if (pos < buf_size()) {
+ if (!content_callback(buf_data(), pos)) {
+ is_valid_ = false;
+ return false;
+ }
+ buf_erase(pos + crlf_dash_boundary_.size());
+ state_ = 4;
+ } else {
+ auto len = buf_size() - crlf_dash_boundary_.size();
+ if (len > 0) {
+ if (!content_callback(buf_data(), len)) {
+ is_valid_ = false;
+ return false;
+ }
+ buf_erase(len);
+ }
+ return true;
+ }
+ break;
+ }
+ case 4: { // Boundary
+ if (crlf_.size() > buf_size()) { return true; }
+ if (buf_start_with(crlf_)) {
+ buf_erase(crlf_.size());
+ state_ = 1;
+ } else {
+ if (dash_crlf_.size() > buf_size()) { return true; }
+ if (buf_start_with(dash_crlf_)) {
+ buf_erase(dash_crlf_.size());
+ is_valid_ = true;
+ buf_erase(buf_size()); // Remove epilogue
+ } else {
+ return true;
+ }
+ }
+ break;
+ }
+ }
+ }
+
+ return true;
+ }
+
+private:
+ void clear_file_info() {
+ file_.name.clear();
+ file_.filename.clear();
+ file_.content_type.clear();
+ }
+
+ bool start_with_case_ignore(const std::string &a,
+ const std::string &b) const {
+ if (a.size() < b.size()) { return false; }
+ for (size_t i = 0; i < b.size(); i++) {
+ if (::tolower(a[i]) != ::tolower(b[i])) { return false; }
+ }
+ return true;
+ }
+
+ const std::string dash_ = "--";
+ const std::string crlf_ = "\r\n";
+ const std::string dash_crlf_ = "--\r\n";
+ std::string boundary_;
+ std::string dash_boundary_crlf_;
+ std::string crlf_dash_boundary_;
+
+ size_t state_ = 0;
+ bool is_valid_ = false;
+ MultipartFormData file_;
+
+ // Buffer
+ bool start_with(const std::string &a, size_t spos, size_t epos,
+ const std::string &b) const {
+ if (epos - spos < b.size()) { return false; }
+ for (size_t i = 0; i < b.size(); i++) {
+ if (a[i + spos] != b[i]) { return false; }
+ }
+ return true;
+ }
+
+ size_t buf_size() const { return buf_epos_ - buf_spos_; }
+
+ const char *buf_data() const { return &buf_[buf_spos_]; }
+
+ std::string buf_head(size_t l) const { return buf_.substr(buf_spos_, l); }
+
+ bool buf_start_with(const std::string &s) const {
+ return start_with(buf_, buf_spos_, buf_epos_, s);
+ }
+
+ size_t buf_find(const std::string &s) const {
+ auto c = s.front();
+
+ size_t off = buf_spos_;
+ while (off < buf_epos_) {
+ auto pos = off;
+ while (true) {
+ if (pos == buf_epos_) { return buf_size(); }
+ if (buf_[pos] == c) { break; }
+ pos++;
+ }
+
+ auto remaining_size = buf_epos_ - pos;
+ if (s.size() > remaining_size) { return buf_size(); }
+
+ if (start_with(buf_, pos, buf_epos_, s)) { return pos - buf_spos_; }
+
+ off = pos + 1;
+ }
+
+ return buf_size();
+ }
+
+ void buf_append(const char *data, size_t n) {
+ auto remaining_size = buf_size();
+ if (remaining_size > 0 && buf_spos_ > 0) {
+ for (size_t i = 0; i < remaining_size; i++) {
+ buf_[i] = buf_[buf_spos_ + i];
+ }
+ }
+ buf_spos_ = 0;
+ buf_epos_ = remaining_size;
+
+ if (remaining_size + n > buf_.size()) { buf_.resize(remaining_size + n); }
+
+ for (size_t i = 0; i < n; i++) {
+ buf_[buf_epos_ + i] = data[i];
+ }
+ buf_epos_ += n;
+ }
+
+ void buf_erase(size_t size) { buf_spos_ += size; }
+
+ std::string buf_;
+ size_t buf_spos_ = 0;
+ size_t buf_epos_ = 0;
+};
+
+inline std::string to_lower(const char *beg, const char *end) {
+ std::string out;
+ auto it = beg;
+ while (it != end) {
+ out += static_cast<char>(::tolower(*it));
+ it++;
+ }
+ return out;
+}
+
+inline std::string make_multipart_data_boundary() {
+ static const char data[] =
+ "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+ // std::random_device might actually be deterministic on some
+ // platforms, but due to lack of support in the c++ standard library,
+ // doing better requires either some ugly hacks or breaking portability.
+ std::random_device seed_gen;
+
+ // Request 128 bits of entropy for initialization
+ std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(), seed_gen()};
+ std::mt19937 engine(seed_sequence);
+
+ std::string result = "--cpp-httplib-multipart-data-";
+
+ for (auto i = 0; i < 16; i++) {
+ result += data[engine() % (sizeof(data) - 1)];
+ }
+
+ return result;
+}
+
+inline bool is_multipart_boundary_chars_valid(const std::string &boundary) {
+ auto valid = true;
+ for (size_t i = 0; i < boundary.size(); i++) {
+ auto c = boundary[i];
+ if (!std::isalnum(c) && c != '-' && c != '_') {
+ valid = false;
+ break;
+ }
+ }
+ return valid;
+}
+
+template <typename T>
+inline std::string
+serialize_multipart_formdata_item_begin(const T &item,
+ const std::string &boundary) {
+ std::string body = "--" + boundary + "\r\n";
+ body += "Content-Disposition: form-data; name=\"" + item.name + "\"";
+ if (!item.filename.empty()) {
+ body += "; filename=\"" + item.filename + "\"";
+ }
+ body += "\r\n";
+ if (!item.content_type.empty()) {
+ body += "Content-Type: " + item.content_type + "\r\n";
+ }
+ body += "\r\n";
+
+ return body;
+}
+
+inline std::string serialize_multipart_formdata_item_end() { return "\r\n"; }
+
+inline std::string
+serialize_multipart_formdata_finish(const std::string &boundary) {
+ return "--" + boundary + "--\r\n";
+}
+
+inline std::string
+serialize_multipart_formdata_get_content_type(const std::string &boundary) {
+ return "multipart/form-data; boundary=" + boundary;
+}
+
+inline std::string
+serialize_multipart_formdata(const MultipartFormDataItems &items,
+ const std::string &boundary, bool finish = true) {
+ std::string body;
+
+ for (const auto &item : items) {
+ body += serialize_multipart_formdata_item_begin(item, boundary);
+ body += item.content + serialize_multipart_formdata_item_end();
+ }
+
+ if (finish) body += serialize_multipart_formdata_finish(boundary);
+
+ return body;
+}
+
+inline std::pair<size_t, size_t>
+get_range_offset_and_length(const Request &req, size_t content_length,
+ size_t index) {
+ auto r = req.ranges[index];
+
+ if (r.first == -1 && r.second == -1) {
+ return std::make_pair(0, content_length);
+ }
+
+ auto slen = static_cast<ssize_t>(content_length);
+
+ if (r.first == -1) {
+ r.first = (std::max)(static_cast<ssize_t>(0), slen - r.second);
+ r.second = slen - 1;
+ }
+
+ if (r.second == -1) { r.second = slen - 1; }
+ return std::make_pair(r.first, static_cast<size_t>(r.second - r.first) + 1);
+}
+
+inline std::string make_content_range_header_field(size_t offset, size_t length,
+ size_t content_length) {
+ std::string field = "bytes ";
+ field += std::to_string(offset);
+ field += "-";
+ field += std::to_string(offset + length - 1);
+ field += "/";
+ field += std::to_string(content_length);
+ return field;
+}
+
+template <typename SToken, typename CToken, typename Content>
+bool process_multipart_ranges_data(const Request &req, Response &res,
+ const std::string &boundary,
+ const std::string &content_type,
+ SToken stoken, CToken ctoken,
+ Content content) {
+ for (size_t i = 0; i < req.ranges.size(); i++) {
+ ctoken("--");
+ stoken(boundary);
+ ctoken("\r\n");
+ if (!content_type.empty()) {
+ ctoken("Content-Type: ");
+ stoken(content_type);
+ ctoken("\r\n");
+ }
+
+ auto offsets = get_range_offset_and_length(req, res.body.size(), i);
+ auto offset = offsets.first;
+ auto length = offsets.second;
+
+ ctoken("Content-Range: ");
+ stoken(make_content_range_header_field(offset, length, res.body.size()));
+ ctoken("\r\n");
+ ctoken("\r\n");
+ if (!content(offset, length)) { return false; }
+ ctoken("\r\n");
+ }
+
+ ctoken("--");
+ stoken(boundary);
+ ctoken("--\r\n");
+
+ return true;
+}
+
+inline bool make_multipart_ranges_data(const Request &req, Response &res,
+ const std::string &boundary,
+ const std::string &content_type,
+ std::string &data) {
+ return process_multipart_ranges_data(
+ req, res, boundary, content_type,
+ [&](const std::string &token) { data += token; },
+ [&](const std::string &token) { data += token; },
+ [&](size_t offset, size_t length) {
+ if (offset < res.body.size()) {
+ data += res.body.substr(offset, length);
+ return true;
+ }
+ return false;
+ });
+}
+
+inline size_t
+get_multipart_ranges_data_length(const Request &req, Response &res,
+ const std::string &boundary,
+ const std::string &content_type) {
+ size_t data_length = 0;
+
+ process_multipart_ranges_data(
+ req, res, boundary, content_type,
+ [&](const std::string &token) { data_length += token.size(); },
+ [&](const std::string &token) { data_length += token.size(); },
+ [&](size_t /*offset*/, size_t length) {
+ data_length += length;
+ return true;
+ });
+
+ return data_length;
+}
+
+template <typename T>
+inline bool write_multipart_ranges_data(Stream &strm, const Request &req,
+ Response &res,
+ const std::string &boundary,
+ const std::string &content_type,
+ const T &is_shutting_down) {
+ return process_multipart_ranges_data(
+ req, res, boundary, content_type,
+ [&](const std::string &token) { strm.write(token); },
+ [&](const std::string &token) { strm.write(token); },
+ [&](size_t offset, size_t length) {
+ return write_content(strm, res.content_provider_, offset, length,
+ is_shutting_down);
+ });
+}
+
+inline std::pair<size_t, size_t>
+get_range_offset_and_length(const Request &req, const Response &res,
+ size_t index) {
+ auto r = req.ranges[index];
+
+ if (r.second == -1) {
+ r.second = static_cast<ssize_t>(res.content_length_) - 1;
+ }
+
+ return std::make_pair(r.first, r.second - r.first + 1);
+}
+
+inline bool expect_content(const Request &req) {
+ if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH" ||
+ req.method == "PRI" || req.method == "DELETE") {
+ return true;
+ }
+ // TODO: check if Content-Length is set
+ return false;
+}
+
+inline bool has_crlf(const std::string &s) {
+ auto p = s.c_str();
+ while (*p) {
+ if (*p == '\r' || *p == '\n') { return true; }
+ p++;
+ }
+ return false;
+}
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+inline std::string message_digest(const std::string &s, const EVP_MD *algo) {
+ auto context = std::unique_ptr<EVP_MD_CTX, decltype(&EVP_MD_CTX_free)>(
+ EVP_MD_CTX_new(), EVP_MD_CTX_free);
+
+ unsigned int hash_length = 0;
+ unsigned char hash[EVP_MAX_MD_SIZE];
+
+ EVP_DigestInit_ex(context.get(), algo, nullptr);
+ EVP_DigestUpdate(context.get(), s.c_str(), s.size());
+ EVP_DigestFinal_ex(context.get(), hash, &hash_length);
+
+ std::stringstream ss;
+ for (auto i = 0u; i < hash_length; ++i) {
+ ss << std::hex << std::setw(2) << std::setfill('0')
+ << (unsigned int)hash[i];
+ }
+
+ return ss.str();
+}
+
+inline std::string MD5(const std::string &s) {
+ return message_digest(s, EVP_md5());
+}
+
+inline std::string SHA_256(const std::string &s) {
+ return message_digest(s, EVP_sha256());
+}
+
+inline std::string SHA_512(const std::string &s) {
+ return message_digest(s, EVP_sha512());
+}
+#endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+#ifdef _WIN32
+// NOTE: This code came up with the following stackoverflow post:
+// https://stackoverflow.com/questions/9507184/can-openssl-on-windows-use-the-system-certificate-store
+inline bool load_system_certs_on_windows(X509_STORE *store) {
+ auto hStore = CertOpenSystemStoreW((HCRYPTPROV_LEGACY)NULL, L"ROOT");
+ if (!hStore) { return false; }
+
+ auto result = false;
+ PCCERT_CONTEXT pContext = NULL;
+ while ((pContext = CertEnumCertificatesInStore(hStore, pContext)) !=
+ nullptr) {
+ auto encoded_cert =
+ static_cast<const unsigned char *>(pContext->pbCertEncoded);
+
+ auto x509 = d2i_X509(NULL, &encoded_cert, pContext->cbCertEncoded);
+ if (x509) {
+ X509_STORE_add_cert(store, x509);
+ X509_free(x509);
+ result = true;
+ }
+ }
+
+ CertFreeCertificateContext(pContext);
+ CertCloseStore(hStore, 0);
+
+ return result;
+}
+#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__)
+#if TARGET_OS_OSX
+template <typename T>
+using CFObjectPtr =
+ std::unique_ptr<typename std::remove_pointer<T>::type, void (*)(CFTypeRef)>;
+
+inline void cf_object_ptr_deleter(CFTypeRef obj) {
+ if (obj) { CFRelease(obj); }
+}
+
+inline bool retrieve_certs_from_keychain(CFObjectPtr<CFArrayRef> &certs) {
+ CFStringRef keys[] = {kSecClass, kSecMatchLimit, kSecReturnRef};
+ CFTypeRef values[] = {kSecClassCertificate, kSecMatchLimitAll,
+ kCFBooleanTrue};
+
+ CFObjectPtr<CFDictionaryRef> query(
+ CFDictionaryCreate(nullptr, reinterpret_cast<const void **>(keys), values,
+ sizeof(keys) / sizeof(keys[0]),
+ &kCFTypeDictionaryKeyCallBacks,
+ &kCFTypeDictionaryValueCallBacks),
+ cf_object_ptr_deleter);
+
+ if (!query) { return false; }
+
+ CFTypeRef security_items = nullptr;
+ if (SecItemCopyMatching(query.get(), &security_items) != errSecSuccess ||
+ CFArrayGetTypeID() != CFGetTypeID(security_items)) {
+ return false;
+ }
+
+ certs.reset(reinterpret_cast<CFArrayRef>(security_items));
+ return true;
+}
+
+inline bool retrieve_root_certs_from_keychain(CFObjectPtr<CFArrayRef> &certs) {
+ CFArrayRef root_security_items = nullptr;
+ if (SecTrustCopyAnchorCertificates(&root_security_items) != errSecSuccess) {
+ return false;
+ }
+
+ certs.reset(root_security_items);
+ return true;
+}
+
+inline bool add_certs_to_x509_store(CFArrayRef certs, X509_STORE *store) {
+ auto result = false;
+ for (int i = 0; i < CFArrayGetCount(certs); ++i) {
+ const auto cert = reinterpret_cast<const __SecCertificate *>(
+ CFArrayGetValueAtIndex(certs, i));
+
+ if (SecCertificateGetTypeID() != CFGetTypeID(cert)) { continue; }
+
+ CFDataRef cert_data = nullptr;
+ if (SecItemExport(cert, kSecFormatX509Cert, 0, nullptr, &cert_data) !=
+ errSecSuccess) {
+ continue;
+ }
+
+ CFObjectPtr<CFDataRef> cert_data_ptr(cert_data, cf_object_ptr_deleter);
+
+ auto encoded_cert = static_cast<const unsigned char *>(
+ CFDataGetBytePtr(cert_data_ptr.get()));
+
+ auto x509 =
+ d2i_X509(NULL, &encoded_cert, CFDataGetLength(cert_data_ptr.get()));
+
+ if (x509) {
+ X509_STORE_add_cert(store, x509);
+ X509_free(x509);
+ result = true;
+ }
+ }
+
+ return result;
+}
+
+inline bool load_system_certs_on_macos(X509_STORE *store) {
+ auto result = false;
+ CFObjectPtr<CFArrayRef> certs(nullptr, cf_object_ptr_deleter);
+ if (retrieve_certs_from_keychain(certs) && certs) {
+ result = add_certs_to_x509_store(certs.get(), store);
+ }
+
+ if (retrieve_root_certs_from_keychain(certs) && certs) {
+ result = add_certs_to_x509_store(certs.get(), store) || result;
+ }
+
+ return result;
+}
+#endif // TARGET_OS_OSX
+#endif // _WIN32
+#endif // CPPHTTPLIB_OPENSSL_SUPPORT
+
+#ifdef _WIN32
+class WSInit {
+public:
+ WSInit() {
+ WSADATA wsaData;
+ if (WSAStartup(0x0002, &wsaData) == 0) is_valid_ = true;
+ }
+
+ ~WSInit() {
+ if (is_valid_) WSACleanup();
+ }
+
+ bool is_valid_ = false;
+};
+
+static WSInit wsinit_;
+#endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+inline std::pair<std::string, std::string> make_digest_authentication_header(
+ const Request &req, const std::map<std::string, std::string> &auth,
+ size_t cnonce_count, const std::string &cnonce, const std::string &username,
+ const std::string &password, bool is_proxy = false) {
+ std::string nc;
+ {
+ std::stringstream ss;
+ ss << std::setfill('0') << std::setw(8) << std::hex << cnonce_count;
+ nc = ss.str();
+ }
+
+ std::string qop;
+ if (auth.find("qop") != auth.end()) {
+ qop = auth.at("qop");
+ if (qop.find("auth-int") != std::string::npos) {
+ qop = "auth-int";
+ } else if (qop.find("auth") != std::string::npos) {
+ qop = "auth";
+ } else {
+ qop.clear();
+ }
+ }
+
+ std::string algo = "MD5";
+ if (auth.find("algorithm") != auth.end()) { algo = auth.at("algorithm"); }
+
+ std::string response;
+ {
+ auto H = algo == "SHA-256" ? detail::SHA_256
+ : algo == "SHA-512" ? detail::SHA_512
+ : detail::MD5;
+
+ auto A1 = username + ":" + auth.at("realm") + ":" + password;
+
+ auto A2 = req.method + ":" + req.path;
+ if (qop == "auth-int") { A2 += ":" + H(req.body); }
+
+ if (qop.empty()) {
+ response = H(H(A1) + ":" + auth.at("nonce") + ":" + H(A2));
+ } else {
+ response = H(H(A1) + ":" + auth.at("nonce") + ":" + nc + ":" + cnonce +
+ ":" + qop + ":" + H(A2));
+ }
+ }
+
+ auto opaque = (auth.find("opaque") != auth.end()) ? auth.at("opaque") : "";
+
+ auto field = "Digest username=\"" + username + "\", realm=\"" +
+ auth.at("realm") + "\", nonce=\"" + auth.at("nonce") +
+ "\", uri=\"" + req.path + "\", algorithm=" + algo +
+ (qop.empty() ? ", response=\""
+ : ", qop=" + qop + ", nc=" + nc + ", cnonce=\"" +
+ cnonce + "\", response=\"") +
+ response + "\"" +
+ (opaque.empty() ? "" : ", opaque=\"" + opaque + "\"");
+
+ auto key = is_proxy ? "Proxy-Authorization" : "Authorization";
+ return std::make_pair(key, field);
+}
+#endif
+
+inline bool parse_www_authenticate(const Response &res,
+ std::map<std::string, std::string> &auth,
+ bool is_proxy) {
+ auto auth_key = is_proxy ? "Proxy-Authenticate" : "WWW-Authenticate";
+ if (res.has_header(auth_key)) {
+ static auto re = std::regex(R"~((?:(?:,\s*)?(.+?)=(?:"(.*?)"|([^,]*))))~");
+ auto s = res.get_header_value(auth_key);
+ auto pos = s.find(' ');
+ if (pos != std::string::npos) {
+ auto type = s.substr(0, pos);
+ if (type == "Basic") {
+ return false;
+ } else if (type == "Digest") {
+ s = s.substr(pos + 1);
+ auto beg = std::sregex_iterator(s.begin(), s.end(), re);
+ for (auto i = beg; i != std::sregex_iterator(); ++i) {
+ auto m = *i;
+ auto key = s.substr(static_cast<size_t>(m.position(1)),
+ static_cast<size_t>(m.length(1)));
+ auto val = m.length(2) > 0
+ ? s.substr(static_cast<size_t>(m.position(2)),
+ static_cast<size_t>(m.length(2)))
+ : s.substr(static_cast<size_t>(m.position(3)),
+ static_cast<size_t>(m.length(3)));
+ auth[key] = val;
+ }
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+// https://stackoverflow.com/questions/440133/how-do-i-create-a-random-alpha-numeric-string-in-c/440240#answer-440240
+inline std::string random_string(size_t length) {
+ auto randchar = []() -> char {
+ const char charset[] = "0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz";
+ const size_t max_index = (sizeof(charset) - 1);
+ return charset[static_cast<size_t>(std::rand()) % max_index];
+ };
+ std::string str(length, 0);
+ std::generate_n(str.begin(), length, randchar);
+ return str;
+}
+
+class ContentProviderAdapter {
+public:
+ explicit ContentProviderAdapter(
+ ContentProviderWithoutLength &&content_provider)
+ : content_provider_(content_provider) {}
+
+ bool operator()(size_t offset, size_t, DataSink &sink) {
+ return content_provider_(offset, sink);
+ }
+
+private:
+ ContentProviderWithoutLength content_provider_;
+};
+
+} // namespace detail
+
+inline std::string hosted_at(const std::string &hostname) {
+ std::vector<std::string> addrs;
+ hosted_at(hostname, addrs);
+ if (addrs.empty()) { return std::string(); }
+ return addrs[0];
+}
+
+inline void hosted_at(const std::string &hostname,
+ std::vector<std::string> &addrs) {
+ struct addrinfo hints;
+ struct addrinfo *result;
+
+ memset(&hints, 0, sizeof(struct addrinfo));
+ hints.ai_family = AF_UNSPEC;
+ hints.ai_socktype = SOCK_STREAM;
+ hints.ai_protocol = 0;
+
+ if (getaddrinfo(hostname.c_str(), nullptr, &hints, &result)) {
+#if defined __linux__ && !defined __ANDROID__
+ res_init();
+#endif
+ return;
+ }
+
+ for (auto rp = result; rp; rp = rp->ai_next) {
+ const auto &addr =
+ *reinterpret_cast<struct sockaddr_storage *>(rp->ai_addr);
+ std::string ip;
+ int dummy = -1;
+ if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip,
+ dummy)) {
+ addrs.push_back(ip);
+ }
+ }
+
+ freeaddrinfo(result);
+}
+
+inline std::string append_query_params(const std::string &path,
+ const Params ¶ms) {
+ std::string path_with_query = path;
+ const static std::regex re("[^?]+\\?.*");
+ auto delm = std::regex_match(path, re) ? '&' : '?';
+ path_with_query += delm + detail::params_to_query_str(params);
+ return path_with_query;
+}
+
+// Header utilities
+inline std::pair<std::string, std::string> make_range_header(Ranges ranges) {
+ std::string field = "bytes=";
+ auto i = 0;
+ for (auto r : ranges) {
+ if (i != 0) { field += ", "; }
+ if (r.first != -1) { field += std::to_string(r.first); }
+ field += '-';
+ if (r.second != -1) { field += std::to_string(r.second); }
+ i++;
+ }
+ return std::make_pair("Range", std::move(field));
+}
+
+inline std::pair<std::string, std::string>
+make_basic_authentication_header(const std::string &username,
+ const std::string &password, bool is_proxy) {
+ auto field = "Basic " + detail::base64_encode(username + ":" + password);
+ auto key = is_proxy ? "Proxy-Authorization" : "Authorization";
+ return std::make_pair(key, std::move(field));
+}
+
+inline std::pair<std::string, std::string>
+make_bearer_token_authentication_header(const std::string &token,
+ bool is_proxy = false) {
+ auto field = "Bearer " + token;
+ auto key = is_proxy ? "Proxy-Authorization" : "Authorization";
+ return std::make_pair(key, std::move(field));
+}
+
+// Request implementation
+inline bool Request::has_header(const std::string &key) const {
+ return detail::has_header(headers, key);
+}
+
+inline std::string Request::get_header_value(const std::string &key,
+ size_t id) const {
+ return detail::get_header_value(headers, key, id, "");
+}
+
+inline size_t Request::get_header_value_count(const std::string &key) const {
+ auto r = headers.equal_range(key);
+ return static_cast<size_t>(std::distance(r.first, r.second));
+}
+
+inline void Request::set_header(const std::string &key,
+ const std::string &val) {
+ if (!detail::has_crlf(key) && !detail::has_crlf(val)) {
+ headers.emplace(key, val);
+ }
+}
+
+inline bool Request::has_param(const std::string &key) const {
+ return params.find(key) != params.end();
+}
+
+inline std::string Request::get_param_value(const std::string &key,
+ size_t id) const {
+ auto rng = params.equal_range(key);
+ auto it = rng.first;
+ std::advance(it, static_cast<ssize_t>(id));
+ if (it != rng.second) { return it->second; }
+ return std::string();
+}
+
+inline size_t Request::get_param_value_count(const std::string &key) const {
+ auto r = params.equal_range(key);
+ return static_cast<size_t>(std::distance(r.first, r.second));
+}
+
+inline bool Request::is_multipart_form_data() const {
+ const auto &content_type = get_header_value("Content-Type");
+ return !content_type.rfind("multipart/form-data", 0);
+}
+
+inline bool Request::has_file(const std::string &key) const {
+ return files.find(key) != files.end();
+}
+
+inline MultipartFormData Request::get_file_value(const std::string &key) const {
+ auto it = files.find(key);
+ if (it != files.end()) { return it->second; }
+ return MultipartFormData();
+}
+
+inline std::vector<MultipartFormData>
+Request::get_file_values(const std::string &key) const {
+ std::vector<MultipartFormData> values;
+ auto rng = files.equal_range(key);
+ for (auto it = rng.first; it != rng.second; it++) {
+ values.push_back(it->second);
+ }
+ return values;
+}
+
+// Response implementation
+inline bool Response::has_header(const std::string &key) const {
+ return headers.find(key) != headers.end();
+}
+
+inline std::string Response::get_header_value(const std::string &key,
+ size_t id) const {
+ return detail::get_header_value(headers, key, id, "");
+}
+
+inline size_t Response::get_header_value_count(const std::string &key) const {
+ auto r = headers.equal_range(key);
+ return static_cast<size_t>(std::distance(r.first, r.second));
+}
+
+inline void Response::set_header(const std::string &key,
+ const std::string &val) {
+ if (!detail::has_crlf(key) && !detail::has_crlf(val)) {
+ headers.emplace(key, val);
+ }
+}
+
+inline void Response::set_redirect(const std::string &url, int stat) {
+ if (!detail::has_crlf(url)) {
+ set_header("Location", url);
+ if (300 <= stat && stat < 400) {
+ this->status = stat;
+ } else {
+ this->status = 302;
+ }
+ }
+}
+
+inline void Response::set_content(const char *s, size_t n,
+ const std::string &content_type) {
+ body.assign(s, n);
+
+ auto rng = headers.equal_range("Content-Type");
+ headers.erase(rng.first, rng.second);
+ set_header("Content-Type", content_type);
+}
+
+inline void Response::set_content(const std::string &s,
+ const std::string &content_type) {
+ set_content(s.data(), s.size(), content_type);
+}
+
+inline void Response::set_content_provider(
+ size_t in_length, const std::string &content_type, ContentProvider provider,
+ ContentProviderResourceReleaser resource_releaser) {
+ set_header("Content-Type", content_type);
+ content_length_ = in_length;
+ if (in_length > 0) { content_provider_ = std::move(provider); }
+ content_provider_resource_releaser_ = resource_releaser;
+ is_chunked_content_provider_ = false;
+}
+
+inline void Response::set_content_provider(
+ const std::string &content_type, ContentProviderWithoutLength provider,
+ ContentProviderResourceReleaser resource_releaser) {
+ set_header("Content-Type", content_type);
+ content_length_ = 0;
+ content_provider_ = detail::ContentProviderAdapter(std::move(provider));
+ content_provider_resource_releaser_ = resource_releaser;
+ is_chunked_content_provider_ = false;
+}
+
+inline void Response::set_chunked_content_provider(
+ const std::string &content_type, ContentProviderWithoutLength provider,
+ ContentProviderResourceReleaser resource_releaser) {
+ set_header("Content-Type", content_type);
+ content_length_ = 0;
+ content_provider_ = detail::ContentProviderAdapter(std::move(provider));
+ content_provider_resource_releaser_ = resource_releaser;
+ is_chunked_content_provider_ = true;
+}
+
+// Result implementation
+inline bool Result::has_request_header(const std::string &key) const {
+ return request_headers_.find(key) != request_headers_.end();
+}
+
+inline std::string Result::get_request_header_value(const std::string &key,
+ size_t id) const {
+ return detail::get_header_value(request_headers_, key, id, "");
+}
+
+inline size_t
+Result::get_request_header_value_count(const std::string &key) const {
+ auto r = request_headers_.equal_range(key);
+ return static_cast<size_t>(std::distance(r.first, r.second));
+}
+
+// Stream implementation
+inline ssize_t Stream::write(const char *ptr) {
+ return write(ptr, strlen(ptr));
+}
+
+inline ssize_t Stream::write(const std::string &s) {
+ return write(s.data(), s.size());
+}
+
+namespace detail {
+
+// Socket stream implementation
+inline SocketStream::SocketStream(socket_t sock, time_t read_timeout_sec,
+ time_t read_timeout_usec,
+ time_t write_timeout_sec,
+ time_t write_timeout_usec)
+ : sock_(sock), read_timeout_sec_(read_timeout_sec),
+ read_timeout_usec_(read_timeout_usec),
+ write_timeout_sec_(write_timeout_sec),
+ write_timeout_usec_(write_timeout_usec), read_buff_(read_buff_size_, 0) {}
+
+inline SocketStream::~SocketStream() {}
+
+inline bool SocketStream::is_readable() const {
+ return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;
+}
+
+inline bool SocketStream::is_writable() const {
+ return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 &&
+ is_socket_alive(sock_);
+}
+
+inline ssize_t SocketStream::read(char *ptr, size_t size) {
+#ifdef _WIN32
+ size =
+ (std::min)(size, static_cast<size_t>((std::numeric_limits<int>::max)()));
+#else
+ size = (std::min)(size,
+ static_cast<size_t>((std::numeric_limits<ssize_t>::max)()));
+#endif
+
+ if (read_buff_off_ < read_buff_content_size_) {
+ auto remaining_size = read_buff_content_size_ - read_buff_off_;
+ if (size <= remaining_size) {
+ memcpy(ptr, read_buff_.data() + read_buff_off_, size);
+ read_buff_off_ += size;
+ return static_cast<ssize_t>(size);
+ } else {
+ memcpy(ptr, read_buff_.data() + read_buff_off_, remaining_size);
+ read_buff_off_ += remaining_size;
+ return static_cast<ssize_t>(remaining_size);
+ }
+ }
+
+ if (!is_readable()) { return -1; }
+
+ read_buff_off_ = 0;
+ read_buff_content_size_ = 0;
+
+ if (size < read_buff_size_) {
+ auto n = read_socket(sock_, read_buff_.data(), read_buff_size_,
+ CPPHTTPLIB_RECV_FLAGS);
+ if (n <= 0) {
+ return n;
+ } else if (n <= static_cast<ssize_t>(size)) {
+ memcpy(ptr, read_buff_.data(), static_cast<size_t>(n));
+ return n;
+ } else {
+ memcpy(ptr, read_buff_.data(), size);
+ read_buff_off_ = size;
+ read_buff_content_size_ = static_cast<size_t>(n);
+ return static_cast<ssize_t>(size);
+ }
+ } else {
+ return read_socket(sock_, ptr, size, CPPHTTPLIB_RECV_FLAGS);
+ }
+}
+
+inline ssize_t SocketStream::write(const char *ptr, size_t size) {
+ if (!is_writable()) { return -1; }
+
+#if defined(_WIN32) && !defined(_WIN64)
+ size =
+ (std::min)(size, static_cast<size_t>((std::numeric_limits<int>::max)()));
+#endif
+
+ return send_socket(sock_, ptr, size, CPPHTTPLIB_SEND_FLAGS);
+}
+
+inline void SocketStream::get_remote_ip_and_port(std::string &ip,
+ int &port) const {
+ return detail::get_remote_ip_and_port(sock_, ip, port);
+}
+
+inline void SocketStream::get_local_ip_and_port(std::string &ip,
+ int &port) const {
+ return detail::get_local_ip_and_port(sock_, ip, port);
+}
+
+inline socket_t SocketStream::socket() const { return sock_; }
+
+// Buffer stream implementation
+inline bool BufferStream::is_readable() const { return true; }
+
+inline bool BufferStream::is_writable() const { return true; }
+
+inline ssize_t BufferStream::read(char *ptr, size_t size) {
+#if defined(_MSC_VER) && _MSC_VER < 1910
+ auto len_read = buffer._Copy_s(ptr, size, size, position);
+#else
+ auto len_read = buffer.copy(ptr, size, position);
+#endif
+ position += static_cast<size_t>(len_read);
+ return static_cast<ssize_t>(len_read);
+}
+
+inline ssize_t BufferStream::write(const char *ptr, size_t size) {
+ buffer.append(ptr, size);
+ return static_cast<ssize_t>(size);
+}
+
+inline void BufferStream::get_remote_ip_and_port(std::string & /*ip*/,
+ int & /*port*/) const {}
+
+inline void BufferStream::get_local_ip_and_port(std::string & /*ip*/,
+ int & /*port*/) const {}
+
+inline socket_t BufferStream::socket() const { return 0; }
+
+inline const std::string &BufferStream::get_buffer() const { return buffer; }
+
+} // namespace detail
+
+// HTTP server implementation
+inline Server::Server()
+ : new_task_queue(
+ [] { return new ThreadPool(CPPHTTPLIB_THREAD_POOL_COUNT); }) {
+#ifndef _WIN32
+ signal(SIGPIPE, SIG_IGN);
+#endif
+}
+
+inline Server::~Server() {}
+
+inline Server &Server::Get(const std::string &pattern, Handler handler) {
+ get_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
+ return *this;
+}
+
+inline Server &Server::Post(const std::string &pattern, Handler handler) {
+ post_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
+ return *this;
+}
+
+inline Server &Server::Post(const std::string &pattern,
+ HandlerWithContentReader handler) {
+ post_handlers_for_content_reader_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
+ return *this;
+}
+
+inline Server &Server::Put(const std::string &pattern, Handler handler) {
+ put_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
+ return *this;
+}
+
+inline Server &Server::Put(const std::string &pattern,
+ HandlerWithContentReader handler) {
+ put_handlers_for_content_reader_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
+ return *this;
+}
+
+inline Server &Server::Patch(const std::string &pattern, Handler handler) {
+ patch_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
+ return *this;
+}
+
+inline Server &Server::Patch(const std::string &pattern,
+ HandlerWithContentReader handler) {
+ patch_handlers_for_content_reader_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
+ return *this;
+}
+
+inline Server &Server::Delete(const std::string &pattern, Handler handler) {
+ delete_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
+ return *this;
+}
+
+inline Server &Server::Delete(const std::string &pattern,
+ HandlerWithContentReader handler) {
+ delete_handlers_for_content_reader_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
+ return *this;
+}
+
+inline Server &Server::Options(const std::string &pattern, Handler handler) {
+ options_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
+ return *this;
+}
+
+inline bool Server::set_base_dir(const std::string &dir,
+ const std::string &mount_point) {
+ return set_mount_point(mount_point, dir);
+}
+
+inline bool Server::set_mount_point(const std::string &mount_point,
+ const std::string &dir, Headers headers) {
+ if (detail::is_dir(dir)) {
+ std::string mnt = !mount_point.empty() ? mount_point : "/";
+ if (!mnt.empty() && mnt[0] == '/') {
+ base_dirs_.push_back({mnt, dir, std::move(headers)});
+ return true;
+ }
+ }
+ return false;
+}
+
+inline bool Server::remove_mount_point(const std::string &mount_point) {
+ for (auto it = base_dirs_.begin(); it != base_dirs_.end(); ++it) {
+ if (it->mount_point == mount_point) {
+ base_dirs_.erase(it);
+ return true;
+ }
+ }
+ return false;
+}
+
+inline Server &
+Server::set_file_extension_and_mimetype_mapping(const std::string &ext,
+ const std::string &mime) {
+ file_extension_and_mimetype_map_[ext] = mime;
+ return *this;
+}
+
+inline Server &Server::set_file_request_handler(Handler handler) {
+ file_request_handler_ = std::move(handler);
+ return *this;
+}
+
+inline Server &Server::set_error_handler(HandlerWithResponse handler) {
+ error_handler_ = std::move(handler);
+ return *this;
+}
+
+inline Server &Server::set_error_handler(Handler handler) {
+ error_handler_ = [handler](const Request &req, Response &res) {
+ handler(req, res);
+ return HandlerResponse::Handled;
+ };
+ return *this;
+}
+
+inline Server &Server::set_exception_handler(ExceptionHandler handler) {
+ exception_handler_ = std::move(handler);
+ return *this;
+}
+
+inline Server &Server::set_pre_routing_handler(HandlerWithResponse handler) {
+ pre_routing_handler_ = std::move(handler);
+ return *this;
+}
+
+inline Server &Server::set_post_routing_handler(Handler handler) {
+ post_routing_handler_ = std::move(handler);
+ return *this;
+}
+
+inline Server &Server::set_logger(Logger logger) {
+ logger_ = std::move(logger);
+ return *this;
+}
+
+inline Server &
+Server::set_expect_100_continue_handler(Expect100ContinueHandler handler) {
+ expect_100_continue_handler_ = std::move(handler);
+
+ return *this;
+}
+
+inline Server &Server::set_address_family(int family) {
+ address_family_ = family;
+ return *this;
+}
+
+inline Server &Server::set_tcp_nodelay(bool on) {
+ tcp_nodelay_ = on;
+ return *this;
+}
+
+inline Server &Server::set_socket_options(SocketOptions socket_options) {
+ socket_options_ = std::move(socket_options);
+ return *this;
+}
+
+inline Server &Server::set_default_headers(Headers headers) {
+ default_headers_ = std::move(headers);
+ return *this;
+}
+
+inline Server &Server::set_keep_alive_max_count(size_t count) {
+ keep_alive_max_count_ = count;
+ return *this;
+}
+
+inline Server &Server::set_keep_alive_timeout(time_t sec) {
+ keep_alive_timeout_sec_ = sec;
+ return *this;
+}
+
+inline Server &Server::set_read_timeout(time_t sec, time_t usec) {
+ read_timeout_sec_ = sec;
+ read_timeout_usec_ = usec;
+ return *this;
+}
+
+inline Server &Server::set_write_timeout(time_t sec, time_t usec) {
+ write_timeout_sec_ = sec;
+ write_timeout_usec_ = usec;
+ return *this;
+}
+
+inline Server &Server::set_idle_interval(time_t sec, time_t usec) {
+ idle_interval_sec_ = sec;
+ idle_interval_usec_ = usec;
+ return *this;
+}
+
+inline Server &Server::set_payload_max_length(size_t length) {
+ payload_max_length_ = length;
+ return *this;
+}
+
+inline bool Server::bind_to_port(const std::string &host, int port,
+ int socket_flags) {
+ if (bind_internal(host, port, socket_flags) < 0) return false;
+ return true;
+}
+inline int Server::bind_to_any_port(const std::string &host, int socket_flags) {
+ return bind_internal(host, 0, socket_flags);
+}
+
+inline bool Server::listen_after_bind() {
+ auto se = detail::scope_exit([&]() { done_ = true; });
+ return listen_internal();
+}
+
+inline bool Server::listen(const std::string &host, int port,
+ int socket_flags) {
+ auto se = detail::scope_exit([&]() { done_ = true; });
+ return bind_to_port(host, port, socket_flags) && listen_internal();
+}
+
+inline bool Server::is_running() const { return is_running_; }
+
+inline void Server::wait_until_ready() const {
+ while (!is_running() && !done_) {
+ std::this_thread::sleep_for(std::chrono::milliseconds{1});
+ }
+}
+
+inline void Server::stop() {
+ if (is_running_) {
+ assert(svr_sock_ != INVALID_SOCKET);
+ std::atomic<socket_t> sock(svr_sock_.exchange(INVALID_SOCKET));
+ detail::shutdown_socket(sock);
+ detail::close_socket(sock);
+ }
+}
+
+inline bool Server::parse_request_line(const char *s, Request &req) {
+ auto len = strlen(s);
+ if (len < 2 || s[len - 2] != '\r' || s[len - 1] != '\n') { return false; }
+ len -= 2;
+
+ {
+ size_t count = 0;
+
+ detail::split(s, s + len, ' ', [&](const char *b, const char *e) {
+ switch (count) {
+ case 0: req.method = std::string(b, e); break;
+ case 1: req.target = std::string(b, e); break;
+ case 2: req.version = std::string(b, e); break;
+ default: break;
+ }
+ count++;
+ });
+
+ if (count != 3) { return false; }
+ }
+
+ static const std::set<std::string> methods{
+ "GET", "HEAD", "POST", "PUT", "DELETE",
+ "CONNECT", "OPTIONS", "TRACE", "PATCH", "PRI"};
+
+ if (methods.find(req.method) == methods.end()) { return false; }
+
+ if (req.version != "HTTP/1.1" && req.version != "HTTP/1.0") { return false; }
+
+ {
+ // Skip URL fragment
+ for (size_t i = 0; i < req.target.size(); i++) {
+ if (req.target[i] == '#') {
+ req.target.erase(i);
+ break;
+ }
+ }
+
+ size_t count = 0;
+
+ detail::split(req.target.data(), req.target.data() + req.target.size(), '?',
+ [&](const char *b, const char *e) {
+ switch (count) {
+ case 0:
+ req.path = detail::decode_url(std::string(b, e), false);
+ break;
+ case 1: {
+ if (e - b > 0) {
+ detail::parse_query_text(std::string(b, e), req.params);
+ }
+ break;
+ }
+ default: break;
+ }
+ count++;
+ });
+
+ if (count > 2) { return false; }
+ }
+
+ return true;
+}
+
+inline bool Server::write_response(Stream &strm, bool close_connection,
+ const Request &req, Response &res) {
+ return write_response_core(strm, close_connection, req, res, false);
+}
+
+inline bool Server::write_response_with_content(Stream &strm,
+ bool close_connection,
+ const Request &req,
+ Response &res) {
+ return write_response_core(strm, close_connection, req, res, true);
+}
+
+inline bool Server::write_response_core(Stream &strm, bool close_connection,
+ const Request &req, Response &res,
+ bool need_apply_ranges) {
+ assert(res.status != -1);
+
+ if (400 <= res.status && error_handler_ &&
+ error_handler_(req, res) == HandlerResponse::Handled) {
+ need_apply_ranges = true;
+ }
+
+ std::string content_type;
+ std::string boundary;
+ if (need_apply_ranges) { apply_ranges(req, res, content_type, boundary); }
+
+ // Prepare additional headers
+ if (close_connection || req.get_header_value("Connection") == "close") {
+ res.set_header("Connection", "close");
+ } else {
+ std::stringstream ss;
+ ss << "timeout=" << keep_alive_timeout_sec_
+ << ", max=" << keep_alive_max_count_;
+ res.set_header("Keep-Alive", ss.str());
+ }
+
+ if (!res.has_header("Content-Type") &&
+ (!res.body.empty() || res.content_length_ > 0 || res.content_provider_)) {
+ res.set_header("Content-Type", "text/plain");
+ }
+
+ if (!res.has_header("Content-Length") && res.body.empty() &&
+ !res.content_length_ && !res.content_provider_) {
+ res.set_header("Content-Length", "0");
+ }
+
+ if (!res.has_header("Accept-Ranges") && req.method == "HEAD") {
+ res.set_header("Accept-Ranges", "bytes");
+ }
+
+ if (post_routing_handler_) { post_routing_handler_(req, res); }
+
+ // Response line and headers
+ {
+ detail::BufferStream bstrm;
+
+ if (!bstrm.write_format("HTTP/1.1 %d %s\r\n", res.status,
+ detail::status_message(res.status))) {
+ return false;
+ }
+
+ if (!detail::write_headers(bstrm, res.headers)) { return false; }
+
+ // Flush buffer
+ auto &data = bstrm.get_buffer();
+ detail::write_data(strm, data.data(), data.size());
+ }
+
+ // Body
+ auto ret = true;
+ if (req.method != "HEAD") {
+ if (!res.body.empty()) {
+ if (!detail::write_data(strm, res.body.data(), res.body.size())) {
+ ret = false;
+ }
+ } else if (res.content_provider_) {
+ if (write_content_with_provider(strm, req, res, boundary, content_type)) {
+ res.content_provider_success_ = true;
+ } else {
+ res.content_provider_success_ = false;
+ ret = false;
+ }
+ }
+ }
+
+ // Log
+ if (logger_) { logger_(req, res); }
+
+ return ret;
+}
+
+inline bool
+Server::write_content_with_provider(Stream &strm, const Request &req,
+ Response &res, const std::string &boundary,
+ const std::string &content_type) {
+ auto is_shutting_down = [this]() {
+ return this->svr_sock_ == INVALID_SOCKET;
+ };
+
+ if (res.content_length_ > 0) {
+ if (req.ranges.empty()) {
+ return detail::write_content(strm, res.content_provider_, 0,
+ res.content_length_, is_shutting_down);
+ } else if (req.ranges.size() == 1) {
+ auto offsets =
+ detail::get_range_offset_and_length(req, res.content_length_, 0);
+ auto offset = offsets.first;
+ auto length = offsets.second;
+ return detail::write_content(strm, res.content_provider_, offset, length,
+ is_shutting_down);
+ } else {
+ return detail::write_multipart_ranges_data(
+ strm, req, res, boundary, content_type, is_shutting_down);
+ }
+ } else {
+ if (res.is_chunked_content_provider_) {
+ auto type = detail::encoding_type(req, res);
+
+ std::unique_ptr<detail::compressor> compressor;
+ if (type == detail::EncodingType::Gzip) {
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+ compressor = detail::make_unique<detail::gzip_compressor>();
+#endif
+ } else if (type == detail::EncodingType::Brotli) {
+#ifdef CPPHTTPLIB_BROTLI_SUPPORT
+ compressor = detail::make_unique<detail::brotli_compressor>();
+#endif
+ } else {
+ compressor = detail::make_unique<detail::nocompressor>();
+ }
+ assert(compressor != nullptr);
+
+ return detail::write_content_chunked(strm, res.content_provider_,
+ is_shutting_down, *compressor);
+ } else {
+ return detail::write_content_without_length(strm, res.content_provider_,
+ is_shutting_down);
+ }
+ }
+}
+
+inline bool Server::read_content(Stream &strm, Request &req, Response &res) {
+ MultipartFormDataMap::iterator cur;
+ auto file_count = 0;
+ if (read_content_core(
+ strm, req, res,
+ // Regular
+ [&](const char *buf, size_t n) {
+ if (req.body.size() + n > req.body.max_size()) { return false; }
+ req.body.append(buf, n);
+ return true;
+ },
+ // Multipart
+ [&](const MultipartFormData &file) {
+ if (file_count++ == CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT) {
+ return false;
+ }
+ cur = req.files.emplace(file.name, file);
+ return true;
+ },
+ [&](const char *buf, size_t n) {
+ auto &content = cur->second.content;
+ if (content.size() + n > content.max_size()) { return false; }
+ content.append(buf, n);
+ return true;
+ })) {
+ const auto &content_type = req.get_header_value("Content-Type");
+ if (!content_type.find("application/x-www-form-urlencoded")) {
+ if (req.body.size() > CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH) {
+ res.status = 413; // NOTE: should be 414?
+ return false;
+ }
+ detail::parse_query_text(req.body, req.params);
+ }
+ return true;
+ }
+ return false;
+}
+
+inline bool Server::read_content_with_content_receiver(
+ Stream &strm, Request &req, Response &res, ContentReceiver receiver,
+ MultipartContentHeader multipart_header,
+ ContentReceiver multipart_receiver) {
+ return read_content_core(strm, req, res, std::move(receiver),
+ std::move(multipart_header),
+ std::move(multipart_receiver));
+}
+
+inline bool Server::read_content_core(Stream &strm, Request &req, Response &res,
+ ContentReceiver receiver,
+ MultipartContentHeader multipart_header,
+ ContentReceiver multipart_receiver) {
+ detail::MultipartFormDataParser multipart_form_data_parser;
+ ContentReceiverWithProgress out;
+
+ if (req.is_multipart_form_data()) {
+ const auto &content_type = req.get_header_value("Content-Type");
+ std::string boundary;
+ if (!detail::parse_multipart_boundary(content_type, boundary)) {
+ res.status = 400;
+ return false;
+ }
+
+ multipart_form_data_parser.set_boundary(std::move(boundary));
+ out = [&](const char *buf, size_t n, uint64_t /*off*/, uint64_t /*len*/) {
+ /* For debug
+ size_t pos = 0;
+ while (pos < n) {
+ auto read_size = (std::min)<size_t>(1, n - pos);
+ auto ret = multipart_form_data_parser.parse(
+ buf + pos, read_size, multipart_receiver, multipart_header);
+ if (!ret) { return false; }
+ pos += read_size;
+ }
+ return true;
+ */
+ return multipart_form_data_parser.parse(buf, n, multipart_receiver,
+ multipart_header);
+ };
+ } else {
+ out = [receiver](const char *buf, size_t n, uint64_t /*off*/,
+ uint64_t /*len*/) { return receiver(buf, n); };
+ }
+
+ if (req.method == "DELETE" && !req.has_header("Content-Length")) {
+ return true;
+ }
+
+ if (!detail::read_content(strm, req, payload_max_length_, res.status, nullptr,
+ out, true)) {
+ return false;
+ }
+
+ if (req.is_multipart_form_data()) {
+ if (!multipart_form_data_parser.is_valid()) {
+ res.status = 400;
+ return false;
+ }
+ }
+
+ return true;
+}
+
+inline bool Server::handle_file_request(const Request &req, Response &res,
+ bool head) {
+ for (const auto &entry : base_dirs_) {
+ // Prefix match
+ if (!req.path.compare(0, entry.mount_point.size(), entry.mount_point)) {
+ std::string sub_path = "/" + req.path.substr(entry.mount_point.size());
+ if (detail::is_valid_path(sub_path)) {
+ auto path = entry.base_dir + sub_path;
+ if (path.back() == '/') { path += "index.html"; }
+
+ if (detail::is_file(path)) {
+ detail::read_file(path, res.body);
+ auto type =
+ detail::find_content_type(path, file_extension_and_mimetype_map_);
+ if (type) { res.set_header("Content-Type", type); }
+ for (const auto &kv : entry.headers) {
+ res.set_header(kv.first.c_str(), kv.second);
+ }
+ res.status = req.has_header("Range") ? 206 : 200;
+ if (!head && file_request_handler_) {
+ file_request_handler_(req, res);
+ }
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+}
+
+inline socket_t
+Server::create_server_socket(const std::string &host, int port,
+ int socket_flags,
+ SocketOptions socket_options) const {
+ return detail::create_socket(
+ host, std::string(), port, address_family_, socket_flags, tcp_nodelay_,
+ std::move(socket_options),
+ [](socket_t sock, struct addrinfo &ai) -> bool {
+ if (::bind(sock, ai.ai_addr, static_cast<socklen_t>(ai.ai_addrlen))) {
+ return false;
+ }
+ if (::listen(sock, CPPHTTPLIB_LISTEN_BACKLOG)) { return false; }
+ return true;
+ });
+}
+
+inline int Server::bind_internal(const std::string &host, int port,
+ int socket_flags) {
+ if (!is_valid()) { return -1; }
+
+ svr_sock_ = create_server_socket(host, port, socket_flags, socket_options_);
+ if (svr_sock_ == INVALID_SOCKET) { return -1; }
+
+ if (port == 0) {
+ struct sockaddr_storage addr;
+ socklen_t addr_len = sizeof(addr);
+ if (getsockname(svr_sock_, reinterpret_cast<struct sockaddr *>(&addr),
+ &addr_len) == -1) {
+ return -1;
+ }
+ if (addr.ss_family == AF_INET) {
+ return ntohs(reinterpret_cast<struct sockaddr_in *>(&addr)->sin_port);
+ } else if (addr.ss_family == AF_INET6) {
+ return ntohs(reinterpret_cast<struct sockaddr_in6 *>(&addr)->sin6_port);
+ } else {
+ return -1;
+ }
+ } else {
+ return port;
+ }
+}
+
+inline bool Server::listen_internal() {
+ auto ret = true;
+ is_running_ = true;
+ auto se = detail::scope_exit([&]() { is_running_ = false; });
+
+ {
+ std::unique_ptr<TaskQueue> task_queue(new_task_queue());
+
+ while (svr_sock_ != INVALID_SOCKET) {
+#ifndef _WIN32
+ if (idle_interval_sec_ > 0 || idle_interval_usec_ > 0) {
+#endif
+ auto val = detail::select_read(svr_sock_, idle_interval_sec_,
+ idle_interval_usec_);
+ if (val == 0) { // Timeout
+ task_queue->on_idle();
+ continue;
+ }
+#ifndef _WIN32
+ }
+#endif
+ socket_t sock = accept(svr_sock_, nullptr, nullptr);
+
+ if (sock == INVALID_SOCKET) {
+ if (errno == EMFILE) {
+ // The per-process limit of open file descriptors has been reached.
+ // Try to accept new connections after a short sleep.
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ continue;
+ } else if (errno == EINTR || errno == EAGAIN) {
+ continue;
+ }
+ if (svr_sock_ != INVALID_SOCKET) {
+ detail::close_socket(svr_sock_);
+ ret = false;
+ } else {
+ ; // The server socket was closed by user.
+ }
+ break;
+ }
+
+ {
+#ifdef _WIN32
+ auto timeout = static_cast<uint32_t>(read_timeout_sec_ * 1000 +
+ read_timeout_usec_ / 1000);
+ setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout,
+ sizeof(timeout));
+#else
+ timeval tv;
+ tv.tv_sec = static_cast<long>(read_timeout_sec_);
+ tv.tv_usec = static_cast<decltype(tv.tv_usec)>(read_timeout_usec_);
+ setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv));
+#endif
+ }
+ {
+
+#ifdef _WIN32
+ auto timeout = static_cast<uint32_t>(write_timeout_sec_ * 1000 +
+ write_timeout_usec_ / 1000);
+ setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout,
+ sizeof(timeout));
+#else
+ timeval tv;
+ tv.tv_sec = static_cast<long>(write_timeout_sec_);
+ tv.tv_usec = static_cast<decltype(tv.tv_usec)>(write_timeout_usec_);
+ setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv));
+#endif
+ }
+
+ task_queue->enqueue([this, sock]() { process_and_close_socket(sock); });
+ }
+
+ task_queue->shutdown();
+ }
+
+ return ret;
+}
+
+inline bool Server::routing(Request &req, Response &res, Stream &strm) {
+ if (pre_routing_handler_ &&
+ pre_routing_handler_(req, res) == HandlerResponse::Handled) {
+ return true;
+ }
+
+ // File handler
+ bool is_head_request = req.method == "HEAD";
+ if ((req.method == "GET" || is_head_request) &&
+ handle_file_request(req, res, is_head_request)) {
+ return true;
+ }
+
+ if (detail::expect_content(req)) {
+ // Content reader handler
+ {
+ ContentReader reader(
+ [&](ContentReceiver receiver) {
+ return read_content_with_content_receiver(
+ strm, req, res, std::move(receiver), nullptr, nullptr);
+ },
+ [&](MultipartContentHeader header, ContentReceiver receiver) {
+ return read_content_with_content_receiver(strm, req, res, nullptr,
+ std::move(header),
+ std::move(receiver));
+ });
+
+ if (req.method == "POST") {
+ if (dispatch_request_for_content_reader(
+ req, res, std::move(reader),
+ post_handlers_for_content_reader_)) {
+ return true;
+ }
+ } else if (req.method == "PUT") {
+ if (dispatch_request_for_content_reader(
+ req, res, std::move(reader),
+ put_handlers_for_content_reader_)) {
+ return true;
+ }
+ } else if (req.method == "PATCH") {
+ if (dispatch_request_for_content_reader(
+ req, res, std::move(reader),
+ patch_handlers_for_content_reader_)) {
+ return true;
+ }
+ } else if (req.method == "DELETE") {
+ if (dispatch_request_for_content_reader(
+ req, res, std::move(reader),
+ delete_handlers_for_content_reader_)) {
+ return true;
+ }
+ }
+ }
+
+ // Read content into `req.body`
+ if (!read_content(strm, req, res)) { return false; }
+ }
+
+ // Regular handler
+ if (req.method == "GET" || req.method == "HEAD") {
+ return dispatch_request(req, res, get_handlers_);
+ } else if (req.method == "POST") {
+ return dispatch_request(req, res, post_handlers_);
+ } else if (req.method == "PUT") {
+ return dispatch_request(req, res, put_handlers_);
+ } else if (req.method == "DELETE") {
+ return dispatch_request(req, res, delete_handlers_);
+ } else if (req.method == "OPTIONS") {
+ return dispatch_request(req, res, options_handlers_);
+ } else if (req.method == "PATCH") {
+ return dispatch_request(req, res, patch_handlers_);
+ }
+
+ res.status = 400;
+ return false;
+}
+
+inline bool Server::dispatch_request(Request &req, Response &res,
+ const Handlers &handlers) {
+ for (const auto &x : handlers) {
+ const auto &pattern = x.first;
+ const auto &handler = x.second;
+
+ if (std::regex_match(req.path, req.matches, pattern)) {
+ handler(req, res);
+ return true;
+ }
+ }
+ return false;
+}
+
+inline void Server::apply_ranges(const Request &req, Response &res,
+ std::string &content_type,
+ std::string &boundary) {
+ if (req.ranges.size() > 1) {
+ boundary = detail::make_multipart_data_boundary();
+
+ auto it = res.headers.find("Content-Type");
+ if (it != res.headers.end()) {
+ content_type = it->second;
+ res.headers.erase(it);
+ }
+
+ res.headers.emplace("Content-Type",
+ "multipart/byteranges; boundary=" + boundary);
+ }
+
+ auto type = detail::encoding_type(req, res);
+
+ if (res.body.empty()) {
+ if (res.content_length_ > 0) {
+ size_t length = 0;
+ if (req.ranges.empty()) {
+ length = res.content_length_;
+ } else if (req.ranges.size() == 1) {
+ auto offsets =
+ detail::get_range_offset_and_length(req, res.content_length_, 0);
+ auto offset = offsets.first;
+ length = offsets.second;
+ auto content_range = detail::make_content_range_header_field(
+ offset, length, res.content_length_);
+ res.set_header("Content-Range", content_range);
+ } else {
+ length = detail::get_multipart_ranges_data_length(req, res, boundary,
+ content_type);
+ }
+ res.set_header("Content-Length", std::to_string(length));
+ } else {
+ if (res.content_provider_) {
+ if (res.is_chunked_content_provider_) {
+ res.set_header("Transfer-Encoding", "chunked");
+ if (type == detail::EncodingType::Gzip) {
+ res.set_header("Content-Encoding", "gzip");
+ } else if (type == detail::EncodingType::Brotli) {
+ res.set_header("Content-Encoding", "br");
+ }
+ }
+ }
+ }
+ } else {
+ if (req.ranges.empty()) {
+ ;
+ } else if (req.ranges.size() == 1) {
+ auto offsets =
+ detail::get_range_offset_and_length(req, res.body.size(), 0);
+ auto offset = offsets.first;
+ auto length = offsets.second;
+ auto content_range = detail::make_content_range_header_field(
+ offset, length, res.body.size());
+ res.set_header("Content-Range", content_range);
+ if (offset < res.body.size()) {
+ res.body = res.body.substr(offset, length);
+ } else {
+ res.body.clear();
+ res.status = 416;
+ }
+ } else {
+ std::string data;
+ if (detail::make_multipart_ranges_data(req, res, boundary, content_type,
+ data)) {
+ res.body.swap(data);
+ } else {
+ res.body.clear();
+ res.status = 416;
+ }
+ }
+
+ if (type != detail::EncodingType::None) {
+ std::unique_ptr<detail::compressor> compressor;
+ std::string content_encoding;
+
+ if (type == detail::EncodingType::Gzip) {
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+ compressor = detail::make_unique<detail::gzip_compressor>();
+ content_encoding = "gzip";
+#endif
+ } else if (type == detail::EncodingType::Brotli) {
+#ifdef CPPHTTPLIB_BROTLI_SUPPORT
+ compressor = detail::make_unique<detail::brotli_compressor>();
+ content_encoding = "br";
+#endif
+ }
+
+ if (compressor) {
+ std::string compressed;
+ if (compressor->compress(res.body.data(), res.body.size(), true,
+ [&](const char *data, size_t data_len) {
+ compressed.append(data, data_len);
+ return true;
+ })) {
+ res.body.swap(compressed);
+ res.set_header("Content-Encoding", content_encoding);
+ }
+ }
+ }
+
+ auto length = std::to_string(res.body.size());
+ res.set_header("Content-Length", length);
+ }
+}
+
+inline bool Server::dispatch_request_for_content_reader(
+ Request &req, Response &res, ContentReader content_reader,
+ const HandlersForContentReader &handlers) {
+ for (const auto &x : handlers) {
+ const auto &pattern = x.first;
+ const auto &handler = x.second;
+
+ if (std::regex_match(req.path, req.matches, pattern)) {
+ handler(req, res, content_reader);
+ return true;
+ }
+ }
+ return false;
+}
+
+inline bool
+Server::process_request(Stream &strm, bool close_connection,
+ bool &connection_closed,
+ const std::function<void(Request &)> &setup_request) {
+ std::array<char, 2048> buf{};
+
+ detail::stream_line_reader line_reader(strm, buf.data(), buf.size());
+
+ // Connection has been closed on client
+ if (!line_reader.getline()) { return false; }
+
+ Request req;
+ Response res;
+
+ res.version = "HTTP/1.1";
+
+ for (const auto &header : default_headers_) {
+ if (res.headers.find(header.first) == res.headers.end()) {
+ res.headers.insert(header);
+ }
+ }
+
+#ifdef _WIN32
+ // TODO: Increase FD_SETSIZE statically (libzmq), dynamically (MySQL).
+#else
+#ifndef CPPHTTPLIB_USE_POLL
+ // Socket file descriptor exceeded FD_SETSIZE...
+ if (strm.socket() >= FD_SETSIZE) {
+ Headers dummy;
+ detail::read_headers(strm, dummy);
+ res.status = 500;
+ return write_response(strm, close_connection, req, res);
+ }
+#endif
+#endif
+
+ // Check if the request URI doesn't exceed the limit
+ if (line_reader.size() > CPPHTTPLIB_REQUEST_URI_MAX_LENGTH) {
+ Headers dummy;
+ detail::read_headers(strm, dummy);
+ res.status = 414;
+ return write_response(strm, close_connection, req, res);
+ }
+
+ // Request line and headers
+ if (!parse_request_line(line_reader.ptr(), req) ||
+ !detail::read_headers(strm, req.headers)) {
+ res.status = 400;
+ return write_response(strm, close_connection, req, res);
+ }
+
+ if (req.get_header_value("Connection") == "close") {
+ connection_closed = true;
+ }
+
+ if (req.version == "HTTP/1.0" &&
+ req.get_header_value("Connection") != "Keep-Alive") {
+ connection_closed = true;
+ }
+
+ strm.get_remote_ip_and_port(req.remote_addr, req.remote_port);
+ req.set_header("REMOTE_ADDR", req.remote_addr);
+ req.set_header("REMOTE_PORT", std::to_string(req.remote_port));
+
+ strm.get_local_ip_and_port(req.local_addr, req.local_port);
+ req.set_header("LOCAL_ADDR", req.local_addr);
+ req.set_header("LOCAL_PORT", std::to_string(req.local_port));
+
+ if (req.has_header("Range")) {
+ const auto &range_header_value = req.get_header_value("Range");
+ if (!detail::parse_range_header(range_header_value, req.ranges)) {
+ res.status = 416;
+ return write_response(strm, close_connection, req, res);
+ }
+ }
+
+ if (setup_request) { setup_request(req); }
+
+ if (req.get_header_value("Expect") == "100-continue") {
+ auto status = 100;
+ if (expect_100_continue_handler_) {
+ status = expect_100_continue_handler_(req, res);
+ }
+ switch (status) {
+ case 100:
+ case 417:
+ strm.write_format("HTTP/1.1 %d %s\r\n\r\n", status,
+ detail::status_message(status));
+ break;
+ default: return write_response(strm, close_connection, req, res);
+ }
+ }
+
+ // Rounting
+ bool routed = false;
+#ifdef CPPHTTPLIB_NO_EXCEPTIONS
+ routed = routing(req, res, strm);
+#else
+ try {
+ routed = routing(req, res, strm);
+ } catch (std::exception &e) {
+ if (exception_handler_) {
+ auto ep = std::current_exception();
+ exception_handler_(req, res, ep);
+ routed = true;
+ } else {
+ res.status = 500;
+ std::string val;
+ auto s = e.what();
+ for (size_t i = 0; s[i]; i++) {
+ switch (s[i]) {
+ case '\r': val += "\\r"; break;
+ case '\n': val += "\\n"; break;
+ default: val += s[i]; break;
+ }
+ }
+ res.set_header("EXCEPTION_WHAT", val);
+ }
+ } catch (...) {
+ if (exception_handler_) {
+ auto ep = std::current_exception();
+ exception_handler_(req, res, ep);
+ routed = true;
+ } else {
+ res.status = 500;
+ res.set_header("EXCEPTION_WHAT", "UNKNOWN");
+ }
+ }
+#endif
+
+ if (routed) {
+ if (res.status == -1) { res.status = req.ranges.empty() ? 200 : 206; }
+ return write_response_with_content(strm, close_connection, req, res);
+ } else {
+ if (res.status == -1) { res.status = 404; }
+ return write_response(strm, close_connection, req, res);
+ }
+}
+
+inline bool Server::is_valid() const { return true; }
+
+inline bool Server::process_and_close_socket(socket_t sock) {
+ auto ret = detail::process_server_socket(
+ svr_sock_, sock, keep_alive_max_count_, keep_alive_timeout_sec_,
+ read_timeout_sec_, read_timeout_usec_, write_timeout_sec_,
+ write_timeout_usec_,
+ [this](Stream &strm, bool close_connection, bool &connection_closed) {
+ return process_request(strm, close_connection, connection_closed,
+ nullptr);
+ });
+
+ detail::shutdown_socket(sock);
+ detail::close_socket(sock);
+ return ret;
+}
+
+// HTTP client implementation
+inline ClientImpl::ClientImpl(const std::string &host)
+ : ClientImpl(host, 80, std::string(), std::string()) {}
+
+inline ClientImpl::ClientImpl(const std::string &host, int port)
+ : ClientImpl(host, port, std::string(), std::string()) {}
+
+inline ClientImpl::ClientImpl(const std::string &host, int port,
+ const std::string &client_cert_path,
+ const std::string &client_key_path)
+ : host_(host), port_(port),
+ host_and_port_(adjust_host_string(host) + ":" + std::to_string(port)),
+ client_cert_path_(client_cert_path), client_key_path_(client_key_path) {}
+
+inline ClientImpl::~ClientImpl() {
+ std::lock_guard<std::mutex> guard(socket_mutex_);
+ shutdown_socket(socket_);
+ close_socket(socket_);
+}
+
+inline bool ClientImpl::is_valid() const { return true; }
+
+inline void ClientImpl::copy_settings(const ClientImpl &rhs) {
+ client_cert_path_ = rhs.client_cert_path_;
+ client_key_path_ = rhs.client_key_path_;
+ connection_timeout_sec_ = rhs.connection_timeout_sec_;
+ read_timeout_sec_ = rhs.read_timeout_sec_;
+ read_timeout_usec_ = rhs.read_timeout_usec_;
+ write_timeout_sec_ = rhs.write_timeout_sec_;
+ write_timeout_usec_ = rhs.write_timeout_usec_;
+ basic_auth_username_ = rhs.basic_auth_username_;
+ basic_auth_password_ = rhs.basic_auth_password_;
+ bearer_token_auth_token_ = rhs.bearer_token_auth_token_;
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ digest_auth_username_ = rhs.digest_auth_username_;
+ digest_auth_password_ = rhs.digest_auth_password_;
+#endif
+ keep_alive_ = rhs.keep_alive_;
+ follow_location_ = rhs.follow_location_;
+ url_encode_ = rhs.url_encode_;
+ address_family_ = rhs.address_family_;
+ tcp_nodelay_ = rhs.tcp_nodelay_;
+ socket_options_ = rhs.socket_options_;
+ compress_ = rhs.compress_;
+ decompress_ = rhs.decompress_;
+ interface_ = rhs.interface_;
+ proxy_host_ = rhs.proxy_host_;
+ proxy_port_ = rhs.proxy_port_;
+ proxy_basic_auth_username_ = rhs.proxy_basic_auth_username_;
+ proxy_basic_auth_password_ = rhs.proxy_basic_auth_password_;
+ proxy_bearer_token_auth_token_ = rhs.proxy_bearer_token_auth_token_;
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ proxy_digest_auth_username_ = rhs.proxy_digest_auth_username_;
+ proxy_digest_auth_password_ = rhs.proxy_digest_auth_password_;
+#endif
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ ca_cert_file_path_ = rhs.ca_cert_file_path_;
+ ca_cert_dir_path_ = rhs.ca_cert_dir_path_;
+ ca_cert_store_ = rhs.ca_cert_store_;
+#endif
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ server_certificate_verification_ = rhs.server_certificate_verification_;
+#endif
+ logger_ = rhs.logger_;
+}
+
+inline socket_t ClientImpl::create_client_socket(Error &error) const {
+ if (!proxy_host_.empty() && proxy_port_ != -1) {
+ return detail::create_client_socket(
+ proxy_host_, std::string(), proxy_port_, address_family_, tcp_nodelay_,
+ socket_options_, connection_timeout_sec_, connection_timeout_usec_,
+ read_timeout_sec_, read_timeout_usec_, write_timeout_sec_,
+ write_timeout_usec_, interface_, error);
+ }
+
+ // Check is custom IP specified for host_
+ std::string ip;
+ auto it = addr_map_.find(host_);
+ if (it != addr_map_.end()) ip = it->second;
+
+ return detail::create_client_socket(
+ host_, ip, port_, address_family_, tcp_nodelay_, socket_options_,
+ connection_timeout_sec_, connection_timeout_usec_, read_timeout_sec_,
+ read_timeout_usec_, write_timeout_sec_, write_timeout_usec_, interface_,
+ error);
+}
+
+inline bool ClientImpl::create_and_connect_socket(Socket &socket,
+ Error &error) {
+ auto sock = create_client_socket(error);
+ if (sock == INVALID_SOCKET) { return false; }
+ socket.sock = sock;
+ return true;
+}
+
+inline void ClientImpl::shutdown_ssl(Socket & /*socket*/,
+ bool /*shutdown_gracefully*/) {
+ // If there are any requests in flight from threads other than us, then it's
+ // a thread-unsafe race because individual ssl* objects are not thread-safe.
+ assert(socket_requests_in_flight_ == 0 ||
+ socket_requests_are_from_thread_ == std::this_thread::get_id());
+}
+
+inline void ClientImpl::shutdown_socket(Socket &socket) {
+ if (socket.sock == INVALID_SOCKET) { return; }
+ detail::shutdown_socket(socket.sock);
+}
+
+inline void ClientImpl::close_socket(Socket &socket) {
+ // If there are requests in flight in another thread, usually closing
+ // the socket will be fine and they will simply receive an error when
+ // using the closed socket, but it is still a bug since rarely the OS
+ // may reassign the socket id to be used for a new socket, and then
+ // suddenly they will be operating on a live socket that is different
+ // than the one they intended!
+ assert(socket_requests_in_flight_ == 0 ||
+ socket_requests_are_from_thread_ == std::this_thread::get_id());
+
+ // It is also a bug if this happens while SSL is still active
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ assert(socket.ssl == nullptr);
+#endif
+ if (socket.sock == INVALID_SOCKET) { return; }
+ detail::close_socket(socket.sock);
+ socket.sock = INVALID_SOCKET;
+}
+
+inline bool ClientImpl::read_response_line(Stream &strm, const Request &req,
+ Response &res) {
+ std::array<char, 2048> buf{};
+
+ detail::stream_line_reader line_reader(strm, buf.data(), buf.size());
+
+ if (!line_reader.getline()) { return false; }
+
+#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR
+ const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n");
+#else
+ const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r?\n");
+#endif
+
+ std::cmatch m;
+ if (!std::regex_match(line_reader.ptr(), m, re)) {
+ return req.method == "CONNECT";
+ }
+ res.version = std::string(m[1]);
+ res.status = std::stoi(std::string(m[2]));
+ res.reason = std::string(m[3]);
+
+ // Ignore '100 Continue'
+ while (res.status == 100) {
+ if (!line_reader.getline()) { return false; } // CRLF
+ if (!line_reader.getline()) { return false; } // next response line
+
+ if (!std::regex_match(line_reader.ptr(), m, re)) { return false; }
+ res.version = std::string(m[1]);
+ res.status = std::stoi(std::string(m[2]));
+ res.reason = std::string(m[3]);
+ }
+
+ return true;
+}
+
+inline bool ClientImpl::send(Request &req, Response &res, Error &error) {
+ std::lock_guard<std::recursive_mutex> request_mutex_guard(request_mutex_);
+ auto ret = send_(req, res, error);
+ if (error == Error::SSLPeerCouldBeClosed_) {
+ assert(!ret);
+ ret = send_(req, res, error);
+ }
+ return ret;
+}
+
+inline bool ClientImpl::send_(Request &req, Response &res, Error &error) {
+ {
+ std::lock_guard<std::mutex> guard(socket_mutex_);
+
+ // Set this to false immediately - if it ever gets set to true by the end of
+ // the request, we know another thread instructed us to close the socket.
+ socket_should_be_closed_when_request_is_done_ = false;
+
+ auto is_alive = false;
+ if (socket_.is_open()) {
+ is_alive = detail::is_socket_alive(socket_.sock);
+ if (!is_alive) {
+ // Attempt to avoid sigpipe by shutting down nongracefully if it seems
+ // like the other side has already closed the connection Also, there
+ // cannot be any requests in flight from other threads since we locked
+ // request_mutex_, so safe to close everything immediately
+ const bool shutdown_gracefully = false;
+ shutdown_ssl(socket_, shutdown_gracefully);
+ shutdown_socket(socket_);
+ close_socket(socket_);
+ }
+ }
+
+ if (!is_alive) {
+ if (!create_and_connect_socket(socket_, error)) { return false; }
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ // TODO: refactoring
+ if (is_ssl()) {
+ auto &scli = static_cast<SSLClient &>(*this);
+ if (!proxy_host_.empty() && proxy_port_ != -1) {
+ auto success = false;
+ if (!scli.connect_with_proxy(socket_, res, success, error)) {
+ return success;
+ }
+ }
+
+ if (!scli.initialize_ssl(socket_, error)) { return false; }
+ }
+#endif
+ }
+
+ // Mark the current socket as being in use so that it cannot be closed by
+ // anyone else while this request is ongoing, even though we will be
+ // releasing the mutex.
+ if (socket_requests_in_flight_ > 1) {
+ assert(socket_requests_are_from_thread_ == std::this_thread::get_id());
+ }
+ socket_requests_in_flight_ += 1;
+ socket_requests_are_from_thread_ = std::this_thread::get_id();
+ }
+
+ for (const auto &header : default_headers_) {
+ if (req.headers.find(header.first) == req.headers.end()) {
+ req.headers.insert(header);
+ }
+ }
+
+ auto ret = false;
+ auto close_connection = !keep_alive_;
+
+ auto se = detail::scope_exit([&]() {
+ // Briefly lock mutex in order to mark that a request is no longer ongoing
+ std::lock_guard<std::mutex> guard(socket_mutex_);
+ socket_requests_in_flight_ -= 1;
+ if (socket_requests_in_flight_ <= 0) {
+ assert(socket_requests_in_flight_ == 0);
+ socket_requests_are_from_thread_ = std::thread::id();
+ }
+
+ if (socket_should_be_closed_when_request_is_done_ || close_connection ||
+ !ret) {
+ shutdown_ssl(socket_, true);
+ shutdown_socket(socket_);
+ close_socket(socket_);
+ }
+ });
+
+ ret = process_socket(socket_, [&](Stream &strm) {
+ return handle_request(strm, req, res, close_connection, error);
+ });
+
+ if (!ret) {
+ if (error == Error::Success) { error = Error::Unknown; }
+ }
+
+ return ret;
+}
+
+inline Result ClientImpl::send(const Request &req) {
+ auto req2 = req;
+ return send_(std::move(req2));
+}
+
+inline Result ClientImpl::send_(Request &&req) {
+ auto res = detail::make_unique<Response>();
+ auto error = Error::Success;
+ auto ret = send(req, *res, error);
+ return Result{ret ? std::move(res) : nullptr, error, std::move(req.headers)};
+}
+
+inline bool ClientImpl::handle_request(Stream &strm, Request &req,
+ Response &res, bool close_connection,
+ Error &error) {
+ if (req.path.empty()) {
+ error = Error::Connection;
+ return false;
+ }
+
+ auto req_save = req;
+
+ bool ret;
+
+ if (!is_ssl() && !proxy_host_.empty() && proxy_port_ != -1) {
+ auto req2 = req;
+ req2.path = "http://" + host_and_port_ + req.path;
+ ret = process_request(strm, req2, res, close_connection, error);
+ req = req2;
+ req.path = req_save.path;
+ } else {
+ ret = process_request(strm, req, res, close_connection, error);
+ }
+
+ if (!ret) { return false; }
+
+ if (300 < res.status && res.status < 400 && follow_location_) {
+ req = req_save;
+ ret = redirect(req, res, error);
+ }
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ if ((res.status == 401 || res.status == 407) &&
+ req.authorization_count_ < 5) {
+ auto is_proxy = res.status == 407;
+ const auto &username =
+ is_proxy ? proxy_digest_auth_username_ : digest_auth_username_;
+ const auto &password =
+ is_proxy ? proxy_digest_auth_password_ : digest_auth_password_;
+
+ if (!username.empty() && !password.empty()) {
+ std::map<std::string, std::string> auth;
+ if (detail::parse_www_authenticate(res, auth, is_proxy)) {
+ Request new_req = req;
+ new_req.authorization_count_ += 1;
+ new_req.headers.erase(is_proxy ? "Proxy-Authorization"
+ : "Authorization");
+ new_req.headers.insert(detail::make_digest_authentication_header(
+ req, auth, new_req.authorization_count_, detail::random_string(10),
+ username, password, is_proxy));
+
+ Response new_res;
+
+ ret = send(new_req, new_res, error);
+ if (ret) { res = new_res; }
+ }
+ }
+ }
+#endif
+
+ return ret;
+}
+
+inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {
+ if (req.redirect_count_ == 0) {
+ error = Error::ExceedRedirectCount;
+ return false;
+ }
+
+ auto location = res.get_header_value("location");
+ if (location.empty()) { return false; }
+
+ const static std::regex re(
+ R"((?:(https?):)?(?://(?:\[([\d:]+)\]|([^:/?#]+))(?::(\d+))?)?([^?#]*)(\?[^#]*)?(?:#.*)?)");
+
+ std::smatch m;
+ if (!std::regex_match(location, m, re)) { return false; }
+
+ auto scheme = is_ssl() ? "https" : "http";
+
+ auto next_scheme = m[1].str();
+ auto next_host = m[2].str();
+ if (next_host.empty()) { next_host = m[3].str(); }
+ auto port_str = m[4].str();
+ auto next_path = m[5].str();
+ auto next_query = m[6].str();
+
+ auto next_port = port_;
+ if (!port_str.empty()) {
+ next_port = std::stoi(port_str);
+ } else if (!next_scheme.empty()) {
+ next_port = next_scheme == "https" ? 443 : 80;
+ }
+
+ if (next_scheme.empty()) { next_scheme = scheme; }
+ if (next_host.empty()) { next_host = host_; }
+ if (next_path.empty()) { next_path = "/"; }
+
+ auto path = detail::decode_url(next_path, true) + next_query;
+
+ if (next_scheme == scheme && next_host == host_ && next_port == port_) {
+ return detail::redirect(*this, req, res, path, location, error);
+ } else {
+ if (next_scheme == "https") {
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ SSLClient cli(next_host.c_str(), next_port);
+ cli.copy_settings(*this);
+ if (ca_cert_store_) { cli.set_ca_cert_store(ca_cert_store_); }
+ return detail::redirect(cli, req, res, path, location, error);
+#else
+ return false;
+#endif
+ } else {
+ ClientImpl cli(next_host.c_str(), next_port);
+ cli.copy_settings(*this);
+ return detail::redirect(cli, req, res, path, location, error);
+ }
+ }
+}
+
+inline bool ClientImpl::write_content_with_provider(Stream &strm,
+ const Request &req,
+ Error &error) {
+ auto is_shutting_down = []() { return false; };
+
+ if (req.is_chunked_content_provider_) {
+ // TODO: Brotli support
+ std::unique_ptr<detail::compressor> compressor;
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+ if (compress_) {
+ compressor = detail::make_unique<detail::gzip_compressor>();
+ } else
+#endif
+ {
+ compressor = detail::make_unique<detail::nocompressor>();
+ }
+
+ return detail::write_content_chunked(strm, req.content_provider_,
+ is_shutting_down, *compressor, error);
+ } else {
+ return detail::write_content(strm, req.content_provider_, 0,
+ req.content_length_, is_shutting_down, error);
+ }
+}
+
+inline bool ClientImpl::write_request(Stream &strm, Request &req,
+ bool close_connection, Error &error) {
+ // Prepare additional headers
+ if (close_connection) {
+ if (!req.has_header("Connection")) {
+ req.headers.emplace("Connection", "close");
+ }
+ }
+
+ if (!req.has_header("Host")) {
+ if (is_ssl()) {
+ if (port_ == 443) {
+ req.headers.emplace("Host", host_);
+ } else {
+ req.headers.emplace("Host", host_and_port_);
+ }
+ } else {
+ if (port_ == 80) {
+ req.headers.emplace("Host", host_);
+ } else {
+ req.headers.emplace("Host", host_and_port_);
+ }
+ }
+ }
+
+ if (!req.has_header("Accept")) { req.headers.emplace("Accept", "*/*"); }
+
+#ifndef CPPHTTPLIB_NO_DEFAULT_USER_AGENT
+ if (!req.has_header("User-Agent")) {
+ auto agent = std::string("cpp-httplib/") + CPPHTTPLIB_VERSION;
+ req.headers.emplace("User-Agent", agent);
+ }
+#endif
+
+ if (req.body.empty()) {
+ if (req.content_provider_) {
+ if (!req.is_chunked_content_provider_) {
+ if (!req.has_header("Content-Length")) {
+ auto length = std::to_string(req.content_length_);
+ req.headers.emplace("Content-Length", length);
+ }
+ }
+ } else {
+ if (req.method == "POST" || req.method == "PUT" ||
+ req.method == "PATCH") {
+ req.headers.emplace("Content-Length", "0");
+ }
+ }
+ } else {
+ if (!req.has_header("Content-Type")) {
+ req.headers.emplace("Content-Type", "text/plain");
+ }
+
+ if (!req.has_header("Content-Length")) {
+ auto length = std::to_string(req.body.size());
+ req.headers.emplace("Content-Length", length);
+ }
+ }
+
+ if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) {
+ if (!req.has_header("Authorization")) {
+ req.headers.insert(make_basic_authentication_header(
+ basic_auth_username_, basic_auth_password_, false));
+ }
+ }
+
+ if (!proxy_basic_auth_username_.empty() &&
+ !proxy_basic_auth_password_.empty()) {
+ if (!req.has_header("Proxy-Authorization")) {
+ req.headers.insert(make_basic_authentication_header(
+ proxy_basic_auth_username_, proxy_basic_auth_password_, true));
+ }
+ }
+
+ if (!bearer_token_auth_token_.empty()) {
+ if (!req.has_header("Authorization")) {
+ req.headers.insert(make_bearer_token_authentication_header(
+ bearer_token_auth_token_, false));
+ }
+ }
+
+ if (!proxy_bearer_token_auth_token_.empty()) {
+ if (!req.has_header("Proxy-Authorization")) {
+ req.headers.insert(make_bearer_token_authentication_header(
+ proxy_bearer_token_auth_token_, true));
+ }
+ }
+
+ // Request line and headers
+ {
+ detail::BufferStream bstrm;
+
+ const auto &path = url_encode_ ? detail::encode_url(req.path) : req.path;
+ bstrm.write_format("%s %s HTTP/1.1\r\n", req.method.c_str(), path.c_str());
+
+ detail::write_headers(bstrm, req.headers);
+
+ // Flush buffer
+ auto &data = bstrm.get_buffer();
+ if (!detail::write_data(strm, data.data(), data.size())) {
+ error = Error::Write;
+ return false;
+ }
+ }
+
+ // Body
+ if (req.body.empty()) {
+ return write_content_with_provider(strm, req, error);
+ }
+
+ if (!detail::write_data(strm, req.body.data(), req.body.size())) {
+ error = Error::Write;
+ return false;
+ }
+
+ return true;
+}
+
+inline std::unique_ptr<Response> ClientImpl::send_with_content_provider(
+ Request &req, const char *body, size_t content_length,
+ ContentProvider content_provider,
+ ContentProviderWithoutLength content_provider_without_length,
+ const std::string &content_type, Error &error) {
+ if (!content_type.empty()) {
+ req.headers.emplace("Content-Type", content_type);
+ }
+
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+ if (compress_) { req.headers.emplace("Content-Encoding", "gzip"); }
+#endif
+
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+ if (compress_ && !content_provider_without_length) {
+ // TODO: Brotli support
+ detail::gzip_compressor compressor;
+
+ if (content_provider) {
+ auto ok = true;
+ size_t offset = 0;
+ DataSink data_sink;
+
+ data_sink.write = [&](const char *data, size_t data_len) -> bool {
+ if (ok) {
+ auto last = offset + data_len == content_length;
+
+ auto ret = compressor.compress(
+ data, data_len, last,
+ [&](const char *compressed_data, size_t compressed_data_len) {
+ req.body.append(compressed_data, compressed_data_len);
+ return true;
+ });
+
+ if (ret) {
+ offset += data_len;
+ } else {
+ ok = false;
+ }
+ }
+ return ok;
+ };
+
+ while (ok && offset < content_length) {
+ if (!content_provider(offset, content_length - offset, data_sink)) {
+ error = Error::Canceled;
+ return nullptr;
+ }
+ }
+ } else {
+ if (!compressor.compress(body, content_length, true,
+ [&](const char *data, size_t data_len) {
+ req.body.append(data, data_len);
+ return true;
+ })) {
+ error = Error::Compression;
+ return nullptr;
+ }
+ }
+ } else
+#endif
+ {
+ if (content_provider) {
+ req.content_length_ = content_length;
+ req.content_provider_ = std::move(content_provider);
+ req.is_chunked_content_provider_ = false;
+ } else if (content_provider_without_length) {
+ req.content_length_ = 0;
+ req.content_provider_ = detail::ContentProviderAdapter(
+ std::move(content_provider_without_length));
+ req.is_chunked_content_provider_ = true;
+ req.headers.emplace("Transfer-Encoding", "chunked");
+ } else {
+ req.body.assign(body, content_length);
+ ;
+ }
+ }
+
+ auto res = detail::make_unique<Response>();
+ return send(req, *res, error) ? std::move(res) : nullptr;
+}
+
+inline Result ClientImpl::send_with_content_provider(
+ const std::string &method, const std::string &path, const Headers &headers,
+ const char *body, size_t content_length, ContentProvider content_provider,
+ ContentProviderWithoutLength content_provider_without_length,
+ const std::string &content_type) {
+ Request req;
+ req.method = method;
+ req.headers = headers;
+ req.path = path;
+
+ auto error = Error::Success;
+
+ auto res = send_with_content_provider(
+ req, body, content_length, std::move(content_provider),
+ std::move(content_provider_without_length), content_type, error);
+
+ return Result{std::move(res), error, std::move(req.headers)};
+}
+
+inline std::string
+ClientImpl::adjust_host_string(const std::string &host) const {
+ if (host.find(':') != std::string::npos) { return "[" + host + "]"; }
+ return host;
+}
+
+inline bool ClientImpl::process_request(Stream &strm, Request &req,
+ Response &res, bool close_connection,
+ Error &error) {
+ // Send request
+ if (!write_request(strm, req, close_connection, error)) { return false; }
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ if (is_ssl()) {
+ auto is_proxy_enabled = !proxy_host_.empty() && proxy_port_ != -1;
+ if (!is_proxy_enabled) {
+ char buf[1];
+ if (SSL_peek(socket_.ssl, buf, 1) == 0 &&
+ SSL_get_error(socket_.ssl, 0) == SSL_ERROR_ZERO_RETURN) {
+ error = Error::SSLPeerCouldBeClosed_;
+ return false;
+ }
+ }
+ }
+#endif
+
+ // Receive response and headers
+ if (!read_response_line(strm, req, res) ||
+ !detail::read_headers(strm, res.headers)) {
+ error = Error::Read;
+ return false;
+ }
+
+ // Body
+ if ((res.status != 204) && req.method != "HEAD" && req.method != "CONNECT") {
+ auto redirect = 300 < res.status && res.status < 400 && follow_location_;
+
+ if (req.response_handler && !redirect) {
+ if (!req.response_handler(res)) {
+ error = Error::Canceled;
+ return false;
+ }
+ }
+
+ auto out =
+ req.content_receiver
+ ? static_cast<ContentReceiverWithProgress>(
+ [&](const char *buf, size_t n, uint64_t off, uint64_t len) {
+ if (redirect) { return true; }
+ auto ret = req.content_receiver(buf, n, off, len);
+ if (!ret) { error = Error::Canceled; }
+ return ret;
+ })
+ : static_cast<ContentReceiverWithProgress>(
+ [&](const char *buf, size_t n, uint64_t /*off*/,
+ uint64_t /*len*/) {
+ if (res.body.size() + n > res.body.max_size()) {
+ return false;
+ }
+ res.body.append(buf, n);
+ return true;
+ });
+
+ auto progress = [&](uint64_t current, uint64_t total) {
+ if (!req.progress || redirect) { return true; }
+ auto ret = req.progress(current, total);
+ if (!ret) { error = Error::Canceled; }
+ return ret;
+ };
+
+ int dummy_status;
+ if (!detail::read_content(strm, res, (std::numeric_limits<size_t>::max)(),
+ dummy_status, std::move(progress), std::move(out),
+ decompress_)) {
+ if (error != Error::Canceled) { error = Error::Read; }
+ return false;
+ }
+ }
+
+ if (res.get_header_value("Connection") == "close" ||
+ (res.version == "HTTP/1.0" && res.reason != "Connection established")) {
+ // TODO this requires a not-entirely-obvious chain of calls to be correct
+ // for this to be safe. Maybe a code refactor (such as moving this out to
+ // the send function and getting rid of the recursiveness of the mutex)
+ // could make this more obvious.
+
+ // This is safe to call because process_request is only called by
+ // handle_request which is only called by send, which locks the request
+ // mutex during the process. It would be a bug to call it from a different
+ // thread since it's a thread-safety issue to do these things to the socket
+ // if another thread is using the socket.
+ std::lock_guard<std::mutex> guard(socket_mutex_);
+ shutdown_ssl(socket_, true);
+ shutdown_socket(socket_);
+ close_socket(socket_);
+ }
+
+ // Log
+ if (logger_) { logger_(req, res); }
+
+ return true;
+}
+
+inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(
+ const std::string &boundary, const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items) {
+ size_t cur_item = 0, cur_start = 0;
+ // cur_item and cur_start are copied to within the std::function and maintain
+ // state between successive calls
+ return [&, cur_item, cur_start](size_t offset,
+ DataSink &sink) mutable -> bool {
+ if (!offset && items.size()) {
+ sink.os << detail::serialize_multipart_formdata(items, boundary, false);
+ return true;
+ } else if (cur_item < provider_items.size()) {
+ if (!cur_start) {
+ const auto &begin = detail::serialize_multipart_formdata_item_begin(
+ provider_items[cur_item], boundary);
+ offset += begin.size();
+ cur_start = offset;
+ sink.os << begin;
+ }
+
+ DataSink cur_sink;
+ bool has_data = true;
+ cur_sink.write = sink.write;
+ cur_sink.done = [&]() { has_data = false; };
+
+ if (!provider_items[cur_item].provider(offset - cur_start, cur_sink))
+ return false;
+
+ if (!has_data) {
+ sink.os << detail::serialize_multipart_formdata_item_end();
+ cur_item++;
+ cur_start = 0;
+ }
+ return true;
+ } else {
+ sink.os << detail::serialize_multipart_formdata_finish(boundary);
+ sink.done();
+ return true;
+ }
+ };
+}
+
+inline bool
+ClientImpl::process_socket(const Socket &socket,
+ std::function<bool(Stream &strm)> callback) {
+ return detail::process_client_socket(
+ socket.sock, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_,
+ write_timeout_usec_, std::move(callback));
+}
+
+inline bool ClientImpl::is_ssl() const { return false; }
+
+inline Result ClientImpl::Get(const std::string &path) {
+ return Get(path, Headers(), Progress());
+}
+
+inline Result ClientImpl::Get(const std::string &path, Progress progress) {
+ return Get(path, Headers(), std::move(progress));
+}
+
+inline Result ClientImpl::Get(const std::string &path, const Headers &headers) {
+ return Get(path, headers, Progress());
+}
+
+inline Result ClientImpl::Get(const std::string &path, const Headers &headers,
+ Progress progress) {
+ Request req;
+ req.method = "GET";
+ req.path = path;
+ req.headers = headers;
+ req.progress = std::move(progress);
+
+ return send_(std::move(req));
+}
+
+inline Result ClientImpl::Get(const std::string &path,
+ ContentReceiver content_receiver) {
+ return Get(path, Headers(), nullptr, std::move(content_receiver), nullptr);
+}
+
+inline Result ClientImpl::Get(const std::string &path,
+ ContentReceiver content_receiver,
+ Progress progress) {
+ return Get(path, Headers(), nullptr, std::move(content_receiver),
+ std::move(progress));
+}
+
+inline Result ClientImpl::Get(const std::string &path, const Headers &headers,
+ ContentReceiver content_receiver) {
+ return Get(path, headers, nullptr, std::move(content_receiver), nullptr);
+}
+
+inline Result ClientImpl::Get(const std::string &path, const Headers &headers,
+ ContentReceiver content_receiver,
+ Progress progress) {
+ return Get(path, headers, nullptr, std::move(content_receiver),
+ std::move(progress));
+}
+
+inline Result ClientImpl::Get(const std::string &path,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver) {
+ return Get(path, Headers(), std::move(response_handler),
+ std::move(content_receiver), nullptr);
+}
+
+inline Result ClientImpl::Get(const std::string &path, const Headers &headers,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver) {
+ return Get(path, headers, std::move(response_handler),
+ std::move(content_receiver), nullptr);
+}
+
+inline Result ClientImpl::Get(const std::string &path,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver,
+ Progress progress) {
+ return Get(path, Headers(), std::move(response_handler),
+ std::move(content_receiver), std::move(progress));
+}
+
+inline Result ClientImpl::Get(const std::string &path, const Headers &headers,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver,
+ Progress progress) {
+ Request req;
+ req.method = "GET";
+ req.path = path;
+ req.headers = headers;
+ req.response_handler = std::move(response_handler);
+ req.content_receiver =
+ [content_receiver](const char *data, size_t data_length,
+ uint64_t /*offset*/, uint64_t /*total_length*/) {
+ return content_receiver(data, data_length);
+ };
+ req.progress = std::move(progress);
+
+ return send_(std::move(req));
+}
+
+inline Result ClientImpl::Get(const std::string &path, const Params ¶ms,
+ const Headers &headers, Progress progress) {
+ if (params.empty()) { return Get(path, headers); }
+
+ std::string path_with_query = append_query_params(path, params);
+ return Get(path_with_query.c_str(), headers, progress);
+}
+
+inline Result ClientImpl::Get(const std::string &path, const Params ¶ms,
+ const Headers &headers,
+ ContentReceiver content_receiver,
+ Progress progress) {
+ return Get(path, params, headers, nullptr, content_receiver, progress);
+}
+
+inline Result ClientImpl::Get(const std::string &path, const Params ¶ms,
+ const Headers &headers,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver,
+ Progress progress) {
+ if (params.empty()) {
+ return Get(path, headers, response_handler, content_receiver, progress);
+ }
+
+ std::string path_with_query = append_query_params(path, params);
+ return Get(path_with_query.c_str(), headers, response_handler,
+ content_receiver, progress);
+}
+
+inline Result ClientImpl::Head(const std::string &path) {
+ return Head(path, Headers());
+}
+
+inline Result ClientImpl::Head(const std::string &path,
+ const Headers &headers) {
+ Request req;
+ req.method = "HEAD";
+ req.headers = headers;
+ req.path = path;
+
+ return send_(std::move(req));
+}
+
+inline Result ClientImpl::Post(const std::string &path) {
+ return Post(path, std::string(), std::string());
+}
+
+inline Result ClientImpl::Post(const std::string &path,
+ const Headers &headers) {
+ return Post(path, headers, nullptr, 0, std::string());
+}
+
+inline Result ClientImpl::Post(const std::string &path, const char *body,
+ size_t content_length,
+ const std::string &content_type) {
+ return Post(path, Headers(), body, content_length, content_type);
+}
+
+inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
+ const char *body, size_t content_length,
+ const std::string &content_type) {
+ return send_with_content_provider("POST", path, headers, body, content_length,
+ nullptr, nullptr, content_type);
+}
+
+inline Result ClientImpl::Post(const std::string &path, const std::string &body,
+ const std::string &content_type) {
+ return Post(path, Headers(), body, content_type);
+}
+
+inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
+ const std::string &body,
+ const std::string &content_type) {
+ return send_with_content_provider("POST", path, headers, body.data(),
+ body.size(), nullptr, nullptr,
+ content_type);
+}
+
+inline Result ClientImpl::Post(const std::string &path, const Params ¶ms) {
+ return Post(path, Headers(), params);
+}
+
+inline Result ClientImpl::Post(const std::string &path, size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return Post(path, Headers(), content_length, std::move(content_provider),
+ content_type);
+}
+
+inline Result ClientImpl::Post(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return Post(path, Headers(), std::move(content_provider), content_type);
+}
+
+inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
+ size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return send_with_content_provider("POST", path, headers, nullptr,
+ content_length, std::move(content_provider),
+ nullptr, content_type);
+}
+
+inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return send_with_content_provider("POST", path, headers, nullptr, 0, nullptr,
+ std::move(content_provider), content_type);
+}
+
+inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
+ const Params ¶ms) {
+ auto query = detail::params_to_query_str(params);
+ return Post(path, headers, query, "application/x-www-form-urlencoded");
+}
+
+inline Result ClientImpl::Post(const std::string &path,
+ const MultipartFormDataItems &items) {
+ return Post(path, Headers(), items);
+}
+
+inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items) {
+ const auto &boundary = detail::make_multipart_data_boundary();
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ const auto &body = detail::serialize_multipart_formdata(items, boundary);
+ return Post(path, headers, body, content_type.c_str());
+}
+
+inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const std::string &boundary) {
+ if (!detail::is_multipart_boundary_chars_valid(boundary)) {
+ return Result{nullptr, Error::UnsupportedMultipartBoundaryChars};
+ }
+
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ const auto &body = detail::serialize_multipart_formdata(items, boundary);
+ return Post(path, headers, body, content_type.c_str());
+}
+
+inline Result
+ClientImpl::Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items) {
+ const auto &boundary = detail::make_multipart_data_boundary();
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ return send_with_content_provider(
+ "POST", path, headers, nullptr, 0, nullptr,
+ get_multipart_content_provider(boundary, items, provider_items),
+ content_type);
+}
+
+inline Result ClientImpl::Put(const std::string &path) {
+ return Put(path, std::string(), std::string());
+}
+
+inline Result ClientImpl::Put(const std::string &path, const char *body,
+ size_t content_length,
+ const std::string &content_type) {
+ return Put(path, Headers(), body, content_length, content_type);
+}
+
+inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
+ const char *body, size_t content_length,
+ const std::string &content_type) {
+ return send_with_content_provider("PUT", path, headers, body, content_length,
+ nullptr, nullptr, content_type);
+}
+
+inline Result ClientImpl::Put(const std::string &path, const std::string &body,
+ const std::string &content_type) {
+ return Put(path, Headers(), body, content_type);
+}
+
+inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
+ const std::string &body,
+ const std::string &content_type) {
+ return send_with_content_provider("PUT", path, headers, body.data(),
+ body.size(), nullptr, nullptr,
+ content_type);
+}
+
+inline Result ClientImpl::Put(const std::string &path, size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return Put(path, Headers(), content_length, std::move(content_provider),
+ content_type);
+}
+
+inline Result ClientImpl::Put(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return Put(path, Headers(), std::move(content_provider), content_type);
+}
+
+inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
+ size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return send_with_content_provider("PUT", path, headers, nullptr,
+ content_length, std::move(content_provider),
+ nullptr, content_type);
+}
+
+inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return send_with_content_provider("PUT", path, headers, nullptr, 0, nullptr,
+ std::move(content_provider), content_type);
+}
+
+inline Result ClientImpl::Put(const std::string &path, const Params ¶ms) {
+ return Put(path, Headers(), params);
+}
+
+inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
+ const Params ¶ms) {
+ auto query = detail::params_to_query_str(params);
+ return Put(path, headers, query, "application/x-www-form-urlencoded");
+}
+
+inline Result ClientImpl::Put(const std::string &path,
+ const MultipartFormDataItems &items) {
+ return Put(path, Headers(), items);
+}
+
+inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items) {
+ const auto &boundary = detail::make_multipart_data_boundary();
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ const auto &body = detail::serialize_multipart_formdata(items, boundary);
+ return Put(path, headers, body, content_type);
+}
+
+inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const std::string &boundary) {
+ if (!detail::is_multipart_boundary_chars_valid(boundary)) {
+ return Result{nullptr, Error::UnsupportedMultipartBoundaryChars};
+ }
+
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ const auto &body = detail::serialize_multipart_formdata(items, boundary);
+ return Put(path, headers, body, content_type);
+}
+
+inline Result
+ClientImpl::Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items) {
+ const auto &boundary = detail::make_multipart_data_boundary();
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ return send_with_content_provider(
+ "PUT", path, headers, nullptr, 0, nullptr,
+ get_multipart_content_provider(boundary, items, provider_items),
+ content_type);
+}
+inline Result ClientImpl::Patch(const std::string &path) {
+ return Patch(path, std::string(), std::string());
+}
+
+inline Result ClientImpl::Patch(const std::string &path, const char *body,
+ size_t content_length,
+ const std::string &content_type) {
+ return Patch(path, Headers(), body, content_length, content_type);
+}
+
+inline Result ClientImpl::Patch(const std::string &path, const Headers &headers,
+ const char *body, size_t content_length,
+ const std::string &content_type) {
+ return send_with_content_provider("PATCH", path, headers, body,
+ content_length, nullptr, nullptr,
+ content_type);
+}
+
+inline Result ClientImpl::Patch(const std::string &path,
+ const std::string &body,
+ const std::string &content_type) {
+ return Patch(path, Headers(), body, content_type);
+}
+
+inline Result ClientImpl::Patch(const std::string &path, const Headers &headers,
+ const std::string &body,
+ const std::string &content_type) {
+ return send_with_content_provider("PATCH", path, headers, body.data(),
+ body.size(), nullptr, nullptr,
+ content_type);
+}
+
+inline Result ClientImpl::Patch(const std::string &path, size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return Patch(path, Headers(), content_length, std::move(content_provider),
+ content_type);
+}
+
+inline Result ClientImpl::Patch(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return Patch(path, Headers(), std::move(content_provider), content_type);
+}
+
+inline Result ClientImpl::Patch(const std::string &path, const Headers &headers,
+ size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return send_with_content_provider("PATCH", path, headers, nullptr,
+ content_length, std::move(content_provider),
+ nullptr, content_type);
+}
+
+inline Result ClientImpl::Patch(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return send_with_content_provider("PATCH", path, headers, nullptr, 0, nullptr,
+ std::move(content_provider), content_type);
+}
+
+inline Result ClientImpl::Delete(const std::string &path) {
+ return Delete(path, Headers(), std::string(), std::string());
+}
+
+inline Result ClientImpl::Delete(const std::string &path,
+ const Headers &headers) {
+ return Delete(path, headers, std::string(), std::string());
+}
+
+inline Result ClientImpl::Delete(const std::string &path, const char *body,
+ size_t content_length,
+ const std::string &content_type) {
+ return Delete(path, Headers(), body, content_length, content_type);
+}
+
+inline Result ClientImpl::Delete(const std::string &path,
+ const Headers &headers, const char *body,
+ size_t content_length,
+ const std::string &content_type) {
+ Request req;
+ req.method = "DELETE";
+ req.headers = headers;
+ req.path = path;
+
+ if (!content_type.empty()) {
+ req.headers.emplace("Content-Type", content_type);
+ }
+ req.body.assign(body, content_length);
+
+ return send_(std::move(req));
+}
+
+inline Result ClientImpl::Delete(const std::string &path,
+ const std::string &body,
+ const std::string &content_type) {
+ return Delete(path, Headers(), body.data(), body.size(), content_type);
+}
+
+inline Result ClientImpl::Delete(const std::string &path,
+ const Headers &headers,
+ const std::string &body,
+ const std::string &content_type) {
+ return Delete(path, headers, body.data(), body.size(), content_type);
+}
+
+inline Result ClientImpl::Options(const std::string &path) {
+ return Options(path, Headers());
+}
+
+inline Result ClientImpl::Options(const std::string &path,
+ const Headers &headers) {
+ Request req;
+ req.method = "OPTIONS";
+ req.headers = headers;
+ req.path = path;
+
+ return send_(std::move(req));
+}
+
+inline size_t ClientImpl::is_socket_open() const {
+ std::lock_guard<std::mutex> guard(socket_mutex_);
+ return socket_.is_open();
+}
+
+inline socket_t ClientImpl::socket() const { return socket_.sock; }
+
+inline void ClientImpl::stop() {
+ std::lock_guard<std::mutex> guard(socket_mutex_);
+
+ // If there is anything ongoing right now, the ONLY thread-safe thing we can
+ // do is to shutdown_socket, so that threads using this socket suddenly
+ // discover they can't read/write any more and error out. Everything else
+ // (closing the socket, shutting ssl down) is unsafe because these actions are
+ // not thread-safe.
+ if (socket_requests_in_flight_ > 0) {
+ shutdown_socket(socket_);
+
+ // Aside from that, we set a flag for the socket to be closed when we're
+ // done.
+ socket_should_be_closed_when_request_is_done_ = true;
+ return;
+ }
+
+ // Otherwise, still holding the mutex, we can shut everything down ourselves
+ shutdown_ssl(socket_, true);
+ shutdown_socket(socket_);
+ close_socket(socket_);
+}
+
+inline void ClientImpl::set_connection_timeout(time_t sec, time_t usec) {
+ connection_timeout_sec_ = sec;
+ connection_timeout_usec_ = usec;
+}
+
+inline void ClientImpl::set_read_timeout(time_t sec, time_t usec) {
+ read_timeout_sec_ = sec;
+ read_timeout_usec_ = usec;
+}
+
+inline void ClientImpl::set_write_timeout(time_t sec, time_t usec) {
+ write_timeout_sec_ = sec;
+ write_timeout_usec_ = usec;
+}
+
+inline void ClientImpl::set_basic_auth(const std::string &username,
+ const std::string &password) {
+ basic_auth_username_ = username;
+ basic_auth_password_ = password;
+}
+
+inline void ClientImpl::set_bearer_token_auth(const std::string &token) {
+ bearer_token_auth_token_ = token;
+}
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+inline void ClientImpl::set_digest_auth(const std::string &username,
+ const std::string &password) {
+ digest_auth_username_ = username;
+ digest_auth_password_ = password;
+}
+#endif
+
+inline void ClientImpl::set_keep_alive(bool on) { keep_alive_ = on; }
+
+inline void ClientImpl::set_follow_location(bool on) { follow_location_ = on; }
+
+inline void ClientImpl::set_url_encode(bool on) { url_encode_ = on; }
+
+inline void
+ClientImpl::set_hostname_addr_map(std::map<std::string, std::string> addr_map) {
+ addr_map_ = std::move(addr_map);
+}
+
+inline void ClientImpl::set_default_headers(Headers headers) {
+ default_headers_ = std::move(headers);
+}
+
+inline void ClientImpl::set_address_family(int family) {
+ address_family_ = family;
+}
+
+inline void ClientImpl::set_tcp_nodelay(bool on) { tcp_nodelay_ = on; }
+
+inline void ClientImpl::set_socket_options(SocketOptions socket_options) {
+ socket_options_ = std::move(socket_options);
+}
+
+inline void ClientImpl::set_compress(bool on) { compress_ = on; }
+
+inline void ClientImpl::set_decompress(bool on) { decompress_ = on; }
+
+inline void ClientImpl::set_interface(const std::string &intf) {
+ interface_ = intf;
+}
+
+inline void ClientImpl::set_proxy(const std::string &host, int port) {
+ proxy_host_ = host;
+ proxy_port_ = port;
+}
+
+inline void ClientImpl::set_proxy_basic_auth(const std::string &username,
+ const std::string &password) {
+ proxy_basic_auth_username_ = username;
+ proxy_basic_auth_password_ = password;
+}
+
+inline void ClientImpl::set_proxy_bearer_token_auth(const std::string &token) {
+ proxy_bearer_token_auth_token_ = token;
+}
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+inline void ClientImpl::set_proxy_digest_auth(const std::string &username,
+ const std::string &password) {
+ proxy_digest_auth_username_ = username;
+ proxy_digest_auth_password_ = password;
+}
+#endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+inline void ClientImpl::set_ca_cert_path(const std::string &ca_cert_file_path,
+ const std::string &ca_cert_dir_path) {
+ ca_cert_file_path_ = ca_cert_file_path;
+ ca_cert_dir_path_ = ca_cert_dir_path;
+}
+
+inline void ClientImpl::set_ca_cert_store(X509_STORE *ca_cert_store) {
+ if (ca_cert_store && ca_cert_store != ca_cert_store_) {
+ ca_cert_store_ = ca_cert_store;
+ }
+}
+#endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+inline void ClientImpl::enable_server_certificate_verification(bool enabled) {
+ server_certificate_verification_ = enabled;
+}
+#endif
+
+inline void ClientImpl::set_logger(Logger logger) {
+ logger_ = std::move(logger);
+}
+
+/*
+ * SSL Implementation
+ */
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+namespace detail {
+
+template <typename U, typename V>
+inline SSL *ssl_new(socket_t sock, SSL_CTX *ctx, std::mutex &ctx_mutex,
+ U SSL_connect_or_accept, V setup) {
+ SSL *ssl = nullptr;
+ {
+ std::lock_guard<std::mutex> guard(ctx_mutex);
+ ssl = SSL_new(ctx);
+ }
+
+ if (ssl) {
+ set_nonblocking(sock, true);
+ auto bio = BIO_new_socket(static_cast<int>(sock), BIO_NOCLOSE);
+ BIO_set_nbio(bio, 1);
+ SSL_set_bio(ssl, bio, bio);
+
+ if (!setup(ssl) || SSL_connect_or_accept(ssl) != 1) {
+ SSL_shutdown(ssl);
+ {
+ std::lock_guard<std::mutex> guard(ctx_mutex);
+ SSL_free(ssl);
+ }
+ set_nonblocking(sock, false);
+ return nullptr;
+ }
+ BIO_set_nbio(bio, 0);
+ set_nonblocking(sock, false);
+ }
+
+ return ssl;
+}
+
+inline void ssl_delete(std::mutex &ctx_mutex, SSL *ssl,
+ bool shutdown_gracefully) {
+ // sometimes we may want to skip this to try to avoid SIGPIPE if we know
+ // the remote has closed the network connection
+ // Note that it is not always possible to avoid SIGPIPE, this is merely a
+ // best-efforts.
+ if (shutdown_gracefully) { SSL_shutdown(ssl); }
+
+ std::lock_guard<std::mutex> guard(ctx_mutex);
+ SSL_free(ssl);
+}
+
+template <typename U>
+bool ssl_connect_or_accept_nonblocking(socket_t sock, SSL *ssl,
+ U ssl_connect_or_accept,
+ time_t timeout_sec,
+ time_t timeout_usec) {
+ int res = 0;
+ while ((res = ssl_connect_or_accept(ssl)) != 1) {
+ auto err = SSL_get_error(ssl, res);
+ switch (err) {
+ case SSL_ERROR_WANT_READ:
+ if (select_read(sock, timeout_sec, timeout_usec) > 0) { continue; }
+ break;
+ case SSL_ERROR_WANT_WRITE:
+ if (select_write(sock, timeout_sec, timeout_usec) > 0) { continue; }
+ break;
+ default: break;
+ }
+ return false;
+ }
+ return true;
+}
+
+template <typename T>
+inline bool process_server_socket_ssl(
+ const std::atomic<socket_t> &svr_sock, SSL *ssl, socket_t sock,
+ size_t keep_alive_max_count, time_t keep_alive_timeout_sec,
+ time_t read_timeout_sec, time_t read_timeout_usec, time_t write_timeout_sec,
+ time_t write_timeout_usec, T callback) {
+ return process_server_socket_core(
+ svr_sock, sock, keep_alive_max_count, keep_alive_timeout_sec,
+ [&](bool close_connection, bool &connection_closed) {
+ SSLSocketStream strm(sock, ssl, read_timeout_sec, read_timeout_usec,
+ write_timeout_sec, write_timeout_usec);
+ return callback(strm, close_connection, connection_closed);
+ });
+}
+
+template <typename T>
+inline bool
+process_client_socket_ssl(SSL *ssl, socket_t sock, time_t read_timeout_sec,
+ time_t read_timeout_usec, time_t write_timeout_sec,
+ time_t write_timeout_usec, T callback) {
+ SSLSocketStream strm(sock, ssl, read_timeout_sec, read_timeout_usec,
+ write_timeout_sec, write_timeout_usec);
+ return callback(strm);
+}
+
+class SSLInit {
+public:
+ SSLInit() {
+ OPENSSL_init_ssl(
+ OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_LOAD_CRYPTO_STRINGS, NULL);
+ }
+};
+
+// SSL socket stream implementation
+inline SSLSocketStream::SSLSocketStream(socket_t sock, SSL *ssl,
+ time_t read_timeout_sec,
+ time_t read_timeout_usec,
+ time_t write_timeout_sec,
+ time_t write_timeout_usec)
+ : sock_(sock), ssl_(ssl), read_timeout_sec_(read_timeout_sec),
+ read_timeout_usec_(read_timeout_usec),
+ write_timeout_sec_(write_timeout_sec),
+ write_timeout_usec_(write_timeout_usec) {
+ SSL_clear_mode(ssl, SSL_MODE_AUTO_RETRY);
+}
+
+inline SSLSocketStream::~SSLSocketStream() {}
+
+inline bool SSLSocketStream::is_readable() const {
+ return detail::select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;
+}
+
+inline bool SSLSocketStream::is_writable() const {
+ return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 &&
+ is_socket_alive(sock_);
+}
+
+inline ssize_t SSLSocketStream::read(char *ptr, size_t size) {
+ if (SSL_pending(ssl_) > 0) {
+ return SSL_read(ssl_, ptr, static_cast<int>(size));
+ } else if (is_readable()) {
+ auto ret = SSL_read(ssl_, ptr, static_cast<int>(size));
+ if (ret < 0) {
+ auto err = SSL_get_error(ssl_, ret);
+ int n = 1000;
+#ifdef _WIN32
+ while (--n >= 0 && (err == SSL_ERROR_WANT_READ ||
+ (err == SSL_ERROR_SYSCALL &&
+ WSAGetLastError() == WSAETIMEDOUT))) {
+#else
+ while (--n >= 0 && err == SSL_ERROR_WANT_READ) {
+#endif
+ if (SSL_pending(ssl_) > 0) {
+ return SSL_read(ssl_, ptr, static_cast<int>(size));
+ } else if (is_readable()) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ ret = SSL_read(ssl_, ptr, static_cast<int>(size));
+ if (ret >= 0) { return ret; }
+ err = SSL_get_error(ssl_, ret);
+ } else {
+ return -1;
+ }
+ }
+ }
+ return ret;
+ }
+ return -1;
+}
+
+inline ssize_t SSLSocketStream::write(const char *ptr, size_t size) {
+ if (is_writable()) {
+ auto handle_size = static_cast<int>(
+ std::min<size_t>(size, (std::numeric_limits<int>::max)()));
+
+ auto ret = SSL_write(ssl_, ptr, static_cast<int>(handle_size));
+ if (ret < 0) {
+ auto err = SSL_get_error(ssl_, ret);
+ int n = 1000;
+#ifdef _WIN32
+ while (--n >= 0 && (err == SSL_ERROR_WANT_WRITE ||
+ (err == SSL_ERROR_SYSCALL &&
+ WSAGetLastError() == WSAETIMEDOUT))) {
+#else
+ while (--n >= 0 && err == SSL_ERROR_WANT_WRITE) {
+#endif
+ if (is_writable()) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ ret = SSL_write(ssl_, ptr, static_cast<int>(handle_size));
+ if (ret >= 0) { return ret; }
+ err = SSL_get_error(ssl_, ret);
+ } else {
+ return -1;
+ }
+ }
+ }
+ return ret;
+ }
+ return -1;
+}
+
+inline void SSLSocketStream::get_remote_ip_and_port(std::string &ip,
+ int &port) const {
+ detail::get_remote_ip_and_port(sock_, ip, port);
+}
+
+inline void SSLSocketStream::get_local_ip_and_port(std::string &ip,
+ int &port) const {
+ detail::get_local_ip_and_port(sock_, ip, port);
+}
+
+inline socket_t SSLSocketStream::socket() const { return sock_; }
+
+static SSLInit sslinit_;
+
+} // namespace detail
+
+// SSL HTTP server implementation
+inline SSLServer::SSLServer(const char *cert_path, const char *private_key_path,
+ const char *client_ca_cert_file_path,
+ const char *client_ca_cert_dir_path,
+ const char *private_key_password) {
+ ctx_ = SSL_CTX_new(TLS_server_method());
+
+ if (ctx_) {
+ SSL_CTX_set_options(ctx_,
+ SSL_OP_NO_COMPRESSION |
+ SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION);
+
+ SSL_CTX_set_min_proto_version(ctx_, TLS1_1_VERSION);
+
+ // add default password callback before opening encrypted private key
+ if (private_key_password != nullptr && (private_key_password[0] != '\0')) {
+ SSL_CTX_set_default_passwd_cb_userdata(ctx_,
+ (char *)private_key_password);
+ }
+
+ if (SSL_CTX_use_certificate_chain_file(ctx_, cert_path) != 1 ||
+ SSL_CTX_use_PrivateKey_file(ctx_, private_key_path, SSL_FILETYPE_PEM) !=
+ 1) {
+ SSL_CTX_free(ctx_);
+ ctx_ = nullptr;
+ } else if (client_ca_cert_file_path || client_ca_cert_dir_path) {
+ SSL_CTX_load_verify_locations(ctx_, client_ca_cert_file_path,
+ client_ca_cert_dir_path);
+
+ SSL_CTX_set_verify(
+ ctx_, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);
+ }
+ }
+}
+
+inline SSLServer::SSLServer(X509 *cert, EVP_PKEY *private_key,
+ X509_STORE *client_ca_cert_store) {
+ ctx_ = SSL_CTX_new(TLS_server_method());
+
+ if (ctx_) {
+ SSL_CTX_set_options(ctx_,
+ SSL_OP_NO_COMPRESSION |
+ SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION);
+
+ SSL_CTX_set_min_proto_version(ctx_, TLS1_1_VERSION);
+
+ if (SSL_CTX_use_certificate(ctx_, cert) != 1 ||
+ SSL_CTX_use_PrivateKey(ctx_, private_key) != 1) {
+ SSL_CTX_free(ctx_);
+ ctx_ = nullptr;
+ } else if (client_ca_cert_store) {
+ SSL_CTX_set_cert_store(ctx_, client_ca_cert_store);
+
+ SSL_CTX_set_verify(
+ ctx_, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);
+ }
+ }
+}
+
+inline SSLServer::SSLServer(
+ const std::function<bool(SSL_CTX &ssl_ctx)> &setup_ssl_ctx_callback) {
+ ctx_ = SSL_CTX_new(TLS_method());
+ if (ctx_) {
+ if (!setup_ssl_ctx_callback(*ctx_)) {
+ SSL_CTX_free(ctx_);
+ ctx_ = nullptr;
+ }
+ }
+}
+
+inline SSLServer::~SSLServer() {
+ if (ctx_) { SSL_CTX_free(ctx_); }
+}
+
+inline bool SSLServer::is_valid() const { return ctx_; }
+
+inline SSL_CTX *SSLServer::ssl_context() const { return ctx_; }
+
+inline bool SSLServer::process_and_close_socket(socket_t sock) {
+ auto ssl = detail::ssl_new(
+ sock, ctx_, ctx_mutex_,
+ [&](SSL *ssl2) {
+ return detail::ssl_connect_or_accept_nonblocking(
+ sock, ssl2, SSL_accept, read_timeout_sec_, read_timeout_usec_);
+ },
+ [](SSL * /*ssl2*/) { return true; });
+
+ auto ret = false;
+ if (ssl) {
+ ret = detail::process_server_socket_ssl(
+ svr_sock_, ssl, sock, keep_alive_max_count_, keep_alive_timeout_sec_,
+ read_timeout_sec_, read_timeout_usec_, write_timeout_sec_,
+ write_timeout_usec_,
+ [this, ssl](Stream &strm, bool close_connection,
+ bool &connection_closed) {
+ return process_request(strm, close_connection, connection_closed,
+ [&](Request &req) { req.ssl = ssl; });
+ });
+
+ // Shutdown gracefully if the result seemed successful, non-gracefully if
+ // the connection appeared to be closed.
+ const bool shutdown_gracefully = ret;
+ detail::ssl_delete(ctx_mutex_, ssl, shutdown_gracefully);
+ }
+
+ detail::shutdown_socket(sock);
+ detail::close_socket(sock);
+ return ret;
+}
+
+// SSL HTTP client implementation
+inline SSLClient::SSLClient(const std::string &host)
+ : SSLClient(host, 443, std::string(), std::string()) {}
+
+inline SSLClient::SSLClient(const std::string &host, int port)
+ : SSLClient(host, port, std::string(), std::string()) {}
+
+inline SSLClient::SSLClient(const std::string &host, int port,
+ const std::string &client_cert_path,
+ const std::string &client_key_path)
+ : ClientImpl(host, port, client_cert_path, client_key_path) {
+ ctx_ = SSL_CTX_new(TLS_client_method());
+
+ detail::split(&host_[0], &host_[host_.size()], '.',
+ [&](const char *b, const char *e) {
+ host_components_.emplace_back(std::string(b, e));
+ });
+
+ if (!client_cert_path.empty() && !client_key_path.empty()) {
+ if (SSL_CTX_use_certificate_file(ctx_, client_cert_path.c_str(),
+ SSL_FILETYPE_PEM) != 1 ||
+ SSL_CTX_use_PrivateKey_file(ctx_, client_key_path.c_str(),
+ SSL_FILETYPE_PEM) != 1) {
+ SSL_CTX_free(ctx_);
+ ctx_ = nullptr;
+ }
+ }
+}
+
+inline SSLClient::SSLClient(const std::string &host, int port,
+ X509 *client_cert, EVP_PKEY *client_key)
+ : ClientImpl(host, port) {
+ ctx_ = SSL_CTX_new(TLS_client_method());
+
+ detail::split(&host_[0], &host_[host_.size()], '.',
+ [&](const char *b, const char *e) {
+ host_components_.emplace_back(std::string(b, e));
+ });
+
+ if (client_cert != nullptr && client_key != nullptr) {
+ if (SSL_CTX_use_certificate(ctx_, client_cert) != 1 ||
+ SSL_CTX_use_PrivateKey(ctx_, client_key) != 1) {
+ SSL_CTX_free(ctx_);
+ ctx_ = nullptr;
+ }
+ }
+}
+
+inline SSLClient::~SSLClient() {
+ if (ctx_) { SSL_CTX_free(ctx_); }
+ // Make sure to shut down SSL since shutdown_ssl will resolve to the
+ // base function rather than the derived function once we get to the
+ // base class destructor, and won't free the SSL (causing a leak).
+ shutdown_ssl_impl(socket_, true);
+}
+
+inline bool SSLClient::is_valid() const { return ctx_; }
+
+inline void SSLClient::set_ca_cert_store(X509_STORE *ca_cert_store) {
+ if (ca_cert_store) {
+ if (ctx_) {
+ if (SSL_CTX_get_cert_store(ctx_) != ca_cert_store) {
+ // Free memory allocated for old cert and use new store `ca_cert_store`
+ SSL_CTX_set_cert_store(ctx_, ca_cert_store);
+ }
+ } else {
+ X509_STORE_free(ca_cert_store);
+ }
+ }
+}
+
+inline long SSLClient::get_openssl_verify_result() const {
+ return verify_result_;
+}
+
+inline SSL_CTX *SSLClient::ssl_context() const { return ctx_; }
+
+inline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) {
+ return is_valid() && ClientImpl::create_and_connect_socket(socket, error);
+}
+
+// Assumes that socket_mutex_ is locked and that there are no requests in flight
+inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,
+ bool &success, Error &error) {
+ success = true;
+ Response res2;
+ if (!detail::process_client_socket(
+ socket.sock, read_timeout_sec_, read_timeout_usec_,
+ write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) {
+ Request req2;
+ req2.method = "CONNECT";
+ req2.path = host_and_port_;
+ return process_request(strm, req2, res2, false, error);
+ })) {
+ // Thread-safe to close everything because we are assuming there are no
+ // requests in flight
+ shutdown_ssl(socket, true);
+ shutdown_socket(socket);
+ close_socket(socket);
+ success = false;
+ return false;
+ }
+
+ if (res2.status == 407) {
+ if (!proxy_digest_auth_username_.empty() &&
+ !proxy_digest_auth_password_.empty()) {
+ std::map<std::string, std::string> auth;
+ if (detail::parse_www_authenticate(res2, auth, true)) {
+ Response res3;
+ if (!detail::process_client_socket(
+ socket.sock, read_timeout_sec_, read_timeout_usec_,
+ write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) {
+ Request req3;
+ req3.method = "CONNECT";
+ req3.path = host_and_port_;
+ req3.headers.insert(detail::make_digest_authentication_header(
+ req3, auth, 1, detail::random_string(10),
+ proxy_digest_auth_username_, proxy_digest_auth_password_,
+ true));
+ return process_request(strm, req3, res3, false, error);
+ })) {
+ // Thread-safe to close everything because we are assuming there are
+ // no requests in flight
+ shutdown_ssl(socket, true);
+ shutdown_socket(socket);
+ close_socket(socket);
+ success = false;
+ return false;
+ }
+ }
+ } else {
+ res = res2;
+ return false;
+ }
+ }
+
+ return true;
+}
+
+inline bool SSLClient::load_certs() {
+ bool ret = true;
+
+ std::call_once(initialize_cert_, [&]() {
+ std::lock_guard<std::mutex> guard(ctx_mutex_);
+ if (!ca_cert_file_path_.empty()) {
+ if (!SSL_CTX_load_verify_locations(ctx_, ca_cert_file_path_.c_str(),
+ nullptr)) {
+ ret = false;
+ }
+ } else if (!ca_cert_dir_path_.empty()) {
+ if (!SSL_CTX_load_verify_locations(ctx_, nullptr,
+ ca_cert_dir_path_.c_str())) {
+ ret = false;
+ }
+ } else {
+ auto loaded = false;
+#ifdef _WIN32
+ loaded =
+ detail::load_system_certs_on_windows(SSL_CTX_get_cert_store(ctx_));
+#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__)
+#if TARGET_OS_OSX
+ loaded = detail::load_system_certs_on_macos(SSL_CTX_get_cert_store(ctx_));
+#endif // TARGET_OS_OSX
+#endif // _WIN32
+ if (!loaded) { SSL_CTX_set_default_verify_paths(ctx_); }
+ }
+ });
+
+ return ret;
+}
+
+inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) {
+ auto ssl = detail::ssl_new(
+ socket.sock, ctx_, ctx_mutex_,
+ [&](SSL *ssl2) {
+ if (server_certificate_verification_) {
+ if (!load_certs()) {
+ error = Error::SSLLoadingCerts;
+ return false;
+ }
+ SSL_set_verify(ssl2, SSL_VERIFY_NONE, nullptr);
+ }
+
+ if (!detail::ssl_connect_or_accept_nonblocking(
+ socket.sock, ssl2, SSL_connect, connection_timeout_sec_,
+ connection_timeout_usec_)) {
+ error = Error::SSLConnection;
+ return false;
+ }
+
+ if (server_certificate_verification_) {
+ verify_result_ = SSL_get_verify_result(ssl2);
+
+ if (verify_result_ != X509_V_OK) {
+ error = Error::SSLServerVerification;
+ return false;
+ }
+
+ auto server_cert = SSL_get1_peer_certificate(ssl2);
+
+ if (server_cert == nullptr) {
+ error = Error::SSLServerVerification;
+ return false;
+ }
+
+ if (!verify_host(server_cert)) {
+ X509_free(server_cert);
+ error = Error::SSLServerVerification;
+ return false;
+ }
+ X509_free(server_cert);
+ }
+
+ return true;
+ },
+ [&](SSL *ssl2) {
+ SSL_set_tlsext_host_name(ssl2, host_.c_str());
+ return true;
+ });
+
+ if (ssl) {
+ socket.ssl = ssl;
+ return true;
+ }
+
+ shutdown_socket(socket);
+ close_socket(socket);
+ return false;
+}
+
+inline void SSLClient::shutdown_ssl(Socket &socket, bool shutdown_gracefully) {
+ shutdown_ssl_impl(socket, shutdown_gracefully);
+}
+
+inline void SSLClient::shutdown_ssl_impl(Socket &socket,
+ bool shutdown_gracefully) {
+ if (socket.sock == INVALID_SOCKET) {
+ assert(socket.ssl == nullptr);
+ return;
+ }
+ if (socket.ssl) {
+ detail::ssl_delete(ctx_mutex_, socket.ssl, shutdown_gracefully);
+ socket.ssl = nullptr;
+ }
+ assert(socket.ssl == nullptr);
+}
+
+inline bool
+SSLClient::process_socket(const Socket &socket,
+ std::function<bool(Stream &strm)> callback) {
+ assert(socket.ssl);
+ return detail::process_client_socket_ssl(
+ socket.ssl, socket.sock, read_timeout_sec_, read_timeout_usec_,
+ write_timeout_sec_, write_timeout_usec_, std::move(callback));
+}
+
+inline bool SSLClient::is_ssl() const { return true; }
+
+inline bool SSLClient::verify_host(X509 *server_cert) const {
+ /* Quote from RFC2818 section 3.1 "Server Identity"
+
+ If a subjectAltName extension of type dNSName is present, that MUST
+ be used as the identity. Otherwise, the (most specific) Common Name
+ field in the Subject field of the certificate MUST be used. Although
+ the use of the Common Name is existing practice, it is deprecated and
+ Certification Authorities are encouraged to use the dNSName instead.
+
+ Matching is performed using the matching rules specified by
+ [RFC2459]. If more than one identity of a given type is present in
+ the certificate (e.g., more than one dNSName name, a match in any one
+ of the set is considered acceptable.) Names may contain the wildcard
+ character * which is considered to match any single domain name
+ component or component fragment. E.g., *.a.com matches foo.a.com but
+ not bar.foo.a.com. f*.com matches foo.com but not bar.com.
+
+ In some cases, the URI is specified as an IP address rather than a
+ hostname. In this case, the iPAddress subjectAltName must be present
+ in the certificate and must exactly match the IP in the URI.
+
+ */
+ return verify_host_with_subject_alt_name(server_cert) ||
+ verify_host_with_common_name(server_cert);
+}
+
+inline bool
+SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {
+ auto ret = false;
+
+ auto type = GEN_DNS;
+
+ struct in6_addr addr6;
+ struct in_addr addr;
+ size_t addr_len = 0;
+
+#ifndef __MINGW32__
+ if (inet_pton(AF_INET6, host_.c_str(), &addr6)) {
+ type = GEN_IPADD;
+ addr_len = sizeof(struct in6_addr);
+ } else if (inet_pton(AF_INET, host_.c_str(), &addr)) {
+ type = GEN_IPADD;
+ addr_len = sizeof(struct in_addr);
+ }
+#endif
+
+ auto alt_names = static_cast<const struct stack_st_GENERAL_NAME *>(
+ X509_get_ext_d2i(server_cert, NID_subject_alt_name, nullptr, nullptr));
+
+ if (alt_names) {
+ auto dsn_matched = false;
+ auto ip_matched = false;
+
+ auto count = sk_GENERAL_NAME_num(alt_names);
+
+ for (decltype(count) i = 0; i < count && !dsn_matched; i++) {
+ auto val = sk_GENERAL_NAME_value(alt_names, i);
+ if (val->type == type) {
+ auto name = (const char *)ASN1_STRING_get0_data(val->d.ia5);
+ auto name_len = (size_t)ASN1_STRING_length(val->d.ia5);
+
+ switch (type) {
+ case GEN_DNS: dsn_matched = check_host_name(name, name_len); break;
+
+ case GEN_IPADD:
+ if (!memcmp(&addr6, name, addr_len) ||
+ !memcmp(&addr, name, addr_len)) {
+ ip_matched = true;
+ }
+ break;
+ }
+ }
+ }
+
+ if (dsn_matched || ip_matched) { ret = true; }
+ }
+
+ GENERAL_NAMES_free((STACK_OF(GENERAL_NAME) *)alt_names);
+ return ret;
+}
+
+inline bool SSLClient::verify_host_with_common_name(X509 *server_cert) const {
+ const auto subject_name = X509_get_subject_name(server_cert);
+
+ if (subject_name != nullptr) {
+ char name[BUFSIZ];
+ auto name_len = X509_NAME_get_text_by_NID(subject_name, NID_commonName,
+ name, sizeof(name));
+
+ if (name_len != -1) {
+ return check_host_name(name, static_cast<size_t>(name_len));
+ }
+ }
+
+ return false;
+}
+
+inline bool SSLClient::check_host_name(const char *pattern,
+ size_t pattern_len) const {
+ if (host_.size() == pattern_len && host_ == pattern) { return true; }
+
+ // Wildcard match
+ // https://bugs.launchpad.net/ubuntu/+source/firefox-3.0/+bug/376484
+ std::vector<std::string> pattern_components;
+ detail::split(&pattern[0], &pattern[pattern_len], '.',
+ [&](const char *b, const char *e) {
+ pattern_components.emplace_back(std::string(b, e));
+ });
+
+ if (host_components_.size() != pattern_components.size()) { return false; }
+
+ auto itr = pattern_components.begin();
+ for (const auto &h : host_components_) {
+ auto &p = *itr;
+ if (p != h && p != "*") {
+ auto partial_match = (p.size() > 0 && p[p.size() - 1] == '*' &&
+ !p.compare(0, p.size() - 1, h));
+ if (!partial_match) { return false; }
+ }
+ ++itr;
+ }
+
+ return true;
+}
+#endif
+
+// Universal client implementation
+inline Client::Client(const std::string &scheme_host_port)
+ : Client(scheme_host_port, std::string(), std::string()) {}
+
+inline Client::Client(const std::string &scheme_host_port,
+ const std::string &client_cert_path,
+ const std::string &client_key_path) {
+ const static std::regex re(
+ R"((?:([a-z]+):\/\/)?(?:\[([\d:]+)\]|([^:/?#]+))(?::(\d+))?)");
+
+ std::smatch m;
+ if (std::regex_match(scheme_host_port, m, re)) {
+ auto scheme = m[1].str();
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ if (!scheme.empty() && (scheme != "http" && scheme != "https")) {
+#else
+ if (!scheme.empty() && scheme != "http") {
+#endif
+#ifndef CPPHTTPLIB_NO_EXCEPTIONS
+ std::string msg = "'" + scheme + "' scheme is not supported.";
+ throw std::invalid_argument(msg);
+#endif
+ return;
+ }
+
+ auto is_ssl = scheme == "https";
+
+ auto host = m[2].str();
+ if (host.empty()) { host = m[3].str(); }
+
+ auto port_str = m[4].str();
+ auto port = !port_str.empty() ? std::stoi(port_str) : (is_ssl ? 443 : 80);
+
+ if (is_ssl) {
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+ cli_ = detail::make_unique<SSLClient>(host, port, client_cert_path,
+ client_key_path);
+ is_ssl_ = is_ssl;
+#endif
+ } else {
+ cli_ = detail::make_unique<ClientImpl>(host, port, client_cert_path,
+ client_key_path);
+ }
+ } else {
+ cli_ = detail::make_unique<ClientImpl>(scheme_host_port, 80,
+ client_cert_path, client_key_path);
+ }
+}
+
+inline Client::Client(const std::string &host, int port)
+ : cli_(detail::make_unique<ClientImpl>(host, port)) {}
+
+inline Client::Client(const std::string &host, int port,
+ const std::string &client_cert_path,
+ const std::string &client_key_path)
+ : cli_(detail::make_unique<ClientImpl>(host, port, client_cert_path,
+ client_key_path)) {}
+
+inline Client::~Client() {}
+
+inline bool Client::is_valid() const {
+ return cli_ != nullptr && cli_->is_valid();
+}
+
+inline Result Client::Get(const std::string &path) { return cli_->Get(path); }
+inline Result Client::Get(const std::string &path, const Headers &headers) {
+ return cli_->Get(path, headers);
+}
+inline Result Client::Get(const std::string &path, Progress progress) {
+ return cli_->Get(path, std::move(progress));
+}
+inline Result Client::Get(const std::string &path, const Headers &headers,
+ Progress progress) {
+ return cli_->Get(path, headers, std::move(progress));
+}
+inline Result Client::Get(const std::string &path,
+ ContentReceiver content_receiver) {
+ return cli_->Get(path, std::move(content_receiver));
+}
+inline Result Client::Get(const std::string &path, const Headers &headers,
+ ContentReceiver content_receiver) {
+ return cli_->Get(path, headers, std::move(content_receiver));
+}
+inline Result Client::Get(const std::string &path,
+ ContentReceiver content_receiver, Progress progress) {
+ return cli_->Get(path, std::move(content_receiver), std::move(progress));
+}
+inline Result Client::Get(const std::string &path, const Headers &headers,
+ ContentReceiver content_receiver, Progress progress) {
+ return cli_->Get(path, headers, std::move(content_receiver),
+ std::move(progress));
+}
+inline Result Client::Get(const std::string &path,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver) {
+ return cli_->Get(path, std::move(response_handler),
+ std::move(content_receiver));
+}
+inline Result Client::Get(const std::string &path, const Headers &headers,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver) {
+ return cli_->Get(path, headers, std::move(response_handler),
+ std::move(content_receiver));
+}
+inline Result Client::Get(const std::string &path,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver, Progress progress) {
+ return cli_->Get(path, std::move(response_handler),
+ std::move(content_receiver), std::move(progress));
+}
+inline Result Client::Get(const std::string &path, const Headers &headers,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver, Progress progress) {
+ return cli_->Get(path, headers, std::move(response_handler),
+ std::move(content_receiver), std::move(progress));
+}
+inline Result Client::Get(const std::string &path, const Params ¶ms,
+ const Headers &headers, Progress progress) {
+ return cli_->Get(path, params, headers, progress);
+}
+inline Result Client::Get(const std::string &path, const Params ¶ms,
+ const Headers &headers,
+ ContentReceiver content_receiver, Progress progress) {
+ return cli_->Get(path, params, headers, content_receiver, progress);
+}
+inline Result Client::Get(const std::string &path, const Params ¶ms,
+ const Headers &headers,
+ ResponseHandler response_handler,
+ ContentReceiver content_receiver, Progress progress) {
+ return cli_->Get(path, params, headers, response_handler, content_receiver,
+ progress);
+}
+
+inline Result Client::Head(const std::string &path) { return cli_->Head(path); }
+inline Result Client::Head(const std::string &path, const Headers &headers) {
+ return cli_->Head(path, headers);
+}
+
+inline Result Client::Post(const std::string &path) { return cli_->Post(path); }
+inline Result Client::Post(const std::string &path, const Headers &headers) {
+ return cli_->Post(path, headers);
+}
+inline Result Client::Post(const std::string &path, const char *body,
+ size_t content_length,
+ const std::string &content_type) {
+ return cli_->Post(path, body, content_length, content_type);
+}
+inline Result Client::Post(const std::string &path, const Headers &headers,
+ const char *body, size_t content_length,
+ const std::string &content_type) {
+ return cli_->Post(path, headers, body, content_length, content_type);
+}
+inline Result Client::Post(const std::string &path, const std::string &body,
+ const std::string &content_type) {
+ return cli_->Post(path, body, content_type);
+}
+inline Result Client::Post(const std::string &path, const Headers &headers,
+ const std::string &body,
+ const std::string &content_type) {
+ return cli_->Post(path, headers, body, content_type);
+}
+inline Result Client::Post(const std::string &path, size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return cli_->Post(path, content_length, std::move(content_provider),
+ content_type);
+}
+inline Result Client::Post(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return cli_->Post(path, std::move(content_provider), content_type);
+}
+inline Result Client::Post(const std::string &path, const Headers &headers,
+ size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return cli_->Post(path, headers, content_length, std::move(content_provider),
+ content_type);
+}
+inline Result Client::Post(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return cli_->Post(path, headers, std::move(content_provider), content_type);
+}
+inline Result Client::Post(const std::string &path, const Params ¶ms) {
+ return cli_->Post(path, params);
+}
+inline Result Client::Post(const std::string &path, const Headers &headers,
+ const Params ¶ms) {
+ return cli_->Post(path, headers, params);
+}
+inline Result Client::Post(const std::string &path,
+ const MultipartFormDataItems &items) {
+ return cli_->Post(path, items);
+}
+inline Result Client::Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items) {
+ return cli_->Post(path, headers, items);
+}
+inline Result Client::Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const std::string &boundary) {
+ return cli_->Post(path, headers, items, boundary);
+}
+inline Result
+Client::Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items) {
+ return cli_->Post(path, headers, items, provider_items);
+}
+inline Result Client::Put(const std::string &path) { return cli_->Put(path); }
+inline Result Client::Put(const std::string &path, const char *body,
+ size_t content_length,
+ const std::string &content_type) {
+ return cli_->Put(path, body, content_length, content_type);
+}
+inline Result Client::Put(const std::string &path, const Headers &headers,
+ const char *body, size_t content_length,
+ const std::string &content_type) {
+ return cli_->Put(path, headers, body, content_length, content_type);
+}
+inline Result Client::Put(const std::string &path, const std::string &body,
+ const std::string &content_type) {
+ return cli_->Put(path, body, content_type);
+}
+inline Result Client::Put(const std::string &path, const Headers &headers,
+ const std::string &body,
+ const std::string &content_type) {
+ return cli_->Put(path, headers, body, content_type);
+}
+inline Result Client::Put(const std::string &path, size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return cli_->Put(path, content_length, std::move(content_provider),
+ content_type);
+}
+inline Result Client::Put(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return cli_->Put(path, std::move(content_provider), content_type);
+}
+inline Result Client::Put(const std::string &path, const Headers &headers,
+ size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return cli_->Put(path, headers, content_length, std::move(content_provider),
+ content_type);
+}
+inline Result Client::Put(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return cli_->Put(path, headers, std::move(content_provider), content_type);
+}
+inline Result Client::Put(const std::string &path, const Params ¶ms) {
+ return cli_->Put(path, params);
+}
+inline Result Client::Put(const std::string &path, const Headers &headers,
+ const Params ¶ms) {
+ return cli_->Put(path, headers, params);
+}
+inline Result Client::Put(const std::string &path,
+ const MultipartFormDataItems &items) {
+ return cli_->Put(path, items);
+}
+inline Result Client::Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items) {
+ return cli_->Put(path, headers, items);
+}
+inline Result Client::Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const std::string &boundary) {
+ return cli_->Put(path, headers, items, boundary);
+}
+inline Result
+Client::Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items) {
+ return cli_->Put(path, headers, items, provider_items);
+}
+inline Result Client::Patch(const std::string &path) {
+ return cli_->Patch(path);
+}
+inline Result Client::Patch(const std::string &path, const char *body,
+ size_t content_length,
+ const std::string &content_type) {
+ return cli_->Patch(path, body, content_length, content_type);
+}
+inline Result Client::Patch(const std::string &path, const Headers &headers,
+ const char *body, size_t content_length,
+ const std::string &content_type) {
+ return cli_->Patch(path, headers, body, content_length, content_type);
+}
+inline Result Client::Patch(const std::string &path, const std::string &body,
+ const std::string &content_type) {
+ return cli_->Patch(path, body, content_type);
+}
+inline Result Client::Patch(const std::string &path, const Headers &headers,
+ const std::string &body,
+ const std::string &content_type) {
+ return cli_->Patch(path, headers, body, content_type);
+}
+inline Result Client::Patch(const std::string &path, size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return cli_->Patch(path, content_length, std::move(content_provider),
+ content_type);
+}
+inline Result Client::Patch(const std::string &path,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return cli_->Patch(path, std::move(content_provider), content_type);
+}
+inline Result Client::Patch(const std::string &path, const Headers &headers,
+ size_t content_length,
+ ContentProvider content_provider,
+ const std::string &content_type) {
+ return cli_->Patch(path, headers, content_length, std::move(content_provider),
+ content_type);
+}
+inline Result Client::Patch(const std::string &path, const Headers &headers,
+ ContentProviderWithoutLength content_provider,
+ const std::string &content_type) {
+ return cli_->Patch(path, headers, std::move(content_provider), content_type);
+}
+inline Result Client::Delete(const std::string &path) {
+ return cli_->Delete(path);
+}
+inline Result Client::Delete(const std::string &path, const Headers &headers) {
+ return cli_->Delete(path, headers);
+}
+inline Result Client::Delete(const std::string &path, const char *body,
+ size_t content_length,
+ const std::string &content_type) {
+ return cli_->Delete(path, body, content_length, content_type);
+}
+inline Result Client::Delete(const std::string &path, const Headers &headers,
+ const char *body, size_t content_length,
+ const std::string &content_type) {
+ return cli_->Delete(path, headers, body, content_length, content_type);
+}
+inline Result Client::Delete(const std::string &path, const std::string &body,
+ const std::string &content_type) {
+ return cli_->Delete(path, body, content_type);
+}
+inline Result Client::Delete(const std::string &path, const Headers &headers,
+ const std::string &body,
+ const std::string &content_type) {
+ return cli_->Delete(path, headers, body, content_type);
+}
+inline Result Client::Options(const std::string &path) {
+ return cli_->Options(path);
+}
+inline Result Client::Options(const std::string &path, const Headers &headers) {
+ return cli_->Options(path, headers);
+}
+
+inline bool Client::send(Request &req, Response &res, Error &error) {
+ return cli_->send(req, res, error);
+}
+
+inline Result Client::send(const Request &req) { return cli_->send(req); }
+
+inline size_t Client::is_socket_open() const { return cli_->is_socket_open(); }
+
+inline socket_t Client::socket() const { return cli_->socket(); }
+
+inline void Client::stop() { cli_->stop(); }
+
+inline void
+Client::set_hostname_addr_map(std::map<std::string, std::string> addr_map) {
+ cli_->set_hostname_addr_map(std::move(addr_map));
+}
+
+inline void Client::set_default_headers(Headers headers) {
+ cli_->set_default_headers(std::move(headers));
+}
+
+inline void Client::set_address_family(int family) {
+ cli_->set_address_family(family);
+}
+
+inline void Client::set_tcp_nodelay(bool on) { cli_->set_tcp_nodelay(on); }
+
+inline void Client::set_socket_options(SocketOptions socket_options) {
+ cli_->set_socket_options(std::move(socket_options));
+}
+
+inline void Client::set_connection_timeout(time_t sec, time_t usec) {
+ cli_->set_connection_timeout(sec, usec);
+}
+
+inline void Client::set_read_timeout(time_t sec, time_t usec) {
+ cli_->set_read_timeout(sec, usec);
+}
+
+inline void Client::set_write_timeout(time_t sec, time_t usec) {
+ cli_->set_write_timeout(sec, usec);
+}
+
+inline void Client::set_basic_auth(const std::string &username,
+ const std::string &password) {
+ cli_->set_basic_auth(username, password);
+}
+inline void Client::set_bearer_token_auth(const std::string &token) {
+ cli_->set_bearer_token_auth(token);
+}
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+inline void Client::set_digest_auth(const std::string &username,
+ const std::string &password) {
+ cli_->set_digest_auth(username, password);
+}
+#endif
+
+inline void Client::set_keep_alive(bool on) { cli_->set_keep_alive(on); }
+inline void Client::set_follow_location(bool on) {
+ cli_->set_follow_location(on);
+}
+
+inline void Client::set_url_encode(bool on) { cli_->set_url_encode(on); }
+
+inline void Client::set_compress(bool on) { cli_->set_compress(on); }
+
+inline void Client::set_decompress(bool on) { cli_->set_decompress(on); }
+
+inline void Client::set_interface(const std::string &intf) {
+ cli_->set_interface(intf);
+}
+
+inline void Client::set_proxy(const std::string &host, int port) {
+ cli_->set_proxy(host, port);
+}
+inline void Client::set_proxy_basic_auth(const std::string &username,
+ const std::string &password) {
+ cli_->set_proxy_basic_auth(username, password);
+}
+inline void Client::set_proxy_bearer_token_auth(const std::string &token) {
+ cli_->set_proxy_bearer_token_auth(token);
+}
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+inline void Client::set_proxy_digest_auth(const std::string &username,
+ const std::string &password) {
+ cli_->set_proxy_digest_auth(username, password);
+}
+#endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+inline void Client::enable_server_certificate_verification(bool enabled) {
+ cli_->enable_server_certificate_verification(enabled);
+}
+#endif
+
+inline void Client::set_logger(Logger logger) { cli_->set_logger(logger); }
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+inline void Client::set_ca_cert_path(const std::string &ca_cert_file_path,
+ const std::string &ca_cert_dir_path) {
+ cli_->set_ca_cert_path(ca_cert_file_path, ca_cert_dir_path);
+}
+
+inline void Client::set_ca_cert_store(X509_STORE *ca_cert_store) {
+ if (is_ssl_) {
+ static_cast<SSLClient &>(*cli_).set_ca_cert_store(ca_cert_store);
+ } else {
+ cli_->set_ca_cert_store(ca_cert_store);
+ }
+}
+
+inline long Client::get_openssl_verify_result() const {
+ if (is_ssl_) {
+ return static_cast<SSLClient &>(*cli_).get_openssl_verify_result();
+ }
+ return -1; // NOTE: -1 doesn't match any of X509_V_ERR_???
+}
+
+inline SSL_CTX *Client::ssl_context() const {
+ if (is_ssl_) { return static_cast<SSLClient &>(*cli_).ssl_context(); }
+ return nullptr;
+}
+#endif
+
+// ----------------------------------------------------------------------------
+
+} // namespace httplib
+
+#if defined(_WIN32) && defined(CPPHTTPLIB_USE_POLL)
+#undef poll
+#endif
+
+#endif // CPPHTTPLIB_HTTPLIB_H
diff --git a/llm/ext_server/server.cpp b/llm/ext_server/server.cpp
new file mode 100644
index 00000000..b43299e9
--- /dev/null
+++ b/llm/ext_server/server.cpp
@@ -0,0 +1,3279 @@
+// MIT License
+
+// Copyright (c) 2023 Georgi Gerganov
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "common.h"
+#include "llama.h"
+#include "grammar-parser.h"
+#include "utils.hpp"
+
+#include "../llava/clip.h"
+#include "../llava/llava.h"
+
+#include "stb_image.h"
+
+#ifndef NDEBUG
+// crash the server in debug mode, otherwise send an http 500 error
+#define CPPHTTPLIB_NO_EXCEPTIONS 1
+#endif
+// increase max payload length to allow use of larger context size
+#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 1048576
+#include "httplib.h"
+#include "json.hpp"
+
+#if defined(_WIN32)
+#include <windows.h>
+#include <errhandlingapi.h>
+#endif
+
+#include <algorithm>
+#include <cstddef>
+#include <thread>
+#include <chrono>
+#include <condition_variable>
+#include <atomic>
+#include <signal.h>
+
+using json = nlohmann::json;
+
+struct server_params {
+ std::string hostname = "127.0.0.1";
+ std::vector<std::string> api_keys;
+ std::string public_path = "examples/server/public";
+ int32_t port = 8080;
+ int32_t read_timeout = 600;
+ int32_t write_timeout = 600;
+ bool slots_endpoint = true;
+ bool metrics_endpoint = false;
+ int n_threads_http = -1;
+};
+
+bool server_verbose = false;
+bool server_log_json = false;
+
+enum stop_type {
+ STOP_FULL,
+ STOP_PARTIAL,
+};
+
+// TODO: can become bool if we can't find use of more states
+enum slot_state {
+ IDLE,
+ PROCESSING,
+};
+
+enum slot_command {
+ NONE,
+ LOAD_PROMPT,
+ RELEASE,
+};
+
+struct slot_params {
+ bool stream = true;
+ bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt
+
+ uint32_t seed = -1; // RNG seed
+ int32_t n_keep = 0; // number of tokens to keep from initial prompt
+ int32_t n_predict = -1; // new tokens to predict
+
+ std::vector<std::string> antiprompt;
+
+ json input_prefix;
+ json input_suffix;
+};
+
+struct slot_image {
+ int32_t id;
+
+ bool request_encode_image = false;
+ float * image_embedding = nullptr;
+ int32_t image_tokens = 0;
+
+ clip_image_u8 * img_data;
+
+ std::string prefix_prompt; // before of this image
+};
+
+struct server_slot {
+ int id;
+ int task_id = -1;
+
+ struct slot_params params;
+
+ slot_state state = IDLE;
+ slot_command command = NONE;
+
+ // used to determine the slot that has been used the longest
+ int64_t t_last_used = -1;
+
+ // generation props
+ int32_t n_ctx = 0; // context size per slot
+ int32_t n_past = 0;
+ int32_t n_decoded = 0;
+ int32_t n_remaining = -1;
+ int32_t i_batch = -1;
+ int32_t n_predict = -1;
+
+ int32_t n_prompt_tokens = 0;
+ int32_t n_prompt_tokens_processed = 0;
+
+ json prompt;
+ std::string generated_text;
+ llama_token sampled;
+ std::vector<llama_token> cache_tokens;
+ std::vector<completion_token_output> generated_token_probs;
+
+ bool embedding = false;
+ bool has_next_token = true;
+ bool truncated = false;
+ bool stopped_eos = false;
+ bool stopped_word = false;
+ bool stopped_limit = false;
+
+ std::string stopping_word;
+
+ // sampling
+ struct llama_sampling_params sparams;
+ llama_sampling_context *ctx_sampling = nullptr;
+
+ int32_t ga_i = 0; // group-attention state
+ int32_t ga_n = 1; // group-attention factor
+ int32_t ga_w = 512; // group-attention width
+
+ int32_t n_past_se = 0; // self-extend
+
+ // multimodal
+ std::vector<slot_image> images;
+
+ // stats
+ size_t n_sent_text = 0; // number of sent text character
+ size_t n_sent_token_probs = 0;
+
+ int64_t t_start_process_prompt;
+ int64_t t_start_genereration;
+
+ double t_prompt_processing; // ms
+ double t_token_generation; // ms
+
+ // multitasks
+ int multitask_id = -1;
+
+ void reset() {
+ n_prompt_tokens = 0;
+ generated_text = "";
+ truncated = false;
+ stopped_eos = false;
+ stopped_word = false;
+ stopped_limit = false;
+ stopping_word = "";
+ n_past = 0;
+ n_sent_text = 0;
+ n_sent_token_probs = 0;
+ ga_i = 0;
+ n_past_se = 0;
+
+ generated_token_probs.clear();
+
+ for (slot_image & img : images) {
+ free(img.image_embedding);
+ if (img.img_data) {
+ clip_image_u8_free(img.img_data);
+ }
+ img.prefix_prompt = "";
+ }
+
+ images.clear();
+ }
+
+ bool has_budget(gpt_params &global_params) {
+ if (params.n_predict == -1 && global_params.n_predict == -1) {
+ return true; // limitless
+ }
+
+ n_remaining = -1;
+
+ if (params.n_predict != -1) {
+ n_remaining = params.n_predict - n_decoded;
+ } else if (global_params.n_predict != -1) {
+ n_remaining = global_params.n_predict - n_decoded;
+ }
+
+ return n_remaining > 0; // no budget
+ }
+
+ bool available() const {
+ return state == IDLE && command == NONE;
+ }
+
+ bool is_processing() const {
+ return (state == IDLE && command == LOAD_PROMPT) || state == PROCESSING;
+ }
+
+ void add_token_string(const completion_token_output &token) {
+ if (command == RELEASE) {
+ return;
+ }
+ cache_tokens.push_back(token.tok);
+ generated_token_probs.push_back(token);
+ }
+
+ void release() {
+ if (state == PROCESSING)
+ {
+ t_token_generation = (ggml_time_us() - t_start_genereration) / 1e3;
+ command = RELEASE;
+ }
+ }
+
+ json get_formated_timings() {
+ return json
+ {
+ {"prompt_n", n_prompt_tokens_processed},
+ {"prompt_ms", t_prompt_processing},
+ {"prompt_per_token_ms", t_prompt_processing / n_prompt_tokens_processed},
+ {"prompt_per_second", 1e3 / t_prompt_processing * n_prompt_tokens_processed},
+
+ {"predicted_n", n_decoded},
+ {"predicted_ms", t_token_generation},
+ {"predicted_per_token_ms", t_token_generation / n_decoded},
+ {"predicted_per_second", 1e3 / t_token_generation * n_decoded},
+ };
+ }
+
+ void print_timings() const {
+ char buffer[512];
+ double t_token = t_prompt_processing / n_prompt_tokens_processed;
+ double n_tokens_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed;
+ snprintf(buffer, sizeof(buffer), "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)",
+ t_prompt_processing, n_prompt_tokens_processed,
+ t_token, n_tokens_second);
+ LOG_DEBUG(buffer, {
+ {"slot_id", id},
+ {"task_id", task_id},
+ {"t_prompt_processing", t_prompt_processing},
+ {"n_prompt_tokens_processed", n_prompt_tokens_processed},
+ {"t_token", t_token},
+ {"n_tokens_second", n_tokens_second},
+ });
+
+ t_token = t_token_generation / n_decoded;
+ n_tokens_second = 1e3 / t_token_generation * n_decoded;
+ snprintf(buffer, sizeof(buffer), "generation eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)",
+ t_token_generation, n_decoded,
+ t_token, n_tokens_second);
+ LOG_DEBUG(buffer, {
+ {"slot_id", id},
+ {"task_id", task_id},
+ {"t_token_generation", t_token_generation},
+ {"n_decoded", n_decoded},
+ {"t_token", t_token},
+ {"n_tokens_second", n_tokens_second},
+ });
+
+ snprintf(buffer, sizeof(buffer), " total time = %10.2f ms", t_prompt_processing + t_token_generation);
+ LOG_DEBUG(buffer, {
+ {"slot_id", id},
+ {"task_id", task_id},
+ {"t_prompt_processing", t_prompt_processing},
+ {"t_token_generation", t_token_generation},
+ {"t_total", t_prompt_processing + t_token_generation},
+ });
+ }
+};
+
+struct server_metrics {
+ uint64_t n_prompt_tokens_processed_total = 0;
+ uint64_t n_tokens_predicted_total = 0;
+
+ uint64_t n_prompt_tokens_processed = 0;
+ uint64_t t_prompt_processing = 0;
+
+ uint64_t n_tokens_predicted = 0;
+ uint64_t t_tokens_generation = 0;
+
+
+ void on_prompt_eval(const server_slot &slot) {
+ n_prompt_tokens_processed_total += slot.n_prompt_tokens_processed;
+ n_prompt_tokens_processed += slot.n_prompt_tokens_processed;
+ t_prompt_processing += slot.t_prompt_processing;
+ }
+
+ void on_prediction(const server_slot &slot) {
+ n_tokens_predicted_total += slot.n_decoded;
+ n_tokens_predicted += slot.n_decoded;
+ t_tokens_generation += slot.t_token_generation;
+ }
+
+ void reset_bucket() {
+ n_prompt_tokens_processed = 0;
+ t_prompt_processing = 0;
+ n_tokens_predicted = 0;
+ t_tokens_generation = 0;
+ }
+};
+
+struct llama_server_context
+{
+ llama_model *model = nullptr;
+ float modelProgress = 0.0;
+ llama_context *ctx = nullptr;
+
+ clip_ctx *clp_ctx = nullptr;
+
+ gpt_params params;
+
+ llama_batch batch;
+
+ bool multimodal = false;
+ bool clean_kv_cache = true;
+ bool all_slots_are_idle = false;
+ bool add_bos_token = true;
+
+ int32_t n_ctx; // total context for all clients / slots
+
+ // system prompt
+ bool system_need_update = false;
+
+ std::string system_prompt;
+ std::vector<llama_token> system_tokens;
+
+ std::string name_user; // this should be the antiprompt
+ std::string name_assistant;
+
+ // slots / clients
+ std::vector<server_slot> slots;
+
+ llama_server_queue queue_tasks;
+ llama_server_response queue_results;
+
+ server_metrics metrics;
+
+ ~llama_server_context()
+ {
+ if (clp_ctx)
+ {
+ LOG_DEBUG("freeing clip model", {});
+ clip_free(clp_ctx);
+ clp_ctx = nullptr;
+ }
+ if (ctx)
+ {
+ llama_free(ctx);
+ ctx = nullptr;
+ }
+ if (model)
+ {
+ llama_free_model(model);
+ model = nullptr;
+ }
+ }
+
+ bool load_model(const gpt_params ¶ms_)
+ {
+ params = params_;
+ if (!params.mmproj.empty()) {
+ multimodal = true;
+ LOG_DEBUG("Multi Modal Mode Enabled", {});
+ clp_ctx = clip_model_load(params.mmproj.c_str(), /*verbosity=*/ 1);
+ if(clp_ctx == nullptr) {
+ LOG_ERROR("unable to load clip model", {{"model", params.mmproj}});
+ return false;
+ }
+
+ if (params.n_ctx < 2048) { // request larger context for the image embedding
+ params.n_ctx = 2048;
+ }
+ }
+
+ auto init_result = llama_init_from_gpt_params(params);
+ model = init_result.model;
+ ctx = init_result.context;
+ if (model == nullptr)
+ {
+ LOG_ERROR("unable to load model", {{"model", params.model}});
+ return false;
+ }
+
+ if (multimodal) {
+ const int n_embd_clip = clip_n_mmproj_embd(clp_ctx);
+ const int n_embd_llm = llama_n_embd(model);
+ if (n_embd_clip != n_embd_llm) {
+ LOG_TEE("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_embd_clip, n_embd_llm);
+ llama_free(ctx);
+ llama_free_model(model);
+ return false;
+ }
+ }
+
+ n_ctx = llama_n_ctx(ctx);
+
+ add_bos_token = llama_add_bos_token(model);
+
+ return true;
+ }
+
+ void initialize() {
+ // create slots
+ all_slots_are_idle = true;
+
+ const int32_t n_ctx_slot = n_ctx / params.n_parallel;
+
+ LOG_DEBUG("initializing slots", {{"n_slots", params.n_parallel}});
+ for (int i = 0; i < params.n_parallel; i++)
+ {
+ server_slot slot;
+
+ slot.id = i;
+ slot.n_ctx = n_ctx_slot;
+ slot.n_predict = params.n_predict;
+
+ LOG_DEBUG("new slot", {
+ {"slot_id", slot.id},
+ {"n_ctx_slot", slot.n_ctx}
+ });
+
+ const int ga_n = params.grp_attn_n;
+ const int ga_w = params.grp_attn_w;
+
+ if (ga_n != 1) {
+ GGML_ASSERT(ga_n > 0 && "ga_n must be positive"); // NOLINT
+ GGML_ASSERT(ga_w % ga_n == 0 && "ga_w must be a multiple of ga_n"); // NOLINT
+ //GGML_ASSERT(n_ctx_train % ga_w == 0 && "n_ctx_train must be a multiple of ga_w"); // NOLINT
+ //GGML_ASSERT(n_ctx >= n_ctx_train * ga_n && "n_ctx must be at least n_ctx_train * ga_n"); // NOLINT
+
+ LOG_DEBUG("slot self-extend", {
+ {"slot_id", slot.id},
+ {"ga_n", ga_n},
+ {"ga_w", ga_w}
+ });
+ }
+
+ slot.ga_i = 0;
+ slot.ga_n = ga_n;
+ slot.ga_w = ga_w;
+
+ slot.reset();
+
+ slots.push_back(slot);
+ }
+
+ batch = llama_batch_init(n_ctx, 0, params.n_parallel);
+ }
+
+ std::vector<llama_token> tokenize(const json & json_prompt, bool add_bos) const
+ {
+ // TODO: currently, we tokenize using special tokens by default
+ // this is not always correct (see https://github.com/ggerganov/llama.cpp/pull/4160#issuecomment-1824826216)
+ // but it's better compared to completely ignoring ChatML and other chat templates
+ const bool TMP_FORCE_SPECIAL = true;
+
+ // If `add_bos` is true, we only add BOS, when json_prompt is a string,
+ // or the first element of the json_prompt array is a string.
+ std::vector<llama_token> prompt_tokens;
+
+ if (json_prompt.is_array())
+ {
+ bool first = true;
+ for (const auto& p : json_prompt)
+ {
+ if (p.is_string())
+ {
+ auto s = p.template get<std::string>();
+ std::vector<llama_token> p;
+ if (first)
+ {
+ p = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL);
+ first = false;
+ }
+ else
+ {
+ p = ::llama_tokenize(ctx, s, false, TMP_FORCE_SPECIAL);
+ }
+ prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());
+ }
+ else
+ {
+ if (first)
+ {
+ first = false;
+ }
+ prompt_tokens.push_back(p.template get<llama_token>());
+ }
+ }
+ }
+ else
+ {
+ auto s = json_prompt.template get<std::string>();
+ prompt_tokens = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL);
+ }
+
+ return prompt_tokens;
+ }
+
+ server_slot* get_slot(int id) {
+ int64_t t_last = ggml_time_us();
+ server_slot *last_used = nullptr;
+
+ for (server_slot & slot : slots)
+ {
+ if (slot.id == id && slot.available())
+ {
+ return &slot;
+ }
+
+ if (slot.available() && slot.t_last_used < t_last)
+ {
+ last_used = &slot;
+ t_last = slot.t_last_used;
+ }
+ }
+
+ return last_used;
+ }
+
+ bool launch_slot_with_data(server_slot* &slot, json data) {
+ slot_params default_params;
+ llama_sampling_params default_sparams;
+
+ slot->params.stream = json_value(data, "stream", false);
+ slot->params.cache_prompt = json_value(data, "cache_prompt", false);
+ slot->params.n_predict = json_value(data, "n_predict", default_params.n_predict);
+ slot->sparams.top_k = json_value(data, "top_k", default_sparams.top_k);
+ slot->sparams.top_p = json_value(data, "top_p", default_sparams.top_p);
+ slot->sparams.min_p = json_value(data, "min_p", default_sparams.min_p);
+ slot->sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z);
+ slot->sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p);
+ slot->sparams.temp = json_value(data, "temperature", default_sparams.temp);
+ slot->sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range);
+ slot->sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent);
+ slot->sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n);
+ slot->sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat);
+ slot->sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq);
+ slot->sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present);
+ slot->sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat);
+ slot->sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau);
+ slot->sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta);
+ slot->sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl);
+ slot->params.n_keep = json_value(data, "n_keep", slot->params.n_keep);
+ slot->sparams.seed = json_value(data, "seed", default_params.seed);
+ slot->sparams.grammar = json_value(data, "grammar", default_sparams.grammar);
+ slot->sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs);
+ slot->sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep);
+
+ if (slot->n_predict > 0 && slot->params.n_predict > slot->n_predict) {
+ // Might be better to reject the request with a 400 ?
+ LOG_WARNING("Max tokens to predict exceeds server configuration", {
+ {"params.n_predict", slot->params.n_predict},
+ {"slot.n_predict", slot->n_predict},
+ });
+ slot->params.n_predict = slot->n_predict;
+ }
+
+ if (data.count("input_suffix") != 0)
+ {
+ slot->params.input_suffix = data["input_suffix"];
+ }
+ else
+ {
+ slot->params.input_suffix = "";
+ }
+
+ if (data.count("prompt") != 0)
+ {
+ slot->prompt = data["prompt"];
+ }
+ else
+ {
+ slot->prompt = "";
+ }
+
+ slot->sparams.penalty_prompt_tokens.clear();
+ slot->sparams.use_penalty_prompt_tokens = false;
+ const auto &penalty_prompt = data.find("penalty_prompt");
+ if (penalty_prompt != data.end())
+ {
+ if (penalty_prompt->is_string())
+ {
+ const auto penalty_prompt_string = penalty_prompt->get<std::string>();
+ auto penalty_tokens = llama_tokenize(model, penalty_prompt_string, false);
+ slot->sparams.penalty_prompt_tokens.swap(penalty_tokens);
+ if (slot->params.n_predict > 0)
+ {
+ slot->sparams.penalty_prompt_tokens.reserve(slot->sparams.penalty_prompt_tokens.size() + slot->params.n_predict);
+ }
+ slot->sparams.use_penalty_prompt_tokens = true;
+ }
+ else if (penalty_prompt->is_array())
+ {
+ const auto n_tokens = penalty_prompt->size();
+ slot->sparams.penalty_prompt_tokens.reserve(n_tokens + std::max(0, slot->params.n_predict));
+ const int n_vocab = llama_n_vocab(model);
+ for (const auto &penalty_token : *penalty_prompt)
+ {
+ if (penalty_token.is_number_integer())
+ {
+ const auto tok = penalty_token.get<llama_token>();
+ if (tok >= 0 && tok < n_vocab)
+ {
+ slot->sparams.penalty_prompt_tokens.push_back(tok);
+ }
+ }
+ }
+ slot->sparams.use_penalty_prompt_tokens = true;
+ }
+ }
+
+ slot->sparams.logit_bias.clear();
+
+ if (json_value(data, "ignore_eos", false))
+ {
+ slot->sparams.logit_bias[llama_token_eos(model)] = -INFINITY;
+ }
+
+ const auto &logit_bias = data.find("logit_bias");
+ if (logit_bias != data.end() && logit_bias->is_array())
+ {
+ const int n_vocab = llama_n_vocab(model);
+ for (const auto &el : *logit_bias)
+ {
+ if (el.is_array() && el.size() == 2)
+ {
+ float bias;
+ if (el[1].is_number())
+ {
+ bias = el[1].get<float>();
+ }
+ else if (el[1].is_boolean() && !el[1].get<bool>())
+ {
+ bias = -INFINITY;
+ }
+ else
+ {
+ continue;
+ }
+
+ if (el[0].is_number_integer())
+ {
+ llama_token tok = el[0].get<llama_token>();
+ if (tok >= 0 && tok < n_vocab)
+ {
+ slot->sparams.logit_bias[tok] = bias;
+ }
+ }
+ else if (el[0].is_string())
+ {
+ auto toks = llama_tokenize(model, el[0].get<std::string>(), false);
+ for (auto tok : toks)
+ {
+ slot->sparams.logit_bias[tok] = bias;
+ }
+ }
+ }
+ }
+ }
+
+ slot->params.antiprompt.clear();
+
+ const auto &stop = data.find("stop");
+ if (stop != data.end() && stop->is_array())
+ {
+ for (const auto &word : *stop)
+ {
+ if (!word.empty())
+ {
+ slot->params.antiprompt.push_back(word);
+ }
+ }
+ }
+
+ const auto &samplers_sequence = data.find("samplers");
+ if (samplers_sequence != data.end() && samplers_sequence->is_array())
+ {
+ std::vector<std::string> sampler_names;
+ for (const auto &sampler_name : *samplers_sequence)
+ {
+ if (sampler_name.is_string())
+ {
+ sampler_names.emplace_back(sampler_name);
+ }
+ }
+ slot->sparams.samplers_sequence = llama_sampling_types_from_names(sampler_names, false);
+ }
+ else
+ {
+ slot->sparams.samplers_sequence = default_sparams.samplers_sequence;
+ }
+
+ if (multimodal)
+ {
+ const auto &images_data = data.find("image_data");
+ if (images_data != data.end() && images_data->is_array())
+ {
+ for (const auto &img : *images_data)
+ {
+ const std::vector<uint8_t> image_buffer = base64_decode(img["data"].get<std::string>());
+
+ slot_image img_sl;
+ img_sl.id = img.count("id") != 0 ? img["id"].get<int>() : slot->images.size();
+ img_sl.img_data = clip_image_u8_init();
+ if (!clip_image_load_from_bytes(image_buffer.data(), image_buffer.size(), img_sl.img_data))
+ {
+ LOG_ERROR("failed to load image", {
+ {"slot_id", slot->id},
+ {"img_sl_id", img_sl.id}
+ });
+ return false;
+ }
+ LOG_VERBOSE("image loaded", {
+ {"slot_id", slot->id},
+ {"img_sl_id", img_sl.id}
+ });
+ img_sl.request_encode_image = true;
+ slot->images.push_back(img_sl);
+ }
+ // process prompt
+ // example: system prompt [img-102] user [img-103] describe [img-134] -> [{id: 102, prefix: 'system prompt '}, {id: 103, prefix: ' user '}, {id: 134, prefix: ' describe '}]}
+ if (slot->images.size() > 0 && !slot->prompt.is_array())
+ {
+ std::string prompt = slot->prompt.get<std::string>();
+ size_t pos = 0, begin_prefix = 0;
+ std::string pattern = "[img-";
+ while ((pos = prompt.find(pattern, pos)) != std::string::npos) {
+ size_t end_prefix = pos;
+ pos += pattern.length();
+ size_t end_pos = prompt.find(']', pos);
+ if (end_pos != std::string::npos)
+ {
+ std::string image_id = prompt.substr(pos, end_pos - pos);
+ try
+ {
+ int img_id = std::stoi(image_id);
+ bool found = false;
+ for (slot_image &img : slot->images)
+ {
+ if (img.id == img_id) {
+ found = true;
+ img.prefix_prompt = prompt.substr(begin_prefix, end_prefix - begin_prefix);
+ begin_prefix = end_pos + 1;
+ break;
+ }
+ }
+ if (!found) {
+ LOG_TEE("ERROR: Image with id: %i, not found.\n", img_id);
+ slot->images.clear();
+ return false;
+ }
+ } catch (const std::invalid_argument& e) {
+ LOG_TEE("Invalid image number id in prompt\n");
+ slot->images.clear();
+ return false;
+ }
+ }
+ }
+ slot->prompt = "";
+ slot->params.input_suffix = prompt.substr(begin_prefix);
+ slot->params.cache_prompt = false; // multimodal doesn't support cache prompt
+ }
+ }
+ }
+
+ if (slot->ctx_sampling != nullptr)
+ {
+ llama_sampling_free(slot->ctx_sampling);
+ }
+ slot->ctx_sampling = llama_sampling_init(slot->sparams);
+ slot->command = LOAD_PROMPT;
+
+ all_slots_are_idle = false;
+
+ LOG_DEBUG("slot is processing task", {
+ {"slot_id", slot->id},
+ {"task_id", slot->task_id},
+ });
+
+ return true;
+ }
+
+ void kv_cache_clear() {
+ // clear the entire KV cache
+ llama_kv_cache_clear(ctx);
+ clean_kv_cache = false;
+ }
+
+ void system_prompt_update() {
+ kv_cache_clear();
+ system_tokens.clear();
+
+ if (!system_prompt.empty()) {
+ system_tokens = ::llama_tokenize(ctx, system_prompt, true);
+
+ llama_batch_clear(batch);
+
+ for (int i = 0; i < (int)system_tokens.size(); ++i)
+ {
+ llama_batch_add(batch, system_tokens[i], i, { 0 }, false);
+ }
+
+ for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += params.n_batch)
+ {
+ const int32_t n_tokens = std::min(params.n_batch, (int32_t) (batch.n_tokens - i));
+ llama_batch batch_view = {
+ n_tokens,
+ batch.token + i,
+ nullptr,
+ batch.pos + i,
+ batch.n_seq_id + i,
+ batch.seq_id + i,
+ batch.logits + i,
+ 0, 0, 0, // unused
+ };
+ if (llama_decode(ctx, batch_view) != 0)
+ {
+ LOG_TEE("%s: llama_decode() failed\n", __func__);
+ return;
+ }
+ }
+
+ // assign the system KV cache to all parallel sequences
+ for (int32_t i = 1; i < params.n_parallel; ++i)
+ {
+ llama_kv_cache_seq_cp(ctx, 0, i, 0, system_tokens.size());
+ }
+ }
+
+ LOG_TEE("system prompt updated\n");
+ system_need_update = false;
+ }
+
+ void system_prompt_notify() {
+ // release all slots
+ for (server_slot &slot : slots)
+ {
+ slot.release();
+ }
+
+ system_need_update = true;
+ }
+
+ static size_t find_stopping_strings(const std::string &text, const size_t last_token_size,
+ const stop_type type, server_slot &slot)
+ {
+ size_t stop_pos = std::string::npos;
+
+ for (const std::string &word : slot.params.antiprompt)
+ {
+ size_t pos;
+ if (type == STOP_FULL)
+ {
+ const size_t tmp = word.size() + last_token_size;
+ const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;
+ pos = text.find(word, from_pos);
+ }
+ else
+ {
+ pos = find_partial_stop_string(word, text);
+ }
+ if (pos != std::string::npos &&
+ (stop_pos == std::string::npos || pos < stop_pos))
+ {
+ if (type == STOP_FULL)
+ {
+ slot.stopped_word = true;
+ slot.stopping_word = word;
+ slot.has_next_token = false;
+ }
+ stop_pos = pos;
+ }
+ }
+
+ return stop_pos;
+ }
+
+ bool process_token(completion_token_output &result, server_slot &slot) {
+ // remember which tokens were sampled - used for repetition penalties during sampling
+ const std::string token_str = llama_token_to_piece(ctx, result.tok);
+ slot.sampled = result.tok;
+
+ // search stop word and delete it
+ if (!llama_token_is_eog(model, result.tok))
+ slot.generated_text += token_str;
+
+ slot.has_next_token = true;
+
+ if (slot.ctx_sampling->params.use_penalty_prompt_tokens && result.tok != -1)
+ {
+ // we can change penalty_prompt_tokens because it is always created from scratch each request
+ slot.ctx_sampling->params.penalty_prompt_tokens.push_back(result.tok);
+ }
+
+ // check if there is incomplete UTF-8 character at the end
+ bool incomplete = false;
+ for (unsigned i = 1; i < 5 && i <= slot.generated_text.size(); ++i)
+ {
+ unsigned char c = slot.generated_text[slot.generated_text.size() - i];
+ if ((c & 0xC0) == 0x80)
+ {
+ // continuation byte: 10xxxxxx
+ continue;
+ }
+ if ((c & 0xE0) == 0xC0)
+ {
+ // 2-byte character: 110xxxxx ...
+ incomplete = i < 2;
+ }
+ else if ((c & 0xF0) == 0xE0)
+ {
+ // 3-byte character: 1110xxxx ...
+ incomplete = i < 3;
+ }
+ else if ((c & 0xF8) == 0xF0)
+ {
+ // 4-byte character: 11110xxx ...
+ incomplete = i < 4;
+ }
+ // else 1-byte character or invalid byte
+ break;
+ }
+
+ if (!incomplete)
+ {
+ size_t pos = std::min(slot.n_sent_text, slot.generated_text.size());
+
+ if (!llama_token_is_eog(model, result.tok)) {
+ const std::string str_test = slot.generated_text.substr(pos);
+ bool is_stop_full = false;
+ size_t stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_FULL, slot);
+ if (stop_pos != std::string::npos)
+ {
+ is_stop_full = true;
+ slot.generated_text.erase(
+ slot.generated_text.begin() + pos + stop_pos,
+ slot.generated_text.end());
+ pos = std::min(slot.n_sent_text, slot.generated_text.size());
+ }
+ else
+ {
+ is_stop_full = false;
+ stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_PARTIAL, slot);
+ }
+
+ // check if there is any token to predict
+ if (stop_pos == std::string::npos || (!slot.has_next_token && !is_stop_full && stop_pos > 0))
+ {
+ // no send the stop word in the response
+ result.text_to_send = slot.generated_text.substr(pos, std::string::npos);
+ slot.n_sent_text += result.text_to_send.size();
+ // add the token to slot queue and cache
+ }
+ } else {
+ result.text_to_send = slot.generated_text.substr(pos, std::string::npos);
+ slot.n_sent_text += result.text_to_send.size();
+ }
+
+ if (slot.params.stream)
+ {
+ send_partial_response(slot, result);
+ }
+ }
+
+ slot.add_token_string(result);
+
+ if (incomplete)
+ {
+ slot.has_next_token = true;
+ }
+
+ // check the limits
+ if (slot.n_decoded > 0 && slot.has_next_token && !slot.has_budget(params))
+ {
+ slot.stopped_limit = true;
+ slot.has_next_token = false;
+ }
+
+ if (!slot.cache_tokens.empty() && llama_token_is_eog(model, result.tok))
+ {
+ slot.stopped_eos = true;
+ slot.has_next_token = false;
+ LOG_VERBOSE("eos token found", {});
+ }
+
+ LOG_VERBOSE("next token", {
+ {"token", result.tok},
+ {"token_text", tokens_to_output_formatted_string(ctx, result.tok)},
+ {"has_next_token", slot.has_next_token},
+ {"n_remain", slot.n_remaining},
+ {"num_tokens_predicted", slot.n_decoded},
+ {"stopped_eos", slot.stopped_eos},
+ {"stopped_word", slot.stopped_word},
+ {"stopped_limit", slot.stopped_limit},
+ {"stopping_word", slot.stopping_word},
+ });
+
+ return slot.has_next_token; // continue
+ }
+
+ bool process_images(server_slot &slot) const
+ {
+ for (slot_image &img : slot.images)
+ {
+ if (!img.request_encode_image)
+ {
+ continue;
+ }
+
+ if (!llava_image_embed_make_with_clip_img(clp_ctx, params.cpuparams.n_threads, img.img_data, &img.image_embedding, &img.image_tokens)) {
+ LOG_TEE("Error processing the given image");
+ return false;
+ }
+
+
+ img.request_encode_image = false;
+ }
+
+ return slot.images.size() > 0;
+ }
+
+ void send_error(task_server& task, const std::string &error)
+ {
+ LOG_TEE("task %i - error: %s\n", task.id, error.c_str());
+ task_result res;
+ res.id = task.id;
+ res.multitask_id = task.multitask_id;
+ res.stop = false;
+ res.error = true;
+ res.result_json = { { "content", error } };
+ queue_results.send(res);
+ }
+
+ json get_formated_generation(server_slot &slot)
+ {
+ const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model));
+ const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() &&
+ eos_bias->second < 0.0f && std::isinf(eos_bias->second);
+ std::vector<std::string> samplers_sequence;
+ for (const auto &sampler_type : slot.sparams.samplers_sequence)
+ {
+ samplers_sequence.emplace_back(llama_sampling_type_to_str(sampler_type));
+ }
+
+ return json {
+ {"n_ctx", slot.n_ctx},
+ {"n_predict", slot.n_predict},
+ {"model", params.model_alias},
+ {"seed", slot.params.seed},
+ {"temperature", slot.sparams.temp},
+ {"dynatemp_range", slot.sparams.dynatemp_range},
+ {"dynatemp_exponent", slot.sparams.dynatemp_exponent},
+ {"top_k", slot.sparams.top_k},
+ {"top_p", slot.sparams.top_p},
+ {"min_p", slot.sparams.min_p},
+ {"tfs_z", slot.sparams.tfs_z},
+ {"typical_p", slot.sparams.typical_p},
+ {"repeat_last_n", slot.sparams.penalty_last_n},
+ {"repeat_penalty", slot.sparams.penalty_repeat},
+ {"presence_penalty", slot.sparams.penalty_present},
+ {"frequency_penalty", slot.sparams.penalty_freq},
+ {"penalty_prompt_tokens", slot.sparams.penalty_prompt_tokens},
+ {"use_penalty_prompt_tokens", slot.sparams.use_penalty_prompt_tokens},
+ {"mirostat", slot.sparams.mirostat},
+ {"mirostat_tau", slot.sparams.mirostat_tau},
+ {"mirostat_eta", slot.sparams.mirostat_eta},
+ {"penalize_nl", slot.sparams.penalize_nl},
+ {"stop", slot.params.antiprompt},
+ {"n_predict", slot.params.n_predict},
+ {"n_keep", params.n_keep},
+ {"ignore_eos", ignore_eos},
+ {"stream", slot.params.stream},
+ {"logit_bias", slot.sparams.logit_bias},
+ {"n_probs", slot.sparams.n_probs},
+ {"min_keep", slot.sparams.min_keep},
+ {"grammar", slot.sparams.grammar},
+ {"samplers", samplers_sequence}
+ };
+ }
+
+ void send_partial_response(server_slot &slot, completion_token_output tkn)
+ {
+ task_result res;
+ res.id = slot.task_id;
+ res.multitask_id = slot.multitask_id;
+ res.error = false;
+ res.stop = false;
+
+ res.result_json = json
+ {
+ {"stop", false},
+ {"slot_id", slot.id},
+ {"multimodal", multimodal}
+ };
+
+ res.result_json["content"] = tkn.text_to_send;
+
+ if (slot.sparams.n_probs > 0)
+ {
+ std::vector<completion_token_output> probs_output = {};
+ const std::vector<llama_token> to_send_toks = llama_tokenize(ctx, tkn.text_to_send, false);
+ size_t probs_pos = std::min(slot.n_sent_token_probs, slot.generated_token_probs.size());
+ size_t probs_stop_pos = std::min(slot.n_sent_token_probs + to_send_toks.size(), slot.generated_token_probs.size());
+ if (probs_pos < probs_stop_pos)
+ {
+ probs_output = std::vector<completion_token_output>(slot.generated_token_probs.begin() + probs_pos, slot.generated_token_probs.begin() + probs_stop_pos);
+ }
+ slot.n_sent_token_probs = probs_stop_pos;
+ res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs_output);
+ }
+
+ queue_results.send(res);
+ }
+
+ void send_final_response(server_slot &slot)
+ {
+ task_result res;
+ res.id = slot.task_id;
+ res.multitask_id = slot.multitask_id;
+ res.error = false;
+ res.stop = true;
+
+ res.result_json = json
+ {
+ {"content", !slot.params.stream ? slot.generated_text : ""},
+ {"slot_id", slot.id},
+ {"stop", true},
+ {"model", params.model_alias},
+ {"tokens_predicted", slot.n_decoded},
+ {"tokens_evaluated", slot.n_prompt_tokens},
+ {"truncated", slot.truncated},
+ {"stopped_eos", slot.stopped_eos},
+ {"stopped_word", slot.stopped_word},
+ {"stopped_limit", slot.stopped_limit},
+ {"stopping_word", slot.stopping_word},
+ {"tokens_cached", slot.n_past},
+ {"timings", slot.get_formated_timings()}
+ };
+
+ if (slot.sparams.n_probs > 0)
+ {
+ std::vector<completion_token_output> probs = {};
+ if (!slot.params.stream && slot.stopped_word)
+ {
+ const std::vector<llama_token> stop_word_toks = llama_tokenize(ctx, slot.stopping_word, false);
+ probs = std::vector<completion_token_output>(slot.generated_token_probs.begin(), slot.generated_token_probs.end() - stop_word_toks.size());
+ }
+ else
+ {
+ probs = std::vector<completion_token_output>(
+ slot.generated_token_probs.begin(),
+ slot.generated_token_probs.end());
+ }
+ res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs);
+ }
+
+ queue_results.send(res);
+ }
+
+ void send_embedding(server_slot & slot, const llama_batch & batch)
+ {
+ task_result res;
+ res.id = slot.task_id;
+ res.multitask_id = slot.multitask_id;
+ res.error = false;
+ res.stop = true;
+
+ const int n_embd = llama_n_embd(model);
+
+ if (!params.embedding)
+ {
+ LOG_WARNING("embedding disabled", {{"params.embedding", params.embedding}});
+ res.result_json = json
+ {
+ {"embedding", std::vector<float>(n_embd, 0.0f)},
+ };
+ }
+ else
+ {
+ for (int i = 0; i < batch.n_tokens; ++i) {
+ if (!batch.logits[i] || batch.seq_id[i][0] != slot.id) {
+ continue;
+ }
+
+ const float * embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]);
+ if (embd == NULL) {
+ embd = llama_get_embeddings_ith(ctx, i);
+ if (embd == NULL) {
+ LOG_ERROR("failed to get embeddings for token", {{"token", batch.token[i]}, {"seq_id", batch.seq_id[i][0]}});
+ res.result_json = json
+ {
+ {"embedding", std::vector<float>(n_embd, 0.0f)},
+ };
+ continue;
+ }
+ }
+
+ res.result_json = json
+ {
+ {"embedding", std::vector<float>(embd, embd + n_embd)},
+ };
+ }
+ }
+ queue_results.send(res);
+ }
+
+ void request_completion(int task_id, json data, bool embedding, int multitask_id)
+ {
+ task_server task;
+ task.id = task_id;
+ task.target_id = 0;
+ task.data = std::move(data);
+ task.embedding_mode = embedding;
+ task.type = TASK_TYPE_COMPLETION;
+ task.multitask_id = multitask_id;
+
+ // when a completion task's prompt array is not a singleton, we split it into multiple requests
+ // otherwise, it's a single-prompt task, we actually queue it
+ // if there's numbers in the prompt array it will be treated as an array of tokens
+ if (task.data.count("prompt") != 0 && task.data.at("prompt").size() > 1) {
+ bool numbers = false;
+ for (const auto& e : task.data.at("prompt")) {
+ if (e.is_number()) {
+ numbers = true;
+ break;
+ }
+ }
+
+ // NOTE: split_multiprompt_task() does not handle a mix of strings and numbers,
+ // it will completely stall the server. I don't know where the bug for this is.
+ //
+ // if there are numbers, it needs to be treated like a single prompt,
+ // queue_tasks handles a mix of strings and numbers just fine.
+ if (numbers) {
+ queue_tasks.post(task);
+ } else {
+ split_multiprompt_task(task_id, task);
+ }
+ } else {
+ // an empty prompt can make slot become buggy
+ if (task.data.contains("prompt") && task.data["prompt"].is_string() && task.data["prompt"].get<std::string>().empty()) {
+ task.data["prompt"] = " "; // add a space so that we have one token
+ }
+ queue_tasks.post(task);
+ }
+ }
+
+ // for multiple images processing
+ bool ingest_images(server_slot &slot, int n_batch)
+ {
+ int image_idx = 0;
+
+ while (image_idx < (int) slot.images.size())
+ {
+ slot_image &img = slot.images[image_idx];
+
+ // process prefix prompt
+ for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch)
+ {
+ const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i));
+ llama_batch batch_view = {
+ n_tokens,
+ batch.token + i,
+ nullptr,
+ batch.pos + i,
+ batch.n_seq_id + i,
+ batch.seq_id + i,
+ batch.logits + i,
+ 0, 0, 0, // unused
+ };
+ if (llama_decode(ctx, batch_view))
+ {
+ LOG_TEE("%s : failed to eval\n", __func__);
+ return false;
+ }
+ }
+
+ // process image with llm
+ for (int i = 0; i < img.image_tokens; i += n_batch)
+ {
+ int n_eval = img.image_tokens - i;
+ if (n_eval > n_batch)
+ {
+ n_eval = n_batch;
+ }
+
+ const int n_embd = llama_n_embd(model);
+ llama_batch batch_img = {
+ n_eval,
+ nullptr,
+ (img.image_embedding + i * n_embd),
+ nullptr,
+ nullptr,
+ nullptr,
+ nullptr,
+ slot.n_past,
+ 1, 0
+ };
+ if (llama_decode(ctx, batch_img))
+ {
+ LOG_TEE("%s : failed to eval image\n", __func__);
+ return false;
+ }
+ slot.n_past += n_eval;
+ }
+ image_idx++;
+
+ llama_batch_clear(batch);
+
+ // append prefix of next image
+ const auto json_prompt = (image_idx >= (int) slot.images.size()) ?
+ slot.params.input_suffix : // no more images, then process suffix prompt
+ (json)(slot.images[image_idx].prefix_prompt);
+
+ std::vector<llama_token> append_tokens = tokenize(json_prompt, false); // has next image
+ for (int i = 0; i < (int) append_tokens.size(); ++i)
+ {
+ llama_batch_add(batch, append_tokens[i], system_tokens.size() + slot.n_past, { slot.id }, true);
+ slot.n_past += 1;
+ }
+ }
+
+ return true;
+ }
+
+ void request_cancel(int task_id)
+ {
+ task_server task;
+ task.type = TASK_TYPE_CANCEL;
+ task.target_id = task_id;
+ queue_tasks.post(task);
+ }
+
+ void split_multiprompt_task(int multitask_id, task_server& multiprompt_task)
+ {
+ int prompt_count = multiprompt_task.data.at("prompt").size();
+ if (prompt_count <= 1) {
+ send_error(multiprompt_task, "error while handling multiple prompts");
+ return;
+ }
+
+ // generate all the ID for subtask
+ std::vector<int> subtask_ids(prompt_count);
+ for (int i = 0; i < prompt_count; i++)
+ {
+ subtask_ids[i] = queue_tasks.get_new_id();
+ }
+
+ // queue up the multitask so we can track its subtask progression
+ queue_tasks.add_multitask(multitask_id, subtask_ids);
+
+ // add subtasks
+ for (int i = 0; i < prompt_count; i++)
+ {
+ json subtask_data = multiprompt_task.data;
+ subtask_data["prompt"] = subtask_data["prompt"][i];
+
+ // subtasks inherit everything else (embedding mode, etc.)
+ request_completion(subtask_ids[i], subtask_data, multiprompt_task.embedding_mode, multitask_id);
+ }
+ }
+
+ std::string common_prefix(const std::string& str1, const std::string& str2) {
+ auto mismatch_pair = std::mismatch(str1.begin(), str1.end(), str2.begin());
+ return std::string(str1.begin(), mismatch_pair.first);
+ }
+
+ // Find the slot that has the greatest common prefix
+ server_slot *prefix_slot(const json &prompt) {
+ if (!prompt.is_string()) {
+ return nullptr;
+ }
+
+ std::string prompt_str = prompt.get<std::string>();
+ server_slot *slot = nullptr;
+ size_t longest = 0;
+
+ for (server_slot &s : slots) {
+ if (s.available() && s.prompt.is_string()) {
+ std::string s_prompt = s.prompt.get<std::string>();
+ std::string prefix = common_prefix(s_prompt, prompt_str);
+
+ if (prefix.size() > longest) {
+ slot = &s;
+ longest = prefix.size();
+ }
+ }
+ }
+
+ if (!slot) {
+ return get_slot(-1);
+ }
+
+ LOG_DEBUG("slot with common prefix found", {{
+ "slot_id", slot->id,
+ "characters", longest
+ }});
+ return slot;
+ }
+
+ void process_single_task(task_server& task)
+ {
+ switch (task.type)
+ {
+ case TASK_TYPE_COMPLETION: {
+ server_slot *slot = nullptr;
+ if (task.embedding_mode) {
+ // Embedding seq_id (aka slot id) must always be <= token length, so always use slot 0
+ slot = slots[0].available() ? &slots[0] : nullptr;
+ } else {
+ slot = prefix_slot(task.data["prompt"]);
+ }
+ if (slot == nullptr)
+ {
+ // if no slot is available, we defer this task for processing later
+ LOG_VERBOSE("no slot is available", {{"task_id", task.id}});
+ queue_tasks.defer(task);
+ break;
+ }
+
+ slot->reset();
+
+ slot->embedding = task.embedding_mode;
+ slot->task_id = task.id;
+ slot->multitask_id = task.multitask_id;
+
+ if (!launch_slot_with_data(slot, task.data))
+ {
+ // send error result
+ send_error(task, "internal_error");
+ break;
+ }
+ } break;
+ case TASK_TYPE_CANCEL: { // release slot linked with the task id
+ for (auto & slot : slots)
+ {
+ if (slot.task_id == task.target_id)
+ {
+ slot.release();
+ break;
+ }
+ }
+ } break;
+ case TASK_TYPE_NEXT_RESPONSE: {
+ // do nothing
+ } break;
+ case TASK_TYPE_METRICS: {
+ json slots_data = json::array();
+ int n_idle_slots = 0;
+ int n_processing_slots = 0;
+
+ for (server_slot &slot: slots) {
+ json slot_data = get_formated_generation(slot);
+ slot_data["id"] = slot.id;
+ slot_data["task_id"] = slot.task_id;
+ slot_data["state"] = slot.state;
+ slot_data["prompt"] = slot.prompt;
+ slot_data["next_token"] = {
+ {"has_next_token", slot.has_next_token},
+ {"n_remain", slot.n_remaining},
+ {"num_tokens_predicted", slot.n_decoded},
+ {"stopped_eos", slot.stopped_eos},
+ {"stopped_word", slot.stopped_word},
+ {"stopped_limit", slot.stopped_limit},
+ {"stopping_word", slot.stopping_word},
+ };
+ if (slot_data["state"] == IDLE) {
+ n_idle_slots++;
+ } else {
+ n_processing_slots++;
+ }
+ slots_data.push_back(slot_data);
+ }
+ LOG_DEBUG("slot data", {
+ {"task_id", task.id},
+ {"n_idle_slots", n_idle_slots},
+ {"n_processing_slots", n_processing_slots}
+ });
+ LOG_VERBOSE("slot data", {
+ {"task_id", task.id},
+ {"n_idle_slots", n_idle_slots},
+ {"n_processing_slots", n_processing_slots},
+ {"slots", slots_data}
+ });
+ task_result res;
+ res.id = task.id;
+ res.multitask_id = task.multitask_id;
+ res.stop = true;
+ res.error = false;
+ res.result_json = {
+ { "idle", n_idle_slots },
+ { "processing", n_processing_slots },
+ { "deferred", queue_tasks.queue_tasks_deferred.size() },
+
+ { "n_prompt_tokens_processed_total", metrics.n_prompt_tokens_processed_total},
+ { "n_tokens_predicted_total", metrics.n_tokens_predicted_total},
+
+ { "n_prompt_tokens_processed", metrics.n_prompt_tokens_processed},
+ { "t_prompt_processing", metrics.t_prompt_processing},
+ { "n_tokens_predicted", metrics.n_tokens_predicted},
+ { "t_tokens_generation", metrics.t_tokens_generation},
+
+ { "kv_cache_tokens_count", llama_get_kv_cache_token_count(ctx)},
+ { "kv_cache_used_cells", llama_get_kv_cache_used_cells(ctx)},
+
+ { "slots", slots_data },
+ };
+ metrics.reset_bucket();
+ queue_results.send(res);
+ } break;
+ }
+ }
+
+ void on_finish_multitask(task_multi& multitask)
+ {
+ // all subtasks done == multitask is done
+ task_result result;
+ result.id = multitask.id;
+ result.stop = true;
+ result.error = false;
+
+ // collect json results into one json result
+ std::vector<json> result_jsons;
+ for (auto& subres : multitask.results)
+ {
+ result_jsons.push_back(subres.result_json);
+ result.error = result.error && subres.error;
+ }
+ result.result_json = json{ { "results", result_jsons } };
+ queue_results.send(result);
+ }
+
+ bool update_slots() {
+ if (system_need_update)
+ {
+ LOG_DEBUG("updating system prompt", {});
+ system_prompt_update();
+ }
+
+ llama_batch_clear(batch);
+
+ if (all_slots_are_idle)
+ {
+ if (system_prompt.empty() && clean_kv_cache)
+ {
+ LOG_DEBUG("all slots are idle and system prompt is empty, clear the KV cache", {});
+ kv_cache_clear();
+ }
+ return true;
+ }
+
+ LOG_VERBOSE("posting NEXT_RESPONSE", {});
+ task_server task;
+ task.type = TASK_TYPE_NEXT_RESPONSE;
+ task.target_id = -1;
+ queue_tasks.post(task);
+
+ for (server_slot &slot : slots)
+ {
+ if (slot.ga_n == 1)
+ {
+ if (slot.is_processing() && system_tokens.size() + slot.cache_tokens.size() >= (size_t) slot.n_ctx)
+ {
+ // Shift context
+ const int n_keep = slot.params.n_keep + add_bos_token;
+ const int n_left = (int) system_tokens.size() + slot.n_past - n_keep;
+ const int n_discard = n_left / 2;
+
+ LOG_DEBUG("slot context shift", {
+ {"slot_id", slot.id},
+ {"task_id", slot.task_id},
+ {"n_keep", n_keep},
+ {"n_left", n_left},
+ {"n_discard", n_discard},
+ {"n_ctx", n_ctx},
+ {"n_past", slot.n_past},
+ {"n_system_tokens", system_tokens.size()},
+ {"n_cache_tokens", slot.cache_tokens.size()}
+ });
+ llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard);
+ llama_kv_cache_seq_add(ctx, slot.id, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard);
+
+ for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++)
+ {
+ slot.cache_tokens[i - n_discard] = slot.cache_tokens[i];
+ }
+
+ slot.cache_tokens.resize(slot.cache_tokens.size() - n_discard);
+
+ slot.n_past -= n_discard;
+
+ slot.truncated = true;
+ }
+ }
+ }
+
+ // decode any currently ongoing sequences
+ LOG_VERBOSE("decoding ongoing sequences", {});
+ for (auto & slot : slots)
+ {
+ // release the slot
+ if (slot.command == RELEASE)
+ {
+ slot.state = IDLE;
+ slot.command = NONE;
+ slot.t_last_used = ggml_time_us();
+
+ LOG_DEBUG("slot released", {
+ {"slot_id", slot.id},
+ {"task_id", slot.task_id},
+ {"n_ctx", n_ctx},
+ {"n_past", slot.n_past},
+ {"n_system_tokens", system_tokens.size()},
+ {"n_cache_tokens", slot.cache_tokens.size()},
+ {"truncated", slot.truncated}
+ });
+ queue_tasks.notify_slot_changed();
+
+ continue;
+ }
+
+ if (slot.state == IDLE)
+ {
+ continue;
+ }
+
+ slot.i_batch = batch.n_tokens;
+
+ const int32_t slot_npast = slot.n_past_se > 0 ? slot.n_past_se : slot.n_past;
+
+ // TODO: we always have to take into account the "system_tokens"
+ // this is not great and needs to be improved somehow
+ llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id }, true);
+ slot.n_past += 1;
+ }
+
+ // process in chunks of params.n_batch
+ int32_t n_batch = params.n_batch;
+
+ // assign workload to the slots
+ if (params.cont_batching || batch.n_tokens == 0)
+ {
+ for (auto & slot : slots)
+ {
+ const bool has_prompt = slot.prompt.is_array() || (slot.prompt.is_string() && !slot.prompt.get<std::string>().empty()) || !slot.images.empty();
+
+ // empty prompt passed -> release the slot and send empty response
+ if (slot.state == IDLE && slot.command == LOAD_PROMPT && !has_prompt)
+ {
+ slot.release();
+ slot.print_timings();
+ send_final_response(slot);
+ continue;
+ }
+
+ // need process the prompt
+ if (slot.state == IDLE && slot.command == LOAD_PROMPT)
+ {
+ slot.state = PROCESSING;
+ slot.command = NONE;
+ std::vector<llama_token> prompt_tokens;
+ slot.t_start_process_prompt = ggml_time_us();
+ slot.t_start_genereration = 0;
+
+ prompt_tokens = tokenize(slot.prompt, system_prompt.empty()); // add BOS if there isn't system prompt
+
+ slot.n_prompt_tokens = prompt_tokens.size();
+
+ if (slot.params.n_keep < 0)
+ {
+ slot.params.n_keep = slot.n_prompt_tokens;
+ }
+ slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep);
+
+ // if input prompt is too big, truncate it, if group attention self-extend is disabled
+ if (slot.ga_n == 1 && slot.n_prompt_tokens >= slot.n_ctx)
+ {
+ const int n_left = slot.n_ctx - slot.params.n_keep;
+ const int n_shift = n_left / 2;
+ const int n_erase = slot.n_prompt_tokens - slot.params.n_keep - n_shift;
+
+ std::vector<llama_token> new_tokens(
+ prompt_tokens.begin(),
+ prompt_tokens.begin() + slot.params.n_keep);
+ new_tokens.insert(
+ new_tokens.end(),
+ prompt_tokens.begin() + slot.params.n_keep + n_erase,
+ prompt_tokens.end());
+
+ LOG_INFO("input truncated", {
+ {"n_ctx", slot.n_ctx},
+ {"n_keep", slot.params.n_keep},
+ {"n_left", n_left},
+ {"n_shift", n_shift},
+ {"n_erase", n_erase},
+ });
+ slot.truncated = true;
+ prompt_tokens = new_tokens;
+
+ slot.n_prompt_tokens = prompt_tokens.size();
+ GGML_ASSERT(slot.n_prompt_tokens < slot.n_ctx);
+ }
+
+ if (!slot.params.cache_prompt)
+ {
+ llama_sampling_reset(slot.ctx_sampling);
+
+ slot.n_past = 0;
+ slot.n_past_se = 0;
+ slot.ga_i = 0;
+ slot.n_prompt_tokens_processed = slot.n_prompt_tokens;
+ }
+ else
+ {
+ // push the prompt into the sampling context (do not apply grammar)
+ for (auto &token : prompt_tokens)
+ {
+ llama_sampling_accept(slot.ctx_sampling, ctx, token, false);
+ }
+
+ slot.n_past = common_part(slot.cache_tokens, prompt_tokens);
+
+ // the last token of the cache is not in the KV cache until the next call to llama_decode
+ // (it was sampled, pushed into the "cache_tokens", but not yet put in the context)
+ if (slot.n_past > 0 && slot.n_past == (int32_t) slot.cache_tokens.size())
+ {
+ slot.n_past -= 1;
+ }
+
+ slot.n_prompt_tokens_processed = slot.n_prompt_tokens;
+
+ if (slot.ga_n != 1)
+ {
+ int ga_i = 0;
+ int32_t ga_n = slot.ga_n;
+ int32_t ga_w = slot.ga_w;
+ int32_t slot_npast = 0;
+ for (int k = 0; k < slot.n_past; ++k)
+ {
+ while (slot_npast >= ga_i + ga_w) {
+ const int bd = (ga_w/ga_n)*(ga_n - 1);
+ slot_npast -= bd;
+ ga_i += ga_w/ga_n;
+ }
+ slot_npast++;
+ }
+ slot.n_past_se = slot_npast;
+ slot.ga_i = ga_i;
+ }
+
+ LOG_DEBUG("slot progression", {
+ { "slot_id", slot.id },
+ { "task_id", slot.task_id },
+ { "n_past", slot.n_past },
+ { "n_past_se", slot.n_past_se },
+ { "ga_i", slot.ga_i },
+ { "n_prompt_tokens_processed", slot.n_prompt_tokens_processed }
+ });
+ }
+
+ slot.cache_tokens = prompt_tokens;
+
+ if (slot.n_past == slot.n_prompt_tokens && slot.n_past > 0)
+ {
+ // we have to evaluate at least 1 token to generate logits.
+ LOG_DEBUG("we have to evaluate at least 1 token to generate logits", {
+ { "slot_id", slot.id },
+ { "task_id", slot.task_id }
+ });
+ slot.n_past--;
+ if (slot.ga_i > 0)
+ {
+ slot.n_past_se--;
+ }
+ }
+
+ int p0 = (int) system_tokens.size() + slot.n_past;
+ LOG_DEBUG("kv cache rm [p0, end)", {
+ { "slot_id", slot.id },
+ { "task_id", slot.task_id },
+ { "p0", p0 }
+ });
+ llama_kv_cache_seq_rm(ctx, slot.id, p0, -1);
+
+ LOG_VERBOSE("prompt ingested", {
+ {"n_past", slot.n_past},
+ {"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)},
+ {"to_eval", tokens_to_str(ctx, slot.cache_tokens.cbegin() + slot.n_past, slot.cache_tokens.cend())},
+ });
+
+ const bool has_images = process_images(slot);
+
+ // process the prefix of first image
+ std::vector<llama_token> prefix_tokens = has_images ? tokenize(slot.images[0].prefix_prompt, add_bos_token) : prompt_tokens;
+
+ int32_t slot_npast = slot.n_past_se > 0 ? slot.n_past_se : slot.n_past;
+
+ int32_t ga_i = slot.ga_i;
+ int32_t ga_n = slot.ga_n;
+ int32_t ga_w = slot.ga_w;
+
+ for (; slot.n_past < (int) prefix_tokens.size(); ++slot.n_past)
+ {
+ if (slot.ga_n != 1)
+ {
+ while (slot_npast >= ga_i + ga_w) {
+ const int bd = (ga_w/ga_n)*(ga_n - 1);
+ slot_npast -= bd;
+ ga_i += ga_w/ga_n;
+ }
+ }
+ llama_batch_add(batch, prefix_tokens[slot.n_past], system_tokens.size() + slot_npast, { slot.id }, false);
+ slot_npast++;
+ }
+
+ if (has_images && !ingest_images(slot, n_batch))
+ {
+ LOG_ERROR("failed processing images", {
+ {"slot_id", slot.id},
+ {"task_id", slot.task_id},
+ });
+ // FIXME @phymbert: to be properly tested
+ // early returning without changing the slot state will block the slot for ever
+ // no one at the moment is checking the return value
+ return false;
+ }
+
+ // extract the logits only for the last token
+ if (batch.n_tokens > 0)
+ {
+ batch.logits[batch.n_tokens - 1] = true;
+ }
+
+ slot.n_decoded = 0;
+ slot.i_batch = batch.n_tokens - 1;
+ }
+ }
+ }
+
+ if (batch.n_tokens == 0)
+ {
+ all_slots_are_idle = true;
+ return true;
+ }
+
+ for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch)
+ {
+ const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i);
+
+ for (auto & slot : slots)
+ {
+ if (slot.ga_n != 1)
+ {
+ // context extension via Self-Extend
+ while (slot.n_past_se >= slot.ga_i + slot.ga_w)
+ {
+ const int ib = (slot.ga_n * slot.ga_i) / slot.ga_w;
+ const int bd = (slot.ga_w / slot.ga_n) * (slot.ga_n - 1);
+ const int dd = (slot.ga_w / slot.ga_n) - ib * bd - slot.ga_w;
+
+ LOG_TEE("\n");
+ LOG_TEE("shift: [%6d, %6d] + %6d -> [%6d, %6d]\n", slot.ga_i, slot.n_past_se, ib * bd, slot.ga_i + ib * bd, slot.n_past_se + ib * bd);
+ LOG_TEE("div: [%6d, %6d] / %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n, (slot.ga_i + ib * bd) / slot.ga_n, (slot.ga_i + ib * bd + slot.ga_w) / slot.ga_n);
+ LOG_TEE("shift: [%6d, %6d] + %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd, slot.ga_i + ib * bd + slot.ga_w + dd, slot.n_past_se + ib * bd + dd);
+
+ llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i, slot.n_past_se, ib * bd);
+ llama_kv_cache_seq_div(ctx, slot.id, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w,slot.ga_n);
+ llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i + ib * bd + slot.ga_w,slot.n_past_se + ib * bd, dd);
+
+ slot.n_past_se -= bd;
+
+ slot.ga_i += slot.ga_w / slot.ga_n;
+
+ LOG_TEE("\nn_past_old = %d, n_past = %d, ga_i = %d\n\n", slot.n_past_se + bd, slot.n_past_se, slot.ga_i);
+ }
+ slot.n_past_se += n_tokens;
+ }
+ }
+
+ llama_batch batch_view =
+ {
+ n_tokens,
+ batch.token + i,
+ nullptr,
+ batch.pos + i,
+ batch.n_seq_id + i,
+ batch.seq_id + i,
+ batch.logits + i,
+ 0, 0, 0, // unused
+ };
+
+ const int ret = llama_decode(ctx, batch_view);
+
+ if (ret != 0)
+ {
+ if (n_batch == 1 || ret < 0)
+ {
+ // if you get here, it means the KV cache is full - try increasing it via the context size
+ LOG_TEE("%s : failed to decode the batch, n_batch = %d, ret = %d\n", __func__, n_batch, ret);
+ return false;
+ }
+
+ LOG_TEE("%s : failed to find free space in the KV cache, retrying with smaller n_batch = %d\n", __func__, n_batch / 2);
+
+ // retry with half the batch size to try to find a free slot in the KV cache
+ n_batch /= 2;
+ i -= n_batch;
+ continue;
+ }
+
+ for (auto & slot : slots)
+ {
+ if (slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens))
+ {
+ continue;
+ }
+
+ // prompt evaluated for embedding
+ if (slot.embedding)
+ {
+ send_embedding(slot, batch_view);
+ slot.release();
+ slot.i_batch = -1;
+ continue;
+ }
+
+ completion_token_output result;
+ const llama_token id = llama_sampling_sample(slot.ctx_sampling, ctx, NULL, slot.i_batch - i);
+
+ llama_sampling_accept(slot.ctx_sampling, ctx, id, true);
+
+ slot.n_decoded += 1;
+ if (slot.n_decoded == 1)
+ {
+ slot.t_start_genereration = ggml_time_us();
+ slot.t_prompt_processing = (slot.t_start_genereration - slot.t_start_process_prompt) / 1e3;
+ metrics.on_prompt_eval(slot);
+ }
+
+ llama_token_data_array cur_p = { slot.ctx_sampling->cur.data(), slot.ctx_sampling->cur.size(), false };
+ result.tok = id;
+
+ const int32_t n_probs = slot.sparams.n_probs;
+ if (slot.sparams.temp <= 0 && n_probs > 0)
+ {
+ // for llama_sample_token_greedy we need to sort candidates
+ llama_sample_softmax(ctx, &cur_p);
+ }
+
+ for (size_t i = 0; i < std::min(cur_p.size, (size_t)n_probs); ++i)
+ {
+ result.probs.push_back({cur_p.data[i].id, cur_p.data[i].p});
+ }
+
+ if (!process_token(result, slot))
+ {
+ slot.release();
+ slot.print_timings();
+ send_final_response(slot);
+ metrics.on_prediction(slot);
+ }
+
+ slot.i_batch = -1;
+ }
+ }
+
+ LOG_VERBOSE("slots updated", {});
+ return true;
+ }
+
+ json model_meta() {
+ return json{
+ {"vocab_type", llama_vocab_type(model)},
+ {"n_vocab", llama_n_vocab(model)},
+ {"n_ctx_train", llama_n_ctx_train(model)},
+ {"n_embd", llama_n_embd(model)},
+ {"n_params", llama_model_n_params(model)},
+ {"size", llama_model_size(model)},
+ };
+ }
+};
+
+static void server_print_usage(const char *argv0, const gpt_params ¶ms,
+ const server_params &sparams)
+{
+ printf("usage: %s [options]\n", argv0);
+ printf("\n");
+ printf("options:\n");
+ printf(" -h, --help show this help message and exit\n");
+ printf(" -v, --verbose verbose output (default: %s)\n", server_verbose ? "enabled" : "disabled");
+ printf(" -t N, --threads N number of threads to use during computation (default: %d)\n", params.cpuparams.n_threads);
+ printf(" -tb N, --threads-batch N number of threads to use during batch and prompt processing (default: same as --threads)\n");
+ printf(" --threads-http N number of threads in the http server pool to process requests (default: max(hardware concurrency - 1, --parallel N + 2))\n");
+ printf(" -c N, --ctx-size N size of the prompt context (default: %d)\n", params.n_ctx);
+ printf(" --rope-scaling {none,linear,yarn}\n");
+ printf(" RoPE frequency scaling method, defaults to linear unless specified by the model\n");
+ printf(" --rope-freq-base N RoPE base frequency (default: loaded from model)\n");
+ printf(" --rope-freq-scale N RoPE frequency scaling factor, expands context by a factor of 1/N\n");
+ printf(" --yarn-ext-factor N YaRN: extrapolation mix factor (default: 1.0, 0.0 = full interpolation)\n");
+ printf(" --yarn-attn-factor N YaRN: scale sqrt(t) or attention magnitude (default: 1.0)\n");
+ printf(" --yarn-beta-slow N YaRN: high correction dim or alpha (default: %.1f)\n", params.yarn_beta_slow);
+ printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast);
+ printf(" --pooling {none,mean,cls}\n");
+ printf(" pooling type for embeddings, use model default if unspecified\n");
+ printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch);
+ printf(" --memory-f32 use f32 instead of f16 for memory key+value (default: disabled)\n");
+ printf(" not recommended: doubles context memory required and no measurable increase in quality\n");
+ if (llama_supports_mlock())
+ {
+ printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n");
+ }
+ if (llama_supports_mmap())
+ {
+ printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n");
+ }
+ printf(" --numa TYPE attempt optimizations that help on some NUMA systems\n");
+ printf(" - distribute: spread execution evenly over all nodes\n");
+ printf(" - isolate: only spawn threads on CPUs on the node that execution started on\n");
+ printf(" - numactl: use the CPU map provided my numactl\n");
+ if (llama_supports_gpu_offload()) {
+ printf(" -ngl N, --n-gpu-layers N\n");
+ printf(" number of layers to store in VRAM\n");
+ printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n");
+ printf(" how to split the model across multiple GPUs, one of:\n");
+ printf(" - none: use one GPU only\n");
+ printf(" - layer (default): split layers and KV across GPUs\n");
+ printf(" - row: split rows across GPUs\n");
+ printf(" -ts SPLIT --tensor-split SPLIT\n");
+ printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n");
+ printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n");
+ printf(" or for intermediate results and KV (with split-mode = row)\n");
+ }
+ printf(" -m FNAME, --model FNAME\n");
+ printf(" model path (default: %s)\n", params.model.c_str());
+ printf(" -a ALIAS, --alias ALIAS\n");
+ printf(" set an alias for the model, will be added as `model` field in completion response\n");
+ printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n");
+ printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n");
+ printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str());
+ printf(" --port PORT port to listen (default (default: %d)\n", sparams.port);
+ printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str());
+ printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n");
+ printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n");
+ printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);
+ printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");
+ printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel);
+ printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n");
+ printf(" -fa, --flash-attn enable Flash Attention (default: %s)\n", params.flash_attn ? "enabled" : "disabled");
+ printf(" -spf FNAME, --system-prompt-file FNAME\n");
+ printf(" set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n");
+ printf(" -ctk TYPE, --cache-type-k TYPE\n");
+ printf(" KV cache data type for K (default: f16)\n");
+ printf(" -ctv TYPE, --cache-type-v TYPE\n");
+ printf(" KV cache data type for V (default: f16)\n");
+ printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n");
+ printf(" --log-format log output format: json or text (default: json)\n");
+ printf(" --log-disable disables logging to a file.\n");
+ printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n");
+ printf(" --metrics enable prometheus compatible metrics endpoint (default: %s).\n", sparams.metrics_endpoint ? "enabled" : "disabled");
+ printf("\n");
+ printf(" -n, --n-predict maximum tokens to predict (default: %d)\n", params.n_predict);
+ printf(" --override-kv KEY=TYPE:VALUE\n");
+ printf(" advanced option to override model metadata by key. may be specified multiple times.\n");
+ printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");
+ printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`\n");
+ printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`\n");
+ printf(" --chat-template JINJA_TEMPLATE\n");
+ printf(" set custom jinja chat template (default: template taken from model's metadata)\n");
+ printf(" Note: only commonly used templates are accepted, since we don't have jinja parser\n");
+ printf("\n");
+}
+
+static void server_params_parse(int argc, char **argv, server_params &sparams, gpt_params ¶ms)
+{
+ gpt_params default_params;
+ server_params default_sparams;
+ std::string arg;
+ bool invalid_param = false;
+
+ for (int i = 1; i < argc; i++)
+ {
+ arg = argv[i];
+ if (arg == "--port")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ sparams.port = std::stoi(argv[i]);
+ }
+ else if (arg == "--host")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ sparams.hostname = argv[i];
+ }
+ else if (arg == "--path")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ sparams.public_path = argv[i];
+ }
+ else if (arg == "--api-key")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ sparams.api_keys.emplace_back(argv[i]);
+ }
+ else if (arg == "--api-key-file")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ std::ifstream key_file(argv[i]);
+ if (!key_file) {
+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+ invalid_param = true;
+ break;
+ }
+ std::string key;
+ while (std::getline(key_file, key)) {
+ if (key.size() > 0) {
+ sparams.api_keys.push_back(key);
+ }
+ }
+ key_file.close();
+ }
+ else if (arg == "--timeout" || arg == "-to")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ sparams.read_timeout = std::stoi(argv[i]);
+ sparams.write_timeout = std::stoi(argv[i]);
+ }
+ else if (arg == "-m" || arg == "--model")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.model = argv[i];
+ }
+ else if (arg == "-a" || arg == "--alias")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.model_alias = argv[i];
+ }
+ else if (arg == "-h" || arg == "--help")
+ {
+ server_print_usage(argv[0], default_params, default_sparams);
+ exit(0);
+ }
+ else if (arg == "-c" || arg == "--ctx-size" || arg == "--ctx_size")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.n_ctx = std::stoi(argv[i]);
+ }
+ else if (arg == "--rope-scaling")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ std::string value(argv[i]);
+ /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE; }
+ else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; }
+ else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; }
+ else { invalid_param = true; break; }
+ }
+ else if (arg == "--rope-freq-base")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.rope_freq_base = std::stof(argv[i]);
+ }
+ else if (arg == "--rope-freq-scale")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.rope_freq_scale = std::stof(argv[i]);
+ }
+ else if (arg == "--yarn-ext-factor")
+ {
+ if (++i >= argc) {
+ invalid_param = true;
+ break;
+ }
+ params.yarn_ext_factor = std::stof(argv[i]);
+ }
+ else if (arg == "--yarn-attn-factor")
+ {
+ if (++i >= argc) {
+ invalid_param = true;
+ break;
+ }
+ params.yarn_attn_factor = std::stof(argv[i]);
+ }
+ else if (arg == "--yarn-beta-fast")
+ {
+ if (++i >= argc) {
+ invalid_param = true;
+ break;
+ }
+ params.yarn_beta_fast = std::stof(argv[i]);
+ }
+ else if (arg == "--yarn-beta-slow")
+ {
+ if (++i >= argc) {
+ invalid_param = true;
+ break;
+ }
+ params.yarn_beta_slow = std::stof(argv[i]);
+ }
+ else if (arg == "--pooling")
+ {
+ if (++i >= argc) {
+ invalid_param = true;
+ break;
+ }
+ std::string value(argv[i]);
+ /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; }
+ else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; }
+ else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; }
+ else { invalid_param = true; break; }
+ }
+ else if (arg == "--threads" || arg == "-t")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.cpuparams.n_threads = std::stoi(argv[i]);
+ }
+ else if (arg == "--grp-attn-n" || arg == "-gan")
+ {
+ if (++i >= argc) {
+ invalid_param = true;
+ break;
+ }
+
+ params.grp_attn_n = std::stoi(argv[i]);
+ }
+ else if (arg == "--grp-attn-w" || arg == "-gaw")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+
+ params.grp_attn_w = std::stoi(argv[i]);
+ }
+ else if (arg == "--threads-batch" || arg == "-tb")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.cpuparams_batch.n_threads = std::stoi(argv[i]);
+ }
+ else if (arg == "--threads-http")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ sparams.n_threads_http = std::stoi(argv[i]);
+ }
+ else if (arg == "-b" || arg == "--batch-size")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.n_batch = std::stoi(argv[i]);
+ }
+ else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ if (llama_supports_gpu_offload()) {
+ params.n_gpu_layers = std::stoi(argv[i]);
+ } else {
+ LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "
+ "See main README.md for information on enabling GPU BLAS support",
+ {{"n_gpu_layers", params.n_gpu_layers}});
+ }
+ }
+ else if (arg == "--split-mode" || arg == "-sm")
+ {
+ if (++i >= argc) {
+ invalid_param = true;
+ break;
+ }
+ std::string arg_next = argv[i];
+ if (arg_next == "none")
+ {
+ params.split_mode = LLAMA_SPLIT_MODE_NONE;
+ }
+ else if (arg_next == "layer")
+ {
+ params.split_mode = LLAMA_SPLIT_MODE_LAYER;
+ }
+ else if (arg_next == "row")
+ {
+ params.split_mode = LLAMA_SPLIT_MODE_ROW;
+ }
+ else {
+ invalid_param = true;
+ break;
+ }
+#ifndef GGML_USE_CUDA
+ fprintf(stderr, "warning: llama.cpp was compiled without CUDA. Setting the split mode has no effect.\n");
+#endif // GGML_USE_CUDA
+ }
+ else if (arg == "--tensor-split" || arg == "-ts")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+#if defined(GGML_USE_CUDA) || defined(GGML_USE_SYCL)
+ std::string arg_next = argv[i];
+
+ // split string by , and /
+ const std::regex regex{R"([,/]+)"};
+ std::sregex_token_iterator it{arg_next.begin(), arg_next.end(), regex, -1};
+ std::vector<std::string> split_arg{it, {}};
+ GGML_ASSERT(split_arg.size() <= llama_max_devices());
+
+ for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device)
+ {
+ if (i_device < split_arg.size())
+ {
+ params.tensor_split[i_device] = std::stof(split_arg[i_device]);
+ }
+ else
+ {
+ params.tensor_split[i_device] = 0.0f;
+ }
+ }
+#else
+ LOG_WARNING("llama.cpp was compiled without CUDA. It is not possible to set a tensor split.\n", {});
+#endif // GGML_USE_CUDA
+ }
+ else if (arg == "--main-gpu" || arg == "-mg")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+#if defined(GGML_USE_CUDA) || defined(GGML_USE_SYCL)
+ params.main_gpu = std::stoi(argv[i]);
+#else
+ LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a main GPU.", {});
+#endif
+ }
+ else if (arg == "--lora")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.lora_adapters.push_back({
+ std::string(argv[i]),
+ 1.0,
+ });
+ params.use_mmap = false;
+ }
+ else if (arg == "--lora-scaled")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ const char * lora_adapter = argv[i];
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.lora_adapters.push_back({
+ lora_adapter,
+ std::stof(argv[i])
+ });
+ params.use_mmap = false;
+ }
+ else if (arg == "-v" || arg == "--verbose")
+ {
+ server_verbose = true;
+ }
+ else if (arg == "--mlock")
+ {
+ params.use_mlock = true;
+ }
+ else if (arg == "--no-mmap")
+ {
+ params.use_mmap = false;
+ }
+ else if (arg == "--numa")
+ {
+ if (++i >= argc) {
+ invalid_param = true;
+ break;
+ } else {
+ std::string value(argv[i]);
+ /**/ if (value == "distribute" || value == "" ) { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; }
+ else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }
+ else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }
+ else { invalid_param = true; break; }
+ }
+ }
+ else if (arg == "--embedding")
+ {
+ params.embedding = true;
+ }
+ else if (arg == "-cb" || arg == "--cont-batching")
+ {
+ params.cont_batching = true;
+ }
+ else if (arg == "-fa" || arg == "--flash-attn")
+ {
+ params.flash_attn = true;
+ }
+ else if (arg == "-np" || arg == "--parallel")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.n_parallel = std::stoi(argv[i]);
+ }
+ else if (arg == "-n" || arg == "--n-predict")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.n_predict = std::stoi(argv[i]);
+ }
+ else if (arg == "-ctk" || arg == "--cache-type-k") {
+ params.cache_type_k = argv[++i];
+ }
+ else if (arg == "-ctv" || arg == "--cache-type-v") {
+ params.cache_type_v = argv[++i];
+ }
+ else if(arg == "--mmproj")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.mmproj = argv[i];
+ }
+ else if (arg == "--log-format")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ if (std::strcmp(argv[i], "json") == 0)
+ {
+ server_log_json = true;
+ }
+ else if (std::strcmp(argv[i], "text") == 0)
+ {
+ server_log_json = false;
+ }
+ else
+ {
+ invalid_param = true;
+ break;
+ }
+ }
+ else if (arg == "--log-disable")
+ {
+ log_set_target(stdout);
+ LOG_DEBUG("logging to file is disabled.", {});
+ }
+ else if (arg == "--slots-endpoint-disable")
+ {
+ sparams.slots_endpoint = false;
+ }
+ else if (arg == "--metrics")
+ {
+ sparams.metrics_endpoint = true;
+ }
+ else if (arg == "--chat-template")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ if (!verify_custom_template(argv[i])) {
+ fprintf(stderr, "error: the supplied chat template is not supported: %s\n", argv[i]);
+ fprintf(stderr, "note: llama.cpp does not use jinja parser, we only support commonly used templates\n");
+ invalid_param = true;
+ break;
+ }
+ }
+ else if (arg == "--override-kv")
+ {
+ if (++i >= argc) {
+ invalid_param = true;
+ break;
+ }
+ char * sep = strchr(argv[i], '=');
+ if (sep == nullptr || sep - argv[i] >= 128) {
+ fprintf(stderr, "error: Malformed KV override: %s\n", argv[i]);
+ invalid_param = true;
+ break;
+ }
+ struct llama_model_kv_override kvo;
+ std::strncpy(kvo.key, argv[i], sep - argv[i]);
+ kvo.key[sep - argv[i]] = 0;
+ sep++;
+ if (strncmp(sep, "int:", 4) == 0) {
+ sep += 4;
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;
+ kvo.val_i64 = std::atol(sep);
+ } else if (strncmp(sep, "float:", 6) == 0) {
+ sep += 6;
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;
+ kvo.val_f64 = std::atof(sep);
+ } else if (strncmp(sep, "bool:", 5) == 0) {
+ sep += 5;
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;
+ if (std::strcmp(sep, "true") == 0) {
+ kvo.val_bool = true;
+ } else if (std::strcmp(sep, "false") == 0) {
+ kvo.val_bool = false;
+ } else {
+ fprintf(stderr, "error: Invalid boolean value for KV override: %s\n", argv[i]);
+ invalid_param = true;
+ break;
+ }
+ } else {
+ fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]);
+ invalid_param = true;
+ break;
+ }
+ params.kv_overrides.push_back(kvo);
+ }
+ else
+ {
+ fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
+ server_print_usage(argv[0], default_params, default_sparams);
+ exit(1);
+ }
+ }
+ if (!params.kv_overrides.empty()) {
+ params.kv_overrides.emplace_back();
+ params.kv_overrides.back().key[0] = 0;
+ }
+
+ postprocess_cpu_params(params.cpuparams, nullptr);
+ postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams);
+ postprocess_cpu_params(params.draft_cpuparams, ¶ms.cpuparams);
+ postprocess_cpu_params(params.draft_cpuparams_batch, ¶ms.cpuparams_batch);
+
+ if (invalid_param)
+ {
+ fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
+ server_print_usage(argv[0], default_params, default_sparams);
+ exit(1);
+ }
+}
+
+/* llama.cpp completion api semantics */
+static json format_partial_response(
+ llama_server_context &llama, server_slot *slot, const std::string &content, const std::vector<completion_token_output> &probs
+) {
+ json res = json
+ {
+ {"content", content },
+ {"stop", false},
+ {"slot_id", slot->id },
+ {"multimodal", llama.multimodal }
+ };
+
+ if (slot->sparams.n_probs > 0)
+ {
+ res["completion_probabilities"] = probs_vector_to_json(llama.ctx, probs);
+ }
+
+ return res;
+}
+
+static json format_tokenizer_response(const std::vector<llama_token> &tokens)
+{
+ return json {
+ {"tokens", tokens}
+ };
+}
+
+static json format_detokenized_response(std::string content)
+{
+ return json {
+ {"content", content}
+ };
+}
+
+
+static void log_server_request(const httplib::Request &req, const httplib::Response &res)
+{
+ // skip GH copilot requests when using default port
+ if (req.path == "/health" || req.path == "/v1/health" || req.path == "/v1/completions")
+ {
+ return;
+ }
+
+ LOG_DEBUG("request", {
+ {"remote_addr", req.remote_addr},
+ {"remote_port", req.remote_port},
+ {"status", res.status},
+ {"method", req.method},
+ {"path", req.path},
+ {"params", req.params},
+ });
+
+ LOG_VERBOSE("request", {
+ {"request", req.body},
+ {"response", res.body},
+ });
+}
+
+static void append_to_generated_text_from_generated_token_probs(llama_server_context &llama, server_slot *slot)
+{
+ auto & gtps = slot->generated_token_probs;
+ auto translator = token_translator{llama.ctx};
+ auto add_strlen = [=](size_t sum, const completion_token_output & cto) { return sum + translator(cto).size(); };
+ const size_t len = std::accumulate(gtps.begin(), gtps.end(), size_t(0), add_strlen);
+ if (slot->generated_text.capacity() < slot->generated_text.size() + len)
+ {
+ slot->generated_text.reserve(slot->generated_text.size() + len);
+ }
+ for (const completion_token_output & cto : gtps)
+ {
+ slot->generated_text += translator(cto);
+ }
+}
+
+std::function<void(int)> shutdown_handler;
+std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
+inline void signal_handler(int signal) {
+ if (is_terminating.test_and_set()) {
+ // in case it hangs, we can force terminate the server by hitting Ctrl+C twice
+ // this is for better developer experience, we can remove when the server is stable enough
+ fprintf(stderr, "Received second interrupt, terminating immediately.\n");
+ exit(1);
+ }
+ shutdown_handler(signal);
+}
+
+static bool update_load_progress(float progress, void *data)
+{
+ ((llama_server_context*)data)->modelProgress = progress;
+ return true;
+}
+
+#if defined(_WIN32)
+char* wchar_to_char(const wchar_t* wstr) {
+ if (wstr == nullptr) return nullptr;
+
+ // Determine the number of bytes needed for the UTF-8 string
+ int bytes = WideCharToMultiByte(CP_UTF8, 0, wstr, -1, nullptr, 0, nullptr, nullptr);
+ char* str = new char[bytes];
+
+ // Convert the wide-character string to a UTF-8 string
+ WideCharToMultiByte(CP_UTF8, 0, wstr, -1, str, bytes, nullptr, nullptr);
+ return str;
+}
+
+int wmain(int argc, wchar_t **wargv) {
+ char** argv = new char*[argc];
+ for (int i = 0; i < argc; ++i) {
+ argv[i] = wchar_to_char(wargv[i]);
+ }
+
+ // Adjust error mode to avoid error dialog after we start.
+ SetErrorMode(SEM_FAILCRITICALERRORS);
+#else
+int main(int argc, char **argv) {
+#endif
+
+#if SERVER_VERBOSE != 1
+ log_disable();
+#endif
+ // own arguments required by this example
+ gpt_params params;
+ server_params sparams;
+
+ // struct that contains llama context and inference
+ llama_server_context llama;
+
+ server_params_parse(argc, argv, sparams, params);
+
+ if (params.model_alias == "unknown")
+ {
+ params.model_alias = params.model;
+ }
+
+ llama_backend_init();
+ llama_numa_init(params.numa);
+
+ LOG_INFO("starting c++ runner", {});
+ LOG_INFO("build info", {{"build", LLAMA_BUILD_NUMBER},
+ {"commit", LLAMA_COMMIT}});
+
+ LOG_INFO("system info", {
+ {"n_threads", params.cpuparams.n_threads},
+ {"n_threads_batch", params.cpuparams_batch.n_threads},
+ {"total_threads", std::thread::hardware_concurrency()},
+ {"system_info", llama_print_system_info()},
+ });
+
+ httplib::Server svr;
+
+ std::atomic<server_state> state{SERVER_STATE_LOADING_MODEL};
+
+ svr.set_default_headers({{"Server", "llama.cpp"}});
+
+ // CORS preflight
+ svr.Options(R"(.*)", [](const httplib::Request &req, httplib::Response &res) {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ res.set_header("Access-Control-Allow-Credentials", "true");
+ res.set_header("Access-Control-Allow-Methods", "POST");
+ res.set_header("Access-Control-Allow-Headers", "*");
+ });
+
+ svr.Get("/health", [&](const httplib::Request& req, httplib::Response& res) {
+ server_state current_state = state.load();
+ switch(current_state) {
+ case SERVER_STATE_READY: {
+ // request slots data using task queue
+ task_server task;
+ task.id = llama.queue_tasks.get_new_id();
+ task.type = TASK_TYPE_METRICS;
+ task.target_id = -1;
+
+ llama.queue_results.add_waiting_task_id(task.id);
+ llama.queue_tasks.post(task);
+
+ // get the result
+ task_result result = llama.queue_results.recv(task.id);
+ llama.queue_results.remove_waiting_task_id(task.id);
+
+ int n_idle_slots = result.result_json["idle"];
+ int n_processing_slots = result.result_json["processing"];
+
+ json health = {
+ {"status", "ok"},
+ {"slots_idle", n_idle_slots},
+ {"slots_processing", n_processing_slots}};
+ res.status = 200; // HTTP OK
+ if (sparams.slots_endpoint && req.has_param("include_slots")) {
+ health["slots"] = result.result_json["slots"];
+ }
+
+ if (n_idle_slots == 0) {
+ health["status"] = "no slot available";
+ if (req.has_param("fail_on_no_slot")) {
+ res.status = 503; // HTTP Service Unavailable
+ }
+ }
+ res.set_content(health.dump(), "application/json");
+ break;
+ }
+ case SERVER_STATE_LOADING_MODEL:
+ char buf[128];
+ snprintf(&buf[0], 128, R"({"status": "loading model", "progress": %0.2f})", llama.modelProgress);
+ res.set_content(buf, "application/json");
+ res.status = 503; // HTTP Service Unavailable
+ break;
+ case SERVER_STATE_ERROR:
+ res.set_content(R"({"status": "error", "error": "Model failed to load"})", "application/json");
+ res.status = 500; // HTTP Internal Server Error
+ break;
+ }
+ });
+
+ if (sparams.slots_endpoint) {
+ svr.Get("/slots", [&](const httplib::Request&, httplib::Response& res) {
+ // request slots data using task queue
+ task_server task;
+ task.id = llama.queue_tasks.get_new_id();
+ task.type = TASK_TYPE_METRICS;
+ task.target_id = -1;
+
+ llama.queue_results.add_waiting_task_id(task.id);
+ llama.queue_tasks.post(task);
+
+ // get the result
+ task_result result = llama.queue_results.recv(task.id);
+ llama.queue_results.remove_waiting_task_id(task.id);
+
+ res.set_content(result.result_json["slots"].dump(), "application/json");
+ res.status = 200; // HTTP OK
+ });
+ }
+
+ if (sparams.metrics_endpoint) {
+ svr.Get("/metrics", [&](const httplib::Request&, httplib::Response& res) {
+ // request slots data using task queue
+ task_server task;
+ task.id = llama.queue_tasks.get_new_id();
+ task.type = TASK_TYPE_METRICS;
+ task.target_id = -1;
+
+ llama.queue_results.add_waiting_task_id(task.id);
+ llama.queue_tasks.post(task);
+
+ // get the result
+ task_result result = llama.queue_results.recv(task.id);
+ llama.queue_results.remove_waiting_task_id(task.id);
+
+ json data = result.result_json;
+
+ uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"];
+ uint64_t t_prompt_processing = data["t_prompt_processing"];
+
+ uint64_t n_tokens_predicted = data["n_tokens_predicted"];
+ uint64_t t_tokens_generation = data["t_tokens_generation"];
+
+ int32_t kv_cache_used_cells = data["kv_cache_used_cells"];
+
+ // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
+ json all_metrics_def = json {
+ {"counter", {{
+ {"name", "prompt_tokens_total"},
+ {"help", "Number of prompt tokens processed."},
+ {"value", data["n_prompt_tokens_processed_total"]}
+ }, {
+ {"name", "tokens_predicted_total"},
+ {"help", "Number of generation tokens processed."},
+ {"value", data["n_tokens_predicted_total"]}
+ }}},
+ {"gauge", {{
+ {"name", "prompt_tokens_seconds"},
+ {"help", "Average prompt throughput in tokens/s."},
+ {"value", n_prompt_tokens_processed ? 1e3 / t_prompt_processing * n_prompt_tokens_processed : 0}
+ },{
+ {"name", "predicted_tokens_seconds"},
+ {"help", "Average generation throughput in tokens/s."},
+ {"value", n_tokens_predicted ? 1e3 / t_tokens_generation * n_tokens_predicted : 0}
+ },{
+ {"name", "kv_cache_usage_ratio"},
+ {"help", "KV-cache usage. 1 means 100 percent usage."},
+ {"value", 1. * kv_cache_used_cells / params.n_ctx}
+ },{
+ {"name", "kv_cache_tokens"},
+ {"help", "KV-cache tokens."},
+ {"value", data["kv_cache_tokens_count"]}
+ },{
+ {"name", "requests_processing"},
+ {"help", "Number of request processing."},
+ {"value", data["processing"]}
+ },{
+ {"name", "requests_deferred"},
+ {"help", "Number of request deferred."},
+ {"value", data["deferred"]}
+ }}}
+ };
+
+ std::stringstream prometheus;
+ for (const auto& el : all_metrics_def.items()) {
+ const auto& type = el.key();
+ const auto& metrics_def = el.value();
+ for (const auto& metric_def : metrics_def) {
+ std::string name = metric_def["name"];
+ std::string help = metric_def["help"];
+ auto value = json_value(metric_def, "value", 0);
+ prometheus << "# HELP llamacpp:" << name << " " << help << "\n"
+ << "# TYPE llamacpp:" << name << " " << type << "\n"
+ << "llamacpp:" << name << " " << value << "\n";
+ }
+ }
+
+ res.set_content(prometheus.str(), "text/plain; version=0.0.4");
+ res.status = 200; // HTTP OK
+ });
+ }
+
+ svr.set_logger(log_server_request);
+
+ svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep)
+ {
+ const char fmt[] = "500 Internal Server Error\n%s";
+ char buf[BUFSIZ];
+ try
+ {
+ std::rethrow_exception(std::move(ep));
+ }
+ catch (std::exception &e)
+ {
+ snprintf(buf, sizeof(buf), fmt, e.what());
+ }
+ catch (...)
+ {
+ snprintf(buf, sizeof(buf), fmt, "Unknown Exception");
+ }
+ res.set_content(buf, "text/plain; charset=utf-8");
+ res.status = 500;
+ });
+
+ svr.set_error_handler([](const httplib::Request &, httplib::Response &res)
+ {
+ if (res.status == 401)
+ {
+ res.set_content("Unauthorized", "text/plain; charset=utf-8");
+ }
+ if (res.status == 400)
+ {
+ res.set_content("Invalid request", "text/plain; charset=utf-8");
+ }
+ else if (res.status == 404)
+ {
+ res.set_content("File Not Found", "text/plain; charset=utf-8");
+ res.status = 404;
+ }
+ });
+
+ // set timeouts and change hostname and port
+ svr.set_read_timeout (sparams.read_timeout);
+ svr.set_write_timeout(sparams.write_timeout);
+
+ if (!svr.bind_to_port(sparams.hostname, sparams.port))
+ {
+ fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port);
+ return 1;
+ }
+
+ // Set the base directory for serving static files
+ svr.set_base_dir(sparams.public_path);
+
+ std::unordered_map<std::string, std::string> log_data;
+ log_data["hostname"] = sparams.hostname;
+ log_data["port"] = std::to_string(sparams.port);
+
+ if (sparams.api_keys.size() == 1) {
+ log_data["api_key"] = "api_key: ****" + sparams.api_keys[0].substr(sparams.api_keys[0].length() - 4);
+ } else if (sparams.api_keys.size() > 1) {
+ log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded";
+ }
+
+ if (sparams.n_threads_http < 1) {
+ // +2 threads for monitoring endpoints
+ sparams.n_threads_http = std::max(params.n_parallel + 2, (int32_t) std::thread::hardware_concurrency() - 1);
+ }
+ log_data["n_threads_http"] = std::to_string(sparams.n_threads_http);
+ svr.new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); };
+
+ LOG_INFO("HTTP server listening", log_data);
+ // run the HTTP server in a thread - see comment below
+ std::thread t([&]()
+ {
+ if (!svr.listen_after_bind())
+ {
+ state.store(SERVER_STATE_ERROR);
+ return 1;
+ }
+
+ return 0;
+ });
+
+ // load the model
+ params.progress_callback = update_load_progress;
+ params.progress_callback_user_data = (void*)&llama;
+
+ if (!llama.load_model(params))
+ {
+ state.store(SERVER_STATE_ERROR);
+ return 1;
+ } else {
+ llama.initialize();
+ state.store(SERVER_STATE_READY);
+ LOG_INFO("model loaded", {});
+ }
+ const auto model_meta = llama.model_meta();
+
+ // Middleware for API key validation
+ auto validate_api_key = [&sparams](const httplib::Request &req, httplib::Response &res) -> bool {
+ // If API key is not set, skip validation
+ if (sparams.api_keys.empty()) {
+ return true;
+ }
+
+ // Check for API key in the header
+ auto auth_header = req.get_header_value("Authorization");
+ std::string prefix = "Bearer ";
+ if (auth_header.substr(0, prefix.size()) == prefix) {
+ std::string received_api_key = auth_header.substr(prefix.size());
+ if (std::find(sparams.api_keys.begin(), sparams.api_keys.end(), received_api_key) != sparams.api_keys.end()) {
+ return true; // API key is valid
+ }
+ }
+
+ // API key is invalid or not provided
+ res.set_content("Unauthorized: Invalid API Key", "text/plain; charset=utf-8");
+ res.status = 401; // Unauthorized
+
+ LOG_WARNING("Unauthorized: Invalid API Key", {});
+
+ return false;
+ };
+
+ // this is only called if no index.html is found in the public --path
+ svr.Get("/", [](const httplib::Request &, httplib::Response &res)
+ {
+ res.set_content("server running", "text/plain; charset=utf-8");
+ res.status = 200; // Unauthorized
+ return true;
+ });
+
+ svr.Post("/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ if (!validate_api_key(req, res)) {
+ return;
+ }
+ json data = json::parse(req.body);
+ const int task_id = llama.queue_tasks.get_new_id();
+ llama.queue_results.add_waiting_task_id(task_id);
+ llama.request_completion(task_id, data, false, -1);
+ if (!json_value(data, "stream", false)) {
+ std::string completion_text;
+ task_result result = llama.queue_results.recv(task_id);
+ if (!result.error && result.stop) {
+ res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");
+ }
+ else
+ {
+ res.status = 404;
+ res.set_content(result.result_json["content"], "text/plain; charset=utf-8");
+ }
+ llama.queue_results.remove_waiting_task_id(task_id);
+ } else {
+ const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink & sink)
+ {
+ while (true)
+ {
+ task_result result = llama.queue_results.recv(task_id);
+ if (!result.error) {
+ const std::string str =
+ "data: " +
+ result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) +
+ "\n\n";
+ LOG_VERBOSE("data stream", {
+ { "to_send", str }
+ });
+ if (!sink.write(str.c_str(), str.size()))
+ {
+ llama.queue_results.remove_waiting_task_id(task_id);
+ return false;
+ }
+ if (result.stop) {
+ break;
+ }
+ } else {
+ const std::string str =
+ "error: " +
+ result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) +
+ "\n\n";
+ LOG_VERBOSE("data stream", {
+ { "to_send", str }
+ });
+ if (!sink.write(str.c_str(), str.size()))
+ {
+ llama.queue_results.remove_waiting_task_id(task_id);
+ return false;
+ }
+ break;
+ }
+ }
+
+ llama.queue_results.remove_waiting_task_id(task_id);
+ sink.done();
+ return true;
+ };
+
+ auto on_complete = [task_id, &llama] (bool)
+ {
+ // cancel
+ llama.request_cancel(task_id);
+ llama.queue_results.remove_waiting_task_id(task_id);
+ };
+
+ res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);
+ }
+ });
+
+ svr.Post("/tokenize", [&llama](const httplib::Request &req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ const json body = json::parse(req.body);
+ std::vector<llama_token> tokens;
+ if (body.count("content") != 0)
+ {
+ tokens = llama.tokenize(body["content"], false);
+ }
+ const json data = format_tokenizer_response(tokens);
+ return res.set_content(data.dump(), "application/json; charset=utf-8");
+ });
+
+ svr.Post("/detokenize", [&llama](const httplib::Request &req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ const json body = json::parse(req.body);
+ std::string content;
+ if (body.count("tokens") != 0)
+ {
+ const std::vector<llama_token> tokens = body["tokens"];
+ content = tokens_to_str(llama.ctx, tokens.cbegin(), tokens.cend());
+ }
+
+ const json data = format_detokenized_response(content);
+ return res.set_content(data.dump(), "application/json; charset=utf-8");
+ });
+
+ svr.Post("/embedding", [&llama](const httplib::Request &req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ const json body = json::parse(req.body);
+ json prompt;
+ if (body.count("content") != 0)
+ {
+ prompt = body["content"];
+ }
+ else
+ {
+ prompt = "";
+ }
+
+ // create and queue the task
+ const int task_id = llama.queue_tasks.get_new_id();
+ llama.queue_results.add_waiting_task_id(task_id);
+ llama.request_completion(task_id, {{"prompt", prompt}}, true, -1);
+
+ // get the result
+ task_result result = llama.queue_results.recv(task_id);
+ llama.queue_results.remove_waiting_task_id(task_id);
+
+ // send the result
+ return res.set_content(result.result_json.dump(), "application/json; charset=utf-8");
+ });
+
+ // GG: if I put the main loop inside a thread, it crashes on the first request when build in Debug!?
+ // "Bus error: 10" - this is on macOS, it does not crash on Linux
+ //std::thread t2([&]()
+ /*{
+ bool running = true;
+ while (running)
+ {
+ running = llama.update_slots();
+ }
+ }*/
+ //);
+
+ llama.queue_tasks.on_new_task(std::bind(
+ &llama_server_context::process_single_task, &llama, std::placeholders::_1));
+ llama.queue_tasks.on_finish_multitask(std::bind(
+ &llama_server_context::on_finish_multitask, &llama, std::placeholders::_1));
+ llama.queue_tasks.on_run_slots(std::bind(
+ &llama_server_context::update_slots, &llama));
+ llama.queue_results.on_multitask_update(std::bind(
+ &llama_server_queue::update_multitask,
+ &llama.queue_tasks,
+ std::placeholders::_1,
+ std::placeholders::_2,
+ std::placeholders::_3
+ ));
+
+ shutdown_handler = [&](int) {
+ llama.queue_tasks.terminate();
+ };
+
+#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
+ struct sigaction sigint_action;
+ sigint_action.sa_handler = signal_handler;
+ sigemptyset (&sigint_action.sa_mask);
+ sigint_action.sa_flags = 0;
+ sigaction(SIGINT, &sigint_action, NULL);
+#elif defined (_WIN32)
+ auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
+ return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;
+ };
+ SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
+
+ for (int i = 0; i < argc; ++i) {
+ delete[] argv[i];
+ }
+ delete[] argv;
+#endif
+ llama.queue_tasks.start_loop();
+ svr.stop();
+ t.join();
+
+ llama_backend_free();
+ return 0;
+}
diff --git a/llm/ext_server/utils.hpp b/llm/ext_server/utils.hpp
new file mode 100644
index 00000000..d63ead04
--- /dev/null
+++ b/llm/ext_server/utils.hpp
@@ -0,0 +1,658 @@
+// MIT License
+
+// Copyright (c) 2023 Georgi Gerganov
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <set>
+#include <mutex>
+#include <condition_variable>
+#include <unordered_map>
+
+#include "json.hpp"
+
+#include "../llava/clip.h"
+
+using json = nlohmann::json;
+
+extern bool server_verbose;
+extern bool server_log_json;
+
+#ifndef SERVER_VERBOSE
+#define SERVER_VERBOSE 1
+#endif
+
+#if SERVER_VERBOSE != 1
+#define LOG_VERBOSE(MSG, ...)
+#else
+#define LOG_VERBOSE(MSG, ...) \
+ do \
+ { \
+ if (server_verbose) \
+ { \
+ server_log("VERB", __func__, __LINE__, MSG, __VA_ARGS__); \
+ } \
+ } while (0)
+#endif
+
+#define LOG_ERROR( MSG, ...) server_log("ERROR", __func__, __LINE__, MSG, __VA_ARGS__)
+#define LOG_WARNING(MSG, ...) server_log("WARN", __func__, __LINE__, MSG, __VA_ARGS__)
+#define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__)
+#define LOG_DEBUG( MSG, ...) server_log("DEBUG", __func__, __LINE__, MSG, __VA_ARGS__)
+
+enum server_state {
+ SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet
+ SERVER_STATE_READY, // Server is ready and model is loaded
+ SERVER_STATE_ERROR // An error occurred, load_model failed
+};
+
+enum task_type {
+ TASK_TYPE_COMPLETION,
+ TASK_TYPE_CANCEL,
+ TASK_TYPE_NEXT_RESPONSE,
+ TASK_TYPE_METRICS
+};
+
+struct task_server {
+ int id = -1; // to be filled by llama_server_queue
+ int target_id;
+ task_type type;
+ json data;
+ bool infill_mode = false;
+ bool embedding_mode = false;
+ int multitask_id = -1;
+};
+
+struct task_result {
+ int id;
+ int multitask_id = -1;
+ bool stop;
+ bool error;
+ json result_json;
+};
+
+struct task_multi {
+ int id;
+ std::set<int> subtasks_remaining{};
+ std::vector<task_result> results{};
+};
+
+// completion token output with probabilities
+struct completion_token_output {
+ struct token_prob
+ {
+ llama_token tok;
+ float prob;
+ };
+
+ std::vector<token_prob> probs;
+ llama_token tok;
+ std::string text_to_send;
+};
+
+struct token_translator {
+ llama_context * ctx;
+ std::string operator()(llama_token tok) const { return llama_token_to_piece(ctx, tok); }
+ std::string operator()(const completion_token_output &cto) const { return (*this)(cto.tok); }
+};
+
+static inline void server_log(const char *level, const char *function, int line, const char *message, const nlohmann::ordered_json &extra) {
+ std::stringstream ss_tid;
+ ss_tid << std::this_thread::get_id();
+ json log = nlohmann::ordered_json{
+ {"tid", ss_tid.str()},
+ {"timestamp", time(nullptr)},
+ };
+
+ if (strncmp("DEBUG", level, strlen(level)) == 0 && !server_verbose) {
+ return;
+ }
+
+ if (server_log_json) {
+ log.merge_patch(
+ {
+ {"level", level},
+ {"function", function},
+ {"line", line},
+ {"msg", message},
+ });
+ if (!extra.empty()) {
+ log.merge_patch(extra);
+ }
+
+ std::cout << log.dump(-1, ' ', false, json::error_handler_t::replace) << "\n" << std::flush;
+ } else {
+ if (!extra.empty()) {
+ log.merge_patch(extra);
+ }
+
+ std::stringstream ss;
+ ss << level << " [" << function << "] " << message << " |";
+ for (const auto& el : log.items())
+ {
+ const std::string value = el.value().dump(-1, ' ', false, json::error_handler_t::replace);
+ ss << " " << el.key() << "=" << value;
+ }
+
+ const std::string str = ss.str();
+ printf("%.*s\n", (int)str.size(), str.data());
+ fflush(stdout);
+ }
+}
+
+//
+// server utils
+//
+
+template <typename T>
+static T json_value(const json &body, const std::string &key, const T &default_value) {
+ // Fallback null to default value
+ return body.contains(key) && !body.at(key).is_null()
+ ? body.value(key, default_value)
+ : default_value;
+}
+
+// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid
+inline bool verify_custom_template(const std::string & tmpl) {
+ llama_chat_message chat[] = {{"user", "test"}};
+ std::vector<char> buf(1);
+ int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, buf.data(), buf.size());
+ return res >= 0;
+}
+
+// Format given chat. If tmpl is empty, we take the template from model metadata
+inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector<json> & messages) {
+ size_t alloc_size = 0;
+ // vector holding all allocated string to be passed to llama_chat_apply_template
+ std::vector<std::string> str(messages.size() * 2);
+ std::vector<llama_chat_message> chat(messages.size());
+
+ for (size_t i = 0; i < messages.size(); ++i) {
+ auto &curr_msg = messages[i];
+ str[i*2 + 0] = json_value(curr_msg, "role", std::string(""));
+ str[i*2 + 1] = json_value(curr_msg, "content", std::string(""));
+ alloc_size += str[i*2 + 1].length();
+ chat[i].role = str[i*2 + 0].c_str();
+ chat[i].content = str[i*2 + 1].c_str();
+ }
+
+ const char * ptr_tmpl = tmpl.empty() ? nullptr : tmpl.c_str();
+ std::vector<char> buf(alloc_size * 2);
+
+ // run the first time to get the total output length
+ int32_t res = llama_chat_apply_template(model, ptr_tmpl, chat.data(), chat.size(), true, buf.data(), buf.size());
+
+ // if it turns out that our buffer is too small, we resize it
+ if ((size_t) res > buf.size()) {
+ buf.resize(res);
+ res = llama_chat_apply_template(model, ptr_tmpl, chat.data(), chat.size(), true, buf.data(), buf.size());
+ }
+
+ std::string formatted_chat(buf.data(), res);
+ LOG_VERBOSE("formatted_chat", {{"text", formatted_chat.c_str()}});
+
+ return formatted_chat;
+}
+
+//
+// work queue utils
+//
+
+struct llama_server_queue {
+ int id = 0;
+ std::mutex mutex_tasks;
+ bool running;
+ // queues
+ std::vector<task_server> queue_tasks;
+ std::vector<task_server> queue_tasks_deferred;
+ std::vector<task_multi> queue_multitasks;
+ std::condition_variable condition_tasks;
+ // callback functions
+ std::function<void(task_server&)> callback_new_task;
+ std::function<void(task_multi&)> callback_finish_multitask;
+ std::function<void(void)> callback_run_slots;
+
+ // Add a new task to the end of the queue
+ int post(task_server task) {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ if (task.id == -1) {
+ task.id = id++;
+ LOG_VERBOSE("new task id", {{"new_id", task.id}});
+ }
+ queue_tasks.push_back(std::move(task));
+ condition_tasks.notify_one();
+ return task.id;
+ }
+
+ // Add a new task, but defer until one slot is available
+ void defer(task_server task) {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ queue_tasks_deferred.push_back(std::move(task));
+ }
+
+ // Get the next id for creating anew task
+ int get_new_id() {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ int new_id = id++;
+ LOG_VERBOSE("new task id", {{"new_id", new_id}});
+ return new_id;
+ }
+
+ // Register function to process a new task
+ void on_new_task(std::function<void(task_server&)> callback) {
+ callback_new_task = callback;
+ }
+
+ // Register function to process a multitask when it is finished
+ void on_finish_multitask(std::function<void(task_multi&)> callback) {
+ callback_finish_multitask = callback;
+ }
+
+ // Register the function to be called when all slots data is ready to be processed
+ void on_run_slots(std::function<void(void)> callback) {
+ callback_run_slots = callback;
+ }
+
+ // Call when the state of one slot is changed
+ void notify_slot_changed() {
+ // move deferred tasks back to main loop
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ for (auto & task : queue_tasks_deferred) {
+ queue_tasks.push_back(std::move(task));
+ }
+ queue_tasks_deferred.clear();
+ }
+
+ // end the start_loop routine
+ void terminate() {
+ {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ running = false;
+ }
+ condition_tasks.notify_all();
+ }
+
+ /**
+ * Main loop consists of these steps:
+ * - Wait until a new task arrives
+ * - Process the task (i.e. maybe copy data into slot)
+ * - Check if multitask is finished
+ * - Run all slots
+ */
+ void start_loop() {
+ running = true;
+ while (true) {
+ LOG_VERBOSE("new task may arrive", {});
+ {
+ while (true)
+ {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ if (queue_tasks.empty()) {
+ lock.unlock();
+ break;
+ }
+ task_server task = queue_tasks.front();
+ queue_tasks.erase(queue_tasks.begin());
+ lock.unlock();
+ LOG_VERBOSE("callback_new_task", {{"task_id", task.id}});
+ callback_new_task(task);
+ }
+ LOG_VERBOSE("update_multitasks", {});
+ // check if we have any finished multitasks
+ auto queue_iterator = queue_multitasks.begin();
+ while (queue_iterator != queue_multitasks.end())
+ {
+ if (queue_iterator->subtasks_remaining.empty())
+ {
+ // all subtasks done == multitask is done
+ task_multi current_multitask = *queue_iterator;
+ callback_finish_multitask(current_multitask);
+ // remove this multitask
+ queue_iterator = queue_multitasks.erase(queue_iterator);
+ }
+ else
+ {
+ ++queue_iterator;
+ }
+ }
+ // all tasks in the current loop is processed, slots data is now ready
+ LOG_VERBOSE("callback_run_slots", {});
+ callback_run_slots();
+ }
+ LOG_VERBOSE("wait for new task", {});
+ // wait for new task
+ {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ if (queue_tasks.empty()) {
+ if (!running) {
+ LOG_VERBOSE("ending start_loop", {});
+ return;
+ }
+ condition_tasks.wait(lock, [&]{
+ return (!queue_tasks.empty() || !running);
+ });
+ }
+ }
+ }
+ }
+
+ //
+ // functions to manage multitasks
+ //
+
+ // add a multitask by specifying the id of all subtask (subtask is a task_server)
+ void add_multitask(int multitask_id, std::vector<int>& sub_ids)
+ {
+ std::lock_guard<std::mutex> lock(mutex_tasks);
+ task_multi multi;
+ multi.id = multitask_id;
+ std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));
+ queue_multitasks.push_back(multi);
+ }
+
+ // updatethe remaining subtasks, while appending results to multitask
+ void update_multitask(int multitask_id, int subtask_id, task_result& result)
+ {
+ std::lock_guard<std::mutex> lock(mutex_tasks);
+ for (auto& multitask : queue_multitasks)
+ {
+ if (multitask.id == multitask_id)
+ {
+ multitask.subtasks_remaining.erase(subtask_id);
+ multitask.results.push_back(result);
+ }
+ }
+ }
+};
+
+struct llama_server_response {
+ typedef std::function<void(int, int, task_result&)> callback_multitask_t;
+ callback_multitask_t callback_update_multitask;
+ // for keeping track of all tasks waiting for the result
+ std::set<int> waiting_task_ids;
+ // the main result queue
+ std::vector<task_result> queue_results;
+ std::mutex mutex_results;
+ std::condition_variable condition_results;
+
+ // add the task_id to the list of tasks waiting for response
+ void add_waiting_task_id(int task_id) {
+ LOG_VERBOSE("waiting for task id", {{"task_id", task_id}});
+ std::unique_lock<std::mutex> lock(mutex_results);
+ waiting_task_ids.insert(task_id);
+ }
+
+ // when the request is finished, we can remove task associated with it
+ void remove_waiting_task_id(int task_id) {
+ LOG_VERBOSE("remove waiting for task id", {{"task_id", task_id}});
+ std::unique_lock<std::mutex> lock(mutex_results);
+ waiting_task_ids.erase(task_id);
+ }
+
+ // This function blocks the thread until there is a response for this task_id
+ task_result recv(int task_id) {
+ while (true)
+ {
+ std::unique_lock<std::mutex> lock(mutex_results);
+ condition_results.wait(lock, [&]{
+ return !queue_results.empty();
+ });
+
+ for (int i = 0; i < (int) queue_results.size(); i++)
+ {
+ if (queue_results[i].id == task_id)
+ {
+ assert(queue_results[i].multitask_id == -1);
+ task_result res = queue_results[i];
+ queue_results.erase(queue_results.begin() + i);
+ return res;
+ }
+ }
+ }
+
+ // should never reach here
+ }
+
+ // Register the function to update multitask
+ void on_multitask_update(callback_multitask_t callback) {
+ callback_update_multitask = callback;
+ }
+
+ // Send a new result to a waiting task_id
+ void send(task_result result) {
+ std::unique_lock<std::mutex> lock(mutex_results);
+ LOG_VERBOSE("send new result", {{"task_id", result.id}});
+ for (auto& task_id : waiting_task_ids) {
+ // LOG_TEE("waiting task id %i \n", task_id);
+ // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result
+ if (result.multitask_id == task_id)
+ {
+ LOG_VERBOSE("callback_update_multitask", {{"task_id", task_id}});
+ callback_update_multitask(task_id, result.id, result);
+ continue;
+ }
+
+ if (result.id == task_id)
+ {
+ LOG_VERBOSE("queue_results.push_back", {{"task_id", task_id}});
+ queue_results.push_back(result);
+ condition_results.notify_all();
+ return;
+ }
+ }
+ }
+};
+
+//
+// base64 utils (TODO: move to common in the future)
+//
+
+static const std::string base64_chars =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz"
+ "0123456789+/";
+
+static inline bool is_base64(uint8_t c)
+{
+ return (isalnum(c) || (c == '+') || (c == '/'));
+}
+
+static inline std::vector<uint8_t> base64_decode(const std::string & encoded_string)
+{
+ int i = 0;
+ int j = 0;
+ int in_ = 0;
+
+ int in_len = encoded_string.size();
+
+ uint8_t char_array_4[4];
+ uint8_t char_array_3[3];
+
+ std::vector<uint8_t> ret;
+
+ while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_]))
+ {
+ char_array_4[i++] = encoded_string[in_]; in_++;
+ if (i == 4)
+ {
+ for (i = 0; i <4; i++)
+ {
+ char_array_4[i] = base64_chars.find(char_array_4[i]);
+ }
+
+ char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);
+ char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
+ char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
+
+ for (i = 0; (i < 3); i++)
+ {
+ ret.push_back(char_array_3[i]);
+ }
+ i = 0;
+ }
+ }
+
+ if (i)
+ {
+ for (j = i; j <4; j++)
+ {
+ char_array_4[j] = 0;
+ }
+
+ for (j = 0; j <4; j++)
+ {
+ char_array_4[j] = base64_chars.find(char_array_4[j]);
+ }
+
+ char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);
+ char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
+ char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
+
+ for (j = 0; (j < i - 1); j++)
+ {
+ ret.push_back(char_array_3[j]);
+ }
+ }
+
+ return ret;
+}
+
+//
+// random string / id
+//
+
+static std::string random_string()
+{
+ static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");
+
+ std::random_device rd;
+ std::mt19937 generator(rd());
+
+ std::string result(32, ' ');
+
+ for (int i = 0; i < 32; ++i) {
+ result[i] = str[generator() % str.size()];
+ }
+
+ return result;
+}
+
+static std::string gen_chatcmplid()
+{
+ std::stringstream chatcmplid;
+ chatcmplid << "chatcmpl-" << random_string();
+ return chatcmplid.str();
+}
+
+//
+// other common utils
+//
+
+static size_t common_part(const std::vector<llama_token> &a, const std::vector<llama_token> &b)
+{
+ size_t i;
+ for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++)
+ {
+ }
+ return i;
+}
+
+static bool ends_with(const std::string &str, const std::string &suffix)
+{
+ return str.size() >= suffix.size() &&
+ 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);
+}
+
+static size_t find_partial_stop_string(const std::string &stop,
+ const std::string &text)
+{
+ if (!text.empty() && !stop.empty())
+ {
+ const char text_last_char = text.back();
+ for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--)
+ {
+ if (stop[char_index] == text_last_char)
+ {
+ const std::string current_partial = stop.substr(0, char_index + 1);
+ if (ends_with(text, current_partial))
+ {
+ return text.size() - char_index - 1;
+ }
+ }
+ }
+ }
+ return std::string::npos;
+}
+
+// TODO: reuse llama_detokenize
+template <class Iter>
+static std::string tokens_to_str(llama_context *ctx, Iter begin, Iter end)
+{
+ std::string ret;
+ for (; begin != end; ++begin)
+ {
+ ret += llama_token_to_piece(ctx, *begin);
+ }
+ return ret;
+}
+
+// format incomplete utf-8 multibyte character for output
+static std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token)
+{
+ std::string out = token == -1 ? "" : llama_token_to_piece(ctx, token);
+ // if the size is 1 and first bit is 1, meaning it's a partial character
+ // (size > 1 meaning it's already a known token)
+ if (out.size() == 1 && (out[0] & 0x80) == 0x80)
+ {
+ std::stringstream ss;
+ ss << std::hex << (out[0] & 0xff);
+ std::string res(ss.str());
+ out = "byte: \\x" + res;
+ }
+ return out;
+}
+
+// convert a vector of completion_token_output to json
+static json probs_vector_to_json(const llama_context *ctx, const std::vector<completion_token_output> &probs)
+{
+ json out = json::array();
+ for (const auto &prob : probs)
+ {
+ json probs_for_token = json::array();
+ for (const auto &p : prob.probs)
+ {
+ std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok);
+ probs_for_token.push_back(json
+ {
+ {"tok_str", tok_str},
+ {"prob", p.prob},
+ });
+ }
+ std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok);
+ out.push_back(json{
+ {"content", tok_str},
+ {"probs", probs_for_token},
+ });
+ }
+ return out;
+}
diff --git a/llm/filetype.go b/llm/filetype.go
new file mode 100644
index 00000000..7a8e9f69
--- /dev/null
+++ b/llm/filetype.go
@@ -0,0 +1,180 @@
+package llm
+
+import "fmt"
+
+type fileType uint32
+
+const (
+ fileTypeF32 fileType = iota
+ fileTypeF16
+ fileTypeQ4_0
+ fileTypeQ4_1
+ fileTypeQ4_1_F16
+ fileTypeQ4_2 // unused
+ fileTypeQ4_3 // unused
+ fileTypeQ8_0
+ fileTypeQ5_0
+ fileTypeQ5_1
+ fileTypeQ2_K
+ fileTypeQ3_K_S
+ fileTypeQ3_K_M
+ fileTypeQ3_K_L
+ fileTypeQ4_K_S
+ fileTypeQ4_K_M
+ fileTypeQ5_K_S
+ fileTypeQ5_K_M
+ fileTypeQ6_K
+ fileTypeIQ2_XXS
+ fileTypeIQ2_XS
+ fileTypeQ2_K_S
+ fileTypeIQ3_XS
+ fileTypeIQ3_XXS
+ fileTypeIQ1_S
+ fileTypeIQ4_NL
+ fileTypeIQ3_S
+ fileTypeIQ2_S
+ fileTypeIQ4_XS
+ fileTypeIQ2_M
+ fileTypeIQ1_M
+ fileTypeBF16
+
+ fileTypeUnknown
+)
+
+func ParseFileType(s string) (fileType, error) {
+ switch s {
+ case "F32":
+ return fileTypeF32, nil
+ case "F16":
+ return fileTypeF16, nil
+ case "Q4_0":
+ return fileTypeQ4_0, nil
+ case "Q4_1":
+ return fileTypeQ4_1, nil
+ case "Q4_1_F16":
+ return fileTypeQ4_1_F16, nil
+ case "Q8_0":
+ return fileTypeQ8_0, nil
+ case "Q5_0":
+ return fileTypeQ5_0, nil
+ case "Q5_1":
+ return fileTypeQ5_1, nil
+ case "Q2_K":
+ return fileTypeQ2_K, nil
+ case "Q3_K_S":
+ return fileTypeQ3_K_S, nil
+ case "Q3_K_M":
+ return fileTypeQ3_K_M, nil
+ case "Q3_K_L":
+ return fileTypeQ3_K_L, nil
+ case "Q4_K_S":
+ return fileTypeQ4_K_S, nil
+ case "Q4_K_M":
+ return fileTypeQ4_K_M, nil
+ case "Q5_K_S":
+ return fileTypeQ5_K_S, nil
+ case "Q5_K_M":
+ return fileTypeQ5_K_M, nil
+ case "Q6_K":
+ return fileTypeQ6_K, nil
+ case "IQ2_XXS":
+ return fileTypeIQ2_XXS, nil
+ case "IQ2_XS":
+ return fileTypeIQ2_XS, nil
+ case "Q2_K_S":
+ return fileTypeQ2_K_S, nil
+ case "IQ3_XS":
+ return fileTypeIQ3_XS, nil
+ case "IQ3_XXS":
+ return fileTypeIQ3_XXS, nil
+ case "IQ1_S":
+ return fileTypeIQ1_S, nil
+ case "IQ4_NL":
+ return fileTypeIQ4_NL, nil
+ case "IQ3_S":
+ return fileTypeIQ3_S, nil
+ case "IQ2_S":
+ return fileTypeIQ2_S, nil
+ case "IQ4_XS":
+ return fileTypeIQ4_XS, nil
+ case "IQ2_M":
+ return fileTypeIQ2_M, nil
+ case "IQ1_M":
+ return fileTypeIQ1_M, nil
+ case "BF16":
+ return fileTypeBF16, nil
+ default:
+ return fileTypeUnknown, fmt.Errorf("unknown fileType: %s", s)
+ }
+}
+
+func (t fileType) String() string {
+ switch t {
+ case fileTypeF32:
+ return "F32"
+ case fileTypeF16:
+ return "F16"
+ case fileTypeQ4_0:
+ return "Q4_0"
+ case fileTypeQ4_1:
+ return "Q4_1"
+ case fileTypeQ4_1_F16:
+ return "Q4_1_F16"
+ case fileTypeQ8_0:
+ return "Q8_0"
+ case fileTypeQ5_0:
+ return "Q5_0"
+ case fileTypeQ5_1:
+ return "Q5_1"
+ case fileTypeQ2_K:
+ return "Q2_K"
+ case fileTypeQ3_K_S:
+ return "Q3_K_S"
+ case fileTypeQ3_K_M:
+ return "Q3_K_M"
+ case fileTypeQ3_K_L:
+ return "Q3_K_L"
+ case fileTypeQ4_K_S:
+ return "Q4_K_S"
+ case fileTypeQ4_K_M:
+ return "Q4_K_M"
+ case fileTypeQ5_K_S:
+ return "Q5_K_S"
+ case fileTypeQ5_K_M:
+ return "Q5_K_M"
+ case fileTypeQ6_K:
+ return "Q6_K"
+ case fileTypeIQ2_XXS:
+ return "IQ2_XXS"
+ case fileTypeIQ2_XS:
+ return "IQ2_XS"
+ case fileTypeQ2_K_S:
+ return "Q2_K_S"
+ case fileTypeIQ3_XS:
+ return "IQ3_XS"
+ case fileTypeIQ3_XXS:
+ return "IQ3_XXS"
+ case fileTypeIQ1_S:
+ return "IQ1_S"
+ case fileTypeIQ4_NL:
+ return "IQ4_NL"
+ case fileTypeIQ3_S:
+ return "IQ3_S"
+ case fileTypeIQ2_S:
+ return "IQ2_S"
+ case fileTypeIQ4_XS:
+ return "IQ4_XS"
+ case fileTypeIQ2_M:
+ return "IQ2_M"
+ case fileTypeIQ1_M:
+ return "IQ1_M"
+ case fileTypeBF16:
+ return "BF16"
+ default:
+ return "unknown"
+ }
+}
+
+func (t fileType) Value() uint32 {
+ return uint32(t)
+}
diff --git a/llm/generate/gen_common.sh b/llm/generate/gen_common.sh
new file mode 100644
index 00000000..e58c2d40
--- /dev/null
+++ b/llm/generate/gen_common.sh
@@ -0,0 +1,137 @@
+# common logic across linux and darwin
+
+init_vars() {
+ case "${GOARCH}" in
+ "amd64")
+ ARCH="x86_64"
+ ;;
+ "arm64")
+ ARCH="arm64"
+ ;;
+ *)
+ echo "GOARCH must be set"
+ echo "this script is meant to be run from within go generate"
+ exit 1
+ ;;
+ esac
+
+ LLAMACPP_DIR=../llama.cpp
+ CMAKE_DEFS="-DCMAKE_SKIP_RPATH=on"
+ CMAKE_TARGETS="--target ollama_llama_server"
+ if echo "${CGO_CFLAGS}" | grep -- '-g' >/dev/null; then
+ CMAKE_DEFS="-DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_VERBOSE_MAKEFILE=on -DLLAMA_GPROF=on -DLLAMA_SERVER_VERBOSE=on ${CMAKE_DEFS}"
+ else
+ # TODO - add additional optimization flags...
+ CMAKE_DEFS="-DCMAKE_BUILD_TYPE=Release -DLLAMA_SERVER_VERBOSE=off ${CMAKE_DEFS}"
+ fi
+ case $(uname -s) in
+ "Darwin")
+ LIB_EXT="dylib"
+ WHOLE_ARCHIVE="-Wl,-force_load"
+ NO_WHOLE_ARCHIVE=""
+ GCC_ARCH="-arch ${ARCH}"
+ DIST_BASE=../../dist/darwin-${GOARCH}/
+ PAYLOAD_BASE=../../build/darwin/${GOARCH}
+ ;;
+ "Linux")
+ LIB_EXT="so"
+ WHOLE_ARCHIVE="-Wl,--whole-archive"
+ NO_WHOLE_ARCHIVE="-Wl,--no-whole-archive"
+
+ # Cross compiling not supported on linux - Use docker
+ GCC_ARCH=""
+ DIST_BASE=../../dist/linux-${GOARCH}/
+ PAYLOAD_BASE=../../build/linux/${GOARCH}
+ ;;
+ *)
+ ;;
+ esac
+ if [ -z "${CMAKE_CUDA_ARCHITECTURES}" ] ; then
+ CMAKE_CUDA_ARCHITECTURES="50;52;61;70;75;80"
+ fi
+ GZIP=$(command -v pigz 2>/dev/null || echo "gzip")
+ RUNNER_BASE="${DIST_BASE}/lib/ollama/runners"
+}
+
+git_module_setup() {
+ if [ -n "${OLLAMA_SKIP_PATCHING}" ]; then
+ echo "Skipping submodule initialization"
+ return
+ fi
+ # Make sure the tree is clean after the directory moves
+ if [ -d "${LLAMACPP_DIR}/gguf" ]; then
+ echo "Cleaning up old submodule"
+ rm -rf ${LLAMACPP_DIR}
+ fi
+ git submodule init
+ git submodule update --force ${LLAMACPP_DIR}
+
+}
+
+apply_patches() {
+ # apply temporary patches until fix is upstream
+ for patch in ../patches/*.patch; do
+ git -c 'user.name=nobody' -c 'user.email=<>' -C ${LLAMACPP_DIR} am ${patch}
+ done
+}
+
+build() {
+ cmake -S ${LLAMACPP_DIR} -B ${BUILD_DIR} ${CMAKE_DEFS}
+ cmake --build ${BUILD_DIR} ${CMAKE_TARGETS} -j8
+ # remove unnecessary build artifacts
+ rm -f ${BUILD_DIR}/bin/ggml-common.h ${BUILD_DIR}/bin/ggml-metal.metal
+}
+
+dist() {
+ [ -z "${RUNNER}" ] && exit 1
+ mkdir -p ${RUNNER_BASE}/${RUNNER}/
+ for f in ${BUILD_DIR}/bin/* ; do
+ cp ${f} ${RUNNER_BASE}/${RUNNER}/
+ done
+ # check for lib directory
+ if [ -d ${BUILD_DIR}/lib ]; then
+ for f in ${BUILD_DIR}/lib/* ; do
+ cp ${f} ${RUNNER_BASE}/${RUNNER}/
+ done
+ fi
+}
+
+# Compress from the build $BUILD_DIR into the $PAYLOAD_BASE/$RUNNER dir
+compress() {
+ [ -z "${RUNNER}" ] && exit 1
+ echo "Compressing payloads with ${GZIP} to reduce overall binary size..."
+ rm -rf "${PAYLOAD_BASE}/${RUNNER}/"
+ mkdir -p "${PAYLOAD_BASE}/${RUNNER}/"
+ for f in ${BUILD_DIR}/bin/* ; do
+ ${GZIP} -c --best ${f} > "${PAYLOAD_BASE}/${RUNNER}/$(basename ${f}).gz" &
+ compress_pids+=" $!"
+ done
+ # check for lib directory
+ if [ -d ${BUILD_DIR}/lib ]; then
+ for f in ${BUILD_DIR}/lib/* ; do
+ ${GZIP} -c --best ${f} > "${PAYLOAD_BASE}/${RUNNER}/$(basename ${f}).gz" &
+ compress_pids+=" $!"
+ done
+ fi
+ echo
+}
+
+wait_for_compress() {
+ for pid in ${compress_pids}; do
+ wait $pid
+ done
+ echo "Finished compression"
+}
+
+install() {
+ echo "Installing libraries to bin dir ${BUILD_DIR}/bin/"
+ for lib in $(find ${BUILD_DIR} -name \*.${LIB_EXT} | grep -v "${BUILD_DIR}/bin/" ); do
+ rm -f "${BUILD_DIR}/bin/$(basename ${lib})"
+ cp -af "${lib}" "${BUILD_DIR}/bin/"
+ done
+}
+
+# Keep the local tree clean after we're done with the build
+cleanup() {
+ git submodule update --force ${LLAMACPP_DIR}
+}
diff --git a/llm/generate/gen_darwin.sh b/llm/generate/gen_darwin.sh
new file mode 100755
index 00000000..3c7c71ea
--- /dev/null
+++ b/llm/generate/gen_darwin.sh
@@ -0,0 +1,91 @@
+#!/bin/bash
+# This script is intended to run inside the go generate
+# working directory must be ./llm/generate/
+
+# TODO - add hardening to detect missing tools (cmake, etc.)
+
+set -ex
+set -o pipefail
+compress_pids=""
+echo "Starting darwin generate script"
+source $(dirname $0)/gen_common.sh
+init_vars
+git_module_setup
+apply_patches
+
+sign() {
+ if [ -n "$APPLE_IDENTITY" ]; then
+ codesign -f --timestamp --deep --options=runtime --sign "$APPLE_IDENTITY" --identifier ai.ollama.ollama $1
+ fi
+}
+
+COMMON_DARWIN_DEFS="-DBUILD_SHARED_LIBS=off -DCMAKE_OSX_DEPLOYMENT_TARGET=11.3 -DGGML_METAL_MACOSX_VERSION_MIN=11.3 -DCMAKE_SYSTEM_NAME=Darwin -DGGML_METAL_EMBED_LIBRARY=on -DGGML_OPENMP=off"
+
+case "${GOARCH}" in
+"amd64")
+ COMMON_CPU_DEFS="${COMMON_DARWIN_DEFS} -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_OSX_ARCHITECTURES=${ARCH} -DGGML_METAL=off -DGGML_NATIVE=off"
+
+ if [ -z "$OLLAMA_SKIP_CPU_GENERATE" ]; then
+ #
+ # CPU first for the default library, set up as lowest common denominator for maximum compatibility (including Rosetta)
+ #
+ init_vars
+ CMAKE_DEFS="${COMMON_CPU_DEFS} -DGGML_ACCELERATE=off -DGGML_BLAS=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off ${CMAKE_DEFS}"
+ RUNNER=cpu
+ BUILD_DIR="../build/darwin/${GOARCH}/${RUNNER}"
+ echo "Building LCD CPU"
+ build
+ sign ${BUILD_DIR}/bin/ollama_llama_server
+ compress
+
+ #
+ # ~2011 CPU Dynamic library with more capabilities turned on to optimize performance
+ # Approximately 400% faster than LCD on same CPU
+ #
+ init_vars
+ CMAKE_DEFS="${COMMON_CPU_DEFS} -DGGML_ACCELERATE=off -DGGML_BLAS=off -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off ${CMAKE_DEFS}"
+ RUNNER=cpu_avx
+ BUILD_DIR="../build/darwin/${GOARCH}/${RUNNER}"
+ echo "Building AVX CPU"
+ build
+ sign ${BUILD_DIR}/bin/ollama_llama_server
+ compress
+
+ #
+ # ~2013 CPU Dynamic library
+ # Approximately 10% faster than AVX on same CPU
+ #
+ init_vars
+ CMAKE_DEFS="${COMMON_CPU_DEFS} -DGGML_ACCELERATE=on -DGGML_BLAS=off -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on ${CMAKE_DEFS}"
+ RUNNER=cpu_avx2
+ BUILD_DIR="../build/darwin/${GOARCH}/${RUNNER}"
+ echo "Building AVX2 CPU"
+ EXTRA_LIBS="${EXTRA_LIBS} -framework Accelerate -framework Foundation"
+ build
+ sign ${BUILD_DIR}/bin/ollama_llama_server
+ compress
+ fi
+ ;;
+"arm64")
+
+ if [ -z "$OLLAMA_SKIP_METAL_GENERATE" ]; then
+ init_vars
+ CMAKE_DEFS="${COMMON_DARWIN_DEFS} -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_OSX_ARCHITECTURES=${ARCH} ${CMAKE_DEFS}"
+ RUNNER="metal"
+ BUILD_DIR="../build/darwin/${GOARCH}/${RUNNER}"
+ EXTRA_LIBS="${EXTRA_LIBS} -framework Accelerate -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders"
+ build
+ sign ${BUILD_DIR}/bin/ollama_llama_server
+ compress
+ fi
+ ;;
+*)
+ echo "GOARCH must be set"
+ echo "this script is meant to be run from within go generate"
+ exit 1
+ ;;
+esac
+
+cleanup
+wait_for_compress
+echo "go generate completed. LLM runners: $(cd ${BUILD_DIR}/..; echo *)"
diff --git a/llm/generate/gen_linux.sh b/llm/generate/gen_linux.sh
new file mode 100755
index 00000000..cacaaf4b
--- /dev/null
+++ b/llm/generate/gen_linux.sh
@@ -0,0 +1,285 @@
+#!/bin/bash
+# This script is intended to run inside the go generate
+# working directory must be llm/generate/
+
+# First we build one or more CPU based LLM libraries
+#
+# Then if we detect CUDA, we build a CUDA dynamic library, and carry the required
+# library dependencies
+#
+# Then if we detect ROCm, we build a dynamically loaded ROCm lib. The ROCM
+# libraries are quite large, and also dynamically load data files at runtime
+# which in turn are large, so we don't attempt to cary them as payload
+
+set -ex
+set -o pipefail
+compress_pids=""
+
+# See https://llvm.org/docs/AMDGPUUsage.html#processors for reference
+amdGPUs() {
+ if [ -n "${AMDGPU_TARGETS}" ]; then
+ echo "${AMDGPU_TARGETS}"
+ return
+ fi
+ GPU_LIST=(
+ "gfx900"
+ "gfx906:xnack-"
+ "gfx908:xnack-"
+ "gfx90a:xnack+"
+ "gfx90a:xnack-"
+ "gfx940"
+ "gfx941"
+ "gfx942"
+ "gfx1010"
+ "gfx1012"
+ "gfx1030"
+ "gfx1100"
+ "gfx1101"
+ "gfx1102"
+ )
+ (
+ IFS=$';'
+ echo "'${GPU_LIST[*]}'"
+ )
+}
+
+echo "Starting linux generate script"
+if [ -z "${CUDACXX}" ]; then
+ if [ -x /usr/local/cuda/bin/nvcc ]; then
+ export CUDACXX=/usr/local/cuda/bin/nvcc
+ else
+ # Try the default location in case it exists
+ export CUDACXX=$(command -v nvcc)
+ fi
+fi
+COMMON_CMAKE_DEFS="-DCMAKE_SKIP_RPATH=on -DBUILD_SHARED_LIBS=on -DCMAKE_POSITION_INDEPENDENT_CODE=on -DGGML_NATIVE=off -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_OPENMP=off"
+source $(dirname $0)/gen_common.sh
+init_vars
+git_module_setup
+apply_patches
+
+init_vars
+if [ -z "${OLLAMA_SKIP_CPU_GENERATE}" ]; then
+ # Users building from source can tune the exact flags we pass to cmake for configuring
+ # llama.cpp, and we'll build only 1 CPU variant in that case as the default.
+ if [ -n "${OLLAMA_CUSTOM_CPU_DEFS}" ]; then
+ init_vars
+ echo "OLLAMA_CUSTOM_CPU_DEFS=\"${OLLAMA_CUSTOM_CPU_DEFS}\""
+ CMAKE_DEFS="${OLLAMA_CUSTOM_CPU_DEFS} -DBUILD_SHARED_LIBS=on -DCMAKE_POSITION_INDEPENDENT_CODE=on ${CMAKE_DEFS}"
+ RUNNER="cpu"
+ BUILD_DIR="../build/linux/${GOARCH}/${RUNNER}"
+ echo "Building custom CPU"
+ build
+ install
+ dist
+ compress
+ else
+ # Darwin Rosetta x86 emulation does NOT support AVX, AVX2, AVX512
+ # -DGGML_AVX -- 2011 Intel Sandy Bridge & AMD Bulldozer
+ # -DGGML_F16C -- 2012 Intel Ivy Bridge & AMD 2011 Bulldozer (No significant improvement over just AVX)
+ # -DGGML_AVX2 -- 2013 Intel Haswell & 2015 AMD Excavator / 2017 AMD Zen
+ # -DGGML_FMA (FMA3) -- 2013 Intel Haswell & 2012 AMD Piledriver
+ # Note: the following seem to yield slower results than AVX2 - ymmv
+ # -DGGML_AVX512 -- 2017 Intel Skylake and High End DeskTop (HEDT)
+ # -DGGML_AVX512_VBMI -- 2018 Intel Cannon Lake
+ # -DGGML_AVX512_VNNI -- 2021 Intel Alder Lake
+
+ COMMON_CPU_DEFS="-DBUILD_SHARED_LIBS=on -DCMAKE_POSITION_INDEPENDENT_CODE=on -DGGML_NATIVE=off -DGGML_OPENMP=off"
+ if [ -z "${OLLAMA_CPU_TARGET}" -o "${OLLAMA_CPU_TARGET}" = "cpu" ]; then
+ #
+ # CPU first for the default library, set up as lowest common denominator for maximum compatibility (including Rosetta)
+ #
+ init_vars
+ CMAKE_DEFS="${COMMON_CPU_DEFS} -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off ${CMAKE_DEFS}"
+ RUNNER=cpu
+ BUILD_DIR="../build/linux/${GOARCH}/${RUNNER}"
+ echo "Building LCD CPU"
+ build
+ install
+ dist
+ compress
+ fi
+
+ if [ "${ARCH}" == "x86_64" ]; then
+ #
+ # ARM chips in M1/M2/M3-based MACs and NVidia Tegra devices do not currently support avx extensions.
+ #
+ if [ -z "${OLLAMA_CPU_TARGET}" -o "${OLLAMA_CPU_TARGET}" = "cpu_avx" ]; then
+ #
+ # ~2011 CPU Dynamic library with more capabilities turned on to optimize performance
+ # Approximately 400% faster than LCD on same CPU
+ #
+ init_vars
+ CMAKE_DEFS="${COMMON_CPU_DEFS} -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off ${CMAKE_DEFS}"
+ RUNNER=cpu_avx
+ BUILD_DIR="../build/linux/${GOARCH}/${RUNNER}"
+ echo "Building AVX CPU"
+ build
+ install
+ dist
+ compress
+ fi
+
+ if [ -z "${OLLAMA_CPU_TARGET}" -o "${OLLAMA_CPU_TARGET}" = "cpu_avx2" ]; then
+ #
+ # ~2013 CPU Dynamic library
+ # Approximately 10% faster than AVX on same CPU
+ #
+ init_vars
+ CMAKE_DEFS="${COMMON_CPU_DEFS} -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on ${CMAKE_DEFS}"
+ RUNNER=cpu_avx2
+ BUILD_DIR="../build/linux/${GOARCH}/${RUNNER}"
+ echo "Building AVX2 CPU"
+ build
+ install
+ dist
+ compress
+ fi
+ fi
+ fi
+else
+ echo "Skipping CPU generation step as requested"
+fi
+
+# If needed, look for the default CUDA toolkit location
+if [ -z "${CUDA_LIB_DIR}" ] && [ -d /usr/local/cuda/lib64 ]; then
+ CUDA_LIB_DIR=/usr/local/cuda/lib64
+fi
+
+# If needed, look for CUDA on Arch Linux
+if [ -z "${CUDA_LIB_DIR}" ] && [ -d /opt/cuda/targets/x86_64-linux/lib ]; then
+ CUDA_LIB_DIR=/opt/cuda/targets/x86_64-linux/lib
+fi
+
+# Allow override in case libcudart is in the wrong place
+if [ -z "${CUDART_LIB_DIR}" ]; then
+ CUDART_LIB_DIR="${CUDA_LIB_DIR}"
+fi
+
+if [ -z "${OLLAMA_SKIP_CUDA_GENERATE}" -a -d "${CUDA_LIB_DIR}" ]; then
+ echo "CUDA libraries detected - building dynamic CUDA library"
+ init_vars
+ CUDA_MAJOR=$(ls "${CUDA_LIB_DIR}"/libcudart.so.* | head -1 | cut -f3 -d. || true)
+ if [ -n "${CUDA_MAJOR}" -a -z "${CUDA_VARIANT}" ]; then
+ CUDA_VARIANT=_v${CUDA_MAJOR}
+ fi
+ if [ "${ARCH}" == "arm64" ]; then
+ echo "ARM CPU detected - disabling unsupported AVX instructions"
+
+ # ARM-based CPUs such as M1 and Tegra do not support AVX extensions.
+ #
+ # CUDA compute < 6.0 lacks proper FP16 support on ARM.
+ # Disabling has minimal performance effect while maintaining compatibility.
+ ARM64_DEFS="-DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_CUDA_F16=off"
+ fi
+ # Users building from source can tune the exact flags we pass to cmake for configuring llama.cpp
+ if [ -n "${OLLAMA_CUSTOM_CUDA_DEFS}" ]; then
+ echo "OLLAMA_CUSTOM_CUDA_DEFS=\"${OLLAMA_CUSTOM_CUDA_DEFS}\""
+ CMAKE_CUDA_DEFS="-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES} ${OLLAMA_CUSTOM_CUDA_DEFS}"
+ echo "Building custom CUDA GPU"
+ else
+ CMAKE_CUDA_DEFS="-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}"
+ fi
+ export CUDAFLAGS="-t8"
+ CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} ${ARM64_DEFS} ${CMAKE_CUDA_DEFS} -DGGML_STATIC=off"
+ RUNNER=cuda${CUDA_VARIANT}
+ BUILD_DIR="../build/linux/${GOARCH}/${RUNNER}"
+ export LLAMA_SERVER_LDFLAGS="-L${CUDA_LIB_DIR} -lcudart -lcublas -lcublasLt -lcuda"
+ CUDA_DIST_DIR="${CUDA_DIST_DIR:-${DIST_BASE}/lib/ollama}"
+ build
+ install
+ dist
+ echo "Installing CUDA dependencies in ${CUDA_DIST_DIR}"
+ mkdir -p "${CUDA_DIST_DIR}"
+ for lib in ${CUDA_LIB_DIR}/libcudart.so* ${CUDA_LIB_DIR}/libcublas.so* ${CUDA_LIB_DIR}/libcublasLt.so* ; do
+ cp -a "${lib}" "${CUDA_DIST_DIR}"
+ done
+ compress
+
+fi
+
+if [ -z "${ONEAPI_ROOT}" ]; then
+ # Try the default location in case it exists
+ ONEAPI_ROOT=/opt/intel/oneapi
+fi
+
+if [ -z "${OLLAMA_SKIP_ONEAPI_GENERATE}" -a -d "${ONEAPI_ROOT}" ]; then
+ echo "OneAPI libraries detected - building dynamic OneAPI library"
+ init_vars
+ source ${ONEAPI_ROOT}/setvars.sh --force # set up environment variables for oneAPI
+ CC=icx
+ CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL=ON -DGGML_SYCL_F16=OFF"
+ RUNNER=oneapi
+ BUILD_DIR="../build/linux/${GOARCH}/${RUNNER}"
+ ONEAPI_DIST_DIR="${DIST_BASE}/lib/ollama"
+ export LLAMA_SERVER_LDFLAGS="-fsycl -lOpenCL -lmkl_core -lmkl_sycl_blas -lmkl_intel_ilp64 -lmkl_tbb_thread -ltbb"
+ DEBUG_FLAGS="" # icx compiles with -O0 if we pass -g, so we must remove it
+ build
+
+ # copy oneAPI dependencies
+ mkdir -p "${ONEAPI_DIST_DIR}"
+ for dep in $(ldd "${BUILD_DIR}/bin/ollama_llama_server" | grep "=>" | cut -f2 -d= | cut -f2 -d' ' | grep -e sycl -e mkl -e tbb); do
+ cp -a "${dep}" "${ONEAPI_DIST_DIR}"
+ done
+ cp "${ONEAPI_ROOT}/compiler/latest/lib/libOpenCL.so" "${ONEAPI_DIST_DIR}"
+ cp "${ONEAPI_ROOT}/compiler/latest/lib/libimf.so" "${ONEAPI_DIST_DIR}"
+ cp "${ONEAPI_ROOT}/compiler/latest/lib/libintlc.so.5" "${ONEAPI_DIST_DIR}"
+ cp "${ONEAPI_ROOT}/compiler/latest/lib/libirng.so" "${ONEAPI_DIST_DIR}"
+ cp "${ONEAPI_ROOT}/compiler/latest/lib/libpi_level_zero.so" "${ONEAPI_DIST_DIR}"
+ cp "${ONEAPI_ROOT}/compiler/latest/lib/libsvml.so" "${ONEAPI_DIST_DIR}"
+ cp "${ONEAPI_ROOT}/compiler/latest/lib/libur_loader.so.0" "${ONEAPI_DIST_DIR}"
+ install
+ dist
+ compress
+fi
+
+if [ -z "${ROCM_PATH}" ]; then
+ # Try the default location in case it exists
+ ROCM_PATH=/opt/rocm
+fi
+
+if [ -z "${CLBlast_DIR}" ]; then
+ # Try the default location in case it exists
+ if [ -d /usr/lib/cmake/CLBlast ]; then
+ export CLBlast_DIR=/usr/lib/cmake/CLBlast
+ fi
+fi
+
+if [ -z "${OLLAMA_SKIP_ROCM_GENERATE}" -a -d "${ROCM_PATH}" ]; then
+ echo "ROCm libraries detected - building dynamic ROCm library"
+ if [ -f ${ROCM_PATH}/lib/librocblas.so.*.*.????? ]; then
+ ROCM_VARIANT=_v$(ls ${ROCM_PATH}/lib/librocblas.so.*.*.????? | cut -f5 -d. || true)
+ fi
+ init_vars
+ CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} -DGGML_HIPBLAS=on -DCMAKE_C_COMPILER=$ROCM_PATH/llvm/bin/clang -DCMAKE_CXX_COMPILER=$ROCM_PATH/llvm/bin/clang++ -DAMDGPU_TARGETS=$(amdGPUs) -DGPU_TARGETS=$(amdGPUs)"
+ # Users building from source can tune the exact flags we pass to cmake for configuring llama.cpp
+ if [ -n "${OLLAMA_CUSTOM_ROCM_DEFS}" ]; then
+ echo "OLLAMA_CUSTOM_ROCM_DEFS=\"${OLLAMA_CUSTOM_ROCM_DEFS}\""
+ CMAKE_DEFS="${CMAKE_DEFS} ${OLLAMA_CUSTOM_ROCM_DEFS}"
+ echo "Building custom ROCM GPU"
+ fi
+ RUNNER=rocm${ROCM_VARIANT}
+ BUILD_DIR="../build/linux/${GOARCH}/${RUNNER}"
+ # ROCm dependencies are too large to fit into a unified bundle
+ ROCM_DIST_DIR="${DIST_BASE}/../linux-${GOARCH}-rocm/lib/ollama"
+ # TODO figure out how to disable runpath (rpath)
+ # export CMAKE_HIP_FLAGS="-fno-rtlib-add-rpath" # doesn't work
+ export LLAMA_SERVER_LDFLAGS="-L${ROCM_PATH}/lib -L/opt/amdgpu/lib/x86_64-linux-gnu/ -lhipblas -lrocblas -lamdhip64 -lrocsolver -lamd_comgr -lhsa-runtime64 -lrocsparse -ldrm -ldrm_amdgpu"
+ build
+
+ # copy the ROCM dependencies
+ mkdir -p "${ROCM_DIST_DIR}"
+ for dep in $(ldd "${BUILD_DIR}/bin/ollama_llama_server" | grep "=>" | cut -f2 -d= | cut -f2 -d' ' | grep -v "${GOARCH}/rocm${ROCM_VARIANT}" | grep -e rocm -e amdgpu -e libtinfo -e libnuma -e libelf ); do
+ cp -a "${dep}"* "${ROCM_DIST_DIR}"
+ if [ $(readlink -f "${dep}") != "${dep}" ] ; then
+ cp $(readlink -f "${dep}") "${ROCM_DIST_DIR}"
+ fi
+ done
+ install
+ dist
+ compress
+fi
+
+cleanup
+wait_for_compress
+echo "go generate completed. LLM runners: $(cd ${PAYLOAD_BASE}; echo *)"
diff --git a/llm/generate/gen_windows.ps1 b/llm/generate/gen_windows.ps1
new file mode 100644
index 00000000..89d7bc25
--- /dev/null
+++ b/llm/generate/gen_windows.ps1
@@ -0,0 +1,403 @@
+#!powershell
+
+$ErrorActionPreference = "Stop"
+
+function amdGPUs {
+ if ($env:AMDGPU_TARGETS) {
+ return $env:AMDGPU_TARGETS
+ }
+ # Current supported rocblas list from ROCm v6.1.2 on windows
+ # https://rocm.docs.amd.com/projects/install-on-windows/en/latest/reference/system-requirements.html#windows-supported-gpus
+ $GPU_LIST = @(
+ "gfx1030"
+ "gfx1100"
+ "gfx1101"
+ "gfx1102"
+ )
+ $GPU_LIST -join ';'
+}
+
+
+function init_vars {
+ write-host "Checking for cmake..."
+ get-command cmake
+ write-host "Checking for ninja..."
+ $d=(get-command -ea 'silentlycontinue' ninja).path
+ if ($null -eq $d) {
+ $MSVC_INSTALL=(Get-CimInstance MSFT_VSInstance -Namespace root/cimv2/vs)[0].InstallLocation
+ $matches=(gci -path $MSVC_INSTALL -r -fi ninja.exe)
+ if ($matches.count -eq 0) {
+ throw "Unable to locate ninja"
+ }
+ $ninjaDir=($matches[0].FullName | split-path -parent)
+ $env:PATH="$env:PATH;$ninjaDir"
+ }
+ if (!$script:SRC_DIR) {
+ $script:SRC_DIR = $(resolve-path "..\..\")
+ }
+ if (!$script:llamacppDir) {
+ $script:llamacppDir = "../llama.cpp"
+ }
+ if (!$script:cmakeTargets) {
+ $script:cmakeTargets = @("ollama_llama_server")
+ }
+ $script:cmakeDefs = @(
+ "-DBUILD_SHARED_LIBS=on",
+ "-DGGML_NATIVE=off",
+ "-DGGML_OPENMP=off"
+ )
+ $script:commonCpuDefs = @("-DCMAKE_POSITION_INDEPENDENT_CODE=on")
+ $script:ARCH = $Env:PROCESSOR_ARCHITECTURE.ToLower()
+ $script:DIST_BASE = "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\runners"
+ md "$script:DIST_BASE" -ea 0 > $null
+ if ($env:CGO_CFLAGS -contains "-g") {
+ $script:cmakeDefs += @("-DCMAKE_VERBOSE_MAKEFILE=on", "-DLLAMA_SERVER_VERBOSE=on", "-DCMAKE_BUILD_TYPE=RelWithDebInfo")
+ $script:config = "RelWithDebInfo"
+ } else {
+ $script:cmakeDefs += @("-DLLAMA_SERVER_VERBOSE=off", "-DCMAKE_BUILD_TYPE=Release")
+ $script:config = "Release"
+ }
+ if ($null -ne $env:CMAKE_SYSTEM_VERSION) {
+ $script:cmakeDefs += @("-DCMAKE_SYSTEM_VERSION=${env:CMAKE_SYSTEM_VERSION}")
+ }
+ # Try to find the CUDA dir
+ if ($env:CUDA_LIB_DIR -eq $null) {
+ $d=(get-command -ea 'silentlycontinue' nvcc).path
+ if ($d -ne $null) {
+ $script:CUDA_LIB_DIR=($d| split-path -parent)
+ $script:CUDA_INCLUDE_DIR=($script:CUDA_LIB_DIR|split-path -parent)+"\include"
+ }
+ } else {
+ $script:CUDA_LIB_DIR=$env:CUDA_LIB_DIR
+ }
+ $script:DUMPBIN=(get-command -ea 'silentlycontinue' dumpbin).path
+ if ($null -eq $env:CMAKE_CUDA_ARCHITECTURES) {
+ $script:CMAKE_CUDA_ARCHITECTURES="50;52;61;70;75;80"
+ } else {
+ $script:CMAKE_CUDA_ARCHITECTURES=$env:CMAKE_CUDA_ARCHITECTURES
+ }
+ # Note: Windows Kits 10 signtool crashes with GCP's plugin
+ if ($null -eq $env:SIGN_TOOL) {
+ ${script:SignTool}="C:\Program Files (x86)\Windows Kits\8.1\bin\x64\signtool.exe"
+ } else {
+ ${script:SignTool}=${env:SIGN_TOOL}
+ }
+ if ("${env:KEY_CONTAINER}") {
+ ${script:OLLAMA_CERT}=$(resolve-path "${script:SRC_DIR}\ollama_inc.crt")
+ }
+}
+
+function git_module_setup {
+ # TODO add flags to skip the init/patch logic to make it easier to mod llama.cpp code in-repo
+ & git submodule init
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ & git submodule update --force "${script:llamacppDir}"
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+}
+
+function apply_patches {
+ # Apply temporary patches until fix is upstream
+ foreach ($patch in $(Get-ChildItem "../patches/*.patch")) {
+ git -c 'user.name=nobody' -c 'user.email=<>' -C "${script:llamacppDir}" am $patch.FullName
+ }
+}
+
+function build {
+ write-host "generating config with: cmake -S ${script:llamacppDir} -B $script:buildDir $script:cmakeDefs"
+ & cmake --version
+ & cmake -S "${script:llamacppDir}" -B $script:buildDir $script:cmakeDefs
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ if ($cmakeDefs -contains "-G") {
+ $extra=@("-j8")
+ } else {
+ $extra= @("--", "/maxCpuCount:8")
+ }
+ write-host "building with: cmake --build $script:buildDir --config $script:config $($script:cmakeTargets | ForEach-Object { `"--target`", $_ }) $extra"
+ & cmake --build $script:buildDir --config $script:config ($script:cmakeTargets | ForEach-Object { "--target", $_ }) $extra
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ # Rearrange output to be consistent between different generators
+ if ($null -ne ${script:config} -And (test-path -path "${script:buildDir}/bin/${script:config}" ) ) {
+ mv -force "${script:buildDir}/bin/${script:config}/*" "${script:buildDir}/bin/"
+ remove-item "${script:buildDir}/bin/${script:config}"
+ }
+}
+
+function sign {
+ if ("${env:KEY_CONTAINER}") {
+ write-host "Signing ${script:buildDir}/bin/*.exe ${script:buildDir}/bin/*.dll"
+ foreach ($file in @(get-childitem "${script:buildDir}/bin/*.exe") + @(get-childitem "${script:buildDir}/bin/*.dll")){
+ & "${script:SignTool}" sign /v /fd sha256 /t http://timestamp.digicert.com /f "${script:OLLAMA_CERT}" `
+ /csp "Google Cloud KMS Provider" /kc "${env:KEY_CONTAINER}" $file
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ }
+ }
+}
+
+function install {
+ write-host "Installing binaries to dist dir ${script:distDir}"
+ mkdir ${script:distDir} -ErrorAction SilentlyContinue
+ $binaries = dir "${script:buildDir}/bin/*.exe"
+ foreach ($file in $binaries) {
+ copy-item -Path $file -Destination ${script:distDir} -Force
+ }
+
+ write-host "Installing dlls to dist dir ${script:distDir}"
+ $dlls = dir "${script:buildDir}/bin/*.dll"
+ foreach ($file in $dlls) {
+ copy-item -Path $file -Destination ${script:distDir} -Force
+ }
+}
+
+function cleanup {
+ $patches = Get-ChildItem "../patches/*.diff"
+ foreach ($patch in $patches) {
+ # Extract file paths from the patch file
+ $filePaths = Get-Content $patch.FullName | Where-Object { $_ -match '^\+\+\+ ' } | ForEach-Object {
+ $parts = $_ -split ' '
+ ($parts[1] -split '/', 2)[1]
+ }
+
+ # Checkout each file
+ foreach ($file in $filePaths) {
+ git -C "${script:llamacppDir}" checkout $file
+ }
+ git -C "${script:llamacppDir}" checkout CMakeLists.txt
+ }
+}
+
+
+# -DGGML_AVX -- 2011 Intel Sandy Bridge & AMD Bulldozer
+# -DGGML_AVX2 -- 2013 Intel Haswell & 2015 AMD Excavator / 2017 AMD Zen
+# -DGGML_FMA (FMA3) -- 2013 Intel Haswell & 2012 AMD Piledriver
+
+
+function build_cpu_x64 {
+ if ((-not "${env:OLLAMA_SKIP_CPU_GENERATE}" ) -and ((-not "${env:OLLAMA_CPU_TARGET}") -or ("${env:OLLAMA_CPU_TARGET}" -eq "cpu"))) {
+ init_vars
+ $script:cmakeDefs = $script:commonCpuDefs + @("-A", "x64", "-DGGML_AVX=off", "-DGGML_AVX2=off", "-DGGML_AVX512=off", "-DGGML_FMA=off", "-DGGML_F16C=off") + $script:cmakeDefs
+ $script:buildDir="../build/windows/${script:ARCH}/cpu"
+ $script:distDir="$script:DIST_BASE\cpu"
+ write-host "Building LCD CPU"
+ build
+ sign
+ install
+ } else {
+ write-host "Skipping CPU generation step as requested"
+ }
+}
+
+function build_cpu_arm64 {
+ if ((-not "${env:OLLAMA_SKIP_CPU_GENERATE}" ) -and ((-not "${env:OLLAMA_CPU_TARGET}") -or ("${env:OLLAMA_CPU_TARGET}" -eq "cpu"))) {
+ init_vars
+ write-host "Checking for clang..."
+ get-command clang
+ $env:CFLAGS="-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only"
+ $env:CXXFLAGS="$env:CFLAGS"
+ $env:LDFLAGS="-static-libstdc++"
+ $script:cmakeDefs = $script:commonCpuDefs + @(
+ "-DCMAKE_VERBOSE_MAKEFILE=on",
+ "-DCMAKE_C_COMPILER=clang.exe",
+ "-DCMAKE_CXX_COMPILER=clang++.exe",
+ "-DMSVC_RUNTIME_LIBRARY=MultiThreaded"
+ ) + $script:cmakeDefs
+ $script:buildDir="../build/windows/${script:ARCH}/cpu"
+ $script:distDir="$script:DIST_BASE\cpu"
+ write-host "Building LCD CPU"
+ build
+ sign
+ install
+ } else {
+ write-host "Skipping CPU generation step as requested"
+ }
+}
+
+
+function build_cpu_avx() {
+ if ((-not "${env:OLLAMA_SKIP_CPU_GENERATE}" ) -and ((-not "${env:OLLAMA_CPU_TARGET}") -or ("${env:OLLAMA_CPU_TARGET}" -eq "cpu_avx"))) {
+ init_vars
+ $script:cmakeDefs = $script:commonCpuDefs + @("-A", "x64", "-DGGML_AVX=on", "-DGGML_AVX2=off", "-DGGML_AVX512=off", "-DGGML_FMA=off", "-DGGML_F16C=off") + $script:cmakeDefs
+ $script:buildDir="../build/windows/${script:ARCH}/cpu_avx"
+ $script:distDir="$script:DIST_BASE\cpu_avx"
+ write-host "Building AVX CPU"
+ build
+ sign
+ install
+ } else {
+ write-host "Skipping CPU AVX generation step as requested"
+ }
+}
+
+function build_cpu_avx2() {
+ if ((-not "${env:OLLAMA_SKIP_CPU_GENERATE}" ) -and ((-not "${env:OLLAMA_CPU_TARGET}") -or ("${env:OLLAMA_CPU_TARGET}" -eq "cpu_avx2"))) {
+ init_vars
+ $script:cmakeDefs = $script:commonCpuDefs + @("-A", "x64", "-DGGML_AVX=on", "-DGGML_AVX2=on", "-DGGML_AVX512=off", "-DGGML_FMA=on", "-DGGML_F16C=on") + $script:cmakeDefs
+ $script:buildDir="../build/windows/${script:ARCH}/cpu_avx2"
+ $script:distDir="$script:DIST_BASE\cpu_avx2"
+ write-host "Building AVX2 CPU"
+ build
+ sign
+ install
+ } else {
+ write-host "Skipping CPU AVX2 generation step as requested"
+ }
+}
+
+function build_cuda() {
+ if ((-not "${env:OLLAMA_SKIP_CUDA_GENERATE}") -and ("${script:CUDA_LIB_DIR}")) {
+ # Then build cuda as a dynamically loaded library
+ $nvcc = "$script:CUDA_LIB_DIR\nvcc.exe"
+ $script:CUDA_VERSION=((get-item ($nvcc | split-path | split-path)).Basename -Split "\.")[0]
+ if ($null -ne $script:CUDA_VERSION) {
+ $script:CUDA_VARIANT="_"+$script:CUDA_VERSION
+ }
+ init_vars
+ $script:buildDir="../build/windows/${script:ARCH}/cuda$script:CUDA_VARIANT"
+ $script:distDir="$script:DIST_BASE\cuda$script:CUDA_VARIANT"
+ $script:cmakeDefs += @(
+ "-A", "x64",
+ "-DGGML_CUDA=ON",
+ "-DGGML_AVX=on",
+ "-DGGML_AVX2=off",
+ "-DCMAKE_CUDA_FLAGS=-t6",
+ "-DCMAKE_CUDA_ARCHITECTURES=${script:CMAKE_CUDA_ARCHITECTURES}",
+ "-DCMAKE_CUDA_COMPILER_TOOLKIT_ROOT=$env:CUDA_PATH"
+ )
+ if ($null -ne $env:OLLAMA_CUSTOM_CUDA_DEFS) {
+ write-host "OLLAMA_CUSTOM_CUDA_DEFS=`"${env:OLLAMA_CUSTOM_CUDA_DEFS}`""
+ $script:cmakeDefs +=@("${env:OLLAMA_CUSTOM_CUDA_DEFS}")
+ write-host "building custom CUDA GPU"
+ }
+ build
+ sign
+ install
+
+ md "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\" -ea 0 > $null
+ write-host "copying CUDA dependencies to ${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${script:CUDA_LIB_DIR}\cudart64_*.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${script:CUDA_LIB_DIR}\cublas64_*.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${script:CUDA_LIB_DIR}\cublasLt64_*.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ } else {
+ write-host "Skipping CUDA generation step"
+ }
+}
+
+function build_oneapi() {
+ if ((-not "${env:OLLAMA_SKIP_ONEAPI_GENERATE}") -and ("${env:ONEAPI_ROOT}")) {
+ # Get oneAPI version
+ $script:ONEAPI_VERSION = icpx --version
+ $script:ONEAPI_VERSION = [regex]::Match($script:ONEAPI_VERSION, '(?<=oneAPI DPC\+\+/C\+\+ Compiler )(?<version>\d+\.\d+\.\d+)').Value
+ if ($null -ne $script:ONEAPI_VERSION) {
+ $script:ONEAPI_VARIANT = "_v" + $script:ONEAPI_VERSION
+ }
+ init_vars
+ $script:buildDir = "../build/windows/${script:ARCH}/oneapi$script:ONEAPI_VARIANT"
+ $script:distDir ="$script:DIST_BASE\oneapi$script:ONEAPI_VARIANT"
+ $script:cmakeDefs += @(
+ "-G", "MinGW Makefiles",
+ "-DGGML_SYCL=ON",
+ "-DCMAKE_C_COMPILER=icx",
+ "-DCMAKE_CXX_COMPILER=icx",
+ "-DCMAKE_BUILD_TYPE=Release"
+ )
+
+ Write-Host "Building oneAPI"
+ build
+ # Ninja doesn't prefix with config name
+ if ($null -ne $script:DUMPBIN) {
+ & "$script:DUMPBIN" /dependents "${script:buildDir}/bin/ollama_llama_server.exe" | Select-String ".dll"
+ }
+ sign
+ install
+
+ md "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\" -ea 0 > $null
+ cp "${env:ONEAPI_ROOT}\compiler\latest\bin\libirngmd.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${env:ONEAPI_ROOT}\compiler\latest\bin\libmmd.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${env:ONEAPI_ROOT}\compiler\latest\bin\pi_level_zero.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${env:ONEAPI_ROOT}\compiler\latest\bin\pi_unified_runtime.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${env:ONEAPI_ROOT}\compiler\latest\bin\pi_win_proxy_loader.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${env:ONEAPI_ROOT}\compiler\latest\bin\svml_dispmd.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${env:ONEAPI_ROOT}\compiler\latest\bin\sycl7.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${env:ONEAPI_ROOT}\mkl\latest\bin\mkl_core.2.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${env:ONEAPI_ROOT}\mkl\latest\bin\mkl_sycl_blas.4.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${env:ONEAPI_ROOT}\mkl\latest\bin\mkl_tbb_thread.2.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ } else {
+ Write-Host "Skipping oneAPI generation step"
+ }
+}
+
+function build_rocm() {
+ if ((-not "${env:OLLAMA_SKIP_ROCM_GENERATE}") -and ("${env:HIP_PATH}")) {
+ $script:ROCM_VERSION=(get-item $env:HIP_PATH).Basename
+ if ($null -ne $script:ROCM_VERSION) {
+ $script:ROCM_VARIANT="_v"+$script:ROCM_VERSION
+ }
+
+ init_vars
+ $script:buildDir="../build/windows/${script:ARCH}/rocm$script:ROCM_VARIANT"
+ $script:distDir="$script:DIST_BASE\rocm$script:ROCM_VARIANT"
+ $script:cmakeDefs += @(
+ "-G", "Ninja",
+ "-DCMAKE_C_COMPILER=clang.exe",
+ "-DCMAKE_CXX_COMPILER=clang++.exe",
+ "-DGGML_HIPBLAS=on",
+ "-DHIP_PLATFORM=amd",
+ "-DGGML_AVX=on",
+ "-DGGML_AVX2=off",
+ "-DCMAKE_POSITION_INDEPENDENT_CODE=on",
+ "-DAMDGPU_TARGETS=$(amdGPUs)",
+ "-DGPU_TARGETS=$(amdGPUs)"
+ )
+
+ # Make sure the ROCm binary dir is first in the path
+ $env:PATH="$env:HIP_PATH\bin;$env:PATH"
+
+ # We have to clobber the LIB var from the developer shell for clang to work properly
+ $env:LIB=""
+ if ($null -ne $env:OLLAMA_CUSTOM_ROCM_DEFS) {
+ write-host "OLLAMA_CUSTOM_ROCM_DEFS=`"${env:OLLAMA_CUSTOM_ROCM_DEFS}`""
+ $script:cmakeDefs += @("${env:OLLAMA_CUSTOM_ROCM_DEFS}")
+ write-host "building custom ROCM GPU"
+ }
+ write-host "Building ROCm"
+ build
+ # Ninja doesn't prefix with config name
+ ${script:config}=""
+ if ($null -ne $script:DUMPBIN) {
+ & "$script:DUMPBIN" /dependents "${script:buildDir}/bin/ollama_llama_server.exe" | select-string ".dll"
+ }
+ sign
+ install
+
+ md "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\rocblas\library\" -ea 0 > $null
+ cp "${env:HIP_PATH}\bin\hipblas.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ cp "${env:HIP_PATH}\bin\rocblas.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
+ # amdhip64.dll dependency comes from the driver and must be installed on the host to use AMD GPUs
+ cp "${env:HIP_PATH}\bin\rocblas\library\*" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\rocblas\library\"
+ } else {
+ write-host "Skipping ROCm generation step"
+ }
+}
+
+init_vars
+if ($($args.count) -eq 0) {
+ git_module_setup
+ apply_patches
+ if ($script:ARCH -eq "arm64") {
+ build_cpu_arm64
+ } else { # amd64
+ build_cpu_x64
+ build_cpu_avx
+ build_cpu_avx2
+ build_cuda
+ build_oneapi
+ build_rocm
+ }
+
+ cleanup
+ write-host "`ngo generate completed. LLM runners: $(get-childitem -path $script:DIST_BASE)"
+} else {
+ for ( $i = 0; $i -lt $args.count; $i++ ) {
+ write-host "performing $($args[$i])"
+ & $($args[$i])
+ }
+}
\ No newline at end of file
diff --git a/llm/generate/generate_darwin.go b/llm/generate/generate_darwin.go
new file mode 100644
index 00000000..77685234
--- /dev/null
+++ b/llm/generate/generate_darwin.go
@@ -0,0 +1,3 @@
+package generate
+
+//go:generate bash ./gen_darwin.sh
diff --git a/llm/llama.cpp/generate_linux.go b/llm/generate/generate_linux.go
similarity index 67%
rename from llm/llama.cpp/generate_linux.go
rename to llm/generate/generate_linux.go
index 119b5c27..2b7e116d 100644
--- a/llm/llama.cpp/generate_linux.go
+++ b/llm/generate/generate_linux.go
@@ -1,3 +1,3 @@
-package llm
+package generate
//go:generate bash ./gen_linux.sh
diff --git a/llm/llama.cpp/generate_windows.go b/llm/generate/generate_windows.go
similarity index 81%
rename from llm/llama.cpp/generate_windows.go
rename to llm/generate/generate_windows.go
index 87acd827..d2ee5428 100644
--- a/llm/llama.cpp/generate_windows.go
+++ b/llm/generate/generate_windows.go
@@ -1,3 +1,3 @@
-package llm
+package generate
//go:generate powershell -ExecutionPolicy Bypass -File ./gen_windows.ps1
diff --git a/llm/ggla.go b/llm/ggla.go
new file mode 100644
index 00000000..831f6071
--- /dev/null
+++ b/llm/ggla.go
@@ -0,0 +1,149 @@
+package llm
+
+import (
+ "encoding/binary"
+ "errors"
+ "io"
+ "slices"
+)
+
+type containerGGLA struct {
+ version uint32
+}
+
+func (c *containerGGLA) Name() string {
+ return "ggla"
+}
+
+func (c *containerGGLA) Decode(rs io.ReadSeeker) (model, error) {
+ if err := binary.Read(rs, binary.LittleEndian, &c.version); err != nil {
+ return nil, err
+ }
+
+ switch c.version {
+ case 1:
+ default:
+ return nil, errors.New("invalid version")
+ }
+
+ model := newGGLA(c)
+ err := model.decode(rs)
+ return model, err
+}
+
+type ggla struct {
+ *containerGGLA
+
+ kv KV
+ tensors []*Tensor
+
+ tensorOffset uint64
+}
+
+func newGGLA(container *containerGGLA) *ggla {
+ return &ggla{
+ containerGGLA: container,
+ kv: make(KV),
+ }
+}
+
+func (llm *ggla) KV() KV {
+ return llm.kv
+}
+
+func (llm *ggla) Tensors() Tensors {
+ return Tensors{
+ Items: llm.tensors,
+ Offset: llm.tensorOffset,
+ }
+}
+
+func (llm *ggla) decode(rs io.ReadSeeker) (retErr error) {
+ var r uint32
+ if err := binary.Read(rs, binary.LittleEndian, &r); err != nil {
+ return err
+ }
+ llm.kv["r"] = r
+
+ var alpha uint32
+ if err := binary.Read(rs, binary.LittleEndian, &alpha); err != nil {
+ return err
+ }
+ llm.kv["alpha"] = alpha
+
+ offset, err := rs.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+
+ llm.tensorOffset = uint64(offset)
+
+ for {
+ var dims uint32
+ if err := binary.Read(rs, binary.LittleEndian, &dims); err != nil {
+ if errors.Is(err, io.EOF) {
+ return nil
+ }
+ return err
+ }
+
+ defer func() {
+ if errors.Is(retErr, io.EOF) {
+ retErr = io.ErrUnexpectedEOF
+ }
+ }()
+
+ var namesize uint32
+ if err := binary.Read(rs, binary.LittleEndian, &namesize); err != nil {
+ return err
+ }
+
+ var t Tensor
+ if err := binary.Read(rs, binary.LittleEndian, &t.Kind); err != nil {
+ return err
+ }
+
+ t.Shape = make([]uint64, dims)
+ for i := 0; uint32(i) < dims; i++ {
+ var shape32 uint32
+ if err := binary.Read(rs, binary.LittleEndian, &shape32); err != nil {
+ return err
+ }
+
+ t.Shape[i] = uint64(shape32)
+ }
+
+ // ggla tensor shape is reversed
+ // ref: https://github.com/ggerganov/llama.cpp/blob/29ae62d2ae163e2b68aa0ad3bf2ab4636de0c957/convert-lora-to-ggml.py#L44
+ slices.Reverse(t.Shape)
+
+ name := make([]byte, namesize)
+ if err := binary.Read(rs, binary.LittleEndian, &name); err != nil {
+ return err
+ }
+
+ t.Name = string(name)
+
+ offset, err := rs.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+
+ if _, err := rs.Seek((offset+31)&-32-offset, io.SeekCurrent); err != nil {
+ return err
+ }
+
+ offset, err = rs.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+
+ t.Offset = uint64(offset)
+
+ if _, err := rs.Seek(int64(t.Size()), io.SeekCurrent); err != nil {
+ return err
+ }
+
+ llm.tensors = append(llm.tensors, &t)
+ }
+}
diff --git a/llm/ggml.go b/llm/ggml.go
index f71328e1..aa846b97 100644
--- a/llm/ggml.go
+++ b/llm/ggml.go
@@ -3,113 +3,269 @@ package llm
import (
"encoding/binary"
"errors"
+ "fmt"
"io"
+ "strings"
+
+ "github.com/ollama/ollama/util/bufioutil"
)
type GGML struct {
container
model
-
- Size int64
-}
-
-const (
- fileTypeF32 uint32 = iota
- fileTypeF16
- fileTypeQ4_0
- fileTypeQ4_1
- fileTypeQ4_1_F16
- fileTypeQ8_0 uint32 = iota + 2
- fileTypeQ5_0
- fileTypeQ5_1
- fileTypeQ2_K
- fileTypeQ3_K_S
- fileTypeQ3_K_M
- fileTypeQ3_K_L
- fileTypeQ4_K_S
- fileTypeQ4_K_M
- fileTypeQ5_K_S
- fileTypeQ5_K_M
- fileTypeQ6_K
-)
-
-func fileType(fileType uint32) string {
- switch fileType {
- case fileTypeF32:
- return "F32"
- case fileTypeF16:
- return "F16"
- case fileTypeQ4_0:
- return "Q4_0"
- case fileTypeQ4_1:
- return "Q4_1"
- case fileTypeQ4_1_F16:
- return "Q4_1_F16"
- case fileTypeQ8_0:
- return "Q8_0"
- case fileTypeQ5_0:
- return "Q5_0"
- case fileTypeQ5_1:
- return "Q5_1"
- case fileTypeQ2_K:
- return "Q2_K"
- case fileTypeQ3_K_S:
- return "Q3_K_S"
- case fileTypeQ3_K_M:
- return "Q3_K_M"
- case fileTypeQ3_K_L:
- return "Q3_K_L"
- case fileTypeQ4_K_S:
- return "Q4_K_S"
- case fileTypeQ4_K_M:
- return "Q4_K_M"
- case fileTypeQ5_K_S:
- return "Q5_K_S"
- case fileTypeQ5_K_M:
- return "Q5_K_M"
- case fileTypeQ6_K:
- return "Q6_K"
- default:
- return "unknown"
- }
}
type model interface {
- ModelFamily() string
- ModelType() string
- FileType() string
- NumLayers() int64
+ KV() KV
+ Tensors() Tensors
+}
+
+type KV map[string]any
+
+func (kv KV) u64(key string) uint64 {
+ switch v := kv[key].(type) {
+ case uint64:
+ return v
+ case uint32:
+ return uint64(v)
+ case float64:
+ return uint64(v)
+ default:
+ return 0
+ }
+}
+
+func (kv KV) Architecture() string {
+ if s, ok := kv["general.architecture"].(string); ok {
+ return s
+ }
+
+ return "unknown"
+}
+
+func (kv KV) Kind() string {
+ if s, ok := kv["general.type"].(string); ok {
+ return s
+ }
+
+ return "unknown"
+}
+
+func (kv KV) ParameterCount() uint64 {
+ return kv.u64("general.parameter_count")
+}
+
+func (kv KV) FileType() fileType {
+ if u64 := kv.u64("general.file_type"); u64 > 0 {
+ return fileType(uint32(u64))
+ }
+
+ return fileTypeUnknown
+}
+
+func (kv KV) BlockCount() uint64 {
+ return kv.u64(fmt.Sprintf("%s.block_count", kv.Architecture()))
+}
+
+func (kv KV) HeadCount() uint64 {
+ return kv.u64(fmt.Sprintf("%s.attention.head_count", kv.Architecture()))
+}
+
+func (kv KV) HeadCountKV() uint64 {
+ if headCountKV := kv.u64(fmt.Sprintf("%s.attention.head_count_kv", kv.Architecture())); headCountKV > 0 {
+ return headCountKV
+ }
+
+ return 1
+}
+
+func (kv KV) EmbeddingHeadCount() uint64 {
+ if heads := kv.HeadCount(); heads > 0 {
+ return kv.EmbeddingLength() / kv.HeadCount()
+ }
+
+ return 0
+}
+
+func (kv KV) EmbeddingHeadCountK() uint64 {
+ if k := kv.u64(fmt.Sprintf("%s.attention.key_length", kv.Architecture())); k > 0 {
+ return k
+ }
+
+ return kv.EmbeddingHeadCount()
+}
+
+func (kv KV) EmbeddingHeadCountV() uint64 {
+ if v := kv.u64(fmt.Sprintf("%s.attention.value_length", kv.Architecture())); v > 0 {
+ return v
+ }
+
+ return kv.EmbeddingHeadCount()
+}
+
+func (kv KV) GQA() uint64 {
+ return kv.HeadCount() / kv.HeadCountKV()
+}
+
+func (kv KV) EmbeddingLength() uint64 {
+ return kv.u64(fmt.Sprintf("%s.embedding_length", kv.Architecture()))
+}
+
+func (kv KV) ContextLength() uint64 {
+ return kv.u64(fmt.Sprintf("%s.context_length", kv.Architecture()))
+}
+
+func (kv KV) ChatTemplate() string {
+ s, _ := kv["tokenizer.chat_template"].(string)
+ return s
+}
+
+type Tensors struct {
+ Items []*Tensor
+ Offset uint64
+}
+
+func (ts Tensors) Layers() map[string]Layer {
+ layers := make(map[string]Layer)
+ for _, t := range ts.Items {
+ parts := strings.Split(t.Name, ".")
+ if parts[0] == "blk" {
+ // join first and second part, e.g. blk.%d
+ parts = append([]string{fmt.Sprintf("%s.%s", parts[0], parts[1])}, parts[2:]...)
+ }
+
+ if _, ok := layers[parts[0]]; !ok {
+ layers[parts[0]] = make(Layer)
+ }
+
+ layers[parts[0]][strings.Join(parts[1:], ".")] = t
+ }
+
+ return layers
+}
+
+type Layer map[string]*Tensor
+
+func (l Layer) size() (size uint64) {
+ for _, t := range l {
+ size += t.Size()
+ }
+
+ return size
+}
+
+type Tensor struct {
+ Name string `json:"name"`
+ Kind uint32 `json:"kind"`
+ Offset uint64 `json:"-"`
+
+ // Shape is the number of elements in each dimension
+ Shape []uint64 `json:"shape"`
+
+ io.WriterTo `json:"-"`
+}
+
+func (t Tensor) block() (n int) {
+ if _, err := fmt.Sscanf(t.Name, "blk.%d.", &n); err != nil {
+ return -1
+ }
+
+ return
+}
+
+func (t Tensor) blockSize() uint64 {
+ switch t.Kind {
+ case 0, 1, 24, 25, 26, 27, 28, 30: // F32, F16, I8, I16, I32, I64, F64, BF16
+ return 1
+ case 2, 3, 4, 5, 6, 7, 8, 9, 20: // Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, Q8_1, IQ4_NL
+ return 32
+ default: // All others
+ return 256
+ }
+}
+
+func (t Tensor) typeSize() uint64 {
+ blockSize := t.blockSize()
+
+ switch t.Kind {
+ case 0: // FP32
+ return 4
+ case 1: // FP16
+ return 2
+ case 2: // Q4_0
+ return 2 + blockSize/2
+ case 3: // Q4_1
+ return 2 + 2 + blockSize/2
+ case 6: // Q5_0
+ return 2 + 4 + blockSize/2
+ case 7: // Q5_1
+ return 2 + 2 + 4 + blockSize/2
+ case 8: // Q8_0
+ return 2 + blockSize
+ case 9: // Q8_1
+ return 4 + 4 + blockSize
+ case 10: // Q2_K
+ return blockSize/16 + blockSize/4 + 2 + 2
+ case 11: // Q3_K
+ return blockSize/8 + blockSize/4 + 12 + 2
+ case 12: // Q4_K
+ return 2 + 2 + 12 + blockSize/2
+ case 13: // Q5_K
+ return 2 + 2 + 12 + blockSize/8 + blockSize/2
+ case 14: // Q6_K
+ return blockSize/2 + blockSize/4 + blockSize/16 + 2
+ case 15: // Q8_K
+ return 2 + blockSize + 2*blockSize/16
+ case 16: // IQ2_XXS
+ return 2 + 2*blockSize/8
+ case 17: // IQ2_XS
+ return 2 + 2*blockSize/8 + blockSize/32
+ case 18: // IQ3_XXS
+ return 2 + blockSize/4 + blockSize/8
+ case 19: // IQ1_S
+ return 2 + blockSize/8 + blockSize/16
+ case 20: // IQ4_NL
+ return 2 + blockSize/2
+ case 21: // IQ3_S
+ return 2 + blockSize/4 + blockSize/8 + blockSize/32 + 4
+ case 22: // IQ2_S
+ return 2 + blockSize/4 + blockSize/16
+ case 23: // IQ4_XS
+ return 2 + 2 + blockSize/2 + blockSize/64
+ case 24: // I8
+ return 1
+ case 25: // I16
+ return 2
+ case 26: // I32
+ return 4
+ case 27: // I64
+ return 8
+ case 28: // F64
+ return 8
+ case 29: // IQ1_M
+ return blockSize/8 + blockSize/16 + blockSize/32
+ case 30: // BF16
+ return 2
+ default:
+ return 0
+ }
+}
+
+func (t Tensor) parameters() uint64 {
+ var count uint64 = 1
+ for _, n := range t.Shape {
+ count *= n
+ }
+ return count
+}
+
+func (t Tensor) Size() uint64 {
+ return t.parameters() * t.typeSize() / t.blockSize()
}
type container interface {
Name() string
- Decode(*readSeekOffset) (model, error)
-}
-
-type containerLORA struct {
- version uint32
-}
-
-func (c *containerLORA) Name() string {
- return "ggla"
-}
-
-func (c *containerLORA) Decode(ro *readSeekOffset) (model, error) {
- var version uint32
- binary.Read(ro, binary.LittleEndian, &version)
-
- switch version {
- case 1:
- default:
- return nil, errors.New("invalid version")
- }
-
- c.version = version
-
- // remaining file contents aren't decoded
- ro.Seek(0, io.SeekEnd)
-
- return nil, nil
+ Decode(io.ReadSeeker) (model, error)
}
const (
@@ -128,58 +284,196 @@ const (
var ErrUnsupportedFormat = errors.New("unsupported model format")
-func DecodeGGML(r io.ReadSeeker) (*GGML, error) {
- ro := readSeekOffset{ReadSeeker: r}
+func DetectGGMLType(b []byte) string {
+ switch binary.LittleEndian.Uint32(b[:4]) {
+ case FILE_MAGIC_GGML:
+ return "ggml"
+ case FILE_MAGIC_GGMF:
+ return "ggmf"
+ case FILE_MAGIC_GGJT:
+ return "ggjt"
+ case FILE_MAGIC_GGLA:
+ return "ggla"
+ case FILE_MAGIC_GGUF_LE, FILE_MAGIC_GGUF_BE:
+ return "gguf"
+ default:
+ return ""
+ }
+}
+
+// DecodeGGML decodes a GGML model from the given reader.
+//
+// It collects array values for arrays with a size less than or equal to
+// maxArraySize. If maxArraySize is 0, the default value of 1024 is used. If
+// the maxArraySize is negative, all arrays are collected.
+func DecodeGGML(rs io.ReadSeeker, maxArraySize int) (*GGML, int64, error) {
+ if maxArraySize == 0 {
+ maxArraySize = 1024
+ }
+
+ rs = bufioutil.NewBufferedSeeker(rs, 32<<10)
var magic uint32
- if err := binary.Read(&ro, binary.LittleEndian, &magic); err != nil {
- return nil, err
+ if err := binary.Read(rs, binary.LittleEndian, &magic); err != nil {
+ return nil, 0, err
}
var c container
switch magic {
case FILE_MAGIC_GGML, FILE_MAGIC_GGMF, FILE_MAGIC_GGJT:
- return nil, ErrUnsupportedFormat
+ return nil, 0, ErrUnsupportedFormat
case FILE_MAGIC_GGLA:
- c = &containerLORA{}
+ c = &containerGGLA{}
case FILE_MAGIC_GGUF_LE:
- c = &containerGGUF{bo: binary.LittleEndian}
+ c = &containerGGUF{ByteOrder: binary.LittleEndian, maxArraySize: maxArraySize}
case FILE_MAGIC_GGUF_BE:
- c = &containerGGUF{bo: binary.BigEndian}
+ c = &containerGGUF{ByteOrder: binary.BigEndian, maxArraySize: maxArraySize}
default:
- return nil, errors.New("invalid file magic")
+ return nil, 0, errors.New("invalid file magic")
}
- model, err := c.Decode(&ro)
+ model, err := c.Decode(rs)
if err != nil {
- return nil, err
+ return nil, 0, err
+ }
+
+ offset, err := rs.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return nil, 0, err
}
// final model type
return &GGML{
container: c,
model: model,
- Size: ro.offset,
- }, nil
+ }, offset, nil
}
-type readSeekOffset struct {
- io.ReadSeeker
- offset int64
-}
+func (llm GGML) GraphSize(context, batch uint64) (partialOffload, fullOffload uint64) {
+ embedding := llm.KV().EmbeddingLength()
+ heads := llm.KV().HeadCount()
+ headsKV := llm.KV().HeadCountKV()
+ vocab := uint64(llm.KV()["tokenizer.ggml.tokens"].(*array).size)
-func (rso *readSeekOffset) Seek(offset int64, whence int) (int64, error) {
- offset, err := rso.ReadSeeker.Seek(offset, whence)
- if err != nil {
- return 0, err
+ embeddingHeads := llm.KV().EmbeddingHeadCount()
+ embeddingHeadsK := llm.KV().EmbeddingHeadCountK()
+
+ layers := llm.Tensors().Layers()
+
+ switch llm.KV().Architecture() {
+ case "llama":
+ fullOffload = max(
+ 4*batch*(1+4*embedding+context*(1+heads)),
+ 4*batch*(embedding+vocab),
+ )
+
+ partialOffload = 4 * batch * embedding
+ partialOffload += max(
+ 4*batch*(1+embedding+max(context, embedding))+embedding*embedding*9/16+4*context*(batch*heads+embeddingHeads*headsKV),
+ 4*batch*(embedding+vocab)+embedding*vocab*105/128,
+ )
+
+ if ffnGateExpsWeight, ok := layers["blk.0"]["ffn_gate_exps.weight"]; ok {
+ // mixtral 8x22b
+ ff := uint64(llm.KV()["llama.feed_forward_length"].(uint32))
+ partialOffload = max(
+ 3*ffnGateExpsWeight.Size()+4*batch*(2*ff+headsKV+embedding+context+embeddingHeads*headsKV),
+ 4*(context*batch*heads+context*embeddingHeads*headsKV+batch*1024+embeddingHeads*headsKV*batch),
+ )
+ } else if ffnGateWeight, ok := layers["blk.0"]["ffn_gate.0.weight"]; ok {
+ // mixtral 8x7b
+ ffnGateWeight1 := ffnGateWeight.Shape[1]
+ fullOffload = 4 * batch * (2 + 3*embedding + context*(1+heads) + 2*headsKV + ffnGateWeight1)
+ partialOffload = max(
+ 4*batch*(3+embeddingHeads*headsKV+embedding+context*(1+heads)+ffnGateWeight1)+(embedding*embedding+3*embedding*headsKV*ffnGateWeight1)*9/16,
+ 4*batch*(1+2*embedding+context*(1+heads))+embedding*(6*context*headsKV/heads+embedding*9/16),
+ )
+ }
+ case "gemma", "gemma2":
+ fullOffload = max(
+ 4*batch*(embedding+vocab),
+ 4*batch*(2+context+context*heads+2*embedding+2*embeddingHeadsK*heads),
+ )
+
+ partialOffload = max(
+ 4*embedding*batch+embedding*vocab*105/128+4*vocab*batch,
+ 4*batch*(2*embedding+1+2*embeddingHeadsK*heads+context+context*heads)+
+ 4*embeddingHeadsK*context*8+
+ embedding*embeddingHeadsK*heads*9/16,
+ )
+ case "command-r":
+ fullOffload = max(
+ 4*batch*(embedding+vocab),
+ 4*batch*(2+4*embedding+context*(1+heads)),
+ )
+
+ partialOffload = max(
+ 4*batch*(embedding+vocab)+embedding*vocab*105/128,
+ 4*batch*(1+2*embedding+context*(1+heads))+4*embedding*context+embedding*embedding*9/16,
+ )
+ case "qwen2":
+ fullOffload = max(
+ 4*batch*(embedding+vocab),
+ 4*batch*(1+2*embedding+context+context*heads),
+ )
+
+ partialOffload = max(
+ 4*batch*(embedding+vocab)+embedding*vocab*105/128,
+ 4*(batch*(1+2*embedding+context*(1+heads))+embedding*(1+context)),
+ )
+ case "phi2":
+ fullOffload = max(
+ 4*batch*(embedding+vocab),
+ 4*batch*(1+4*embedding+context+context*heads),
+ )
+
+ partialOffload = max(
+ 4*batch*(2*embedding+vocab)+embedding*vocab*105/128,
+ 4*batch*(2+3*embedding+context+context*heads),
+ )
+ case "stablelm":
+ fullOffload = 4 * batch * (context*(1+heads) + 3*embedding + 2)
+ partialOffload = max(
+ 4*batch*(vocab+2*embedding),
+ fullOffload,
+ )
+ case "deepseek2":
+ fullOffload = max(
+ 4*batch*(3*embedding+vocab),
+ 4*batch*(3*embedding+2+context*(1+headsKV)+2*embeddingHeadsK*headsKV),
+ )
+
+ partialOffload = max(
+ 4*batch*(3*embedding+vocab)+embedding*vocab*105/128,
+ 4*batch*(2*embedding+1+2*embeddingHeadsK*headsKV+context+context*headsKV)+4*embeddingHeadsK*context*headsKV+embedding*embeddingHeadsK*headsKV*9/16,
+ )
+ case "chatglm":
+ fullOffload = 4 * batch * (embedding + vocab)
+ partialOffload = 4*batch*(embedding+vocab) + embedding*vocab*105/128
+ if qkvBias, ok := layers["blk.0"]["attn_qkv.bias"]; ok {
+ fullOffload = max(
+ fullOffload,
+ 4*batch*(2+
+ 2*embedding+
+ context+
+ context*heads+
+ embeddingHeadsK*heads+
+ qkvBias.Shape[0]),
+ )
+
+ partialOffload = max(
+ partialOffload,
+ 4*batch*(1+
+ 2*embedding+
+ embeddingHeadsK*heads+
+ context+
+ context*heads)+
+ 4*embeddingHeadsK*context+
+ 4*context*embeddingHeadsK+
+ 4*qkvBias.Shape[0],
+ )
+ }
}
- rso.offset = offset
- return offset, nil
-}
-
-func (rso *readSeekOffset) Read(p []byte) (int, error) {
- n, err := rso.ReadSeeker.Read(p)
- rso.offset += int64(n)
- return n, err
+ return
}
diff --git a/llm/ggml_test.go b/llm/ggml_test.go
new file mode 100644
index 00000000..006c3ded
--- /dev/null
+++ b/llm/ggml_test.go
@@ -0,0 +1 @@
+package llm
diff --git a/llm/gguf.go b/llm/gguf.go
index d92932c5..2e6bc542 100644
--- a/llm/gguf.go
+++ b/llm/gguf.go
@@ -2,15 +2,20 @@ package llm
import (
"bytes"
+ "cmp"
"encoding/binary"
+ "encoding/json"
"fmt"
"io"
+ "log/slog"
+ "slices"
+ "strings"
- "github.com/jmorganca/ollama/format"
+ "golang.org/x/exp/maps"
)
type containerGGUF struct {
- bo binary.ByteOrder
+ ByteOrder binary.ByteOrder
Version uint32
@@ -23,24 +28,43 @@ type containerGGUF struct {
NumTensor uint64
NumKV uint64
}
+
+ V3 struct {
+ NumTensor uint64
+ NumKV uint64
+ }
+
+ maxArraySize int
+}
+
+func (c *containerGGUF) canCollectArray(size int) bool {
+ return c.maxArraySize < 0 || size <= c.maxArraySize
}
func (c *containerGGUF) Name() string {
return "gguf"
}
-func (c *containerGGUF) Decode(rso *readSeekOffset) (model, error) {
- binary.Read(rso, c.bo, &c.Version)
-
- switch c.Version {
- case 1:
- binary.Read(rso, c.bo, &c.V1)
- default:
- binary.Read(rso, c.bo, &c.V2)
+func (c *containerGGUF) Decode(rs io.ReadSeeker) (model, error) {
+ if err := binary.Read(rs, c.ByteOrder, &c.Version); err != nil {
+ return nil, err
}
- model := newGGUFModel(c)
- if err := model.Decode(rso); err != nil {
+ var err error
+ switch c.Version {
+ case 1:
+ err = binary.Read(rs, c.ByteOrder, &c.V1)
+ case 2:
+ err = binary.Read(rs, c.ByteOrder, &c.V2)
+ default:
+ err = binary.Read(rs, c.ByteOrder, &c.V3)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ model := newGGUF(c)
+ if err := model.Decode(rs); err != nil {
return nil, err
}
@@ -63,297 +87,209 @@ const (
ggufTypeFloat64
)
-type kv map[string]any
-
-type tensor struct {
- name string
- kind uint32
- offset uint64
- size uint64
-
- // shape is the number of elements in each dimension
- shape [4]uint64
-}
-
-type ggufModel struct {
+type gguf struct {
*containerGGUF
- kv
- tensors []tensor
+ kv KV
+ tensors []*Tensor
- parameters uint64
+ parameters uint64
+ tensorOffset uint64
+
+ scratch [16 << 10]byte
}
-func newGGUFModel(container *containerGGUF) *ggufModel {
- return &ggufModel{
+func newGGUF(container *containerGGUF) *gguf {
+ return &gguf{
containerGGUF: container,
- kv: make(kv),
+ kv: make(KV),
}
}
-func (llm *ggufModel) NumTensor() uint64 {
- if llm.Version == 1 {
+func (llm *gguf) KV() KV {
+ return llm.kv
+}
+
+func (llm *gguf) Tensors() Tensors {
+ return Tensors{
+ Items: llm.tensors,
+ Offset: llm.tensorOffset,
+ }
+}
+
+func (llm *gguf) numTensor() uint64 {
+ switch llm.Version {
+ case 1:
return uint64(llm.V1.NumTensor)
+ case 2:
+ return llm.V2.NumTensor
+ default:
+ return llm.V3.NumTensor
}
-
- return llm.V2.NumTensor
}
-func (llm *ggufModel) NumKV() uint64 {
- if llm.Version == 1 {
+func (llm *gguf) numKV() uint64 {
+ switch llm.Version {
+ case 1:
return uint64(llm.V1.NumKV)
+ case 2:
+ return llm.V2.NumKV
+ default:
+ return llm.V3.NumKV
}
-
- return llm.V2.NumKV
}
-func (llm *ggufModel) ModelFamily() string {
- if t, ok := llm.kv["general.architecture"].(string); ok {
- return t
- }
-
- return "unknown"
-}
-
-func (llm *ggufModel) ModelType() string {
- if llm.parameters > 0 {
- return format.HumanNumber(llm.parameters)
- }
-
- return "unknown"
-}
-
-func (llm *ggufModel) FileType() string {
- if t, ok := llm.kv["general.file_type"].(uint32); ok {
- return fileType(t)
- }
-
- return "unknown"
-}
-
-func (llm *ggufModel) Decode(rso *readSeekOffset) error {
+func (llm *gguf) Decode(rs io.ReadSeeker) error {
// decode key-values
- for i := 0; uint64(i) < llm.NumKV(); i++ {
- k, err := llm.readString(rso)
+ for i := 0; uint64(i) < llm.numKV(); i++ {
+ k, err := readGGUFString(llm, rs)
if err != nil {
return err
}
- vtype := llm.readU32(rso)
+ t, err := readGGUF[uint32](llm, rs)
+ if err != nil {
+ return err
+ }
var v any
- switch vtype {
+ switch t {
case ggufTypeUint8:
- v = llm.readU8(rso)
+ v, err = readGGUF[uint8](llm, rs)
case ggufTypeInt8:
- v = llm.readI8(rso)
+ v, err = readGGUF[int8](llm, rs)
case ggufTypeUint16:
- v = llm.readU16(rso)
+ v, err = readGGUF[uint16](llm, rs)
case ggufTypeInt16:
- v = llm.readI16(rso)
+ v, err = readGGUF[int16](llm, rs)
case ggufTypeUint32:
- v = llm.readU32(rso)
+ v, err = readGGUF[uint32](llm, rs)
case ggufTypeInt32:
- v = llm.readI32(rso)
+ v, err = readGGUF[int32](llm, rs)
case ggufTypeUint64:
- v = llm.readU64(rso)
+ v, err = readGGUF[uint64](llm, rs)
case ggufTypeInt64:
- v = llm.readI64(rso)
+ v, err = readGGUF[int64](llm, rs)
case ggufTypeFloat32:
- v = llm.readF32(rso)
+ v, err = readGGUF[float32](llm, rs)
case ggufTypeFloat64:
- v = llm.readF64(rso)
+ v, err = readGGUF[float64](llm, rs)
case ggufTypeBool:
- v = llm.readBool(rso)
+ v, err = readGGUF[bool](llm, rs)
case ggufTypeString:
- s, err := llm.readString(rso)
- if err != nil {
- return err
- }
-
- v = s
+ v, err = readGGUFString(llm, rs)
case ggufTypeArray:
- a, err := llm.readArray(rso)
- if err != nil {
- return err
- }
-
- v = a
+ v, err = readGGUFArray(llm, rs)
default:
- return fmt.Errorf("invalid type: %d", vtype)
+ return fmt.Errorf("invalid type: %d", t)
+ }
+
+ if err != nil {
+ return err
}
llm.kv[k] = v
}
// decode tensors
- for i := 0; uint64(i) < llm.NumTensor(); i++ {
- name, err := llm.readString(rso)
+ for range llm.numTensor() {
+ name, err := readGGUFString(llm, rs)
if err != nil {
- return err
+ return fmt.Errorf("failed to read tensor name: %w", err)
}
// dims is the number of dimensions in the tensor
- dims := llm.readU32(rso)
+ dims, err := readGGUF[uint32](llm, rs)
+ if err != nil {
+ return fmt.Errorf("failed to read tensor dimensions: %w", err)
+ }
- shape := [4]uint64{1, 1, 1, 1}
+ shape := make([]uint64, dims)
for i := 0; uint32(i) < dims; i++ {
- shape[i] = llm.readU64(rso)
+ shape[i], err = readGGUF[uint64](llm, rs)
+ if err != nil {
+ return fmt.Errorf("failed to read tensor shape: %w", err)
+ }
}
- kind := llm.readU32(rso)
- offset := llm.readU64(rso)
-
- var blockSize uint64
- switch {
- case kind < 2:
- blockSize = 1
- case kind < 10:
- blockSize = 32
- default:
- blockSize = 256
+ kind, err := readGGUF[uint32](llm, rs)
+ if err != nil {
+ return fmt.Errorf("failed to read tensor kind: %w", err)
}
- var typeSize uint64
- switch kind {
- case 0: // FP32
- typeSize = 4
- case 1: // FP16
- typeSize = 2
- case 2: // Q4_0
- typeSize = 2 + blockSize/2
- case 3: // Q4_1
- typeSize = 2 + 2 + blockSize/2
- case 6: // Q5_0
- typeSize = 2 + 4 + blockSize/2
- case 7: // Q5_1
- typeSize = 2 + 2 + 4 + blockSize/2
- case 8: // Q8_0
- typeSize = 2 + blockSize
- case 9: // Q8_1
- typeSize = 4 + 4 + blockSize
- case 10: // Q2_K
- typeSize = blockSize/16 + blockSize/4 + 2 + 2
- case 11: // Q3_K
- typeSize = blockSize/8 + blockSize/4 + 12 + 2
- case 12: // Q4_K
- typeSize = 2 + 2 + 12 + blockSize/2
- case 13: // Q5_K
- typeSize = 2 + 2 + 12 + blockSize/8 + blockSize/2
- case 14: // Q6_K
- typeSize = blockSize/2 + blockSize/4 + blockSize/16 + 2
+ offset, err := readGGUF[uint64](llm, rs)
+ if err != nil {
+ return fmt.Errorf("failed to read tensor offset: %w", err)
}
- parameters := shape[0] * shape[1] * shape[2] * shape[3]
- size := parameters * typeSize / blockSize
+ tensor := Tensor{
+ Name: name,
+ Kind: kind,
+ Offset: offset,
+ Shape: shape[:],
+ }
- llm.tensors = append(llm.tensors, tensor{
- name: name,
- kind: kind,
- offset: offset,
- size: size,
- shape: shape,
- })
-
- llm.parameters += parameters
+ llm.tensors = append(llm.tensors, &tensor)
+ llm.parameters += tensor.parameters()
}
+ // patch KV with parameter count
+ llm.kv["general.parameter_count"] = llm.parameters
+
alignment, ok := llm.kv["general.alignment"].(uint32)
if !ok {
alignment = 32
}
- rso.Seek(int64(alignment)-rso.offset%int64(alignment), io.SeekCurrent)
+ offset, err := rs.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+
+ padding := ggufPadding(offset, int64(alignment))
+ llm.tensorOffset = uint64(offset + padding)
+
for _, tensor := range llm.tensors {
- padded := (int64(tensor.size) + int64(alignment) - 1) & ^(int64(alignment) - 1)
- rso.Seek(padded, io.SeekCurrent)
+ offset, err := rs.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return fmt.Errorf("failed to get current offset: %w", err)
+ }
+
+ padding := ggufPadding(offset, int64(alignment))
+ if _, err := rs.Seek(padding, io.SeekCurrent); err != nil {
+ return fmt.Errorf("failed to seek to init padding: %w", err)
+ }
+
+ if _, err := rs.Seek(int64(tensor.Size()), io.SeekCurrent); err != nil {
+ return fmt.Errorf("failed to seek to tensor: %w", err)
+ }
}
return nil
}
-func (llm *ggufModel) NumLayers() int64 {
- value, exists := llm.kv[fmt.Sprintf("%s.block_count", llm.ModelFamily())]
- if !exists {
- return 0
+func readGGUF[T any](llm *gguf, r io.Reader) (T, error) {
+ var t T
+ err := binary.Read(r, llm.ByteOrder, &t)
+ return t, err
+}
+
+func writeGGUF[V any](w io.Writer, t uint32, v V) error {
+ if err := binary.Write(w, binary.LittleEndian, t); err != nil {
+ return err
}
- v := value.(uint32)
- return int64(v)
+ return binary.Write(w, binary.LittleEndian, v)
}
-func (llm ggufModel) readU8(r io.Reader) uint8 {
- var u8 uint8
- binary.Read(r, llm.bo, &u8)
- return u8
-}
-
-func (llm ggufModel) readI8(r io.Reader) int8 {
- var i8 int8
- binary.Read(r, llm.bo, &i8)
- return i8
-}
-
-func (llm ggufModel) readU16(r io.Reader) uint16 {
- var u16 uint16
- binary.Read(r, llm.bo, &u16)
- return u16
-}
-
-func (llm ggufModel) readI16(r io.Reader) int16 {
- var i16 int16
- binary.Read(r, llm.bo, &i16)
- return i16
-}
-
-func (llm ggufModel) readU32(r io.Reader) uint32 {
- var u32 uint32
- binary.Read(r, llm.bo, &u32)
- return u32
-}
-
-func (llm ggufModel) readI32(r io.Reader) int32 {
- var i32 int32
- binary.Read(r, llm.bo, &i32)
- return i32
-}
-
-func (llm ggufModel) readU64(r io.Reader) uint64 {
- var u64 uint64
- binary.Read(r, llm.bo, &u64)
- return u64
-}
-
-func (llm ggufModel) readI64(r io.Reader) int64 {
- var i64 int64
- binary.Read(r, llm.bo, &i64)
- return i64
-}
-
-func (llm ggufModel) readF32(r io.Reader) float32 {
- var f32 float32
- binary.Read(r, llm.bo, &f32)
- return f32
-}
-
-func (llm ggufModel) readF64(r io.Reader) float64 {
- var f64 float64
- binary.Read(r, llm.bo, &f64)
- return f64
-}
-
-func (llm ggufModel) readBool(r io.Reader) bool {
- var b bool
- binary.Read(r, llm.bo, &b)
- return b
-}
-
-func (llm ggufModel) readStringV1(r io.Reader) (string, error) {
- var nameLength uint32
- binary.Read(r, llm.bo, &nameLength)
+func readGGUFV1String(llm *gguf, r io.Reader) (string, error) {
+ var length uint64
+ if err := binary.Read(r, llm.ByteOrder, &length); err != nil {
+ return "", err
+ }
var b bytes.Buffer
- if _, err := io.CopyN(&b, r, int64(nameLength)); err != nil {
+ if _, err := io.CopyN(&b, r, int64(length)); err != nil {
return "", err
}
@@ -363,102 +299,364 @@ func (llm ggufModel) readStringV1(r io.Reader) (string, error) {
return b.String(), nil
}
-func (llm ggufModel) readString(r io.Reader) (string, error) {
- if llm.Version == 1 {
- return llm.readStringV1(r)
+func discardGGUFString(llm *gguf, r io.Reader) error {
+ buf := llm.scratch[:8]
+ _, err := io.ReadFull(r, buf)
+ if err != nil {
+ return err
}
- var nameLength uint64
- binary.Read(r, llm.bo, &nameLength)
+ size := int(llm.ByteOrder.Uint64(buf))
+ for size > 0 {
+ n, err := r.Read(llm.scratch[:min(size, cap(llm.scratch))])
+ if err != nil {
+ return err
+ }
+ size -= n
+ }
+ return nil
+}
- var b bytes.Buffer
- if _, err := io.CopyN(&b, r, int64(nameLength)); err != nil {
+func readGGUFString(llm *gguf, r io.Reader) (string, error) {
+ if llm.Version == 1 {
+ return readGGUFV1String(llm, r)
+ }
+
+ buf := llm.scratch[:8]
+ _, err := io.ReadFull(r, buf)
+ if err != nil {
return "", err
}
- return b.String(), nil
+ length := int(llm.ByteOrder.Uint64(buf))
+ if length > len(llm.scratch) {
+ buf = make([]byte, length)
+ } else {
+ buf = llm.scratch[:length]
+ }
+ clear(buf)
+
+ _, err = io.ReadFull(r, buf)
+ if err != nil {
+ return "", err
+ }
+ return string(buf), nil
}
-func (llm *ggufModel) readArrayV1(r io.Reader) (arr []any, err error) {
- atype := llm.readU32(r)
- n := llm.readU32(r)
-
- for i := 0; uint32(i) < n; i++ {
- switch atype {
- case ggufTypeUint8:
- arr = append(arr, llm.readU8(r))
- case ggufTypeInt8:
- arr = append(arr, llm.readI8(r))
- case ggufTypeUint16:
- arr = append(arr, llm.readU16(r))
- case ggufTypeInt16:
- arr = append(arr, llm.readI16(r))
- case ggufTypeUint32:
- arr = append(arr, llm.readU32(r))
- case ggufTypeInt32:
- arr = append(arr, llm.readI32(r))
- case ggufTypeFloat32:
- arr = append(arr, llm.readF32(r))
- case ggufTypeBool:
- arr = append(arr, llm.readBool(r))
- case ggufTypeString:
- s, err := llm.readStringV1(r)
- if err != nil {
- return nil, err
- }
-
- arr = append(arr, s)
- default:
- return nil, fmt.Errorf("invalid array type: %d", atype)
- }
+func writeGGUFString(w io.Writer, s string) error {
+ if err := binary.Write(w, binary.LittleEndian, ggufTypeString); err != nil {
+ return err
}
- return
+ if err := binary.Write(w, binary.LittleEndian, uint64(len(s))); err != nil {
+ return err
+ }
+
+ _, err := io.Copy(w, strings.NewReader(s))
+ return err
}
-func (llm *ggufModel) readArray(r io.Reader) (arr []any, err error) {
- if llm.Version == 1 {
- return llm.readArrayV1(r)
+type array struct {
+ size int
+ values []any
+}
+
+func (a *array) MarshalJSON() ([]byte, error) {
+ return json.Marshal(a.values)
+}
+
+func readGGUFV1Array(llm *gguf, r io.Reader) (*array, error) {
+ t, err := readGGUF[uint32](llm, r)
+ if err != nil {
+ return nil, err
}
- atype := llm.readU32(r)
- n := llm.readU64(r)
+ n, err := readGGUF[uint32](llm, r)
+ if err != nil {
+ return nil, err
+ }
- for i := 0; uint64(i) < n; i++ {
- switch atype {
+ a := &array{size: int(n)}
+ if llm.canCollectArray(int(n)) {
+ a.values = make([]any, 0, int(n))
+ }
+
+ for i := range n {
+ var e any
+ switch t {
case ggufTypeUint8:
- arr = append(arr, llm.readU8(r))
+ e, err = readGGUF[uint8](llm, r)
case ggufTypeInt8:
- arr = append(arr, llm.readI8(r))
+ e, err = readGGUF[int8](llm, r)
case ggufTypeUint16:
- arr = append(arr, llm.readU16(r))
+ e, err = readGGUF[uint16](llm, r)
case ggufTypeInt16:
- arr = append(arr, llm.readI16(r))
+ e, err = readGGUF[int16](llm, r)
case ggufTypeUint32:
- arr = append(arr, llm.readU32(r))
+ e, err = readGGUF[uint32](llm, r)
case ggufTypeInt32:
- arr = append(arr, llm.readI32(r))
+ e, err = readGGUF[int32](llm, r)
case ggufTypeUint64:
- arr = append(arr, llm.readU64(r))
+ e, err = readGGUF[uint64](llm, r)
case ggufTypeInt64:
- arr = append(arr, llm.readI64(r))
+ e, err = readGGUF[int64](llm, r)
case ggufTypeFloat32:
- arr = append(arr, llm.readF32(r))
+ e, err = readGGUF[float32](llm, r)
case ggufTypeFloat64:
- arr = append(arr, llm.readF64(r))
+ e, err = readGGUF[float64](llm, r)
case ggufTypeBool:
- arr = append(arr, llm.readBool(r))
+ e, err = readGGUF[bool](llm, r)
case ggufTypeString:
- s, err := llm.readString(r)
- if err != nil {
- return nil, err
- }
-
- arr = append(arr, s)
+ e, err = readGGUFV1String(llm, r)
default:
- return nil, fmt.Errorf("invalid array type: %d", atype)
+ return nil, fmt.Errorf("invalid array type: %d", t)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ if a.values != nil {
+ a.values[i] = e
}
}
- return
+ return a, nil
+}
+
+func readGGUFArray(llm *gguf, r io.Reader) (*array, error) {
+ if llm.Version == 1 {
+ return readGGUFV1Array(llm, r)
+ }
+
+ t, err := readGGUF[uint32](llm, r)
+ if err != nil {
+ return nil, err
+ }
+
+ n, err := readGGUF[uint64](llm, r)
+ if err != nil {
+ return nil, err
+ }
+
+ a := &array{size: int(n)}
+ if llm.canCollectArray(int(n)) {
+ a.values = make([]any, int(n))
+ }
+
+ for i := range n {
+ var e any
+ switch t {
+ case ggufTypeUint8:
+ e, err = readGGUF[uint8](llm, r)
+ case ggufTypeInt8:
+ e, err = readGGUF[int8](llm, r)
+ case ggufTypeUint16:
+ e, err = readGGUF[uint16](llm, r)
+ case ggufTypeInt16:
+ e, err = readGGUF[int16](llm, r)
+ case ggufTypeUint32:
+ e, err = readGGUF[uint32](llm, r)
+ case ggufTypeInt32:
+ e, err = readGGUF[int32](llm, r)
+ case ggufTypeUint64:
+ e, err = readGGUF[uint64](llm, r)
+ case ggufTypeInt64:
+ e, err = readGGUF[int64](llm, r)
+ case ggufTypeFloat32:
+ e, err = readGGUF[float32](llm, r)
+ case ggufTypeFloat64:
+ e, err = readGGUF[float64](llm, r)
+ case ggufTypeBool:
+ e, err = readGGUF[bool](llm, r)
+ case ggufTypeString:
+ if a.values != nil {
+ e, err = readGGUFString(llm, r)
+ } else {
+ err = discardGGUFString(llm, r)
+ }
+ default:
+ return nil, fmt.Errorf("invalid array type: %d", t)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ if a.values != nil {
+ a.values[i] = e
+ }
+ }
+
+ return a, nil
+}
+
+// writeGGUFArray writes a slice s of type E to the write with a gguf type of t
+func writeGGUFArray[S ~[]E, E any](w io.Writer, t uint32, s S) error {
+ if err := binary.Write(w, binary.LittleEndian, ggufTypeArray); err != nil {
+ return err
+ }
+
+ if err := binary.Write(w, binary.LittleEndian, t); err != nil {
+ return err
+ }
+
+ if err := binary.Write(w, binary.LittleEndian, uint64(len(s))); err != nil {
+ return err
+ }
+
+ return binary.Write(w, binary.LittleEndian, s)
+}
+
+func WriteGGUF(ws io.WriteSeeker, kv KV, ts []Tensor) error {
+ if err := binary.Write(ws, binary.LittleEndian, []byte("GGUF")); err != nil {
+ return err
+ }
+
+ if err := binary.Write(ws, binary.LittleEndian, uint32(3)); err != nil {
+ return err
+ }
+
+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(ts))); err != nil {
+ return err
+ }
+
+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(kv))); err != nil {
+ return err
+ }
+
+ keys := maps.Keys(kv)
+ slices.Sort(keys)
+
+ for _, key := range keys {
+ if err := ggufWriteKV(ws, key, kv[key]); err != nil {
+ return err
+ }
+ }
+
+ slices.SortStableFunc(ts, func(a, b Tensor) int {
+ if i, j := a.block(), b.block(); i < 0 && j > 0 {
+ return 1
+ } else if i > 0 && j < 0 {
+ return -1
+ } else {
+ return cmp.Compare(i, j)
+ }
+ })
+
+ var s uint64
+ for _, t := range ts {
+ t.Offset = s
+ if err := ggufWriteTensorInfo(ws, t); err != nil {
+ return err
+ }
+ s += t.Size()
+ }
+
+ var alignment int64 = 32
+ for _, t := range ts {
+ if err := ggufWriteTensor(ws, t, alignment); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func ggufWriteKV(ws io.WriteSeeker, k string, v any) error {
+ slog.Debug(k, "type", fmt.Sprintf("%T", v))
+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(k))); err != nil {
+ return err
+ }
+
+ if err := binary.Write(ws, binary.LittleEndian, []byte(k)); err != nil {
+ return err
+ }
+
+ var err error
+ switch v := v.(type) {
+ case uint32:
+ err = writeGGUF(ws, ggufTypeUint32, v)
+ case float32:
+ err = writeGGUF(ws, ggufTypeFloat32, v)
+ case bool:
+ err = writeGGUF(ws, ggufTypeBool, v)
+ case string:
+ err = writeGGUFString(ws, v)
+ case []int32:
+ err = writeGGUFArray(ws, ggufTypeInt32, v)
+ case []uint32:
+ err = writeGGUFArray(ws, ggufTypeUint32, v)
+ case []float32:
+ err = writeGGUFArray(ws, ggufTypeFloat32, v)
+ case []string:
+ if err := binary.Write(ws, binary.LittleEndian, ggufTypeArray); err != nil {
+ return err
+ }
+
+ if err := binary.Write(ws, binary.LittleEndian, ggufTypeString); err != nil {
+ return err
+ }
+
+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(v))); err != nil {
+ return err
+ }
+
+ for _, e := range v {
+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(e))); err != nil {
+ return err
+ }
+
+ if err := binary.Write(ws, binary.LittleEndian, []byte(e)); err != nil {
+ return err
+ }
+ }
+ default:
+ return fmt.Errorf("improper type for '%s'", k)
+ }
+
+ return err
+}
+
+func ggufWriteTensorInfo(ws io.WriteSeeker, t Tensor) error {
+ slog.Debug(t.Name, "kind", t.Kind, "shape", t.Shape, "offset", t.Offset)
+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(t.Name))); err != nil {
+ return err
+ }
+
+ if err := binary.Write(ws, binary.LittleEndian, []byte(t.Name)); err != nil {
+ return err
+ }
+
+ if err := binary.Write(ws, binary.LittleEndian, uint32(len(t.Shape))); err != nil {
+ return err
+ }
+
+ for i := range len(t.Shape) {
+ if err := binary.Write(ws, binary.LittleEndian, t.Shape[len(t.Shape)-i-1]); err != nil {
+ return err
+ }
+ }
+
+ if err := binary.Write(ws, binary.LittleEndian, t.Kind); err != nil {
+ return err
+ }
+
+ return binary.Write(ws, binary.LittleEndian, t.Offset)
+}
+
+func ggufWriteTensor(ws io.WriteSeeker, t Tensor, alignment int64) error {
+ offset, err := ws.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+
+ if err := binary.Write(ws, binary.LittleEndian, bytes.Repeat([]byte{0}, int(ggufPadding(offset, alignment)))); err != nil {
+ return err
+ }
+
+ _, err = t.WriteTo(ws)
+ return err
+}
+
+func ggufPadding(offset, align int64) int64 {
+ return (align - offset%align) % align
}
diff --git a/llm/llama.cpp b/llm/llama.cpp
new file mode 160000
index 00000000..8962422b
--- /dev/null
+++ b/llm/llama.cpp
@@ -0,0 +1 @@
+Subproject commit 8962422b1c6f9b8b15f5aeaea42600bcc2d44177
diff --git a/llm/llama.cpp/gen_common.sh b/llm/llama.cpp/gen_common.sh
deleted file mode 100644
index 0bfd8d8f..00000000
--- a/llm/llama.cpp/gen_common.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-# common logic accross linux and darwin
-
-init_vars() {
- LLAMACPP_DIR=gguf
- PATCHES="0001-Expose-callable-API-for-server.patch"
- CMAKE_DEFS="-DLLAMA_ACCELERATE=on"
- # TODO - LLAMA_K_QUANTS is stale and needs to be mapped to newer cmake settings
- CMAKE_TARGETS="--target ggml --target ggml_static --target llama --target build_info --target common --target ext_server --target llava_static"
- if echo "${CGO_CFLAGS}" | grep -- '-g' >/dev/null; then
- CMAKE_DEFS="-DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_VERBOSE_MAKEFILE=on -DLLAMA_GPROF=on -DLLAMA_SERVER_VERBOSE=on ${CMAKE_DEFS}"
- else
- # TODO - add additional optimization flags...
- CMAKE_DEFS="-DCMAKE_BUILD_TYPE=Release -DLLAMA_SERVER_VERBOSE=off ${CMAKE_DEFS}"
- fi
-}
-
-git_module_setup() {
- if [ -n "${OLLAMA_SKIP_PATCHING}" ]; then
- echo "Skipping submodule initialization"
- return
- fi
- git submodule init
- git submodule update --force gguf
-
-}
-
-apply_patches() {
- if [ -n "${OLLAMA_SKIP_PATCHING}" ]; then
- echo "Skipping submodule patching"
- return
- fi
- # Workaround git apply not handling creation well for iteration
- rm -f gguf/examples/server/server.h
- for patch in ${PATCHES}; do
- git -C gguf apply ../patches/${patch}
- done
-}
-
-build() {
- cmake -S ${LLAMACPP_DIR} -B ${BUILD_DIR} ${CMAKE_DEFS}
- cmake --build ${BUILD_DIR} ${CMAKE_TARGETS} -j8
-}
diff --git a/llm/llama.cpp/gen_darwin.sh b/llm/llama.cpp/gen_darwin.sh
deleted file mode 100755
index 1364e9d1..00000000
--- a/llm/llama.cpp/gen_darwin.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-# This script is intended to run inside the go generate
-# working directory must be ../llm/llama.cpp
-
-# TODO - add hardening to detect missing tools (cmake, etc.)
-
-set -ex
-set -o pipefail
-echo "Starting darwin generate script"
-source $(dirname $0)/gen_common.sh
-init_vars
-CMAKE_DEFS="-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 -DLLAMA_METAL=on ${CMAKE_DEFS}"
-BUILD_DIR="gguf/build/metal"
-case "${GOARCH}" in
- "amd64")
- CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 ${CMAKE_DEFS}"
- ;;
- "arm64")
- CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 ${CMAKE_DEFS}"
- ;;
- *)
- echo "GOARCH must be set"
- echo "this script is meant to be run from within go generate"
- exit 1
- ;;
-esac
-
-git_module_setup
-apply_patches
-build
\ No newline at end of file
diff --git a/llm/llama.cpp/gen_linux.sh b/llm/llama.cpp/gen_linux.sh
deleted file mode 100755
index 3d659fff..00000000
--- a/llm/llama.cpp/gen_linux.sh
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/bin/bash
-# This script is intended to run inside the go generate
-# working directory must be llm/llama.cpp
-
-# First we build our default built-in library which will be linked into the CGO
-# binary as a normal dependency. This default build is CPU based.
-#
-# Then we build a CUDA dynamic library (although statically linked with the CUDA
-# library dependencies for maximum portability)
-#
-# Then if we detect ROCm, we build a dynamically loaded ROCm lib. ROCm is particularly
-# important to be a dynamic lib even if it's the only GPU library detected because
-# we can't redistribute the objectfiles but must rely on dynamic libraries at
-# runtime, which could lead the server not to start if not present.
-
-set -ex
-set -o pipefail
-
-echo "Starting linux generate script"
-if [ -z "${CUDACXX}" -a -x /usr/local/cuda/bin/nvcc ]; then
- export CUDACXX=/usr/local/cuda/bin/nvcc
-fi
-COMMON_CMAKE_DEFS="-DCMAKE_POSITION_INDEPENDENT_CODE=on -DLLAMA_ACCELERATE=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off"
-OLLAMA_DYN_LIB_DIR="gguf/build/lib"
-source $(dirname $0)/gen_common.sh
-init_vars
-git_module_setup
-apply_patches
-
-mkdir -p ${OLLAMA_DYN_LIB_DIR}
-touch ${OLLAMA_DYN_LIB_DIR}/.generated
-
-#
-# CPU first for the default library
-#
-CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS}"
-BUILD_DIR="gguf/build/cpu"
-build
-
-if [ -d /usr/local/cuda/lib64/ ]; then
- echo "CUDA libraries detected - building dynamic CUDA library"
- init_vars
- CMAKE_DEFS="-DLLAMA_CUBLAS=on ${COMMON_CMAKE_DEFS} ${CMAKE_DEFS}"
- BUILD_DIR="gguf/build/cuda"
- CUDA_LIB_DIR=/usr/local/cuda/lib64
- build
- gcc -fPIC -g -shared -o ${OLLAMA_DYN_LIB_DIR}/libcuda_server.so \
- -Wl,--whole-archive \
- ${BUILD_DIR}/examples/server/libext_server.a \
- ${BUILD_DIR}/common/libcommon.a \
- ${BUILD_DIR}/libllama.a \
- -Wl,--no-whole-archive \
- ${CUDA_LIB_DIR}/libcudart_static.a \
- ${CUDA_LIB_DIR}/libcublas_static.a \
- ${CUDA_LIB_DIR}/libcublasLt_static.a \
- ${CUDA_LIB_DIR}/libcudadevrt.a \
- ${CUDA_LIB_DIR}/libculibos.a \
- -lrt -lpthread -ldl -lstdc++ -lm
-fi
-
-if [ -z "${ROCM_PATH}" ]; then
- # Try the default location in case it exists
- ROCM_PATH=/opt/rocm
-fi
-
-if [ -z "${CLBlast_DIR}" ]; then
- # Try the default location in case it exists
- if [ -d /usr/lib/cmake/CLBlast ]; then
- export CLBlast_DIR=/usr/lib/cmake/CLBlast
- fi
-fi
-
-if [ -d "${ROCM_PATH}" ]; then
- echo "ROCm libraries detected - building dynamic ROCm library"
- init_vars
- CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} -DLLAMA_HIPBLAS=on -DCMAKE_C_COMPILER=$ROCM_PATH/llvm/bin/clang -DCMAKE_CXX_COMPILER=$ROCM_PATH/llvm/bin/clang++ -DAMDGPU_TARGETS='gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102' -DGPU_TARGETS='gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102'"
- BUILD_DIR="gguf/build/rocm"
- build
- gcc -fPIC -g -shared -o ${OLLAMA_DYN_LIB_DIR}/librocm_server.so \
- -Wl,--whole-archive \
- ${BUILD_DIR}/examples/server/libext_server.a \
- ${BUILD_DIR}/common/libcommon.a \
- ${BUILD_DIR}/libllama.a \
- -Wl,--no-whole-archive \
- -lrt -lpthread -ldl -lstdc++ -lm \
- -L/opt/rocm/lib -L/opt/amdgpu/lib/x86_64-linux-gnu/ \
- -Wl,-rpath,/opt/rocm/lib,-rpath,/opt/amdgpu/lib/x86_64-linux-gnu/ \
- -lhipblas -lrocblas -lamdhip64 -lrocsolver -lamd_comgr -lhsa-runtime64 -lrocsparse -ldrm -ldrm_amdgpu
-fi
diff --git a/llm/llama.cpp/gen_windows.ps1 b/llm/llama.cpp/gen_windows.ps1
deleted file mode 100644
index 2f2f856d..00000000
--- a/llm/llama.cpp/gen_windows.ps1
+++ /dev/null
@@ -1,93 +0,0 @@
-#!powershell
-
-$ErrorActionPreference = "Stop"
-
-function init_vars {
- $script:patches = @("0001-Expose-callable-API-for-server.patch")
- $script:cmakeDefs = @("-DBUILD_SHARED_LIBS=on", "-DLLAMA_NATIVE=off", "-DLLAMA_F16C=off", "-DLLAMA_FMA=off", "-DLLAMA_AVX512=off", "-DLLAMA_AVX2=off", "-DLLAMA_AVX=on", "-DLLAMA_K_QUANTS=on", "-DLLAMA_ACCELERATE=on", "-A","x64")
-
- if ($env:CGO_CFLAGS -contains "-g") {
- $script:cmakeDefs += @("-DCMAKE_VERBOSE_MAKEFILE=on", "-DLLAMA_SERVER_VERBOSE=on")
- $script:config = "RelWithDebInfo"
- } else {
- $script:cmakeDefs += @("-DLLAMA_SERVER_VERBOSE=off")
- $script:config = "Release"
- }
-}
-
-function git_module_setup {
- # TODO add flags to skip the init/patch logic to make it easier to mod llama.cpp code in-repo
- & git submodule init
- if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
- & git submodule update --force gguf
- if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-}
-
-function apply_patches {
- rm -erroraction ignore -path "gguf/examples/server/server.h"
- foreach ($patch in $script:patches) {
- write-host "Applying patch $patch"
- & git -C gguf apply ../patches/$patch
- if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
- }
-}
-
-function build {
- write-host "generating config with: cmake -S gguf -B $script:buildDir $script:cmakeDefs"
- & cmake --version
- & cmake -S gguf -B $script:buildDir $script:cmakeDefs
- if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
- write-host "building with: cmake --build $script:buildDir --config $script:config"
- & cmake --build $script:buildDir --config $script:config
- if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-}
-
-function install {
- rm -erroraction ignore -recurse -force -path $script:installDir
- & cmake --install $script:buildDir --prefix $script:installDir --config $script:config
- if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-
-}
-
-init_vars
-git_module_setup
-apply_patches
-
-# first build CPU based
-$script:buildDir="gguf/build/wincpu"
-$script:installDir="gguf/build/wincpu/dist"
-
-build
-# install
-
-md gguf/build/lib -ea 0
-md gguf/build/wincpu/dist/lib -ea 0
-mv gguf/build/wincpu/bin/$script:config/ext_server_shared.dll gguf/build/wincpu/dist/lib/cpu_server.dll
-
-
-# Nope, this barfs on lots of symbol problems
-#mv gguf/build/wincpu/examples/server/$script:config/ext_server_shared.dll gguf/build/wincpu/dist/lib/cpu_server.lib
-# Nope: this needs lots of include paths to pull in things like msvcprt.lib and other deps
-# & cl.exe `
-# gguf/build/wincpu/examples/server/$script:config/ext_server.lib `
-# gguf/build/wincpu/common/$script:config/common.lib `
-# gguf/build/wincpu/$script:config/llama.lib `
-# gguf/build/wincpu/$script:config/ggml_static.lib `
-# /link /DLL /DEF:cpu_server.def /NOENTRY /MACHINE:X64 /OUT:gguf/build/wincpu/dist/lib/cpu_server.dll
-# if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-
-# Then build cuda as a dynamically loaded library
-init_vars
-$script:buildDir="gguf/build/wincuda"
-$script:installDir="gguf/build/wincuda/dist"
-$script:cmakeDefs += @("-DLLAMA_CUBLAS=ON", "-DBUILD_SHARED_LIBS=on")
-build
-install
-cp gguf/build/wincuda/dist/bin/ext_server_shared.dll gguf/build/lib/cuda_server.dll
-
-# TODO - more to do here to create a usable dll
-
-
-# TODO - implement ROCm support on windows
-md gguf/build/winrocm/lib -ea 0
-echo $null >> gguf/build/winrocm/lib/.generated
diff --git a/llm/llama.cpp/generate_darwin.go b/llm/llama.cpp/generate_darwin.go
deleted file mode 100644
index 498e5005..00000000
--- a/llm/llama.cpp/generate_darwin.go
+++ /dev/null
@@ -1,3 +0,0 @@
-package llm
-
-//go:generate sh ./gen_darwin.sh
diff --git a/llm/llama.cpp/gguf b/llm/llama.cpp/gguf
deleted file mode 160000
index 328b83de..00000000
--- a/llm/llama.cpp/gguf
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 328b83de23b33240e28f4e74900d1d06726f5eb1
diff --git a/llm/llama.cpp/patches/0001-Expose-callable-API-for-server.patch b/llm/llama.cpp/patches/0001-Expose-callable-API-for-server.patch
deleted file mode 100644
index e1c1b141..00000000
--- a/llm/llama.cpp/patches/0001-Expose-callable-API-for-server.patch
+++ /dev/null
@@ -1,464 +0,0 @@
-From 90c332fe2ef61149b38561d02836e66715df214d Mon Sep 17 00:00:00 2001
-From: Daniel Hiltgen <daniel@ollama.com>
-Date: Mon, 13 Nov 2023 12:25:58 -0800
-Subject: [PATCH] Expose callable API for server
-
-This adds an extern "C" interface within the example server
----
- examples/server/CMakeLists.txt | 27 ++++
- examples/server/server.cpp | 280 +++++++++++++++++++++++++++++++++
- examples/server/server.h | 89 +++++++++++
- ggml-cuda.cu | 1 +
- 4 files changed, 397 insertions(+)
- create mode 100644 examples/server/server.h
-
-diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt
-index 859cd12..da2b9bf 100644
---- a/examples/server/CMakeLists.txt
-+++ b/examples/server/CMakeLists.txt
-@@ -11,3 +11,30 @@ if (WIN32)
- TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ws2_32)
- endif()
- target_compile_features(${TARGET} PRIVATE cxx_std_11)
-+
-+set(TARGET ext_server)
-+option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON)
-+add_library(${TARGET} STATIC server.cpp)
-+target_include_directories(${TARGET} PRIVATE ../../common)
-+target_include_directories(${TARGET} PRIVATE ../..)
-+target_compile_features(${TARGET} PRIVATE cxx_std_11)
-+target_compile_definitions(${TARGET} PUBLIC LLAMA_SERVER_LIBRARY=1)
-+target_link_libraries(${TARGET} PRIVATE common llama llava ${CMAKE_THREAD_LIBS_INIT})
-+target_compile_definitions(${TARGET} PRIVATE
-+ SERVER_VERBOSE=$<BOOL:${LLAMA_SERVER_VERBOSE}>
-+)
-+
-+if (BUILD_SHARED_LIBS)
-+ set_target_properties(ext_server PROPERTIES POSITION_INDEPENDENT_CODE ON)
-+ target_compile_definitions(ext_server PRIVATE LLAMA_SHARED LLAMA_BUILD)
-+ add_library(ext_server_shared SHARED $<TARGET_OBJECTS:ext_server>)
-+ target_link_libraries(ext_server_shared PRIVATE ggml llama llava common ${CMAKE_THREAD_LIBS_INIT})
-+ install(TARGETS ext_server_shared LIBRARY)
-+endif()
-+
-+if (CUDAToolkit_FOUND)
-+ target_include_directories(${TARGET} PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-+ if (WIN32)
-+ target_link_libraries(ext_server_shared PRIVATE nvml)
-+ endif()
-+endif()
-\ No newline at end of file
-diff --git a/examples/server/server.cpp b/examples/server/server.cpp
-index 0403853..07fb05c 100644
---- a/examples/server/server.cpp
-+++ b/examples/server/server.cpp
-@@ -5,6 +5,9 @@
- #include "../llava/clip.h"
-
- #include "stb_image.h"
-+#if defined(LLAMA_SERVER_LIBRARY)
-+#include "server.h"
-+#endif
-
- #ifndef NDEBUG
- // crash the server in debug mode, otherwise send an http 500 error
-@@ -2643,6 +2646,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con
- }
- }
-
-+#ifndef LLAMA_SERVER_LIBRARY
- int main(int argc, char **argv)
- {
- #if SERVER_VERBOSE != 1
-@@ -3123,3 +3127,279 @@ int main(int argc, char **argv)
- llama_backend_free();
- return 0;
- }
-+
-+#else // LLAMA_SERVER_LIBRARY
-+// Expose the llama server as a callable extern "C" API
-+llama_server_context *llama = NULL;
-+std::atomic<bool> ext_server_running(false);
-+std::thread ext_server_thread;
-+
-+void llama_server_init(ext_server_params *sparams, ext_server_resp_t *err)
-+{
-+#if SERVER_VERBOSE != 1
-+ LOG_TEE("disabling verbose llm logging\n");
-+ log_disable();
-+#endif
-+ assert(err != NULL && sparams != NULL);
-+ err->id = 0;
-+ err->msg[0] = '\0';
-+ try {
-+ llama = new llama_server_context;
-+ log_set_target(stdout);
-+ gpt_params params;
-+ params.n_ctx = sparams->n_ctx;
-+ params.n_batch = sparams->n_batch;
-+ if (sparams->n_threads > 0) {
-+ params.n_threads = sparams->n_threads;
-+ }
-+ params.n_parallel = sparams->n_parallel;
-+ params.rope_freq_base = sparams->rope_freq_base;
-+ params.rope_freq_scale = sparams->rope_freq_scale;
-+
-+ if (sparams->memory_f16) {
-+ params.cache_type_k = "f16";
-+ params.cache_type_v = "f16";
-+ } else {
-+ params.cache_type_k = "f32";
-+ params.cache_type_v = "f32";
-+ }
-+
-+ params.n_gpu_layers = sparams->n_gpu_layers;
-+ params.main_gpu = sparams->main_gpu;
-+ params.use_mlock = sparams->use_mlock;
-+ params.use_mmap = sparams->use_mmap;
-+ params.numa = sparams->numa;
-+ params.embedding = sparams->embedding;
-+ if (sparams->model != NULL) {
-+ params.model = sparams->model;
-+ }
-+
-+ for (ext_server_lora_adapter *la = sparams->lora_adapters; la != NULL; la = la->next) {
-+ params.lora_adapter.push_back(std::make_tuple(la->adapter, la->scale));
-+ }
-+
-+ if (sparams->mmproj != NULL) {
-+ params.mmproj = std::string(sparams->mmproj);
-+ }
-+
-+ llama_backend_init(params.numa);
-+
-+ // load the model
-+ if (!llama->load_model(params))
-+ {
-+ // TODO - consider modifying the logging logic or patching load_model so we can capture more detailed error messages
-+ // and pass them back to the caller for better UX
-+ err->id = -1;
-+ snprintf(err->msg, err->msg_len, "error loading model %s", params.model.c_str());
-+ return;
-+ }
-+
-+ llama->initialize();
-+ } catch (std::exception &e) {
-+ err->id = -1;
-+ snprintf(err->msg, err->msg_len, "exception %s", e.what());
-+ } catch (...) {
-+ err->id = -1;
-+ snprintf(err->msg, err->msg_len, "Unknown exception initializing llama server");
-+ }
-+}
-+
-+void llama_server_start()
-+{
-+ assert(llama != NULL);
-+ // TODO mutex to protect thread creation
-+ ext_server_thread = std::thread([&]()
-+ {
-+ ext_server_running = true;
-+ try {
-+ LOG_TEE("llama server main loop starting\n");
-+ ggml_time_init();
-+ while (ext_server_running.load())
-+ {
-+ if (!llama->update_slots()) {
-+ LOG_TEE("unexpected error in llama server update_slots - exiting main loop\n");
-+ break;
-+ }
-+ }
-+ } catch (std::exception &e) {
-+ LOG_TEE("caught exception in llama server main loop: %s\n", e.what());
-+ } catch (...) {
-+ LOG_TEE("caught unknown exception in llama server main loop\n");
-+ }
-+ LOG_TEE("\nllama server shutting down\n");
-+ llama_backend_free();
-+ });
-+}
-+
-+void llama_server_stop() {
-+ assert(llama != NULL);
-+ // TODO - too verbose, remove once things are solid
-+ LOG_TEE("requesting llama server shutdown\n");
-+ ext_server_running = false;
-+ ext_server_thread.join();
-+ delete llama;
-+ llama = NULL;
-+ LOG_TEE("llama server shutdown complete\n");
-+}
-+
-+void llama_server_completion(const char *json_req, ext_server_resp_t *resp) {
-+ assert(llama != NULL && json_req != NULL && resp != NULL);
-+ resp->id = -1;
-+ resp->msg[0] = '\0';
-+ try {
-+ json data = json::parse(json_req);
-+ resp->id = llama->request_completion(data, false, false, -1);
-+ } catch (std::exception &e) {
-+ snprintf(resp->msg, resp->msg_len, "exception %s", e.what());
-+ } catch (...) {
-+ snprintf(resp->msg, resp->msg_len, "Unknown exception during completion");
-+ }
-+}
-+
-+void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *resp) {
-+ assert(llama != NULL && resp != NULL);
-+ std::string msg;
-+ resp->id = -1;
-+ resp->stop = false;
-+ resp->error = false;
-+ resp->json_resp = NULL;
-+ std::string result_json;
-+ try {
-+ task_result result = llama->next_result(task_id);
-+ result_json = result.result_json.dump(-1, ' ', false, json::error_handler_t::replace);
-+ resp->id = result.id;
-+ resp->stop = result.stop;
-+ resp->error = result.error;
-+ if (result.error) {
-+ llama->request_cancel(task_id);
-+ } else if (result.stop) {
-+ llama->request_cancel(task_id);
-+ }
-+ } catch (std::exception &e) {
-+ resp->error = true;
-+ resp->id = -1;
-+ result_json = "{\"error\":\"exception " + std::string(e.what()) + "\"}";
-+ } catch (...) {
-+ resp->error = true;
-+ resp->id = -1;
-+ result_json = "{\"error\":\"Unknown exception during completion\"}";
-+ }
-+ const std::string::size_type size = result_json.size() + 1;
-+ resp->json_resp = new char[size];
-+ snprintf(resp->json_resp, size, "%s", result_json.c_str());
-+}
-+
-+void llama_server_release_task_result(ext_server_task_result_t *result) {
-+ if (result == NULL || result->json_resp == NULL) {
-+ return;
-+ }
-+ delete[] result->json_resp;
-+}
-+
-+void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err) {
-+ assert(llama != NULL && err != NULL);
-+ err->id = 0;
-+ err->msg[0] = '\0';
-+ try {
-+ llama->request_cancel(task_id);
-+ } catch (std::exception &e) {
-+ err->id = -1;
-+ snprintf(err->msg, err->msg_len, "exception %s", e.what());
-+ } catch (...) {
-+ err->id = -1;
-+ snprintf(err->msg, err->msg_len, "Unknown exception completion cancel in llama server");
-+ }
-+}
-+
-+void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) {
-+ assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
-+ *json_resp = NULL;
-+ err->id = 0;
-+ err->msg[0] = '\0';
-+ try {
-+ const json body = json::parse(json_req);
-+ std::vector<llama_token> tokens;
-+ if (body.count("content") != 0)
-+ {
-+ tokens = llama->tokenize(body["content"], false);
-+ }
-+ const json data = format_tokenizer_response(tokens);
-+ std::string result_json = data.dump();
-+ const std::string::size_type size = result_json.size() + 1;
-+ *json_resp = new char[size];
-+ snprintf(*json_resp, size, "%s", result_json.c_str());
-+ } catch (std::exception &e) {
-+ err->id = -1;
-+ snprintf(err->msg, err->msg_len, "exception %s", e.what());
-+ } catch (...) {
-+ err->id = -1;
-+ snprintf(err->msg, err->msg_len, "Unknown exception during tokenize");
-+ }
-+}
-+
-+void llama_server_release_json_resp(char **json_resp) {
-+ if (json_resp == NULL || *json_resp == NULL) {
-+ return;
-+ }
-+ delete[] *json_resp;
-+}
-+
-+void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) {
-+ assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
-+ *json_resp = NULL;
-+ err->id = 0;
-+ err->msg[0] = '\0';
-+ try {
-+ const json body = json::parse(json_req);
-+ std::string content;
-+ if (body.count("tokens") != 0)
-+ {
-+ const std::vector<llama_token> tokens = body["tokens"];
-+ content = tokens_to_str(llama->ctx, tokens.cbegin(), tokens.cend());
-+ }
-+ const json data = format_detokenized_response(content);
-+ std::string result_json = data.dump();
-+ const std::string::size_type size = result_json.size() + 1;
-+ *json_resp = new char[size];
-+ snprintf(*json_resp, size, "%s", result_json.c_str());
-+ } catch (std::exception &e) {
-+ err->id = -1;
-+ snprintf(err->msg, err->msg_len, "exception %s", e.what());
-+ } catch (...) {
-+ err->id = -1;
-+ snprintf(err->msg, err->msg_len, "Unknown exception during detokenize");
-+ }
-+}
-+
-+void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err) {
-+ assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
-+ *json_resp = NULL;
-+ err->id = 0;
-+ err->msg[0] = '\0';
-+ try {
-+ const json body = json::parse(json_req);
-+ json prompt;
-+ if (body.count("content") != 0)
-+ {
-+ prompt = body["content"];
-+ }
-+ else
-+ {
-+ prompt = "";
-+ }
-+ const int task_id = llama->request_completion({ {"prompt", prompt}, { "n_predict", 0} }, false, true, -1);
-+ task_result result = llama->next_result(task_id);
-+ std::string result_json = result.result_json.dump();
-+ const std::string::size_type size = result_json.size() + 1;
-+ *json_resp = new char[size];
-+ snprintf(*json_resp, size, "%s", result_json.c_str());
-+ } catch (std::exception &e) {
-+ err->id = -1;
-+ snprintf(err->msg, err->msg_len, "exception %s", e.what());
-+ } catch (...) {
-+ err->id = -1;
-+ snprintf(err->msg, err->msg_len, "Unknown exception during embedding");
-+ }
-+}
-+
-+#endif // LLAMA_SERVER_LIBRARY
-\ No newline at end of file
-diff --git a/examples/server/server.h b/examples/server/server.h
-new file mode 100644
-index 0000000..d22f1b6
---- /dev/null
-+++ b/examples/server/server.h
-@@ -0,0 +1,89 @@
-+#if defined(LLAMA_SERVER_LIBRARY)
-+#ifndef LLAMA_SERVER_H
-+#define LLAMA_SERVER_H
-+#include <stddef.h>
-+#include <stdint.h>
-+#include <stdio.h>
-+#include <stdbool.h>
-+
-+// This exposes extern C entrypoints into the llama_server
-+// To enable the server compile with LLAMA_SERVER_LIBRARY
-+
-+#ifdef __cplusplus
-+extern "C"
-+{
-+#endif
-+ typedef struct ext_server_resp {
-+ int id; // < 0 on error
-+ size_t msg_len; // caller must allocate msg and set msg_len
-+ char *msg;
-+ } ext_server_resp_t;
-+
-+ // Allocated and freed by caller
-+ typedef struct ext_server_lora_adapter {
-+ char *adapter;
-+ float scale;
-+ struct ext_server_lora_adapter *next;
-+ } ext_server_lora_adapter_t;
-+
-+ // Allocated and freed by caller
-+ typedef struct ext_server_params
-+ {
-+ char *model;
-+ uint32_t n_ctx; // text context, 0 = from model
-+ uint32_t n_batch; // prompt processing maximum batch size
-+ uint32_t n_threads; // number of threads to use for generation
-+ int32_t n_parallel; // number of parallel sequences to decodewra
-+ float rope_freq_base; // RoPE base frequency, 0 = from model
-+ float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model
-+ bool memory_f16; // use f16 instead of f32 for memory kv
-+ int32_t n_gpu_layers; // number of layers to store in VRAM (-1 - use default)
-+ int32_t main_gpu; // the GPU that is used for scratch and small tensors
-+ bool use_mlock; // force system to keep model in RAM
-+ bool use_mmap; // use mmap if possible
-+ bool numa; // attempt optimizations that help on some NUMA systems
-+ bool embedding; // get only sentence embedding
-+ ext_server_lora_adapter_t* lora_adapters;
-+ char *mmproj;
-+ } ext_server_params_t;
-+
-+ typedef struct ext_server_task_result
-+ {
-+ int id;
-+ bool stop;
-+ bool error;
-+ char* json_resp; // null terminated, memory managed by ext_server
-+ } ext_server_task_result_t;
-+
-+ // Initialize the server once per process
-+ // err->id = 0 for success and err->msg[0] = NULL
-+ // err->id != 0 for failure, and err->msg contains error message
-+ void llama_server_init(ext_server_params_t *sparams, ext_server_resp_t *err);
-+
-+ // Run the main loop, called once per init
-+ void llama_server_start();
-+ // Stop the main loop and free up resources allocated in init and start. Init must be called again to reuse
-+ void llama_server_stop();
-+
-+ // json_req null terminated string, memory managed by caller
-+ // resp->id >= 0 on success (task ID)
-+ // resp->id < 0 on error, and resp->msg contains error message
-+ void llama_server_completion(const char *json_req, ext_server_resp_t *resp);
-+
-+ // Caller must call llama_server_release_task_result to free resp->json_resp
-+ void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *result);
-+ void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err);
-+ void llama_server_release_task_result(ext_server_task_result_t *result);
-+
-+ // Caller must call llama_server_releaes_json_resp to free json_resp if err.id < 0
-+ void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err);
-+ void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err);
-+ void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err);
-+ void llama_server_release_json_resp(char **json_resp);
-+
-+#ifdef __cplusplus
-+}
-+#endif
-+
-+#endif
-+#endif // LLAMA_SERVER_LIBRARY
-\ No newline at end of file
-diff --git a/ggml-cuda.cu b/ggml-cuda.cu
-index f20846f..9640cf3 100644
---- a/ggml-cuda.cu
-+++ b/ggml-cuda.cu
-@@ -6757,6 +6757,7 @@ static cudaError_t ggml_cuda_cpy_tensor_2d(
- CUDA_CHECK(cudaGetDevice(&id));
- src_ptr = (char *) extra->data_device[id];
- } else {
-+ fprintf(stderr, "ggml_cuda_cpy_tensor_2d assert: backend: %d\n", src->backend);
- GGML_ASSERT(false);
- }
- char * dst_ptr = (char *) dst;
---
-2.39.3 (Apple Git-145)
-
diff --git a/llm/llama.go b/llm/llama.go
deleted file mode 100644
index 6278abc3..00000000
--- a/llm/llama.go
+++ /dev/null
@@ -1,246 +0,0 @@
-package llm
-
-import (
- "bytes"
- "context"
- _ "embed"
- "errors"
- "fmt"
- "io"
- "io/fs"
- "os"
- "os/exec"
- "path/filepath"
- "sync"
- "time"
-
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/format"
-)
-
-const jsonGrammar = `
-root ::= object
-value ::= object | array | string | number | ("true" | "false" | "null") ws
-
-object ::=
- "{" ws (
- string ":" ws value
- ("," ws string ":" ws value)*
- )? "}" ws
-
-array ::=
- "[" ws (
- value
- ("," ws value)*
- )? "]" ws
-
-string ::=
- "\"" (
- [^"\\] |
- "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
- )* "\"" ws
-
-number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
-
-# Optional space: by convention, applied in this grammar after literal chars when allowed
-ws ::= ([ \t\n] ws)?
-`
-
-type llamaModel struct {
- hyperparameters llamaHyperparameters
-}
-
-func (llm *llamaModel) ModelFamily() string {
- return "llama"
-}
-
-func llamaModelType(numLayer uint32) string {
- switch numLayer {
- case 26:
- return "3B"
- case 32:
- return "7B"
- case 40:
- return "13B"
- case 48:
- return "34B"
- case 60:
- return "30B"
- case 80:
- return "65B"
- default:
- return "unknown"
- }
-}
-
-func (llm *llamaModel) ModelType() string {
- return llamaModelType(llm.hyperparameters.NumLayer)
-}
-
-func (llm *llamaModel) FileType() string {
- return fileType(llm.hyperparameters.FileType)
-}
-
-func (llm *llamaModel) NumLayers() int64 {
- return int64(llm.hyperparameters.NumLayer)
-}
-
-type llamaHyperparameters struct {
- // NumVocab is the size of the model's vocabulary.
- NumVocab uint32
-
- // NumEmbd is the size of the model's embedding layer.
- NumEmbd uint32
- NumMult uint32
- NumHead uint32
-
- // NumLayer is the number of layers in the model.
- NumLayer uint32
- NumRot uint32
-
- // FileType describes the quantization level of the model, e.g. Q4_0, Q5_K, etc.
- FileType uint32
-}
-
-type Running struct {
- Port int
- Cmd *exec.Cmd
- Cancel context.CancelFunc
- exitOnce sync.Once
- exitCh chan error // channel to receive the exit status of the subprocess
- *StatusWriter // captures error messages from the llama runner process
-}
-
-type ImageData struct {
- Data []byte `json:"data"`
- ID int `json:"id"`
-}
-
-var (
- errNvidiaSMI = errors.New("warning: gpu support may not be enabled, check that you have installed GPU drivers: nvidia-smi command failed")
- errAvailableVRAM = errors.New("not enough VRAM available, falling back to CPU only")
- payloadMissing = fmt.Errorf("expected dynamic library payloads not included in this build of ollama")
-)
-
-// StatusWriter is a writer that captures error messages from the llama runner process
-type StatusWriter struct {
- ErrCh chan error
- LastErrMsg string
-}
-
-func NewStatusWriter() *StatusWriter {
- return &StatusWriter{
- ErrCh: make(chan error, 1),
- }
-}
-
-func (w *StatusWriter) Write(b []byte) (int, error) {
- var errMsg string
- if _, after, ok := bytes.Cut(b, []byte("error:")); ok {
- errMsg = string(bytes.TrimSpace(after))
- } else if _, after, ok := bytes.Cut(b, []byte("CUDA error")); ok {
- errMsg = string(bytes.TrimSpace(after))
- }
-
- if errMsg != "" {
- w.LastErrMsg = errMsg
- w.ErrCh <- fmt.Errorf("llama runner: %s", errMsg)
- }
-
- return os.Stderr.Write(b)
-}
-
-type prediction struct {
- Content string `json:"content"`
- Model string `json:"model"`
- Prompt string `json:"prompt"`
- Stop bool `json:"stop"`
-
- Timings struct {
- PredictedN int `json:"predicted_n"`
- PredictedMS float64 `json:"predicted_ms"`
- PromptN int `json:"prompt_n"`
- PromptMS float64 `json:"prompt_ms"`
- }
-}
-
-const maxBufferSize = 512 * format.KiloByte
-const maxRetries = 3
-const retryDelay = 1 * time.Second
-
-type PredictOpts struct {
- Prompt string
- Format string
- Images []api.ImageData
-}
-
-type PredictResult struct {
- Content string
- Done bool
- PromptEvalCount int
- PromptEvalDuration time.Duration
- EvalCount int
- EvalDuration time.Duration
-}
-
-type TokenizeRequest struct {
- Content string `json:"content"`
-}
-
-type TokenizeResponse struct {
- Tokens []int `json:"tokens"`
-}
-
-type DetokenizeRequest struct {
- Tokens []int `json:"tokens"`
-}
-
-type DetokenizeResponse struct {
- Content string `json:"content"`
-}
-
-type EmbeddingRequest struct {
- Content string `json:"content"`
-}
-
-type EmbeddingResponse struct {
- Embedding []float64 `json:"embedding"`
-}
-
-func extractDynamicLibs(workDir, glob string) ([]string, error) {
- files, err := fs.Glob(libEmbed, glob)
- if err != nil || len(files) == 0 {
- return nil, payloadMissing
- }
- libs := make([]string, len(files))
-
- for i, file := range files {
- srcFile, err := libEmbed.Open(file)
- if err != nil {
- return nil, fmt.Errorf("read payload %s: %v", file, err)
- }
- defer srcFile.Close()
- if err := os.MkdirAll(workDir, 0o755); err != nil {
- return nil, fmt.Errorf("create payload temp dir %s: %v", workDir, err)
- }
-
- destFile := filepath.Join(workDir, filepath.Base(file))
- libs[i] = destFile
-
- _, err = os.Stat(destFile)
- switch {
- case errors.Is(err, os.ErrNotExist):
- destFile, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
- if err != nil {
- return nil, fmt.Errorf("write payload %s: %v", file, err)
- }
- defer destFile.Close()
- if _, err := io.Copy(destFile, srcFile); err != nil {
- return nil, fmt.Errorf("copy payload %s: %v", file, err)
- }
- case err != nil:
- return nil, fmt.Errorf("stat payload %s: %v", file, err)
- }
- }
- return libs, nil
-}
diff --git a/llm/llm.go b/llm/llm.go
deleted file mode 100644
index d581c7ac..00000000
--- a/llm/llm.go
+++ /dev/null
@@ -1,103 +0,0 @@
-package llm
-
-import (
- "context"
- "fmt"
- "log"
- "os"
- "runtime"
-
- "github.com/pbnjay/memory"
-
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/format"
- "github.com/jmorganca/ollama/gpu"
-)
-
-type LLM interface {
- Predict(context.Context, PredictOpts, func(PredictResult)) error
- Embedding(context.Context, string) ([]float64, error)
- Encode(context.Context, string) ([]int, error)
- Decode(context.Context, []int) (string, error)
- Close()
-}
-
-var AvailableShims = map[string]string{}
-
-func New(workDir, model string, adapters, projectors []string, opts api.Options) (LLM, error) {
- if _, err := os.Stat(model); err != nil {
- return nil, err
- }
-
- f, err := os.Open(model)
- if err != nil {
- return nil, err
- }
- defer f.Close()
-
- ggml, err := DecodeGGML(f)
- if err != nil {
- return nil, err
- }
-
- if runtime.GOOS == "darwin" {
- switch ggml.FileType() {
- case "F32", "Q5_0", "Q5_1", "Q8_0":
- if ggml.Name() != "gguf" && opts.NumGPU != 0 {
- // GGML Q8_0 do not support Metal API and will
- // cause the runner to segmentation fault so disable GPU
- log.Printf("WARNING: GPU disabled for F32, Q5_0, Q5_1, and Q8_0")
- opts.NumGPU = 0
- }
- }
-
- var requiredMemory int64
- var f16Multiplier int64 = 2
-
- switch ggml.ModelType() {
- case "3B", "7B":
- requiredMemory = 8 * format.GigaByte
- case "13B":
- requiredMemory = 16 * format.GigaByte
- case "30B", "34B", "40B":
- requiredMemory = 32 * format.GigaByte
- case "65B", "70B":
- requiredMemory = 64 * format.GigaByte
- case "180B":
- requiredMemory = 128 * format.GigaByte
- f16Multiplier = 4
- }
-
- systemMemory := int64(memory.TotalMemory())
-
- if ggml.FileType() == "F16" && requiredMemory*f16Multiplier > systemMemory {
- return nil, fmt.Errorf("F16 model requires at least %s of total memory", format.HumanBytes(requiredMemory))
- } else if requiredMemory > systemMemory {
- return nil, fmt.Errorf("model requires at least %s of total memory", format.HumanBytes(requiredMemory))
- }
- }
-
- opts.NumGQA = 0
- opts.RopeFrequencyBase = 0.0
- opts.RopeFrequencyScale = 0.0
- gpuInfo := gpu.GetGPUInfo()
- return newLlmServer(gpuInfo.Library, model, adapters, projectors, ggml.NumLayers(), opts)
-}
-
-// Give any native cgo implementations an opportunity to initialize
-func Init(workdir string) error {
- return nativeInit(workdir)
-}
-
-func newLlmServer(library, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
- if _, libPresent := AvailableShims[library]; libPresent && library != "default" {
- srv, err := newDynamicShimExtServer(AvailableShims[library], model, adapters, projectors, numLayers, opts)
- if err == nil {
- return srv, nil
- }
- log.Printf("Failed to load dynamic library - falling back to CPU mode %s", err)
- }
-
- return newDefaultExtServer(model, adapters, projectors, numLayers, opts)
-
-}
diff --git a/llm/llm_darwin.go b/llm/llm_darwin.go
new file mode 100644
index 00000000..60837ed0
--- /dev/null
+++ b/llm/llm_darwin.go
@@ -0,0 +1,7 @@
+package llm
+
+import (
+ "syscall"
+)
+
+var LlamaServerSysProcAttr = &syscall.SysProcAttr{}
diff --git a/llm/llm_linux.go b/llm/llm_linux.go
new file mode 100644
index 00000000..60837ed0
--- /dev/null
+++ b/llm/llm_linux.go
@@ -0,0 +1,7 @@
+package llm
+
+import (
+ "syscall"
+)
+
+var LlamaServerSysProcAttr = &syscall.SysProcAttr{}
diff --git a/llm/llm_windows.go b/llm/llm_windows.go
new file mode 100644
index 00000000..915355a2
--- /dev/null
+++ b/llm/llm_windows.go
@@ -0,0 +1,22 @@
+package llm
+
+import (
+ "syscall"
+)
+
+const (
+ CREATE_DEFAULT_ERROR_MODE = 0x04000000
+ ABOVE_NORMAL_PRIORITY_CLASS = 0x00008000
+)
+
+var LlamaServerSysProcAttr = &syscall.SysProcAttr{
+ // Wire up the default error handling logic If for some reason a DLL is
+ // missing in the path this will pop up a GUI Dialog explaining the fault so
+ // the user can either fix their PATH, or report a bug. Without this
+ // setting, the process exits immediately with a generic exit status but no
+ // way to (easily) figure out what the actual missing DLL was.
+ //
+ // Setting Above Normal priority class ensures when running as a "background service"
+ // with "programs" given best priority, we aren't starved of cpu cycles
+ CreationFlags: CREATE_DEFAULT_ERROR_MODE | ABOVE_NORMAL_PRIORITY_CLASS,
+}
diff --git a/llm/memory.go b/llm/memory.go
new file mode 100644
index 00000000..99db7629
--- /dev/null
+++ b/llm/memory.go
@@ -0,0 +1,373 @@
+package llm
+
+import (
+ "fmt"
+ "log/slog"
+ "strconv"
+ "strings"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/format"
+ "github.com/ollama/ollama/gpu"
+)
+
+// This algorithm looks for a complete fit to determine if we need to unload other models
+func PredictServerFit(allGpus gpu.GpuInfoList, ggml *GGML, adapters, projectors []string, opts api.Options) (bool, uint64) {
+ // Split up the GPUs by type and try them
+ var estimatedVRAM uint64
+ for _, gpus := range allGpus.ByLibrary() {
+ var layerCount int
+ estimate := EstimateGPULayers(gpus, ggml, projectors, opts)
+ layerCount, estimatedVRAM = estimate.Layers, estimate.VRAMSize
+ if opts.NumGPU < 0 {
+ if layerCount > 0 && layerCount >= int(ggml.KV().BlockCount()+1) {
+ return true, estimatedVRAM
+ }
+ } else {
+ if layerCount > 0 && layerCount >= opts.NumGPU {
+ return true, estimatedVRAM
+ }
+ }
+ }
+ return false, estimatedVRAM
+}
+
+type MemoryEstimate struct {
+ // How many layers we predict we can load
+ Layers int
+
+ // The size of the graph which occupies the main GPU
+ Graph uint64
+
+ // How much VRAM will be allocated given the number of layers we predict
+ VRAMSize uint64
+
+ // The total size of the model if loaded into VRAM. If all layers are loaded, VRAMSize == TotalSize
+ TotalSize uint64
+
+ // For multi-GPU scenarios, this provides the tensor split parameter
+ TensorSplit string
+
+ // For multi-GPU scenarios, this is the size in bytes per GPU
+ GPUSizes []uint64
+
+ // internal fields for logging purposes
+ inferenceLibrary string
+ layersRequested int
+ layersModel int
+ availableList []string
+ kv uint64
+ allocationsList []string
+ memoryWeights uint64
+ memoryLayerOutput uint64
+ graphFullOffload uint64
+ graphPartialOffload uint64
+}
+
+// Given a model and one or more GPU targets, predict how many layers and bytes we can load, and the total size
+// The GPUs provided must all be the same Library
+func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts api.Options) MemoryEstimate {
+ // Graph size for a partial offload, applies to all GPUs
+ var graphPartialOffload uint64
+
+ // Graph size when all layers are offloaded, applies to all GPUs
+ var graphFullOffload uint64
+
+ // Final graph offload once we know full or partial
+ var graphOffload uint64
+
+ // Projectors loaded into GPU0 only
+ var projectorSize uint64
+
+ // Conditional output size on GPU 0
+ var memoryLayerOutput uint64
+
+ // The sizes of a layer
+ var layerSize uint64
+
+ // The sum of all the layer sizes (just for logging)
+ var memoryWeights uint64
+
+ // True if all the layers are loaded
+ var fullyLoaded bool
+
+ // Overflow that didn't fit into the GPU
+ var overflow uint64
+
+ overhead := envconfig.GpuOverhead()
+ availableList := make([]string, len(gpus))
+ for i, gpu := range gpus {
+ availableList[i] = format.HumanBytes2(gpu.FreeMemory)
+ }
+ slog.Debug("evaluating", "library", gpus[0].Library, "gpu_count", len(gpus), "available", availableList)
+
+ for _, projector := range projectors {
+ projectorSize += projectorMemoryRequirements(projector)
+
+ // multimodal models require at least 2048 context
+ opts.NumCtx = max(opts.NumCtx, 2048)
+ }
+
+ layers := ggml.Tensors().Layers()
+ // add one layer worth of memory as a buffer
+ if blk0, ok := layers["blk.0"]; ok {
+ layerSize = blk0.size()
+ } else {
+ slog.Warn("model missing blk.0 layer size")
+ }
+
+ // fp16 k,v = sizeof(float16) * n_ctx * n_layer * (n_embd_head_k + n_embd_head_v) * n_head_kv
+ var kv uint64 = 2 * uint64(opts.NumCtx) * ggml.KV().BlockCount() * (ggml.KV().EmbeddingHeadCountK() + ggml.KV().EmbeddingHeadCountV()) * ggml.KV().HeadCountKV()
+
+ // KV is proportional to the number of layers
+ layerSize += kv / ggml.KV().BlockCount()
+
+ graphPartialOffload, graphFullOffload = ggml.GraphSize(uint64(opts.NumCtx), uint64(min(opts.NumCtx, opts.NumBatch)))
+ if graphPartialOffload == 0 {
+ graphPartialOffload = ggml.KV().GQA() * kv / 6
+ }
+ if graphFullOffload == 0 {
+ graphFullOffload = graphPartialOffload
+ }
+
+ // on metal there's no partial offload overhead
+ if gpus[0].Library == "metal" {
+ graphPartialOffload = graphFullOffload
+ } else if len(gpus) > 1 {
+ // multigpu should always use the partial graph size
+ graphFullOffload = graphPartialOffload
+ }
+
+ if layer, ok := layers["output_norm"]; ok {
+ memoryLayerOutput += layer.size()
+ }
+ if layer, ok := layers["output"]; ok {
+ memoryLayerOutput += layer.size()
+ } else if layer, ok := layers["token_embd"]; ok {
+ memoryLayerOutput += layer.size()
+ }
+
+ // Output layer handled at the end if we have space
+ gpuZeroOverhead := projectorSize
+
+ // Reduce set of GPUs to only those that have sufficient space to fit overhead and at least one layer
+ var layerCount int
+ layerCounts := make([]int, len(gpus))
+ gpuAllocations := make([]uint64, len(gpus))
+ type gs struct {
+ i int
+ g *gpu.GpuInfo
+ }
+ gpusWithSpace := []gs{}
+ for i := range gpus {
+ var gzo uint64
+ if len(gpusWithSpace) == 0 {
+ gzo = gpuZeroOverhead
+ }
+ // Only include GPUs that can fit the graph, gpu minimum, the layer buffer and at least more layer
+ if (gpus[i].FreeMemory - overhead) < gzo+max(graphPartialOffload, graphFullOffload)+gpus[i].MinimumMemory+2*layerSize {
+ slog.Debug("gpu has too little memory to allocate any layers",
+ "id", gpus[i].ID,
+ "library", gpus[i].Library,
+ "variant", gpus[i].Variant,
+ "compute", gpus[i].Compute,
+ "driver", fmt.Sprintf("%d.%d", gpus[i].DriverMajor, gpus[i].DriverMinor),
+ "name", gpus[i].Name,
+ "total", format.HumanBytes2(gpus[i].TotalMemory),
+ "available", format.HumanBytes2(gpus[i].FreeMemory),
+ "minimum_memory", gpus[i].MinimumMemory,
+ "layer_size", format.HumanBytes2(layerSize),
+ "gpu_zer_overhead", format.HumanBytes2(gzo),
+ "partial_offload", format.HumanBytes2(graphPartialOffload),
+ "full_offload", format.HumanBytes2(graphFullOffload),
+ )
+ continue
+ }
+ gpusWithSpace = append(gpusWithSpace, gs{i, &gpus[i]})
+ gpuAllocations[i] += gpus[i].MinimumMemory + layerSize // We hold off on graph until we know partial vs. full
+ }
+
+ var gpuZeroID int
+ if len(gpusWithSpace) > 0 {
+ gpuZeroID = gpusWithSpace[0].i
+ gpuAllocations[gpuZeroID] += gpuZeroOverhead
+ }
+
+ // For all the layers, find where they can fit on the GPU(s)
+ for i := range int(ggml.KV().BlockCount()) {
+ // Some models have inconsistent layer sizes
+ if blk, ok := layers[fmt.Sprintf("blk.%d", i)]; ok {
+ layerSize = blk.size()
+ layerSize += kv / ggml.KV().BlockCount()
+ }
+ memoryWeights += layerSize
+
+ if opts.NumGPU >= 0 && layerCount >= opts.NumGPU {
+ // Stop allocating on GPU(s) once we hit the users target NumGPU
+ continue
+ }
+
+ // distribute the layers across the GPU(s) that have space
+ for j := len(gpusWithSpace); j > 0; j-- {
+ g := gpusWithSpace[i%j]
+ used := gpuAllocations[g.i] + max(graphPartialOffload, graphFullOffload)
+ if (g.g.FreeMemory - overhead) > used+layerSize {
+ gpuAllocations[g.i] += layerSize
+ layerCounts[g.i]++
+ layerCount++
+ break
+ } else {
+ gpusWithSpace = append(gpusWithSpace[:i%j], gpusWithSpace[i%j+1:]...)
+ }
+ }
+ }
+ if layerCount >= int(ggml.KV().BlockCount()) {
+ fullyLoaded = true
+ } else {
+ for i := layerCount; i < int(ggml.KV().BlockCount()); i++ {
+ overflow += layerSize
+ }
+ }
+
+ // Determine if we need to consider output then find where it fits
+ if memoryLayerOutput > 0 && (opts.NumGPU < 0 || layerCount < opts.NumGPU) {
+ for j := len(gpusWithSpace); j > 0; j-- {
+ g := gpusWithSpace[layerCount%j]
+ used := gpuAllocations[g.i] + max(graphPartialOffload, graphFullOffload)
+ if (g.g.FreeMemory - overhead) > used+memoryLayerOutput {
+ gpuAllocations[g.i] += memoryLayerOutput
+ layerCounts[g.i]++
+ layerCount++
+ break
+ }
+ }
+
+ if layerCount < int(ggml.KV().BlockCount())+1 {
+ fullyLoaded = false
+ overflow += memoryLayerOutput
+ }
+ }
+
+ // Add the applicable (full or partial) graph allocations
+ for i := range gpus {
+ if layerCounts[i] <= 0 {
+ continue
+ }
+ if fullyLoaded {
+ gpuAllocations[i] += graphFullOffload
+ } else {
+ gpuAllocations[i] += graphPartialOffload
+ }
+ }
+ if fullyLoaded {
+ graphOffload = graphFullOffload
+ } else {
+ graphOffload = graphPartialOffload
+ }
+
+ // Summaries for the log
+ var memoryRequiredPartial, memoryRequiredTotal uint64
+ for i := range gpuAllocations {
+ memoryRequiredPartial += gpuAllocations[i]
+ }
+ memoryRequiredTotal = memoryRequiredPartial + overflow
+
+ tensorSplit := ""
+ if len(gpus) > 1 {
+ splits := make([]string, len(gpus))
+ for i, count := range layerCounts {
+ splits[i] = strconv.Itoa(count)
+ }
+ tensorSplit = strings.Join(splits, ",")
+ }
+ allocationsList := []string{}
+ for _, a := range gpuAllocations {
+ allocationsList = append(allocationsList, format.HumanBytes2(a))
+ }
+
+ estimate := MemoryEstimate{
+ TotalSize: memoryRequiredTotal,
+ Layers: 0,
+ Graph: 0,
+ VRAMSize: 0,
+ GPUSizes: []uint64{},
+
+ inferenceLibrary: gpus[0].Library,
+ layersRequested: opts.NumGPU,
+ layersModel: int(ggml.KV().BlockCount()) + 1,
+ availableList: availableList,
+ kv: kv,
+ allocationsList: allocationsList,
+ memoryWeights: memoryWeights,
+ memoryLayerOutput: memoryLayerOutput,
+ graphFullOffload: graphFullOffload,
+ graphPartialOffload: graphPartialOffload,
+ }
+
+ if gpus[0].Library == "cpu" {
+ return estimate
+ }
+ if layerCount == 0 {
+ slog.Debug("insufficient VRAM to load any model layers")
+ return estimate
+ }
+ estimate.Layers = layerCount
+ estimate.Graph = graphOffload
+ estimate.VRAMSize = memoryRequiredPartial
+ estimate.TotalSize = memoryRequiredTotal
+ estimate.TensorSplit = tensorSplit
+ estimate.GPUSizes = gpuAllocations
+ return estimate
+}
+
+func (m MemoryEstimate) log() {
+ overhead := envconfig.GpuOverhead()
+ slog.Info(
+ "offload to "+m.inferenceLibrary,
+ slog.Group(
+ "layers",
+ // requested number of layers to offload
+ "requested", m.layersRequested,
+ // The number of layers the model has (including output)
+ "model", m.layersModel,
+ // estimated number of layers that can be offloaded
+ "offload", m.Layers,
+ // multi-gpu split for tensors
+ "split", m.TensorSplit,
+ ),
+ slog.Group(
+ "memory",
+ // memory available by GPU for offloading
+ "available", m.availableList,
+ "gpu_overhead", format.HumanBytes2(overhead),
+ slog.Group(
+ "required",
+ // memory required for full offloading
+ "full", format.HumanBytes2(m.TotalSize),
+ // memory required to offload layers.estimate layers
+ "partial", format.HumanBytes2(m.VRAMSize),
+ // memory of KV cache
+ "kv", format.HumanBytes2(m.kv),
+ // Allocations across the GPUs
+ "allocations", m.allocationsList,
+ ),
+ slog.Group(
+ "weights",
+ // memory of the weights
+ "total", format.HumanBytes2(m.memoryWeights),
+ // memory of repeating layers
+ "repeating", format.HumanBytes2(m.memoryWeights-m.memoryLayerOutput),
+ // memory of non-repeating layers
+ "nonrepeating", format.HumanBytes2(m.memoryLayerOutput),
+ ),
+ slog.Group(
+ "graph",
+ // memory of graph when fully offloaded
+ "full", format.HumanBytes2(m.graphFullOffload),
+ // memory of graph when not fully offloaded
+ "partial", format.HumanBytes2(m.graphPartialOffload),
+ ),
+ ),
+ )
+}
diff --git a/llm/memory_test.go b/llm/memory_test.go
new file mode 100644
index 00000000..ffb14286
--- /dev/null
+++ b/llm/memory_test.go
@@ -0,0 +1,128 @@
+package llm
+
+import (
+ "bytes"
+ "fmt"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/gpu"
+)
+
+func TestEstimateGPULayers(t *testing.T) {
+ t.Setenv("OLLAMA_DEBUG", "1")
+
+ modelName := "dummy"
+ f, err := os.CreateTemp(t.TempDir(), modelName)
+ require.NoError(t, err)
+ defer f.Close()
+ inputLayerCount := 5
+
+ tensors := []Tensor{
+ {Name: "blk.0.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
+ {Name: "blk.1.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
+ {Name: "blk.2.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
+ {Name: "blk.3.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
+ {Name: "blk.4.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
+ {Name: "output.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
+ }
+ assert.Len(t, tensors, inputLayerCount+1)
+ err = WriteGGUF(f, KV{
+ "general.architecture": "llama",
+ "llama.context_length": uint32(32),
+ "llama.embedding_length": uint32(4096),
+ "llama.block_count": uint32(inputLayerCount),
+ "llama.attention.head_count": uint32(32),
+ "llama.attention.head_count_kv": uint32(32),
+ "tokenizer.ggml.tokens": []string{" "},
+ "tokenizer.ggml.scores": []float32{0},
+ "tokenizer.ggml.token_type": []int32{0},
+ }, tensors)
+ require.NoError(t, err)
+
+ ggml, err := LoadModel(f.Name(), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Simple CPU scenario
+ gpus := []gpu.GpuInfo{
+ {
+ Library: "cpu",
+ },
+ }
+ projectors := []string{}
+ opts := api.DefaultOptions()
+ t.Run("cpu", func(t *testing.T) {
+ estimate := EstimateGPULayers(gpus, ggml, projectors, opts)
+ assert.Equal(t, 0, estimate.Layers)
+ assert.Equal(t, uint64(0), estimate.Graph)
+ })
+
+ // derived from the dummy ggml file above
+ graphPartialOffload := uint64(202377216)
+ graphFullOffload := uint64(171968512)
+ layerSize := uint64(33554436)
+ projectorSize := uint64(0)
+ memoryLayerOutput := uint64(4)
+
+ // Dual CUDA scenario with assymetry
+ gpuMinimumMemory := uint64(2048)
+ gpus = []gpu.GpuInfo{
+ {
+ Library: "cuda",
+ MinimumMemory: gpuMinimumMemory,
+ },
+ {
+ Library: "cuda",
+ MinimumMemory: gpuMinimumMemory,
+ },
+ }
+ // Nested array: GPU0 layer space, GPU1 layer space, expected gpu0, expected gpu1
+ for i, s := range []struct {
+ layer0, layer1 uint64
+ expect0, expect1 uint64
+ }{
+ {1, 1, 1, 1},
+ {2, 1, 2, 1},
+ {2, 2, 2, 2},
+ {1, 2, 1, 2},
+ {3, 3, 3, 3},
+ {4, 4, 3, 3},
+ {6, 6, 3, 3},
+ {0, 3, 0, 3},
+ } {
+ t.Run(fmt.Sprintf("%v", s), func(t *testing.T) {
+ gpus[0].FreeMemory = 0
+ gpus[1].FreeMemory = 0
+ gpus[0].FreeMemory += projectorSize
+ if s.layer0 > 0 {
+ gpus[0].FreeMemory += memoryLayerOutput
+ } else {
+ gpus[1].FreeMemory += memoryLayerOutput
+ }
+ gpus[0].FreeMemory += gpuMinimumMemory + layerSize + s.layer0*layerSize + 1
+ gpus[1].FreeMemory += gpuMinimumMemory + layerSize + s.layer1*layerSize + 1
+ gpus[0].FreeMemory += max(graphFullOffload, graphPartialOffload)
+ gpus[1].FreeMemory += max(graphFullOffload, graphPartialOffload)
+ estimate := EstimateGPULayers(gpus, ggml, projectors, opts)
+ assert.Equal(t, int(s.expect0+s.expect1), estimate.Layers, "scenario %d: %v", i, s)
+ assert.Equal(t, fmt.Sprintf("%d,%d", s.expect0, s.expect1), estimate.TensorSplit, "scenario %d: %v", i, s)
+ var layerSums uint64
+ for _, b := range estimate.GPUSizes {
+ layerSums += b
+ }
+ if estimate.Layers < inputLayerCount+1 {
+ assert.Less(t, estimate.VRAMSize, estimate.TotalSize, "scenario %d: %v %+v", i, s, estimate)
+ assert.Equal(t, estimate.VRAMSize, layerSums, "scenario %d: %v %+v", i, s, estimate)
+ } else {
+ assert.Equal(t, estimate.VRAMSize, estimate.TotalSize, "scenario %d: %v %+v", i, s, estimate)
+ assert.Equal(t, estimate.TotalSize, layerSums, "scenario %d: %v %+v", i, s, estimate)
+ }
+ })
+ }
+}
diff --git a/llm/patches/0000-cmakelist.patch b/llm/patches/0000-cmakelist.patch
new file mode 100644
index 00000000..54e9b602
--- /dev/null
+++ b/llm/patches/0000-cmakelist.patch
@@ -0,0 +1,22 @@
+From 8b8d83ffca775840acc5dc700f3b3703e9f5cfe4 Mon Sep 17 00:00:00 2001
+From: Michael Yang <mxyng@pm.me>
+Date: Fri, 23 Aug 2024 11:27:48 -0700
+Subject: [PATCH] patch cmakelist
+
+---
+ CMakeLists.txt | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index a3132063..6a2a9912 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -199,3 +199,5 @@ if (LLAMA_BUILD_EXAMPLES)
+ add_subdirectory(examples)
+ add_subdirectory(pocs)
+ endif()
++
++add_subdirectory(../ext_server ext_server) # ollama
+--
+2.45.2
+
diff --git a/llm/patches/0001-load-progress.patch b/llm/patches/0001-load-progress.patch
new file mode 100644
index 00000000..0ddabc80
--- /dev/null
+++ b/llm/patches/0001-load-progress.patch
@@ -0,0 +1,44 @@
+From 2cfaa0a04faa9c87ba8f1ac8527eb953e69c6cde Mon Sep 17 00:00:00 2001
+From: Michael Yang <mxyng@pm.me>
+Date: Mon, 16 Sep 2024 15:53:10 -0700
+Subject: [PATCH] 01-load-progress.diff
+
+---
+ common/common.cpp | 2 ++
+ common/common.h | 7 +++++++
+ 2 files changed, 9 insertions(+)
+
+diff --git a/common/common.cpp b/common/common.cpp
+index 9fa18472..48ff41e9 100644
+--- a/common/common.cpp
++++ b/common/common.cpp
+@@ -2573,6 +2573,8 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &
+ mparams.use_mmap = params.use_mmap;
+ mparams.use_mlock = params.use_mlock;
+ mparams.check_tensors = params.check_tensors;
++ mparams.progress_callback = params.progress_callback;
++ mparams.progress_callback_user_data = params.progress_callback_user_data;
+ if (params.kv_overrides.empty()) {
+ mparams.kv_overrides = NULL;
+ } else {
+diff --git a/common/common.h b/common/common.h
+index cb5e7f6d..d8f043f7 100644
+--- a/common/common.h
++++ b/common/common.h
+@@ -204,6 +204,13 @@ struct gpt_params {
+ std::string mmproj = ""; // path to multimodal projector
+ std::vector<std::string> image; // path to image file(s)
+
++ // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.
++ // If the provided progress_callback returns true, model loading continues.
++ // If it returns false, model loading is immediately aborted.
++ llama_progress_callback progress_callback = NULL;
++ // context pointer passed to the progress callback
++ void * progress_callback_user_data;
++
+ // embedding
+ bool embedding = false; // get only sentence embedding
+ int32_t embd_normalize = 2; // normalisation for embendings (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)
+--
+2.46.0
+
diff --git a/llm/patches/0002-clip-log.patch b/llm/patches/0002-clip-log.patch
new file mode 100644
index 00000000..8df0da17
--- /dev/null
+++ b/llm/patches/0002-clip-log.patch
@@ -0,0 +1,24 @@
+From ba4bba80a744f76ac67b8234451c259a3c5da83b Mon Sep 17 00:00:00 2001
+From: Michael Yang <mxyng@pm.me>
+Date: Mon, 16 Sep 2024 15:53:11 -0700
+Subject: [PATCH] 02-clip-log.diff
+
+---
+ examples/llava/clip.cpp | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp
+index 9b890571..cb51793d 100644
+--- a/examples/llava/clip.cpp
++++ b/examples/llava/clip.cpp
+@@ -3,6 +3,7 @@
+ // I'll gradually clean and extend it
+ // Note: Even when using identical normalized image inputs (see normalize_image_u8_to_f32()) we have a significant difference in resulting embeddings compared to pytorch
+ #include "clip.h"
++#include "common.h"
+ #include "log.h"
+ #include "ggml.h"
+ #include "ggml-alloc.h"
+--
+2.46.0
+
diff --git a/llm/patches/0003-load_exception.patch b/llm/patches/0003-load_exception.patch
new file mode 100644
index 00000000..3d858ebb
--- /dev/null
+++ b/llm/patches/0003-load_exception.patch
@@ -0,0 +1,57 @@
+From e43bfd3f607a6dfcaba2d490d35f412a52e55e30 Mon Sep 17 00:00:00 2001
+From: Michael Yang <mxyng@pm.me>
+Date: Mon, 16 Sep 2024 15:53:12 -0700
+Subject: [PATCH] 03-load_exception.diff
+
+---
+ src/llama.cpp | 25 ++++++++++++++++---------
+ 1 file changed, 16 insertions(+), 9 deletions(-)
+
+diff --git a/src/llama.cpp b/src/llama.cpp
+index 88355971..926bb71a 100644
+--- a/src/llama.cpp
++++ b/src/llama.cpp
+@@ -8635,7 +8635,7 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam
+ }
+ } catch (const std::exception & err) {
+ LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what());
+- return -1;
++ throw;
+ }
+
+ return 0;
+@@ -18022,16 +18022,23 @@ struct llama_model * llama_load_model_from_file(
+ }
+ model->rpc_servers.push_back(servers);
+ }
+- int status = llama_model_load(path_model, *model, params);
+- GGML_ASSERT(status <= 0);
+- if (status < 0) {
+- if (status == -1) {
+- LLAMA_LOG_ERROR("%s: failed to load model\n", __func__);
+- } else if (status == -2) {
+- LLAMA_LOG_INFO("%s: cancelled model load\n", __func__);
++
++ try {
++ int status = llama_model_load(path_model, *model, params);
++ GGML_ASSERT(status <= 0);
++ if (status < 0) {
++ if (status == -1) {
++ LLAMA_LOG_ERROR("%s: failed to load model\n", __func__);
++ } else if (status == -2) {
++ LLAMA_LOG_INFO("%s: cancelled model load\n", __func__);
++ }
++ delete model;
++ return nullptr;
+ }
++ } catch (...) {
++ LLAMA_LOG_ERROR("%s: exception loading model\n", __func__);
+ delete model;
+- return nullptr;
++ throw;
+ }
+
+ return model;
+--
+2.46.0
+
diff --git a/llm/patches/0004-metal.patch b/llm/patches/0004-metal.patch
new file mode 100644
index 00000000..4cfa407e
--- /dev/null
+++ b/llm/patches/0004-metal.patch
@@ -0,0 +1,57 @@
+From 29411d9a9d2b6a0af6425ffe88498f17f71f7d5d Mon Sep 17 00:00:00 2001
+From: Michael Yang <mxyng@pm.me>
+Date: Mon, 16 Sep 2024 15:53:12 -0700
+Subject: [PATCH] 04-metal.diff
+
+---
+ ggml/src/ggml-metal.m | 30 +++++++++++++-----------------
+ 1 file changed, 13 insertions(+), 17 deletions(-)
+
+diff --git a/ggml/src/ggml-metal.m b/ggml/src/ggml-metal.m
+index 91b5e61b..9cfa72ac 100644
+--- a/ggml/src/ggml-metal.m
++++ b/ggml/src/ggml-metal.m
+@@ -1734,27 +1734,23 @@ static enum ggml_status ggml_metal_graph_compute(
+ // to the matrix-vector kernel
+ int ne11_mm_min = 1;
+
+-#if 0
+ // the numbers below are measured on M2 Ultra for 7B and 13B models
+ // these numbers do not translate to other devices or model sizes
+ // TODO: need to find a better approach
+- if ([ctx->device.name isEqualToString:@"Apple M2 Ultra"]) {
+- switch (src0t) {
+- case GGML_TYPE_F16: ne11_mm_min = 2; break;
+- case GGML_TYPE_Q8_0: ne11_mm_min = 7; break;
+- case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;
+- case GGML_TYPE_Q3_K: ne11_mm_min = 7; break;
+- case GGML_TYPE_Q4_0:
+- case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;
+- case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;
+- case GGML_TYPE_Q5_0: // not tested yet
+- case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet
+- case GGML_TYPE_Q5_K: ne11_mm_min = 7; break;
+- case GGML_TYPE_Q6_K: ne11_mm_min = 7; break;
+- default: ne11_mm_min = 1; break;
+- }
++ switch (src0t) {
++ case GGML_TYPE_F16: ne11_mm_min = 2; break;
++ case GGML_TYPE_Q8_0: ne11_mm_min = 7; break;
++ case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;
++ case GGML_TYPE_Q3_K: ne11_mm_min = 7; break;
++ case GGML_TYPE_Q4_0:
++ case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;
++ case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;
++ case GGML_TYPE_Q5_0: // not tested yet
++ case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet
++ case GGML_TYPE_Q5_K: ne11_mm_min = 7; break;
++ case GGML_TYPE_Q6_K: ne11_mm_min = 7; break;
++ default: ne11_mm_min = 1; break;
+ }
+-#endif
+
+ // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
+ // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
+--
+2.46.0
+
diff --git a/llm/patches/0005-default-pretokenizer.patch b/llm/patches/0005-default-pretokenizer.patch
new file mode 100644
index 00000000..6ad0ee97
--- /dev/null
+++ b/llm/patches/0005-default-pretokenizer.patch
@@ -0,0 +1,44 @@
+From b298ac8614d1e38da28f760eb1d2ae8af0fbbe62 Mon Sep 17 00:00:00 2001
+From: Michael Yang <mxyng@pm.me>
+Date: Mon, 16 Sep 2024 15:53:13 -0700
+Subject: [PATCH] 05-default-pretokenizer.diff
+
+---
+ src/llama.cpp | 14 +++-----------
+ 1 file changed, 3 insertions(+), 11 deletions(-)
+
+diff --git a/src/llama.cpp b/src/llama.cpp
+index 926bb71a..d1e959fc 100644
+--- a/src/llama.cpp
++++ b/src/llama.cpp
+@@ -6083,16 +6083,7 @@ static void llm_load_vocab(
+ if (vocab.type == LLAMA_VOCAB_TYPE_BPE) {
+ vocab.tokenizer_add_space_prefix = false;
+ vocab.tokenizer_clean_spaces = true;
+- if (tokenizer_pre.empty()) {
+- LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__);
+- LLAMA_LOG_WARN("%s: \n", __func__);
+- LLAMA_LOG_WARN("%s: ************************************ \n", __func__);
+- LLAMA_LOG_WARN("%s: GENERATION QUALITY WILL BE DEGRADED! \n", __func__);
+- LLAMA_LOG_WARN("%s: CONSIDER REGENERATING THE MODEL \n", __func__);
+- LLAMA_LOG_WARN("%s: ************************************ \n", __func__);
+- LLAMA_LOG_WARN("%s: \n", __func__);
+- vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+- } else if (tokenizer_pre == "default") {
++ if (tokenizer_pre == "default") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ } else if (
+ tokenizer_pre == "llama3" ||
+@@ -6188,7 +6179,8 @@ static void llm_load_vocab(
+ tokenizer_pre == "exaone") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_EXAONE;
+ } else {
+- throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
++ LLAMA_LOG_WARN("%s: missing or unrecognized pre-tokenizer type, using: 'default'\n", __func__);
++ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ }
+ } else if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+--
+2.46.0
+
diff --git a/llm/patches/0006-embeddings.patch b/llm/patches/0006-embeddings.patch
new file mode 100644
index 00000000..8f89ffeb
--- /dev/null
+++ b/llm/patches/0006-embeddings.patch
@@ -0,0 +1,55 @@
+From c9a6ca9fc039233dee746a4da9705762cd9e515d Mon Sep 17 00:00:00 2001
+From: Michael Yang <mxyng@pm.me>
+Date: Mon, 16 Sep 2024 15:53:14 -0700
+Subject: [PATCH] 06-embeddings.diff
+
+---
+ src/llama.cpp | 17 ++++++++++-------
+ 1 file changed, 10 insertions(+), 7 deletions(-)
+
+diff --git a/src/llama.cpp b/src/llama.cpp
+index d1e959fc..f79bd782 100644
+--- a/src/llama.cpp
++++ b/src/llama.cpp
+@@ -15898,7 +15898,7 @@ static size_t llama_output_reserve(llama_context & lctx, size_t n_outputs) {
+ const auto n_embd = hparams.n_embd;
+
+ // TODO: use a per-batch flag for logits presence instead
+- const bool has_logits = !cparams.embeddings;
++ const bool has_logits = cparams.causal_attn;
+ const bool has_embd = cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE);
+
+ const size_t logits_size = has_logits ? n_vocab*n_outputs_max : 0;
+@@ -16167,20 +16167,23 @@ static int llama_decode_internal(
+ // no output
+ res = nullptr;
+ embd = nullptr;
+- } else if (cparams.embeddings) {
+- res = nullptr; // do not extract logits for embedding case
+- embd = nullptr;
++ }
++
++ if (cparams.embeddings) {
+ for (int i = gf->n_nodes - 1; i >= 0; --i) {
+- if (strcmp(gf->nodes[i]->name, "result_embd_pooled") == 0) {
+- embd = gf->nodes[i];
++ embd = gf->nodes[i];
++ if (strcmp(embd->name, "result_embd_pooled") == 0) {
+ break;
+ }
+ }
+- GGML_ASSERT(embd != nullptr && "missing embeddings tensor");
+ } else {
+ embd = nullptr; // do not extract embeddings when not needed
+ GGML_ASSERT(strcmp(res->name, "result_output") == 0 && "missing result_output tensor");
+ }
++
++ if (!cparams.causal_attn) {
++ res = nullptr; // do not extract logits when not needed
++ }
+ // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);
+
+ ggml_backend_sched_alloc_graph(lctx.sched, gf);
+--
+2.46.0
+
diff --git a/llm/patches/0007-clip-unicode.patch b/llm/patches/0007-clip-unicode.patch
new file mode 100644
index 00000000..72c061cb
--- /dev/null
+++ b/llm/patches/0007-clip-unicode.patch
@@ -0,0 +1,54 @@
+From ae2b188a679c83ce105aa1e823499441dfab3c57 Mon Sep 17 00:00:00 2001
+From: Michael Yang <mxyng@pm.me>
+Date: Mon, 16 Sep 2024 15:53:15 -0700
+Subject: [PATCH] 07-clip-unicode.diff
+
+---
+ examples/llava/clip.cpp | 23 +++++++++++++++++++++++
+ 1 file changed, 23 insertions(+)
+
+diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp
+index cb51793d..8716472b 100644
+--- a/examples/llava/clip.cpp
++++ b/examples/llava/clip.cpp
+@@ -41,6 +41,14 @@
+ #include <cinttypes>
+ #include <limits>
+
++#if defined(_WIN32)
++#define WIN32_LEAN_AND_MEAN
++#ifndef NOMINMAX
++ #define NOMINMAX
++#endif
++#include <windows.h>
++#endif
++
+ //#define CLIP_DEBUG_FUNCTIONS
+
+ // RGB uint8 image
+@@ -1223,7 +1231,22 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
+ return nullptr;
+ }
+
++#ifdef _WIN32
++ int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
++ if (!wlen) {
++ return NULL;
++ }
++ wchar_t * wbuf = (wchar_t *) malloc(wlen * sizeof(wchar_t));
++ wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, wbuf, wlen);
++ if (!wlen) {
++ free(wbuf);
++ return NULL;
++ }
++ auto fin = std::ifstream(wbuf, std::ios::binary);
++ free(wbuf);
++#else
+ auto fin = std::ifstream(fname, std::ios::binary);
++#endif
+ if (!fin) {
+ LOG_TEE("cannot open model file for loading tensors\n");
+ clip_free(new_clip);
+--
+2.46.0
+
diff --git a/llm/patches/0008-solar-pro.patch b/llm/patches/0008-solar-pro.patch
new file mode 100644
index 00000000..54f18457
--- /dev/null
+++ b/llm/patches/0008-solar-pro.patch
@@ -0,0 +1,402 @@
+From 8313ce5f43f11f3d84f352f97f3802792e90e18c Mon Sep 17 00:00:00 2001
+From: Michael Yang <mxyng@pm.me>
+Date: Mon, 16 Sep 2024 15:53:16 -0700
+Subject: [PATCH] add solar-pro support
+
+solar-pro introduces block skip connections where blocks are connected
+to other, non-sequential blocks with a scale multiple
+
+this change adds 4 new keys to store the skip connections and one new
+tensor to store the scalar. the scalar is implemented a 1-dimensional
+tensor with 2 elements dervied from the model's bskcn_tv configuration.
+in general, the values are (bskcn_tv, 1 - bskcn_tv)
+---
+ src/llama.cpp | 267 +++++++++++++++++++++++++++++++++++++++++++++++---
+ 1 file changed, 254 insertions(+), 13 deletions(-)
+
+diff --git a/src/llama.cpp b/src/llama.cpp
+index f79bd782..b7771f53 100644
+--- a/src/llama.cpp
++++ b/src/llama.cpp
+@@ -213,6 +213,7 @@ enum llm_arch {
+ LLM_ARCH_NEMOTRON,
+ LLM_ARCH_EXAONE,
+ LLM_ARCH_RWKV6,
++ LLM_ARCH_SOLAR,
+ LLM_ARCH_UNKNOWN,
+ };
+
+@@ -261,6 +262,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
+ { LLM_ARCH_NEMOTRON, "nemotron" },
+ { LLM_ARCH_EXAONE, "exaone" },
+ { LLM_ARCH_RWKV6, "rwkv6" },
++ { LLM_ARCH_SOLAR, "solar" },
+ { LLM_ARCH_UNKNOWN, "(unknown)" },
+ };
+
+@@ -314,6 +316,7 @@ enum llm_kv {
+ LLM_KV_ATTENTION_KV_LORA_RANK,
+ LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT,
+ LLM_KV_ATTENTION_SLIDING_WINDOW,
++ LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION,
+
+ LLM_KV_ROPE_DIMENSION_COUNT,
+ LLM_KV_ROPE_FREQ_BASE,
+@@ -405,19 +408,20 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
+ { LLM_KV_TIME_MIX_EXTRA_DIM, "%s.time_mix_extra_dim" },
+ { LLM_KV_TIME_DECAY_EXTRA_DIM, "%s.time_decay_extra_dim" },
+
+- { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },
+- { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },
+- { LLM_KV_ATTENTION_MAX_ALIBI_BIAS, "%s.attention.max_alibi_bias" },
+- { LLM_KV_ATTENTION_CLAMP_KQV, "%s.attention.clamp_kqv" },
+- { LLM_KV_ATTENTION_KEY_LENGTH, "%s.attention.key_length" },
+- { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" },
+- { LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" },
+- { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" },
+- { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" },
+- { LLM_KV_ATTENTION_Q_LORA_RANK, "%s.attention.q_lora_rank" },
+- { LLM_KV_ATTENTION_KV_LORA_RANK, "%s.attention.kv_lora_rank" },
+- { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" },
+- { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" },
++ { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },
++ { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },
++ { LLM_KV_ATTENTION_MAX_ALIBI_BIAS, "%s.attention.max_alibi_bias" },
++ { LLM_KV_ATTENTION_CLAMP_KQV, "%s.attention.clamp_kqv" },
++ { LLM_KV_ATTENTION_KEY_LENGTH, "%s.attention.key_length" },
++ { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" },
++ { LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" },
++ { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" },
++ { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" },
++ { LLM_KV_ATTENTION_Q_LORA_RANK, "%s.attention.q_lora_rank" },
++ { LLM_KV_ATTENTION_KV_LORA_RANK, "%s.attention.kv_lora_rank" },
++ { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" },
++ { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" },
++ { LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION, "%s.attention.block_skip_connection.%d" },
+
+ { LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" },
+ { LLM_KV_ROPE_FREQ_BASE, "%s.rope.freq_base" },
+@@ -589,6 +593,7 @@ enum llm_tensor {
+ LLM_TENSOR_ENC_FFN_DOWN,
+ LLM_TENSOR_ENC_FFN_UP,
+ LLM_TENSOR_ENC_OUTPUT_NORM,
++ LLM_TENSOR_BSKCN_TV,
+ };
+
+ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES = {
+@@ -1408,6 +1413,24 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
+ { LLM_TENSOR_CHANNEL_MIX_RECEPTANCE, "blk.%d.channel_mix_receptance" },
+ },
+ },
++ {
++ LLM_ARCH_SOLAR,
++ {
++ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
++ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
++ { LLM_TENSOR_OUTPUT, "output" },
++ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
++ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
++ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
++ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
++ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
++ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
++ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
++ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
++ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
++ { LLM_TENSOR_BSKCN_TV, "bskcn_tv" },
++ },
++ },
+ {
+ LLM_ARCH_UNKNOWN,
+ {
+@@ -2237,6 +2260,7 @@ enum e_model {
+ MODEL_15B,
+ MODEL_16B,
+ MODEL_20B,
++ MODEL_22B,
+ MODEL_30B,
+ MODEL_34B,
+ MODEL_35B,
+@@ -2284,6 +2308,8 @@ struct llama_hparams {
+ std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_kv_arr;
+ std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_arr;
+
++ std::array<std::array<uint32_t, LLAMA_MAX_LAYERS>, 4> n_bskcn_arr;
++
+ uint32_t n_layer_dense_lead = 0;
+ uint32_t n_lora_q = 0;
+ uint32_t n_lora_kv = 0;
+@@ -2349,6 +2375,7 @@ struct llama_hparams {
+ if (this->n_head_arr != other.n_head_arr) return true;
+ if (this->n_head_kv_arr != other.n_head_kv_arr) return true;
+ if (this->n_ff_arr != other.n_ff_arr) return true;
++ if (this->n_bskcn_arr != other.n_bskcn_arr) return true;
+
+ if (this->n_rel_attn_bkts != other.n_rel_attn_bkts) return true;
+ if (this->n_layer_dense_lead != other.n_layer_dense_lead) return true;
+@@ -2455,6 +2482,14 @@ struct llama_hparams {
+ return ssm_d_state * ssm_d_inner;
+ }
+ }
++
++ bool n_bskcn(uint32_t n, uint32_t il = 0) const {
++ if (il < n_layer) {
++ return n_bskcn_arr[n][il] > 0;
++ }
++
++ GGML_ABORT("fatal error");
++ }
+ };
+
+ static_assert(std::is_trivially_copyable<llama_hparams>::value, "llama_hparams must be trivially copyable");
+@@ -2635,6 +2670,8 @@ struct llama_layer {
+ struct ggml_tensor * ffn_gate_scale;
+ struct ggml_tensor * ffn_up_scale;
+ struct ggml_tensor * ffn_down_scale;
++
++ struct ggml_tensor * bskcn_tv;
+ };
+
+ // very similar to llama_batch,
+@@ -5937,6 +5974,21 @@ static void llm_load_hparams(
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
++ case LLM_ARCH_SOLAR:
++ {
++ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
++
++ for (int i = 0; i < hparams.n_bskcn_arr.max_size(); ++i) {
++ auto & bskcn = hparams.n_bskcn_arr.at(i);
++ bskcn.fill(0);
++ ml.get_key_or_arr(::format(LLM_KV_NAMES.at(LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION), LLM_ARCH_NAMES.at(ml.llm_kv.arch), i), bskcn, hparams.n_layer, false);
++ }
++
++ switch (hparams.n_layer) {
++ case 64: model.type = e_model::MODEL_22B; break;
++ default: model.type = e_model::MODEL_UNKNOWN;
++ }
++ }
+ default: (void)0;
+ }
+
+@@ -8420,6 +8472,38 @@ static bool llm_load_tensors(
+ }
+
+ } break;
++ case LLM_ARCH_SOLAR:
++ {
++ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
++
++ // output
++ {
++ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
++ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
++ }
++
++ for (int i = 0; i < n_layer; ++i) {
++ ggml_context * ctx_layer = ctx_for_layer(i);
++ ggml_context * ctx_split = ctx_for_layer_split(i);
++
++ auto & layer = model.layers[i];
++
++ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
++
++ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head});
++ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
++ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
++ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});
++
++ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
++
++ layer.bskcn_tv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_BSKCN_TV, "weight"), {2}, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));
++
++ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
++ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
++ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
++ }
++ } break;
+ default:
+ throw std::runtime_error("unknown architecture");
+ }
+@@ -15173,6 +15257,158 @@ struct llm_build_context {
+
+ return gf;
+ }
++
++ ggml_cgraph * build_solar() {
++ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
++
++ // mutable variable, needed during the last layer of the computation to skip unused tokens
++ int32_t n_tokens = this->n_tokens;
++
++ const int64_t n_embd_head = hparams.n_embd_head_v;
++ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
++ GGML_ASSERT(n_embd_head == hparams.n_rot);
++
++ struct ggml_tensor * cur;
++ struct ggml_tensor * inpL;
++
++ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
++
++ // inp_pos - contains the positions
++ struct ggml_tensor * inp_pos = build_inp_pos();
++
++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
++ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
++
++ struct ggml_tensor * bskcn_1;
++ struct ggml_tensor * bskcn_2;
++
++ for (int il = 0; il < n_layer; ++il) {
++ struct ggml_tensor * inpSA = inpL;
++
++ if (hparams.n_bskcn(0, il)) {
++ bskcn_1 = inpSA;
++ }
++
++ if (hparams.n_bskcn(1, il)) {
++ bskcn_2 = inpSA;
++ }
++
++ if (hparams.n_bskcn(2, il)) {
++ inpSA = ggml_add(
++ ctx0,
++ ggml_mul(ctx0, bskcn_1, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, 0)),
++ ggml_mul(ctx0, inpSA, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, ggml_element_size(model.layers[il].bskcn_tv))));
++ }
++
++ if (hparams.n_bskcn(3, il)) {
++ inpSA = ggml_add(
++ ctx0,
++ ggml_mul(ctx0, bskcn_2, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, 0)),
++ ggml_mul(ctx0, inpSA, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, ggml_element_size(model.layers[il].bskcn_tv))));
++ }
++
++ // norm
++ cur = llm_build_norm(ctx0, inpL, hparams,
++ model.layers[il].attn_norm, NULL,
++ LLM_NORM_RMS, cb, il);
++ cb(cur, "attn_norm", il);
++
++ // self-attention
++ {
++ // rope freq factors for llama3; may return nullptr for llama2 and other models
++ struct ggml_tensor * rope_factors = build_rope_factors(il);
++
++ // compute Q and K and RoPE them
++ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
++ cb(Qcur, "Qcur", il);
++ if (model.layers[il].bq) {
++ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
++ cb(Qcur, "Qcur", il);
++ }
++
++ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
++ cb(Kcur, "Kcur", il);
++ if (model.layers[il].bk) {
++ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
++ cb(Kcur, "Kcur", il);
++ }
++
++ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
++ cb(Vcur, "Vcur", il);
++ if (model.layers[il].bv) {
++ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
++ cb(Vcur, "Vcur", il);
++ }
++
++ Qcur = ggml_rope_ext(
++ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors,
++ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
++ ext_factor, attn_factor, beta_fast, beta_slow
++ );
++ cb(Qcur, "Qcur", il);
++
++ Kcur = ggml_rope_ext(
++ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors,
++ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
++ ext_factor, attn_factor, beta_fast, beta_slow
++ );
++ cb(Kcur, "Kcur", il);
++
++ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
++ model.layers[il].wo, model.layers[il].bo,
++ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
++ }
++
++ if (il == n_layer - 1) {
++ // skip computing output for unused tokens
++ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
++ n_tokens = n_outputs;
++ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
++ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
++ }
++
++ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
++ cb(ffn_inp, "ffn_inp", il);
++
++ // feed-forward network
++ cur = llm_build_norm(ctx0, ffn_inp, hparams,
++ model.layers[il].ffn_norm, NULL,
++ LLM_NORM_RMS, cb, il);
++ cb(cur, "ffn_norm", il);
++
++ cur = llm_build_ffn(ctx0, lctx, cur,
++ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
++ model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,
++ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
++ NULL,
++ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
++ cb(cur, "ffn_out", il);
++
++ cur = ggml_add(ctx0, cur, ffn_inp);
++ cb(cur, "ffn_out", il);
++
++ cur = lctx.cvec.apply_to(ctx0, cur, il);
++ cb(cur, "l_out", il);
++
++ // input for next layer
++ inpL = cur;
++ }
++
++ cur = inpL;
++
++ cur = llm_build_norm(ctx0, cur, hparams,
++ model.output_norm, NULL,
++ LLM_NORM_RMS, cb, -1);
++ cb(cur, "result_norm", -1);
++
++ // lm_head
++ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
++ cb(cur, "result_output", -1);
++
++ ggml_build_forward_expand(gf, cur);
++
++ return gf;
++ }
+ };
+
+ static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector<uint32_t> & ids) {
+@@ -15423,6 +15659,10 @@ static struct ggml_cgraph * llama_build_graph(
+ {
+ result = llm.build_rwkv6();
+ } break;
++ case LLM_ARCH_SOLAR:
++ {
++ result = llm.build_solar();
++ } break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+@@ -18503,6 +18743,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {
+ case LLM_ARCH_ARCTIC:
+ case LLM_ARCH_DEEPSEEK2:
+ case LLM_ARCH_CHATGLM:
++ case LLM_ARCH_SOLAR:
+ return LLAMA_ROPE_TYPE_NORM;
+
+ // the pairs of head values are offset by n_rot/2
+--
+2.46.0
+
diff --git a/llm/server.go b/llm/server.go
new file mode 100644
index 00000000..36bcfad2
--- /dev/null
+++ b/llm/server.go
@@ -0,0 +1,1128 @@
+package llm
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "log/slog"
+ "math/rand"
+ "net"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "golang.org/x/sync/semaphore"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/build"
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/format"
+ "github.com/ollama/ollama/gpu"
+ "github.com/ollama/ollama/llama"
+ "github.com/ollama/ollama/runners"
+)
+
+type LlamaServer interface {
+ Ping(ctx context.Context) error
+ WaitUntilRunning(ctx context.Context) error
+ Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error
+ Embedding(ctx context.Context, input string) ([]float32, error)
+ Tokenize(ctx context.Context, content string) ([]int, error)
+ Detokenize(ctx context.Context, tokens []int) (string, error)
+ Close() error
+ EstimatedVRAM() uint64 // Total VRAM across all GPUs
+ EstimatedTotal() uint64
+ EstimatedVRAMByGPU(gpuID string) uint64
+}
+
+// llmServer is an instance of the llama.cpp server
+type llmServer struct {
+ port int
+ cmd *exec.Cmd
+ done chan error // Channel to signal when the process exits
+ status *StatusWriter
+ options api.Options
+ numParallel int
+ modelPath string
+ modelLock sync.Mutex // Temporary until we switch fully to Go server
+ model *llama.Model // If non-nil, the runner is a new Go server
+
+ estimate MemoryEstimate
+ totalLayers uint64
+ // gpuCount int
+ gpus gpu.GpuInfoList // Recorded just before the model loaded, free space will be incorrect
+ loadDuration time.Duration // Record how long it took the model to load
+ loadProgress float32
+
+ sem *semaphore.Weighted
+}
+
+// LoadModel will load a model from disk. The model must be in the GGML format.
+//
+// It collects array values for arrays with a size less than or equal to
+// maxArraySize. If maxArraySize is 0, the default value of 1024 is used. If
+// the maxArraySize is negative, all arrays are collected.
+func LoadModel(model string, maxArraySize int) (*GGML, error) {
+ if _, err := os.Stat(model); err != nil {
+ return nil, err
+ }
+
+ f, err := os.Open(model)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ ggml, _, err := DecodeGGML(f, maxArraySize)
+ return ggml, err
+}
+
+// NewLlamaServer will run a server for the given GPUs
+// The gpu list must be a single family.
+func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, projectors []string, opts api.Options, numParallel int) (LlamaServer, error) {
+ var err error
+ var cpuRunner string
+ var estimate MemoryEstimate
+ var systemTotalMemory uint64
+ var systemFreeMemory uint64
+ var systemSwapFreeMemory uint64
+
+ systemInfo := gpu.GetSystemInfo()
+ systemTotalMemory = systemInfo.System.TotalMemory
+ systemFreeMemory = systemInfo.System.FreeMemory
+ systemSwapFreeMemory = systemInfo.System.FreeSwap
+ slog.Info("system memory", "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "free_swap", format.HumanBytes2(systemSwapFreeMemory))
+
+ // If the user wants zero GPU layers, reset the gpu list to be CPU/system ram info
+ if opts.NumGPU == 0 {
+ gpus = gpu.GetCPUInfo()
+ }
+ if len(gpus) == 1 && gpus[0].Library == "cpu" {
+ cpuRunner = runners.ServerForCpu()
+ estimate = EstimateGPULayers(gpus, ggml, projectors, opts)
+ } else {
+ estimate = EstimateGPULayers(gpus, ggml, projectors, opts)
+
+ switch {
+ case gpus[0].Library == "metal" && estimate.VRAMSize > systemTotalMemory:
+ // disable partial offloading when model is greater than total system memory as this
+ // can lead to locking up the system
+ opts.NumGPU = 0
+ case gpus[0].Library != "metal" && estimate.Layers == 0:
+ // Don't bother loading into the GPU if no layers can fit
+ cpuRunner = runners.ServerForCpu()
+ gpus = gpu.GetCPUInfo()
+ case opts.NumGPU < 0 && estimate.Layers > 0 && gpus[0].Library != "cpu":
+ opts.NumGPU = estimate.Layers
+ }
+ }
+
+ // On linux and windows, over-allocating CPU memory will almost always result in an error
+ // Darwin has fully dynamic swap so has no direct concept of free swap space
+ if runtime.GOOS != "darwin" {
+ systemMemoryRequired := estimate.TotalSize - estimate.VRAMSize
+ available := systemFreeMemory + systemSwapFreeMemory
+ if systemMemoryRequired > available {
+ slog.Warn("model request too large for system", "requested", format.HumanBytes2(systemMemoryRequired), "available", available, "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "swap", format.HumanBytes2(systemSwapFreeMemory))
+ return nil, fmt.Errorf("model requires more system memory (%s) than is available (%s)", format.HumanBytes2(systemMemoryRequired), format.HumanBytes2(available))
+ }
+ }
+
+ estimate.log()
+
+ // Loop through potential servers
+ finalErr := errors.New("no suitable llama servers found")
+
+ if len(adapters) > 1 {
+ return nil, errors.New("ollama supports only one lora adapter, but multiple were provided")
+ }
+
+ rDir, err := runners.Refresh(build.EmbedFS)
+ if err != nil {
+ return nil, err
+ }
+
+ availableServers := runners.GetAvailableServers(rDir)
+ if len(availableServers) == 0 {
+ return nil, finalErr
+ }
+ var servers []string
+ if cpuRunner != "" {
+ servers = []string{cpuRunner}
+ } else {
+ servers = runners.ServersForGpu(gpus[0]) // All GPUs in the list are matching Library and Variant
+ }
+ demandLib := envconfig.LLMLibrary()
+ if demandLib != "" {
+ serverPath := availableServers[demandLib]
+ if serverPath == "" {
+ slog.Info(fmt.Sprintf("Invalid OLLAMA_LLM_LIBRARY %s - not found", demandLib))
+ } else {
+ slog.Info("user override", "OLLAMA_LLM_LIBRARY", demandLib, "path", serverPath)
+ servers = []string{demandLib}
+ if strings.HasPrefix(demandLib, "cpu") {
+ // Omit the GPU flag to silence the warning
+ opts.NumGPU = -1
+ }
+ }
+ }
+
+ if len(servers) == 0 {
+ return nil, fmt.Errorf("no servers found for %v", gpus)
+ }
+
+ params := []string{
+ "--model", model,
+ "--ctx-size", strconv.Itoa(opts.NumCtx),
+ "--batch-size", strconv.Itoa(opts.NumBatch),
+ "--embedding",
+ }
+
+ params = append(params, "--log-disable")
+
+ if opts.NumGPU >= 0 {
+ params = append(params, "--n-gpu-layers", strconv.Itoa(opts.NumGPU))
+ }
+
+ if envconfig.Debug() {
+ params = append(params, "--verbose")
+ }
+
+ if opts.MainGPU > 0 {
+ params = append(params, "--main-gpu", strconv.Itoa(opts.MainGPU))
+ }
+
+ if len(adapters) > 0 {
+ // TODO: applying multiple adapters is not supported by the llama.cpp server yet
+ params = append(params, "--lora", adapters[0])
+ }
+
+ if len(projectors) > 0 {
+ // TODO: applying multiple projectors is not supported by the llama.cpp server yet
+ params = append(params, "--mmproj", projectors[0])
+ }
+
+ defaultThreads := systemInfo.GetOptimalThreadCount()
+ if opts.NumThread > 0 {
+ params = append(params, "--threads", strconv.Itoa(opts.NumThread))
+ } else if defaultThreads > 0 {
+ params = append(params, "--threads", strconv.Itoa(defaultThreads))
+ }
+
+ if !opts.F16KV {
+ params = append(params, "--memory-f32")
+ }
+
+ flashAttnEnabled := envconfig.FlashAttention()
+
+ for _, g := range gpus {
+ // only cuda (compute capability 7+) and metal support flash attention
+ if g.Library != "metal" && (g.Library != "cuda" || g.DriverMajor < 7) {
+ flashAttnEnabled = false
+ }
+
+ // mmap has issues with partial offloading on metal
+ if g.Library == "metal" &&
+ uint64(opts.NumGPU) > 0 &&
+ uint64(opts.NumGPU) < ggml.KV().BlockCount()+1 {
+ opts.UseMMap = new(bool)
+ *opts.UseMMap = false
+ }
+ }
+
+ if flashAttnEnabled {
+ params = append(params, "--flash-attn")
+ }
+
+ // Windows CUDA should not use mmap for best performance
+ // Linux with a model larger than free space, mmap leads to thrashing
+ // For CPU loads we want the memory to be allocated, not FS cache
+ if (runtime.GOOS == "windows" && gpus[0].Library == "cuda" && opts.UseMMap == nil) ||
+ (runtime.GOOS == "linux" && systemFreeMemory < estimate.TotalSize && opts.UseMMap == nil) ||
+ (gpus[0].Library == "cpu" && opts.UseMMap == nil) ||
+ (opts.UseMMap != nil && !*opts.UseMMap) {
+ params = append(params, "--no-mmap")
+ }
+
+ if opts.UseMLock {
+ params = append(params, "--mlock")
+ }
+
+ // TODO - NUMA support currently doesn't work properly
+
+ params = append(params, "--parallel", strconv.Itoa(numParallel))
+
+ if estimate.TensorSplit != "" {
+ params = append(params, "--tensor-split", estimate.TensorSplit)
+ }
+
+ if envconfig.MultiUserCache() {
+ params = append(params, "--multiuser-cache")
+ }
+
+ for i := range servers {
+ dir := availableServers[servers[i]]
+ if dir == "" {
+ // Shouldn't happen
+ finalErr = fmt.Errorf("[%d] server %s not listed in available servers %v", i, servers[i], availableServers)
+ slog.Error("server list inconsistent", "error", finalErr)
+ continue
+ }
+
+ if strings.HasPrefix(servers[i], "cpu") {
+ gpus = gpu.GetCPUInfo()
+ }
+
+ // Find an availableServers port, retry on each iteration in case the failure was a port conflict race
+ port := 0
+ if a, err := net.ResolveTCPAddr("tcp", "localhost:0"); err == nil {
+ var l *net.TCPListener
+ if l, err = net.ListenTCP("tcp", a); err == nil {
+ port = l.Addr().(*net.TCPAddr).Port
+ l.Close()
+ }
+ }
+ if port == 0 {
+ slog.Debug("ResolveTCPAddr failed ", "error", err)
+ port = rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range
+ }
+ finalParams := append(params, "--port", strconv.Itoa(port))
+
+ pathEnv := "LD_LIBRARY_PATH"
+ if runtime.GOOS == "windows" {
+ pathEnv = "PATH"
+ }
+ // Start with the server directory for the LD_LIBRARY_PATH/PATH
+ libraryPaths := []string{dir}
+
+ if libraryPath, ok := os.LookupEnv(pathEnv); ok {
+ // favor our bundled library dependencies over system libraries
+ libraryPaths = append(libraryPaths, filepath.SplitList(libraryPath)...)
+ }
+
+ // Note: we always put the dependency path first
+ // since this was the exact version we compiled/linked against
+ if gpus[0].DependencyPath != "" {
+ // assume gpus from the same library have the same dependency path
+ libraryPaths = append([]string{gpus[0].DependencyPath}, libraryPaths...)
+ }
+
+ server := filepath.Join(dir, "ollama_llama_server")
+ if runtime.GOOS == "windows" {
+ server += ".exe"
+ }
+
+ // Detect tmp cleaners wiping out the file
+ _, err := os.Stat(server)
+ if errors.Is(err, os.ErrNotExist) {
+ slog.Warn("llama server disappeared, reinitializing payloads", "path", server, "error", err)
+ _, err = runners.Refresh(build.EmbedFS)
+ if err != nil {
+ slog.Warn("failed to reinitialize payloads", "error", err)
+ return nil, err
+ }
+ }
+
+ // TODO - once fully switched to the Go runner, load the model here for tokenize/detokenize cgo access
+ s := &llmServer{
+ port: port,
+ cmd: exec.Command(server, finalParams...),
+ status: NewStatusWriter(os.Stderr),
+ options: opts,
+ modelPath: model,
+ estimate: estimate,
+ numParallel: numParallel,
+ sem: semaphore.NewWeighted(int64(numParallel)),
+ totalLayers: ggml.KV().BlockCount() + 1,
+ gpus: gpus,
+ done: make(chan error, 1),
+ }
+
+ s.cmd.Env = os.Environ()
+ s.cmd.Stdout = os.Stdout
+ s.cmd.Stderr = s.status
+ s.cmd.SysProcAttr = LlamaServerSysProcAttr
+
+ envWorkarounds := [][2]string{}
+ for _, gpu := range gpus {
+ envWorkarounds = append(envWorkarounds, gpu.EnvWorkarounds...)
+ }
+ visibleDevicesEnv, visibleDevicesEnvVal := gpus.GetVisibleDevicesEnv()
+ pathEnvVal := strings.Join(libraryPaths, string(filepath.ListSeparator))
+
+ // Update or add the path and visible devices variable with our adjusted version
+ pathNeeded := true
+ devicesNeeded := visibleDevicesEnv != ""
+ for i := range s.cmd.Env {
+ cmp := strings.SplitN(s.cmd.Env[i], "=", 2)
+ if strings.EqualFold(cmp[0], pathEnv) {
+ s.cmd.Env[i] = pathEnv + "=" + pathEnvVal
+ pathNeeded = false
+ } else if devicesNeeded && strings.EqualFold(cmp[0], visibleDevicesEnv) {
+ s.cmd.Env[i] = visibleDevicesEnv + "=" + visibleDevicesEnvVal
+ devicesNeeded = false
+ } else if len(envWorkarounds) != 0 {
+ for _, kv := range envWorkarounds {
+ if strings.EqualFold(cmp[0], kv[0]) {
+ s.cmd.Env[i] = kv[0] + "=" + kv[1]
+ }
+ }
+ }
+ }
+ if pathNeeded {
+ s.cmd.Env = append(s.cmd.Env, pathEnv+"="+pathEnvVal)
+ }
+ if devicesNeeded {
+ s.cmd.Env = append(s.cmd.Env, visibleDevicesEnv+"="+visibleDevicesEnvVal)
+ }
+
+ slog.Info("starting llama server", "cmd", s.cmd.String())
+ if envconfig.Debug() {
+ filteredEnv := []string{}
+ for _, ev := range s.cmd.Env {
+ if strings.HasPrefix(ev, "CUDA_") ||
+ strings.HasPrefix(ev, "ROCR_") ||
+ strings.HasPrefix(ev, "ROCM_") ||
+ strings.HasPrefix(ev, "HIP_") ||
+ strings.HasPrefix(ev, "GPU_") ||
+ strings.HasPrefix(ev, "HSA_") ||
+ strings.HasPrefix(ev, "GGML_") ||
+ strings.HasPrefix(ev, "PATH=") ||
+ strings.HasPrefix(ev, "LD_LIBRARY_PATH=") {
+ filteredEnv = append(filteredEnv, ev)
+ }
+ }
+ // Log at debug as the environment is inherited and might contain sensitive information
+ slog.Debug("subprocess", "environment", filteredEnv)
+ }
+
+ if err = s.cmd.Start(); err != nil {
+ // Detect permission denied and augment the message about noexec
+ if errors.Is(err, os.ErrPermission) {
+ finalErr = fmt.Errorf("unable to start server %w. %s may have noexec set. Set OLLAMA_TMPDIR for server to a writable executable directory", err, dir)
+ continue
+ }
+ msg := ""
+ if s.status != nil && s.status.LastErrMsg != "" {
+ msg = s.status.LastErrMsg
+ }
+ err = fmt.Errorf("error starting the external llama server: %v %s", err, msg)
+ finalErr = err
+ continue
+ }
+
+ // reap subprocess when it exits
+ go func() {
+ err := s.cmd.Wait()
+ // Favor a more detailed message over the process exit status
+ if err != nil && s.status != nil && s.status.LastErrMsg != "" {
+ slog.Debug("llama runner terminated", "error", err)
+ if strings.Contains(s.status.LastErrMsg, "unknown model") {
+ s.status.LastErrMsg = "this model is not supported by your version of Ollama. You may need to upgrade"
+ }
+ s.done <- errors.New(s.status.LastErrMsg)
+ } else {
+ s.done <- err
+ }
+ }()
+
+ return s, nil
+ }
+
+ slog.Error("unable to load any llama server", "error", finalErr)
+ return nil, finalErr
+}
+
+func projectorMemoryRequirements(filename string) uint64 {
+ file, err := os.Open(filename)
+ if err != nil {
+ return 0
+ }
+ defer file.Close()
+
+ ggml, _, err := DecodeGGML(file, 0)
+ if err != nil {
+ return 0
+ }
+
+ var mem uint64
+ for _, layer := range ggml.Tensors().Layers() {
+ mem += layer.size()
+ }
+
+ return mem
+}
+
+type ServerStatus int
+
+const ( // iota is reset to 0
+ ServerStatusReady ServerStatus = iota
+ ServerStatusNoSlotsAvailable
+ ServerStatusLoadingModel
+ ServerStatusNotResponding
+ ServerStatusError
+)
+
+func (s ServerStatus) ToString() string {
+ switch s {
+ case ServerStatusReady:
+ return "llm server ready"
+ case ServerStatusNoSlotsAvailable:
+ return "llm busy - no slots available"
+ case ServerStatusLoadingModel:
+ return "llm server loading model"
+ case ServerStatusNotResponding:
+ return "llm server not responding"
+ default:
+ return "llm server error"
+ }
+}
+
+type ServerStatusResp struct {
+ Status string `json:"status"`
+ SlotsIdle int `json:"slots_idle"`
+ SlotsProcessing int `json:"slots_processing"`
+ Error string `json:"error"`
+ Progress float32 `json:"progress"`
+}
+
+func (s *llmServer) getServerStatus(ctx context.Context) (ServerStatus, error) {
+ // Fail fast if its exited
+ if s.cmd.ProcessState != nil {
+ msg := ""
+ if s.status != nil && s.status.LastErrMsg != "" {
+ msg = s.status.LastErrMsg
+ }
+ if s.cmd.ProcessState.ExitCode() == -1 {
+ // Most likely a signal killed it, log some more details to try to help troubleshoot
+ slog.Warn("llama runner process no longer running", "sys", s.cmd.ProcessState.Sys(), "string", s.cmd.ProcessState.String())
+ }
+ return ServerStatusError, fmt.Errorf("llama runner process no longer running: %d %s", s.cmd.ProcessState.ExitCode(), msg)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", s.port), nil)
+ if err != nil {
+ return ServerStatusError, fmt.Errorf("error creating GET request: %v", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ if errors.Is(err, context.DeadlineExceeded) {
+ return ServerStatusNotResponding, errors.New("server not responding")
+ }
+ return ServerStatusError, fmt.Errorf("health resp: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return ServerStatusError, fmt.Errorf("read health request: %w", err)
+ }
+
+ var status ServerStatusResp
+ if err := json.Unmarshal(body, &status); err != nil {
+ return ServerStatusError, fmt.Errorf("health unmarshal encode response: %w", err)
+ }
+
+ switch status.Status {
+ case "ok":
+ return ServerStatusReady, nil
+ case "no slot available":
+ return ServerStatusNoSlotsAvailable, nil
+ case "loading model":
+ s.loadProgress = status.Progress
+ return ServerStatusLoadingModel, nil
+ default:
+ return ServerStatusError, fmt.Errorf("server error: %+v", status)
+ }
+}
+
+// getServerStatusRetry will retry if ServerStatusNoSlotsAvailable is received
+func (s *llmServer) getServerStatusRetry(ctx context.Context) (ServerStatus, error) {
+ var retries int
+ for {
+ status, err := s.getServerStatus(ctx)
+ if err != nil {
+ return status, err
+ }
+
+ if status == ServerStatusNoSlotsAvailable {
+ if retries >= 10 {
+ return status, fmt.Errorf("no slots available after %d retries", retries)
+ }
+
+ time.Sleep(5 * time.Millisecond)
+ retries++
+ continue
+ }
+
+ return status, nil
+ }
+}
+
+func (s *llmServer) Ping(ctx context.Context) error {
+ _, err := s.getServerStatus(ctx)
+ if err != nil {
+ slog.Debug("server unhealthy", "error", err)
+ return err
+ }
+ return nil
+}
+
+func (s *llmServer) WaitUntilRunning(ctx context.Context) error {
+ start := time.Now()
+ stallDuration := envconfig.LoadTimeout() // If no progress happens
+ stallTimer := time.Now().Add(stallDuration) // give up if we stall
+
+ slog.Info("waiting for llama runner to start responding")
+ var lastStatus ServerStatus = -1
+ fullyLoaded := false
+
+ for {
+ select {
+ case <-ctx.Done():
+ slog.Warn("client connection closed before server finished loading, aborting load")
+ return fmt.Errorf("timed out waiting for llama runner to start: %w", ctx.Err())
+ case err := <-s.done:
+ return fmt.Errorf("llama runner process has terminated: %w", err)
+ default:
+ }
+ if time.Now().After(stallTimer) {
+ // timeout
+ msg := ""
+ if s.status != nil && s.status.LastErrMsg != "" {
+ msg = s.status.LastErrMsg
+ }
+ return fmt.Errorf("timed out waiting for llama runner to start - progress %0.2f - %s", s.loadProgress, msg)
+ }
+ if s.cmd.ProcessState != nil {
+ msg := ""
+ if s.status != nil && s.status.LastErrMsg != "" {
+ msg = s.status.LastErrMsg
+ }
+ return fmt.Errorf("llama runner process no longer running: %d %s", s.cmd.ProcessState.ExitCode(), msg)
+ }
+ ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
+ defer cancel()
+ priorProgress := s.loadProgress
+ status, _ := s.getServerStatus(ctx)
+ if lastStatus != status && status != ServerStatusReady {
+ // Only log on status changes
+ slog.Info("waiting for server to become available", "status", status.ToString())
+ }
+ switch status {
+ case ServerStatusReady:
+ s.loadDuration = time.Since(start)
+ slog.Info(fmt.Sprintf("llama runner started in %0.2f seconds", s.loadDuration.Seconds()))
+ return nil
+ default:
+ lastStatus = status
+ // Reset the timer as long as we're making forward progress on the load
+ if priorProgress != s.loadProgress {
+ slog.Debug(fmt.Sprintf("model load progress %0.2f", s.loadProgress))
+ stallTimer = time.Now().Add(stallDuration)
+ } else if !fullyLoaded && int(s.loadProgress*100.0) >= 100 {
+ slog.Debug("model load completed, waiting for server to become available", "status", status.ToString())
+ stallTimer = time.Now().Add(stallDuration)
+ fullyLoaded = true
+ }
+ time.Sleep(time.Millisecond * 250)
+ continue
+ }
+ }
+}
+
+const jsonGrammar = `
+root ::= object
+value ::= object | array | string | number | ("true" | "false" | "null") ws
+
+object ::=
+ "{" ws (
+ string ":" ws value
+ ("," ws string ":" ws value)*
+ )? "}" ws
+
+array ::=
+ "[" ws (
+ value
+ ("," ws value)*
+ )? "]" ws
+
+string ::=
+ "\"" (
+ [^"\\\x7F\x00-\x1F] |
+ "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+ )* "\"" ws
+
+number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
+
+# Optional space: by convention, applied in this grammar after literal chars when allowed
+ws ::= ([ \t\n] ws)?
+`
+
+const maxBufferSize = 512 * format.KiloByte
+
+type ImageData struct {
+ Data []byte `json:"data"`
+ ID int `json:"id"`
+}
+
+type completion struct {
+ Content string `json:"content"`
+ Model string `json:"model"`
+ Prompt string `json:"prompt"`
+ Stop bool `json:"stop"`
+ StoppedLimit bool `json:"stopped_limit"`
+
+ Timings struct {
+ PredictedN int `json:"predicted_n"`
+ PredictedMS float64 `json:"predicted_ms"`
+ PromptN int `json:"prompt_n"`
+ PromptMS float64 `json:"prompt_ms"`
+ }
+}
+
+type CompletionRequest struct {
+ Prompt string
+ Format string
+ Images []ImageData
+ Options *api.Options
+}
+
+type CompletionResponse struct {
+ Content string
+ DoneReason string
+ Done bool
+ PromptEvalCount int
+ PromptEvalDuration time.Duration
+ EvalCount int
+ EvalDuration time.Duration
+}
+
+func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error {
+ if err := s.sem.Acquire(ctx, 1); err != nil {
+ slog.Error("Failed to acquire semaphore", "error", err)
+ return err
+ }
+ defer s.sem.Release(1)
+
+ // put an upper limit on num_predict to avoid the model running on forever
+ if req.Options.NumPredict < 0 || req.Options.NumPredict > 10*s.options.NumCtx {
+ req.Options.NumPredict = 10 * s.options.NumCtx
+ }
+
+ request := map[string]any{
+ "prompt": req.Prompt,
+ "stream": true,
+ "n_predict": req.Options.NumPredict,
+ "n_keep": req.Options.NumKeep,
+ "main_gpu": req.Options.MainGPU,
+ "temperature": req.Options.Temperature,
+ "top_k": req.Options.TopK,
+ "top_p": req.Options.TopP,
+ "min_p": req.Options.MinP,
+ "tfs_z": req.Options.TFSZ,
+ "typical_p": req.Options.TypicalP,
+ "repeat_last_n": req.Options.RepeatLastN,
+ "repeat_penalty": req.Options.RepeatPenalty,
+ "presence_penalty": req.Options.PresencePenalty,
+ "frequency_penalty": req.Options.FrequencyPenalty,
+ "mirostat": req.Options.Mirostat,
+ "mirostat_tau": req.Options.MirostatTau,
+ "mirostat_eta": req.Options.MirostatEta,
+ "penalize_nl": req.Options.PenalizeNewline,
+ "seed": req.Options.Seed,
+ "stop": req.Options.Stop,
+ "image_data": req.Images,
+ "cache_prompt": true,
+ }
+
+ // Make sure the server is ready
+ status, err := s.getServerStatusRetry(ctx)
+ if err != nil {
+ return err
+ } else if status != ServerStatusReady {
+ return fmt.Errorf("unexpected server status: %s", status.ToString())
+ }
+
+ if req.Format == "json" {
+ request["grammar"] = jsonGrammar
+ if !strings.Contains(strings.ToLower(req.Prompt), "json") {
+ slog.Warn("Prompt does not specify that the LLM should response in JSON, but JSON format is expected. For best results specify that JSON is expected in the system prompt.")
+ }
+ }
+
+ // Handling JSON marshaling with special characters unescaped.
+ buffer := &bytes.Buffer{}
+ enc := json.NewEncoder(buffer)
+ enc.SetEscapeHTML(false)
+
+ if err := enc.Encode(request); err != nil {
+ return fmt.Errorf("failed to marshal data: %v", err)
+ }
+
+ endpoint := fmt.Sprintf("http://127.0.0.1:%d/completion", s.port)
+ serverReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, buffer)
+ if err != nil {
+ return fmt.Errorf("error creating POST request: %v", err)
+ }
+ serverReq.Header.Set("Content-Type", "application/json")
+
+ res, err := http.DefaultClient.Do(serverReq)
+ if err != nil {
+ return fmt.Errorf("POST predict: %v", err)
+ }
+ defer res.Body.Close()
+
+ if res.StatusCode >= 400 {
+ bodyBytes, err := io.ReadAll(res.Body)
+ if err != nil {
+ return fmt.Errorf("failed reading llm error response: %w", err)
+ }
+ log.Printf("llm predict error: %s", bodyBytes)
+ return fmt.Errorf("%s", bodyBytes)
+ }
+
+ scanner := bufio.NewScanner(res.Body)
+ buf := make([]byte, 0, maxBufferSize)
+ scanner.Buffer(buf, maxBufferSize)
+
+ // keep track of the last token generated, this is used to abort if the model starts looping
+ var lastToken string
+ var tokenRepeat int
+
+ for scanner.Scan() {
+ select {
+ case <-ctx.Done():
+ // This handles the request cancellation
+ return ctx.Err()
+ default:
+ line := scanner.Bytes()
+ if len(line) == 0 {
+ continue
+ }
+
+ // slog.Debug("got line", "line", string(line))
+ evt, ok := bytes.CutPrefix(line, []byte("data: "))
+ if !ok {
+ evt = line
+ }
+
+ var c completion
+ if err := json.Unmarshal(evt, &c); err != nil {
+ return fmt.Errorf("error unmarshalling llm prediction response: %v", err)
+ }
+ switch {
+ case strings.TrimSpace(c.Content) == lastToken:
+ tokenRepeat++
+ default:
+ lastToken = strings.TrimSpace(c.Content)
+ tokenRepeat = 0
+ }
+
+ // 30 picked as an arbitrary max token repeat limit, modify as needed
+ if tokenRepeat > 30 {
+ slog.Debug("prediction aborted, token repeat limit reached")
+ return ctx.Err()
+ }
+
+ if c.Content != "" {
+ fn(CompletionResponse{
+ Content: c.Content,
+ })
+ }
+
+ if c.Stop {
+ doneReason := "stop"
+ if c.StoppedLimit {
+ doneReason = "length"
+ }
+
+ fn(CompletionResponse{
+ Done: true,
+ DoneReason: doneReason,
+ PromptEvalCount: c.Timings.PromptN,
+ PromptEvalDuration: parseDurationMs(c.Timings.PromptMS),
+ EvalCount: c.Timings.PredictedN,
+ EvalDuration: parseDurationMs(c.Timings.PredictedMS),
+ })
+ return nil
+ }
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ if strings.Contains(err.Error(), "unexpected EOF") {
+ s.Close()
+ msg := ""
+ if s.status != nil && s.status.LastErrMsg != "" {
+ msg = s.status.LastErrMsg
+ }
+ return fmt.Errorf("an unknown error was encountered while running the model %s", msg)
+ }
+
+ return fmt.Errorf("error reading llm response: %v", err)
+ }
+
+ return nil
+}
+
+type EmbeddingRequest struct {
+ Content string `json:"content"`
+}
+
+type EmbeddingResponse struct {
+ Embedding []float32 `json:"embedding"`
+}
+
+func (s *llmServer) Embedding(ctx context.Context, input string) ([]float32, error) {
+ if err := s.sem.Acquire(ctx, 1); err != nil {
+ slog.Error("Failed to acquire semaphore", "error", err)
+ return nil, err
+ }
+ defer s.sem.Release(1)
+
+ // Make sure the server is ready
+ status, err := s.getServerStatusRetry(ctx)
+ if err != nil {
+ return nil, err
+ } else if status != ServerStatusReady {
+ return nil, fmt.Errorf("unexpected server status: %s", status.ToString())
+ }
+
+ data, err := json.Marshal(EmbeddingRequest{Content: input})
+ if err != nil {
+ return nil, fmt.Errorf("error marshaling embed data: %w", err)
+ }
+
+ r, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/embedding", s.port), bytes.NewBuffer(data))
+ if err != nil {
+ return nil, fmt.Errorf("error creating embed request: %w", err)
+ }
+ r.Header.Set("Content-Type", "application/json")
+
+ resp, err := http.DefaultClient.Do(r)
+ if err != nil {
+ return nil, fmt.Errorf("do embedding request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("error reading embed response: %w", err)
+ }
+
+ if resp.StatusCode >= 400 {
+ log.Printf("llm embedding error: %s", body)
+ return nil, fmt.Errorf("%s", body)
+ }
+
+ var e EmbeddingResponse
+ if err := json.Unmarshal(body, &e); err != nil {
+ return nil, fmt.Errorf("unmarshal tokenize response: %w", err)
+ }
+
+ return e.Embedding, nil
+}
+
+type TokenizeRequest struct {
+ Content string `json:"content"`
+}
+
+type TokenizeResponse struct {
+ Tokens []int `json:"tokens"`
+}
+
+func (s *llmServer) Tokenize(ctx context.Context, content string) ([]int, error) {
+ s.modelLock.Lock()
+ defer s.modelLock.Unlock()
+ if s.model != nil {
+ return s.model.Tokenize(content, false, true)
+ }
+
+ // Make sure the server is ready
+ status, err := s.getServerStatus(ctx)
+ if err != nil {
+ return nil, err
+ } else if status != ServerStatusReady && status != ServerStatusNoSlotsAvailable {
+ return nil, fmt.Errorf("unexpected server status: %s", status.ToString())
+ }
+
+ data, err := json.Marshal(TokenizeRequest{Content: content})
+ if err != nil {
+ return nil, fmt.Errorf("marshaling encode data: %w", err)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/tokenize", s.port), bytes.NewBuffer(data))
+ if err != nil {
+ return nil, fmt.Errorf("encode request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("do encode request: %w", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode == http.StatusNotFound {
+ if s.model == nil {
+ slog.Debug("new runner detected, loading model for cgo tokenization")
+ m := llama.LoadModelFromFile(s.modelPath, llama.ModelParams{VocabOnly: true})
+ s.model = m
+ }
+ return s.model.Tokenize(content, false, true)
+ }
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("read encode request: %w", err)
+ }
+
+ if resp.StatusCode >= 400 {
+ log.Printf("llm encode error: %s", body)
+ return nil, fmt.Errorf("%s", body)
+ }
+
+ var encoded TokenizeResponse
+ if err := json.Unmarshal(body, &encoded); err != nil {
+ return nil, fmt.Errorf("unmarshal encode response: %w", err)
+ }
+
+ return encoded.Tokens, nil
+}
+
+type DetokenizeRequest struct {
+ Tokens []int `json:"tokens"`
+}
+
+type DetokenizeResponse struct {
+ Content string `json:"content"`
+}
+
+func (s *llmServer) Detokenize(ctx context.Context, tokens []int) (string, error) {
+ s.modelLock.Lock()
+ defer s.modelLock.Unlock()
+ if s.model != nil {
+ var resp string
+ for _, token := range tokens {
+ resp += s.model.TokenToPiece(token)
+ }
+ return resp, nil
+ }
+ // Make sure the server is ready
+ status, err := s.getServerStatus(ctx)
+ if err != nil {
+ return "", err
+ } else if status != ServerStatusReady && status != ServerStatusNoSlotsAvailable {
+ return "", fmt.Errorf("unexpected server status: %s", status.ToString())
+ }
+
+ data, err := json.Marshal(DetokenizeRequest{Tokens: tokens})
+ if err != nil {
+ return "", fmt.Errorf("marshaling decode data: %w", err)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/detokenize", s.port), bytes.NewBuffer(data))
+ if err != nil {
+ return "", fmt.Errorf("decode request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return "", fmt.Errorf("do decode request: %w", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode == http.StatusNotFound {
+ if s.model == nil {
+ slog.Debug("new runner detected, loading model for cgo tokenization")
+ m := llama.LoadModelFromFile(s.modelPath, llama.ModelParams{VocabOnly: true})
+ s.model = m
+ }
+ var resp string
+ for _, token := range tokens {
+ resp += s.model.TokenToPiece(token)
+ }
+ return resp, nil
+ }
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return "", fmt.Errorf("read decode request: %w", err)
+ }
+
+ if resp.StatusCode >= 400 {
+ log.Printf("llm decode error: %s", body)
+ return "", fmt.Errorf("%s", body)
+ }
+
+ var decoded DetokenizeResponse
+ if err := json.Unmarshal(body, &decoded); err != nil {
+ return "", fmt.Errorf("unmarshal encode response: %w", err)
+ }
+
+ return decoded.Content, nil
+}
+
+func (s *llmServer) Close() error {
+ s.modelLock.Lock()
+ if s.model != nil {
+ llama.FreeModel(s.model)
+ s.model = nil
+ }
+ s.modelLock.Unlock()
+
+ if s.cmd != nil {
+ slog.Debug("stopping llama server")
+ if err := s.cmd.Process.Kill(); err != nil {
+ return err
+ }
+ // if ProcessState is already populated, Wait already completed, no need to wait again
+ if s.cmd.ProcessState == nil {
+ slog.Debug("waiting for llama server to exit")
+ <-s.done
+ }
+
+ slog.Debug("llama server stopped")
+ }
+
+ return nil
+}
+
+func (s *llmServer) EstimatedVRAM() uint64 {
+ return s.estimate.VRAMSize
+}
+
+func (s *llmServer) EstimatedTotal() uint64 {
+ return s.estimate.TotalSize
+}
+
+func (s *llmServer) EstimatedVRAMByGPU(gpuID string) uint64 {
+ for i, gpu := range s.gpus {
+ if gpu.ID == gpuID {
+ return s.estimate.GPUSizes[i]
+ }
+ }
+ return 0
+}
+
+func parseDurationMs(ms float64) time.Duration {
+ dur, err := time.ParseDuration(fmt.Sprintf("%fms", ms))
+ if err != nil {
+ panic(err)
+ }
+
+ return dur
+}
diff --git a/llm/shim_darwin.go b/llm/shim_darwin.go
deleted file mode 100644
index 98e7a7d5..00000000
--- a/llm/shim_darwin.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package llm
-
-import (
- "embed"
- "fmt"
- "log"
- "os"
-
- "github.com/jmorganca/ollama/api"
-)
-
-//go:embed llama.cpp/gguf/ggml-metal.metal
-var libEmbed embed.FS
-
-func newDynamicShimExtServer(library, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
- // should never happen...
- return nil, fmt.Errorf("Dynamic library loading not supported on Mac")
-}
-
-func nativeInit(workdir string) error {
- _, err := extractDynamicLibs(workdir, "llama.cpp/gguf/ggml-metal.metal")
- if err != nil {
- if err == payloadMissing {
- // TODO perhaps consider this a hard failure on arm macs?
- log.Printf("ggml-meta.metal payload missing")
- return nil
- }
- return err
- }
- os.Setenv("GGML_METAL_PATH_RESOURCES", workdir)
- return nil
-}
diff --git a/llm/shim_ext_server.go b/llm/shim_ext_server.go
deleted file mode 100644
index d9c2df46..00000000
--- a/llm/shim_ext_server.go
+++ /dev/null
@@ -1,152 +0,0 @@
-//go:build !darwin
-
-package llm
-
-/*
-
-#include <stdlib.h>
-#include "dynamic_shim.h"
-
-*/
-import "C"
-import (
- "context"
- "embed"
- "errors"
- "fmt"
- "io/fs"
- "log"
- "os"
- "path/filepath"
- "strings"
- "sync"
- "unsafe"
-
- "github.com/jmorganca/ollama/api"
-)
-
-//go:embed llama.cpp/gguf/build/lib/*
-var libEmbed embed.FS
-
-var RocmShimMissing = fmt.Errorf("ROCm shim library not included in this build of ollama. Radeon GPUs are not supported")
-
-type shimExtServer struct {
- s C.struct_dynamic_llama_server
- options api.Options
-}
-
-// Note: current implementation does not support concurrent instantiations
-var shimMutex sync.Mutex
-var llm *shimExtServer
-
-func (llm *shimExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) {
- C.dynamic_shim_llama_server_init(llm.s, sparams, err)
-}
-func (llm *shimExtServer) llama_server_start() {
- C.dynamic_shim_llama_server_start(llm.s)
-}
-func (llm *shimExtServer) llama_server_stop() {
- C.dynamic_shim_llama_server_stop(llm.s)
-}
-
-func (llm *shimExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) {
- C.dynamic_shim_llama_server_completion(llm.s, json_req, resp)
-}
-func (llm *shimExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) {
- C.dynamic_shim_llama_server_completion_next_result(llm.s, task_id, resp)
-}
-func (llm *shimExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) {
- C.dynamic_shim_llama_server_completion_cancel(llm.s, task_id, err)
-}
-func (llm *shimExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) {
- C.dynamic_shim_llama_server_release_task_result(llm.s, result)
-}
-
-func (llm *shimExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
- C.dynamic_shim_llama_server_tokenize(llm.s, json_req, json_resp, err)
-}
-func (llm *shimExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
- C.dynamic_shim_llama_server_detokenize(llm.s, json_req, json_resp, err)
-}
-func (llm *shimExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
- C.dynamic_shim_llama_server_embedding(llm.s, json_req, json_resp, err)
-}
-func (llm *shimExtServer) llama_server_release_json_resp(json_resp **C.char) {
- C.dynamic_shim_llama_server_release_json_resp(llm.s, json_resp)
-}
-
-func newDynamicShimExtServer(library, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
- shimMutex.Lock()
- defer shimMutex.Unlock()
- libPath := C.CString(library)
- defer C.free(unsafe.Pointer(libPath))
- resp := newExtServerResp(128)
- defer freeExtServerResp(resp)
- var srv C.struct_dynamic_llama_server
- C.dynamic_shim_init(libPath, &srv, &resp)
- if resp.id < 0 {
- return nil, fmt.Errorf("Unable to load dynamic library: %s", C.GoString(resp.msg))
- }
- llm = &shimExtServer{
- s: srv,
- options: opts,
- }
- log.Printf("Loading Dynamic Shim llm server: %s", library)
- return newExtServer(llm, model, adapters, projectors, numLayers, opts)
-}
-
-func (llm *shimExtServer) Predict(ctx context.Context, pred PredictOpts, fn func(PredictResult)) error {
- return predict(llm, llm.options, ctx, pred, fn)
-}
-
-func (llm *shimExtServer) Encode(ctx context.Context, prompt string) ([]int, error) {
- return encode(llm, ctx, prompt)
-}
-
-func (llm *shimExtServer) Decode(ctx context.Context, tokens []int) (string, error) {
- return decode(llm, ctx, tokens)
-}
-
-func (llm *shimExtServer) Embedding(ctx context.Context, input string) ([]float64, error) {
- return embedding(llm, ctx, input)
-}
-
-func (llm *shimExtServer) Close() {
- close(llm)
-}
-
-func nativeInit(workdir string) error {
- libs, err := extractDynamicLibs(workdir, "llama.cpp/gguf/build/lib/*server*")
- if err != nil {
- if err == payloadMissing {
- log.Printf("%s", payloadMissing)
- return nil
- }
- return err
- }
- for _, lib := range libs {
- libName := strings.Split(strings.TrimPrefix(filepath.Base(lib), "lib"), ".")[0]
- AvailableShims[libName] = lib
- }
-
- // Only check ROCm access if we have the dynamic lib loaded
- if _, rocmPresent := AvailableShims["rocm_server"]; rocmPresent {
- // Verify we have permissions - either running as root, or we have group access to the driver
- fd, err := os.OpenFile("/dev/kfd", os.O_RDWR, 0666)
- if err != nil {
- if errors.Is(err, fs.ErrPermission) {
- log.Fatalf("Radeon card detected, but permissions not set up properly. Either run ollama as root, or add you user account to the render group.")
- return err
- } else if errors.Is(err, fs.ErrNotExist) {
- // expected behavior without a radeon card
- return nil
- }
-
- return fmt.Errorf("failed to check permission on /dev/kfd: %w", err)
- }
- fd.Close()
-
- }
-
- return nil
-}
diff --git a/llm/status.go b/llm/status.go
new file mode 100644
index 00000000..604fe9e0
--- /dev/null
+++ b/llm/status.go
@@ -0,0 +1,44 @@
+package llm
+
+import (
+ "bytes"
+ "os"
+)
+
+// StatusWriter is a writer that captures error messages from the llama runner process
+type StatusWriter struct {
+ LastErrMsg string
+ out *os.File
+}
+
+func NewStatusWriter(out *os.File) *StatusWriter {
+ return &StatusWriter{
+ out: out,
+ }
+}
+
+// TODO - regex matching to detect errors like
+// libcublasLt.so.11: cannot open shared object file: No such file or directory
+
+var errorPrefixes = []string{
+ "error:",
+ "CUDA error",
+ "cudaMalloc failed",
+ "\"ERR\"",
+ "error loading model",
+ "GGML_ASSERT",
+}
+
+func (w *StatusWriter) Write(b []byte) (int, error) {
+ var errMsg string
+ for _, prefix := range errorPrefixes {
+ if _, after, ok := bytes.Cut(b, []byte(prefix)); ok {
+ errMsg = prefix + string(bytes.TrimSpace(after))
+ }
+ }
+ if errMsg != "" {
+ w.LastErrMsg = errMsg
+ }
+
+ return w.out.Write(b)
+}
diff --git a/llm/utils.go b/llm/utils.go
deleted file mode 100644
index 4dc03c80..00000000
--- a/llm/utils.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package llm
-
-import (
- "fmt"
- "time"
-)
-
-func parseDurationMs(ms float64) time.Duration {
- dur, err := time.ParseDuration(fmt.Sprintf("%fms", ms))
- if err != nil {
- panic(err)
- }
-
- return dur
-}
diff --git a/app/.eslintrc.json b/macapp/.eslintrc.json
similarity index 100%
rename from app/.eslintrc.json
rename to macapp/.eslintrc.json
diff --git a/macapp/.gitignore b/macapp/.gitignore
new file mode 100644
index 00000000..8296128d
--- /dev/null
+++ b/macapp/.gitignore
@@ -0,0 +1,92 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+.DS_Store
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# next.js build output
+.next
+
+# nuxt.js build output
+.nuxt
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# Webpack
+.webpack/
+
+# Vite
+.vite/
+
+# Electron-Forge
+out/
diff --git a/macapp/README.md b/macapp/README.md
new file mode 100644
index 00000000..8bde06e2
--- /dev/null
+++ b/macapp/README.md
@@ -0,0 +1,21 @@
+# Desktop
+
+This app builds upon Ollama to provide a desktop experience for running models.
+
+## Developing
+
+First, build the `ollama` binary:
+
+```
+cd ..
+go build .
+```
+
+Then run the desktop app with `npm start`:
+
+```
+cd macapp
+npm install
+npm start
+```
+
diff --git a/app/assets/icon.icns b/macapp/assets/icon.icns
similarity index 100%
rename from app/assets/icon.icns
rename to macapp/assets/icon.icns
diff --git a/app/assets/iconDarkTemplate.png b/macapp/assets/iconDarkTemplate.png
similarity index 100%
rename from app/assets/iconDarkTemplate.png
rename to macapp/assets/iconDarkTemplate.png
diff --git a/app/assets/iconDarkTemplate@2x.png b/macapp/assets/iconDarkTemplate@2x.png
similarity index 100%
rename from app/assets/iconDarkTemplate@2x.png
rename to macapp/assets/iconDarkTemplate@2x.png
diff --git a/app/assets/iconDarkUpdateTemplate.png b/macapp/assets/iconDarkUpdateTemplate.png
similarity index 100%
rename from app/assets/iconDarkUpdateTemplate.png
rename to macapp/assets/iconDarkUpdateTemplate.png
diff --git a/app/assets/iconDarkUpdateTemplate@2x.png b/macapp/assets/iconDarkUpdateTemplate@2x.png
similarity index 100%
rename from app/assets/iconDarkUpdateTemplate@2x.png
rename to macapp/assets/iconDarkUpdateTemplate@2x.png
diff --git a/app/assets/iconTemplate.png b/macapp/assets/iconTemplate.png
similarity index 100%
rename from app/assets/iconTemplate.png
rename to macapp/assets/iconTemplate.png
diff --git a/app/assets/iconTemplate@2x.png b/macapp/assets/iconTemplate@2x.png
similarity index 100%
rename from app/assets/iconTemplate@2x.png
rename to macapp/assets/iconTemplate@2x.png
diff --git a/app/assets/iconUpdateTemplate.png b/macapp/assets/iconUpdateTemplate.png
similarity index 100%
rename from app/assets/iconUpdateTemplate.png
rename to macapp/assets/iconUpdateTemplate.png
diff --git a/app/assets/iconUpdateTemplate@2x.png b/macapp/assets/iconUpdateTemplate@2x.png
similarity index 100%
rename from app/assets/iconUpdateTemplate@2x.png
rename to macapp/assets/iconUpdateTemplate@2x.png
diff --git a/app/forge.config.ts b/macapp/forge.config.ts
similarity index 100%
rename from app/forge.config.ts
rename to macapp/forge.config.ts
diff --git a/app/package-lock.json b/macapp/package-lock.json
similarity index 100%
rename from app/package-lock.json
rename to macapp/package-lock.json
diff --git a/app/package.json b/macapp/package.json
similarity index 100%
rename from app/package.json
rename to macapp/package.json
diff --git a/app/postcss.config.js b/macapp/postcss.config.js
similarity index 100%
rename from app/postcss.config.js
rename to macapp/postcss.config.js
diff --git a/app/src/app.css b/macapp/src/app.css
similarity index 100%
rename from app/src/app.css
rename to macapp/src/app.css
diff --git a/app/src/app.tsx b/macapp/src/app.tsx
similarity index 99%
rename from app/src/app.tsx
rename to macapp/src/app.tsx
index fc1df21c..449fc851 100644
--- a/app/src/app.tsx
+++ b/macapp/src/app.tsx
@@ -19,7 +19,7 @@ export default function () {
const [step, setStep] = useState<Step>(Step.WELCOME)
const [commandCopied, setCommandCopied] = useState<boolean>(false)
- const command = 'ollama run llama2'
+ const command = 'ollama run llama3.2'
return (
<div className='drag'>
diff --git a/app/src/declarations.d.ts b/macapp/src/declarations.d.ts
similarity index 100%
rename from app/src/declarations.d.ts
rename to macapp/src/declarations.d.ts
diff --git a/app/src/index.html b/macapp/src/index.html
similarity index 100%
rename from app/src/index.html
rename to macapp/src/index.html
diff --git a/app/src/index.ts b/macapp/src/index.ts
similarity index 98%
rename from app/src/index.ts
rename to macapp/src/index.ts
index 28dac136..a5d04d5f 100644
--- a/app/src/index.ts
+++ b/macapp/src/index.ts
@@ -162,7 +162,7 @@ app.on('before-quit', () => {
}
})
-const updateURL = `https://ollama.ai/api/update?os=${process.platform}&arch=${
+const updateURL = `https://ollama.com/api/update?os=${process.platform}&arch=${
process.arch
}&version=${app.getVersion()}&id=${id()}`
diff --git a/app/src/install.ts b/macapp/src/install.ts
similarity index 100%
rename from app/src/install.ts
rename to macapp/src/install.ts
diff --git a/app/src/ollama.svg b/macapp/src/ollama.svg
similarity index 100%
rename from app/src/ollama.svg
rename to macapp/src/ollama.svg
diff --git a/app/src/preload.ts b/macapp/src/preload.ts
similarity index 100%
rename from app/src/preload.ts
rename to macapp/src/preload.ts
diff --git a/app/src/renderer.tsx b/macapp/src/renderer.tsx
similarity index 100%
rename from app/src/renderer.tsx
rename to macapp/src/renderer.tsx
diff --git a/app/tailwind.config.js b/macapp/tailwind.config.js
similarity index 100%
rename from app/tailwind.config.js
rename to macapp/tailwind.config.js
diff --git a/app/tsconfig.json b/macapp/tsconfig.json
similarity index 100%
rename from app/tsconfig.json
rename to macapp/tsconfig.json
diff --git a/app/webpack.main.config.ts b/macapp/webpack.main.config.ts
similarity index 100%
rename from app/webpack.main.config.ts
rename to macapp/webpack.main.config.ts
diff --git a/app/webpack.plugins.ts b/macapp/webpack.plugins.ts
similarity index 100%
rename from app/webpack.plugins.ts
rename to macapp/webpack.plugins.ts
diff --git a/app/webpack.renderer.config.ts b/macapp/webpack.renderer.config.ts
similarity index 100%
rename from app/webpack.renderer.config.ts
rename to macapp/webpack.renderer.config.ts
diff --git a/app/webpack.rules.ts b/macapp/webpack.rules.ts
similarity index 100%
rename from app/webpack.rules.ts
rename to macapp/webpack.rules.ts
diff --git a/main.go b/main.go
index ecdcb71d..650e03a6 100644
--- a/main.go
+++ b/main.go
@@ -3,8 +3,9 @@ package main
import (
"context"
- "github.com/jmorganca/ollama/cmd"
"github.com/spf13/cobra"
+
+ "github.com/ollama/ollama/cmd"
)
func main() {
diff --git a/openai/openai.go b/openai/openai.go
new file mode 100644
index 00000000..2bf9b9f9
--- /dev/null
+++ b/openai/openai.go
@@ -0,0 +1,913 @@
+// openai package provides middleware for partial compatibility with the OpenAI REST API
+package openai
+
+import (
+ "bytes"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "math/rand"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/gin-gonic/gin"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/types/model"
+)
+
+type Error struct {
+ Message string `json:"message"`
+ Type string `json:"type"`
+ Param interface{} `json:"param"`
+ Code *string `json:"code"`
+}
+
+type ErrorResponse struct {
+ Error Error `json:"error"`
+}
+
+type Message struct {
+ Role string `json:"role"`
+ Content any `json:"content"`
+ ToolCalls []ToolCall `json:"tool_calls,omitempty"`
+}
+
+type Choice struct {
+ Index int `json:"index"`
+ Message Message `json:"message"`
+ FinishReason *string `json:"finish_reason"`
+}
+
+type ChunkChoice struct {
+ Index int `json:"index"`
+ Delta Message `json:"delta"`
+ FinishReason *string `json:"finish_reason"`
+}
+
+type CompleteChunkChoice struct {
+ Text string `json:"text"`
+ Index int `json:"index"`
+ FinishReason *string `json:"finish_reason"`
+}
+
+type Usage struct {
+ PromptTokens int `json:"prompt_tokens"`
+ CompletionTokens int `json:"completion_tokens"`
+ TotalTokens int `json:"total_tokens"`
+}
+
+type ResponseFormat struct {
+ Type string `json:"type"`
+}
+
+type EmbedRequest struct {
+ Input any `json:"input"`
+ Model string `json:"model"`
+}
+
+type ChatCompletionRequest struct {
+ Model string `json:"model"`
+ Messages []Message `json:"messages"`
+ Stream bool `json:"stream"`
+ MaxTokens *int `json:"max_tokens"`
+ Seed *int `json:"seed"`
+ Stop any `json:"stop"`
+ Temperature *float64 `json:"temperature"`
+ FrequencyPenalty *float64 `json:"frequency_penalty"`
+ PresencePenalty *float64 `json:"presence_penalty"`
+ TopP *float64 `json:"top_p"`
+ ResponseFormat *ResponseFormat `json:"response_format"`
+ Tools []api.Tool `json:"tools"`
+}
+
+type ChatCompletion struct {
+ Id string `json:"id"`
+ Object string `json:"object"`
+ Created int64 `json:"created"`
+ Model string `json:"model"`
+ SystemFingerprint string `json:"system_fingerprint"`
+ Choices []Choice `json:"choices"`
+ Usage Usage `json:"usage,omitempty"`
+}
+
+type ChatCompletionChunk struct {
+ Id string `json:"id"`
+ Object string `json:"object"`
+ Created int64 `json:"created"`
+ Model string `json:"model"`
+ SystemFingerprint string `json:"system_fingerprint"`
+ Choices []ChunkChoice `json:"choices"`
+}
+
+// TODO (https://github.com/ollama/ollama/issues/5259): support []string, []int and [][]int
+type CompletionRequest struct {
+ Model string `json:"model"`
+ Prompt string `json:"prompt"`
+ FrequencyPenalty float32 `json:"frequency_penalty"`
+ MaxTokens *int `json:"max_tokens"`
+ PresencePenalty float32 `json:"presence_penalty"`
+ Seed *int `json:"seed"`
+ Stop any `json:"stop"`
+ Stream bool `json:"stream"`
+ Temperature *float32 `json:"temperature"`
+ TopP float32 `json:"top_p"`
+ Suffix string `json:"suffix"`
+}
+
+type Completion struct {
+ Id string `json:"id"`
+ Object string `json:"object"`
+ Created int64 `json:"created"`
+ Model string `json:"model"`
+ SystemFingerprint string `json:"system_fingerprint"`
+ Choices []CompleteChunkChoice `json:"choices"`
+ Usage Usage `json:"usage,omitempty"`
+}
+
+type CompletionChunk struct {
+ Id string `json:"id"`
+ Object string `json:"object"`
+ Created int64 `json:"created"`
+ Choices []CompleteChunkChoice `json:"choices"`
+ Model string `json:"model"`
+ SystemFingerprint string `json:"system_fingerprint"`
+}
+
+type ToolCall struct {
+ ID string `json:"id"`
+ Type string `json:"type"`
+ Function struct {
+ Name string `json:"name"`
+ Arguments string `json:"arguments"`
+ } `json:"function"`
+}
+
+type Model struct {
+ Id string `json:"id"`
+ Object string `json:"object"`
+ Created int64 `json:"created"`
+ OwnedBy string `json:"owned_by"`
+}
+
+type Embedding struct {
+ Object string `json:"object"`
+ Embedding []float32 `json:"embedding"`
+ Index int `json:"index"`
+}
+
+type ListCompletion struct {
+ Object string `json:"object"`
+ Data []Model `json:"data"`
+}
+
+type EmbeddingList struct {
+ Object string `json:"object"`
+ Data []Embedding `json:"data"`
+ Model string `json:"model"`
+ Usage EmbeddingUsage `json:"usage,omitempty"`
+}
+
+type EmbeddingUsage struct {
+ PromptTokens int `json:"prompt_tokens"`
+ TotalTokens int `json:"total_tokens"`
+}
+
+func NewError(code int, message string) ErrorResponse {
+ var etype string
+ switch code {
+ case http.StatusBadRequest:
+ etype = "invalid_request_error"
+ case http.StatusNotFound:
+ etype = "not_found_error"
+ default:
+ etype = "api_error"
+ }
+
+ return ErrorResponse{Error{Type: etype, Message: message}}
+}
+
+func toolCallId() string {
+ const letterBytes = "abcdefghijklmnopqrstuvwxyz0123456789"
+ b := make([]byte, 8)
+ for i := range b {
+ b[i] = letterBytes[rand.Intn(len(letterBytes))]
+ }
+ return "call_" + strings.ToLower(string(b))
+}
+
+func toChatCompletion(id string, r api.ChatResponse) ChatCompletion {
+ toolCalls := make([]ToolCall, len(r.Message.ToolCalls))
+ for i, tc := range r.Message.ToolCalls {
+ toolCalls[i].ID = toolCallId()
+ toolCalls[i].Type = "function"
+ toolCalls[i].Function.Name = tc.Function.Name
+
+ args, err := json.Marshal(tc.Function.Arguments)
+ if err != nil {
+ slog.Error("could not marshall function arguments to json", "error", err)
+ continue
+ }
+
+ toolCalls[i].Function.Arguments = string(args)
+ }
+
+ return ChatCompletion{
+ Id: id,
+ Object: "chat.completion",
+ Created: r.CreatedAt.Unix(),
+ Model: r.Model,
+ SystemFingerprint: "fp_ollama",
+ Choices: []Choice{{
+ Index: 0,
+ Message: Message{Role: r.Message.Role, Content: r.Message.Content, ToolCalls: toolCalls},
+ FinishReason: func(reason string) *string {
+ if len(toolCalls) > 0 {
+ reason = "tool_calls"
+ }
+ if len(reason) > 0 {
+ return &reason
+ }
+ return nil
+ }(r.DoneReason),
+ }},
+ Usage: Usage{
+ PromptTokens: r.PromptEvalCount,
+ CompletionTokens: r.EvalCount,
+ TotalTokens: r.PromptEvalCount + r.EvalCount,
+ },
+ }
+}
+
+func toChunk(id string, r api.ChatResponse) ChatCompletionChunk {
+ return ChatCompletionChunk{
+ Id: id,
+ Object: "chat.completion.chunk",
+ Created: time.Now().Unix(),
+ Model: r.Model,
+ SystemFingerprint: "fp_ollama",
+ Choices: []ChunkChoice{{
+ Index: 0,
+ Delta: Message{Role: "assistant", Content: r.Message.Content},
+ FinishReason: func(reason string) *string {
+ if len(reason) > 0 {
+ return &reason
+ }
+ return nil
+ }(r.DoneReason),
+ }},
+ }
+}
+
+func toCompletion(id string, r api.GenerateResponse) Completion {
+ return Completion{
+ Id: id,
+ Object: "text_completion",
+ Created: r.CreatedAt.Unix(),
+ Model: r.Model,
+ SystemFingerprint: "fp_ollama",
+ Choices: []CompleteChunkChoice{{
+ Text: r.Response,
+ Index: 0,
+ FinishReason: func(reason string) *string {
+ if len(reason) > 0 {
+ return &reason
+ }
+ return nil
+ }(r.DoneReason),
+ }},
+ Usage: Usage{
+ PromptTokens: r.PromptEvalCount,
+ CompletionTokens: r.EvalCount,
+ TotalTokens: r.PromptEvalCount + r.EvalCount,
+ },
+ }
+}
+
+func toCompleteChunk(id string, r api.GenerateResponse) CompletionChunk {
+ return CompletionChunk{
+ Id: id,
+ Object: "text_completion",
+ Created: time.Now().Unix(),
+ Model: r.Model,
+ SystemFingerprint: "fp_ollama",
+ Choices: []CompleteChunkChoice{{
+ Text: r.Response,
+ Index: 0,
+ FinishReason: func(reason string) *string {
+ if len(reason) > 0 {
+ return &reason
+ }
+ return nil
+ }(r.DoneReason),
+ }},
+ }
+}
+
+func toListCompletion(r api.ListResponse) ListCompletion {
+ var data []Model
+ for _, m := range r.Models {
+ data = append(data, Model{
+ Id: m.Name,
+ Object: "model",
+ Created: m.ModifiedAt.Unix(),
+ OwnedBy: model.ParseName(m.Name).Namespace,
+ })
+ }
+
+ return ListCompletion{
+ Object: "list",
+ Data: data,
+ }
+}
+
+func toEmbeddingList(model string, r api.EmbedResponse) EmbeddingList {
+ if r.Embeddings != nil {
+ var data []Embedding
+ for i, e := range r.Embeddings {
+ data = append(data, Embedding{
+ Object: "embedding",
+ Embedding: e,
+ Index: i,
+ })
+ }
+
+ return EmbeddingList{
+ Object: "list",
+ Data: data,
+ Model: model,
+ Usage: EmbeddingUsage{
+ PromptTokens: r.PromptEvalCount,
+ TotalTokens: r.PromptEvalCount,
+ },
+ }
+ }
+
+ return EmbeddingList{}
+}
+
+func toModel(r api.ShowResponse, m string) Model {
+ return Model{
+ Id: m,
+ Object: "model",
+ Created: r.ModifiedAt.Unix(),
+ OwnedBy: model.ParseName(m).Namespace,
+ }
+}
+
+func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {
+ var messages []api.Message
+ for _, msg := range r.Messages {
+ switch content := msg.Content.(type) {
+ case string:
+ messages = append(messages, api.Message{Role: msg.Role, Content: content})
+ case []any:
+ for _, c := range content {
+ data, ok := c.(map[string]any)
+ if !ok {
+ return nil, errors.New("invalid message format")
+ }
+ switch data["type"] {
+ case "text":
+ text, ok := data["text"].(string)
+ if !ok {
+ return nil, errors.New("invalid message format")
+ }
+ messages = append(messages, api.Message{Role: msg.Role, Content: text})
+ case "image_url":
+ var url string
+ if urlMap, ok := data["image_url"].(map[string]any); ok {
+ if url, ok = urlMap["url"].(string); !ok {
+ return nil, errors.New("invalid message format")
+ }
+ } else {
+ if url, ok = data["image_url"].(string); !ok {
+ return nil, errors.New("invalid message format")
+ }
+ }
+
+ types := []string{"jpeg", "jpg", "png"}
+ valid := false
+ for _, t := range types {
+ prefix := "data:image/" + t + ";base64,"
+ if strings.HasPrefix(url, prefix) {
+ url = strings.TrimPrefix(url, prefix)
+ valid = true
+ break
+ }
+ }
+
+ if !valid {
+ return nil, errors.New("invalid image input")
+ }
+
+ img, err := base64.StdEncoding.DecodeString(url)
+ if err != nil {
+ return nil, errors.New("invalid message format")
+ }
+
+ messages = append(messages, api.Message{Role: msg.Role, Images: []api.ImageData{img}})
+ default:
+ return nil, errors.New("invalid message format")
+ }
+ }
+ default:
+ if msg.ToolCalls == nil {
+ return nil, fmt.Errorf("invalid message content type: %T", content)
+ }
+
+ toolCalls := make([]api.ToolCall, len(msg.ToolCalls))
+ for i, tc := range msg.ToolCalls {
+ toolCalls[i].Function.Name = tc.Function.Name
+ err := json.Unmarshal([]byte(tc.Function.Arguments), &toolCalls[i].Function.Arguments)
+ if err != nil {
+ return nil, errors.New("invalid tool call arguments")
+ }
+ }
+ messages = append(messages, api.Message{Role: msg.Role, ToolCalls: toolCalls})
+ }
+ }
+
+ options := make(map[string]interface{})
+
+ switch stop := r.Stop.(type) {
+ case string:
+ options["stop"] = []string{stop}
+ case []any:
+ var stops []string
+ for _, s := range stop {
+ if str, ok := s.(string); ok {
+ stops = append(stops, str)
+ }
+ }
+ options["stop"] = stops
+ }
+
+ if r.MaxTokens != nil {
+ options["num_predict"] = *r.MaxTokens
+ }
+
+ if r.Temperature != nil {
+ options["temperature"] = *r.Temperature
+ } else {
+ options["temperature"] = 1.0
+ }
+
+ if r.Seed != nil {
+ options["seed"] = *r.Seed
+ }
+
+ if r.FrequencyPenalty != nil {
+ options["frequency_penalty"] = *r.FrequencyPenalty
+ }
+
+ if r.PresencePenalty != nil {
+ options["presence_penalty"] = *r.PresencePenalty
+ }
+
+ if r.TopP != nil {
+ options["top_p"] = *r.TopP
+ } else {
+ options["top_p"] = 1.0
+ }
+
+ var format string
+ if r.ResponseFormat != nil && r.ResponseFormat.Type == "json_object" {
+ format = "json"
+ }
+
+ return &api.ChatRequest{
+ Model: r.Model,
+ Messages: messages,
+ Format: format,
+ Options: options,
+ Stream: &r.Stream,
+ Tools: r.Tools,
+ }, nil
+}
+
+func fromCompleteRequest(r CompletionRequest) (api.GenerateRequest, error) {
+ options := make(map[string]any)
+
+ switch stop := r.Stop.(type) {
+ case string:
+ options["stop"] = []string{stop}
+ case []any:
+ var stops []string
+ for _, s := range stop {
+ if str, ok := s.(string); ok {
+ stops = append(stops, str)
+ } else {
+ return api.GenerateRequest{}, fmt.Errorf("invalid type for 'stop' field: %T", s)
+ }
+ }
+ options["stop"] = stops
+ }
+
+ if r.MaxTokens != nil {
+ options["num_predict"] = *r.MaxTokens
+ }
+
+ if r.Temperature != nil {
+ options["temperature"] = *r.Temperature
+ } else {
+ options["temperature"] = 1.0
+ }
+
+ if r.Seed != nil {
+ options["seed"] = *r.Seed
+ }
+
+ options["frequency_penalty"] = r.FrequencyPenalty
+
+ options["presence_penalty"] = r.PresencePenalty
+
+ if r.TopP != 0.0 {
+ options["top_p"] = r.TopP
+ } else {
+ options["top_p"] = 1.0
+ }
+
+ return api.GenerateRequest{
+ Model: r.Model,
+ Prompt: r.Prompt,
+ Options: options,
+ Stream: &r.Stream,
+ Suffix: r.Suffix,
+ }, nil
+}
+
+type BaseWriter struct {
+ gin.ResponseWriter
+}
+
+type ChatWriter struct {
+ stream bool
+ id string
+ BaseWriter
+}
+
+type CompleteWriter struct {
+ stream bool
+ id string
+ BaseWriter
+}
+
+type ListWriter struct {
+ BaseWriter
+}
+
+type RetrieveWriter struct {
+ BaseWriter
+ model string
+}
+
+type EmbedWriter struct {
+ BaseWriter
+ model string
+}
+
+func (w *BaseWriter) writeError(code int, data []byte) (int, error) {
+ var serr api.StatusError
+ err := json.Unmarshal(data, &serr)
+ if err != nil {
+ return 0, err
+ }
+
+ w.ResponseWriter.Header().Set("Content-Type", "application/json")
+ err = json.NewEncoder(w.ResponseWriter).Encode(NewError(http.StatusInternalServerError, serr.Error()))
+ if err != nil {
+ return 0, err
+ }
+
+ return len(data), nil
+}
+
+func (w *ChatWriter) writeResponse(data []byte) (int, error) {
+ var chatResponse api.ChatResponse
+ err := json.Unmarshal(data, &chatResponse)
+ if err != nil {
+ return 0, err
+ }
+
+ // chat chunk
+ if w.stream {
+ d, err := json.Marshal(toChunk(w.id, chatResponse))
+ if err != nil {
+ return 0, err
+ }
+
+ w.ResponseWriter.Header().Set("Content-Type", "text/event-stream")
+ _, err = w.ResponseWriter.Write([]byte(fmt.Sprintf("data: %s\n\n", d)))
+ if err != nil {
+ return 0, err
+ }
+
+ if chatResponse.Done {
+ _, err = w.ResponseWriter.Write([]byte("data: [DONE]\n\n"))
+ if err != nil {
+ return 0, err
+ }
+ }
+
+ return len(data), nil
+ }
+
+ // chat completion
+ w.ResponseWriter.Header().Set("Content-Type", "application/json")
+ err = json.NewEncoder(w.ResponseWriter).Encode(toChatCompletion(w.id, chatResponse))
+ if err != nil {
+ return 0, err
+ }
+
+ return len(data), nil
+}
+
+func (w *ChatWriter) Write(data []byte) (int, error) {
+ code := w.ResponseWriter.Status()
+ if code != http.StatusOK {
+ return w.writeError(code, data)
+ }
+
+ return w.writeResponse(data)
+}
+
+func (w *CompleteWriter) writeResponse(data []byte) (int, error) {
+ var generateResponse api.GenerateResponse
+ err := json.Unmarshal(data, &generateResponse)
+ if err != nil {
+ return 0, err
+ }
+
+ // completion chunk
+ if w.stream {
+ d, err := json.Marshal(toCompleteChunk(w.id, generateResponse))
+ if err != nil {
+ return 0, err
+ }
+
+ w.ResponseWriter.Header().Set("Content-Type", "text/event-stream")
+ _, err = w.ResponseWriter.Write([]byte(fmt.Sprintf("data: %s\n\n", d)))
+ if err != nil {
+ return 0, err
+ }
+
+ if generateResponse.Done {
+ _, err = w.ResponseWriter.Write([]byte("data: [DONE]\n\n"))
+ if err != nil {
+ return 0, err
+ }
+ }
+
+ return len(data), nil
+ }
+
+ // completion
+ w.ResponseWriter.Header().Set("Content-Type", "application/json")
+ err = json.NewEncoder(w.ResponseWriter).Encode(toCompletion(w.id, generateResponse))
+ if err != nil {
+ return 0, err
+ }
+
+ return len(data), nil
+}
+
+func (w *CompleteWriter) Write(data []byte) (int, error) {
+ code := w.ResponseWriter.Status()
+ if code != http.StatusOK {
+ return w.writeError(code, data)
+ }
+
+ return w.writeResponse(data)
+}
+
+func (w *ListWriter) writeResponse(data []byte) (int, error) {
+ var listResponse api.ListResponse
+ err := json.Unmarshal(data, &listResponse)
+ if err != nil {
+ return 0, err
+ }
+
+ w.ResponseWriter.Header().Set("Content-Type", "application/json")
+ err = json.NewEncoder(w.ResponseWriter).Encode(toListCompletion(listResponse))
+ if err != nil {
+ return 0, err
+ }
+
+ return len(data), nil
+}
+
+func (w *ListWriter) Write(data []byte) (int, error) {
+ code := w.ResponseWriter.Status()
+ if code != http.StatusOK {
+ return w.writeError(code, data)
+ }
+
+ return w.writeResponse(data)
+}
+
+func (w *RetrieveWriter) writeResponse(data []byte) (int, error) {
+ var showResponse api.ShowResponse
+ err := json.Unmarshal(data, &showResponse)
+ if err != nil {
+ return 0, err
+ }
+
+ // retrieve completion
+ w.ResponseWriter.Header().Set("Content-Type", "application/json")
+ err = json.NewEncoder(w.ResponseWriter).Encode(toModel(showResponse, w.model))
+ if err != nil {
+ return 0, err
+ }
+
+ return len(data), nil
+}
+
+func (w *RetrieveWriter) Write(data []byte) (int, error) {
+ code := w.ResponseWriter.Status()
+ if code != http.StatusOK {
+ return w.writeError(code, data)
+ }
+
+ return w.writeResponse(data)
+}
+
+func (w *EmbedWriter) writeResponse(data []byte) (int, error) {
+ var embedResponse api.EmbedResponse
+ err := json.Unmarshal(data, &embedResponse)
+ if err != nil {
+ return 0, err
+ }
+
+ w.ResponseWriter.Header().Set("Content-Type", "application/json")
+ err = json.NewEncoder(w.ResponseWriter).Encode(toEmbeddingList(w.model, embedResponse))
+ if err != nil {
+ return 0, err
+ }
+
+ return len(data), nil
+}
+
+func (w *EmbedWriter) Write(data []byte) (int, error) {
+ code := w.ResponseWriter.Status()
+ if code != http.StatusOK {
+ return w.writeError(code, data)
+ }
+
+ return w.writeResponse(data)
+}
+
+func ListMiddleware() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ w := &ListWriter{
+ BaseWriter: BaseWriter{ResponseWriter: c.Writer},
+ }
+
+ c.Writer = w
+
+ c.Next()
+ }
+}
+
+func RetrieveMiddleware() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ var b bytes.Buffer
+ if err := json.NewEncoder(&b).Encode(api.ShowRequest{Name: c.Param("model")}); err != nil {
+ c.AbortWithStatusJSON(http.StatusInternalServerError, NewError(http.StatusInternalServerError, err.Error()))
+ return
+ }
+
+ c.Request.Body = io.NopCloser(&b)
+
+ // response writer
+ w := &RetrieveWriter{
+ BaseWriter: BaseWriter{ResponseWriter: c.Writer},
+ model: c.Param("model"),
+ }
+
+ c.Writer = w
+
+ c.Next()
+ }
+}
+
+func CompletionsMiddleware() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ var req CompletionRequest
+ err := c.ShouldBindJSON(&req)
+ if err != nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, NewError(http.StatusBadRequest, err.Error()))
+ return
+ }
+
+ var b bytes.Buffer
+ genReq, err := fromCompleteRequest(req)
+ if err != nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, NewError(http.StatusBadRequest, err.Error()))
+ return
+ }
+
+ if err := json.NewEncoder(&b).Encode(genReq); err != nil {
+ c.AbortWithStatusJSON(http.StatusInternalServerError, NewError(http.StatusInternalServerError, err.Error()))
+ return
+ }
+
+ c.Request.Body = io.NopCloser(&b)
+
+ w := &CompleteWriter{
+ BaseWriter: BaseWriter{ResponseWriter: c.Writer},
+ stream: req.Stream,
+ id: fmt.Sprintf("cmpl-%d", rand.Intn(999)),
+ }
+
+ c.Writer = w
+ c.Next()
+ }
+}
+
+func EmbeddingsMiddleware() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ var req EmbedRequest
+ err := c.ShouldBindJSON(&req)
+ if err != nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, NewError(http.StatusBadRequest, err.Error()))
+ return
+ }
+
+ if req.Input == "" {
+ req.Input = []string{""}
+ }
+
+ if req.Input == nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, NewError(http.StatusBadRequest, "invalid input"))
+ return
+ }
+
+ if v, ok := req.Input.([]any); ok && len(v) == 0 {
+ c.AbortWithStatusJSON(http.StatusBadRequest, NewError(http.StatusBadRequest, "invalid input"))
+ return
+ }
+
+ var b bytes.Buffer
+ if err := json.NewEncoder(&b).Encode(api.EmbedRequest{Model: req.Model, Input: req.Input}); err != nil {
+ c.AbortWithStatusJSON(http.StatusInternalServerError, NewError(http.StatusInternalServerError, err.Error()))
+ return
+ }
+
+ c.Request.Body = io.NopCloser(&b)
+
+ w := &EmbedWriter{
+ BaseWriter: BaseWriter{ResponseWriter: c.Writer},
+ model: req.Model,
+ }
+
+ c.Writer = w
+
+ c.Next()
+ }
+}
+
+func ChatMiddleware() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ var req ChatCompletionRequest
+ err := c.ShouldBindJSON(&req)
+ if err != nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, NewError(http.StatusBadRequest, err.Error()))
+ return
+ }
+
+ if len(req.Messages) == 0 {
+ c.AbortWithStatusJSON(http.StatusBadRequest, NewError(http.StatusBadRequest, "[] is too short - 'messages'"))
+ return
+ }
+
+ var b bytes.Buffer
+
+ chatReq, err := fromChatRequest(req)
+ if err != nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, NewError(http.StatusBadRequest, err.Error()))
+ return
+ }
+
+ if err := json.NewEncoder(&b).Encode(chatReq); err != nil {
+ c.AbortWithStatusJSON(http.StatusInternalServerError, NewError(http.StatusInternalServerError, err.Error()))
+ return
+ }
+
+ c.Request.Body = io.NopCloser(&b)
+
+ w := &ChatWriter{
+ BaseWriter: BaseWriter{ResponseWriter: c.Writer},
+ stream: req.Stream,
+ id: fmt.Sprintf("chatcmpl-%d", rand.Intn(999)),
+ }
+
+ c.Writer = w
+
+ c.Next()
+ }
+}
diff --git a/openai/openai_test.go b/openai/openai_test.go
new file mode 100644
index 00000000..eabf5b66
--- /dev/null
+++ b/openai/openai_test.go
@@ -0,0 +1,561 @@
+package openai
+
+import (
+ "bytes"
+ "encoding/base64"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/gin-gonic/gin"
+
+ "github.com/ollama/ollama/api"
+)
+
+const (
+ prefix = `data:image/jpeg;base64,`
+ image = `iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=`
+)
+
+var (
+ False = false
+ True = true
+)
+
+func captureRequestMiddleware(capturedRequest any) gin.HandlerFunc {
+ return func(c *gin.Context) {
+ bodyBytes, _ := io.ReadAll(c.Request.Body)
+ c.Request.Body = io.NopCloser(bytes.NewReader(bodyBytes))
+ err := json.Unmarshal(bodyBytes, capturedRequest)
+ if err != nil {
+ c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to unmarshal request")
+ }
+ c.Next()
+ }
+}
+
+func TestChatMiddleware(t *testing.T) {
+ type testCase struct {
+ name string
+ body string
+ req api.ChatRequest
+ err ErrorResponse
+ }
+
+ var capturedRequest *api.ChatRequest
+
+ testCases := []testCase{
+ {
+ name: "chat handler",
+ body: `{
+ "model": "test-model",
+ "messages": [
+ {"role": "user", "content": "Hello"}
+ ]
+ }`,
+ req: api.ChatRequest{
+ Model: "test-model",
+ Messages: []api.Message{
+ {
+ Role: "user",
+ Content: "Hello",
+ },
+ },
+ Options: map[string]any{
+ "temperature": 1.0,
+ "top_p": 1.0,
+ },
+ Stream: &False,
+ },
+ },
+ {
+ name: "chat handler with options",
+ body: `{
+ "model": "test-model",
+ "messages": [
+ {"role": "user", "content": "Hello"}
+ ],
+ "stream": true,
+ "max_tokens": 999,
+ "seed": 123,
+ "stop": ["\n", "stop"],
+ "temperature": 3.0,
+ "frequency_penalty": 4.0,
+ "presence_penalty": 5.0,
+ "top_p": 6.0,
+ "response_format": {"type": "json_object"}
+ }`,
+ req: api.ChatRequest{
+ Model: "test-model",
+ Messages: []api.Message{
+ {
+ Role: "user",
+ Content: "Hello",
+ },
+ },
+ Options: map[string]any{
+ "num_predict": 999.0, // float because JSON doesn't distinguish between float and int
+ "seed": 123.0,
+ "stop": []any{"\n", "stop"},
+ "temperature": 3.0,
+ "frequency_penalty": 4.0,
+ "presence_penalty": 5.0,
+ "top_p": 6.0,
+ },
+ Format: "json",
+ Stream: &True,
+ },
+ },
+ {
+ name: "chat handler with image content",
+ body: `{
+ "model": "test-model",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Hello"
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "` + prefix + image + `"
+ }
+ }
+ ]
+ }
+ ]
+ }`,
+ req: api.ChatRequest{
+ Model: "test-model",
+ Messages: []api.Message{
+ {
+ Role: "user",
+ Content: "Hello",
+ },
+ {
+ Role: "user",
+ Images: []api.ImageData{
+ func() []byte {
+ img, _ := base64.StdEncoding.DecodeString(image)
+ return img
+ }(),
+ },
+ },
+ },
+ Options: map[string]any{
+ "temperature": 1.0,
+ "top_p": 1.0,
+ },
+ Stream: &False,
+ },
+ },
+ {
+ name: "chat handler with tools",
+ body: `{
+ "model": "test-model",
+ "messages": [
+ {"role": "user", "content": "What's the weather like in Paris Today?"},
+ {"role": "assistant", "tool_calls": [{"id": "id", "type": "function", "function": {"name": "get_current_weather", "arguments": "{\"location\": \"Paris, France\", \"format\": \"celsius\"}"}}]}
+ ]
+ }`,
+ req: api.ChatRequest{
+ Model: "test-model",
+ Messages: []api.Message{
+ {
+ Role: "user",
+ Content: "What's the weather like in Paris Today?",
+ },
+ {
+ Role: "assistant",
+ ToolCalls: []api.ToolCall{
+ {
+ Function: api.ToolCallFunction{
+ Name: "get_current_weather",
+ Arguments: map[string]interface{}{
+ "location": "Paris, France",
+ "format": "celsius",
+ },
+ },
+ },
+ },
+ },
+ },
+ Options: map[string]any{
+ "temperature": 1.0,
+ "top_p": 1.0,
+ },
+ Stream: &False,
+ },
+ },
+
+ {
+ name: "chat handler error forwarding",
+ body: `{
+ "model": "test-model",
+ "messages": [
+ {"role": "user", "content": 2}
+ ]
+ }`,
+ err: ErrorResponse{
+ Error: Error{
+ Message: "invalid message content type: float64",
+ Type: "invalid_request_error",
+ },
+ },
+ },
+ }
+
+ endpoint := func(c *gin.Context) {
+ c.Status(http.StatusOK)
+ }
+
+ gin.SetMode(gin.TestMode)
+ router := gin.New()
+ router.Use(ChatMiddleware(), captureRequestMiddleware(&capturedRequest))
+ router.Handle(http.MethodPost, "/api/chat", endpoint)
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ req, _ := http.NewRequest(http.MethodPost, "/api/chat", strings.NewReader(tc.body))
+ req.Header.Set("Content-Type", "application/json")
+
+ defer func() { capturedRequest = nil }()
+
+ resp := httptest.NewRecorder()
+ router.ServeHTTP(resp, req)
+
+ var errResp ErrorResponse
+ if resp.Code != http.StatusOK {
+ if err := json.Unmarshal(resp.Body.Bytes(), &errResp); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if capturedRequest != nil && !reflect.DeepEqual(tc.req, *capturedRequest) {
+ t.Fatal("requests did not match")
+ }
+
+ if !reflect.DeepEqual(tc.err, errResp) {
+ t.Fatal("errors did not match")
+ }
+ })
+ }
+}
+
+func TestCompletionsMiddleware(t *testing.T) {
+ type testCase struct {
+ name string
+ body string
+ req api.GenerateRequest
+ err ErrorResponse
+ }
+
+ var capturedRequest *api.GenerateRequest
+
+ testCases := []testCase{
+ {
+ name: "completions handler",
+ body: `{
+ "model": "test-model",
+ "prompt": "Hello",
+ "temperature": 0.8,
+ "stop": ["\n", "stop"],
+ "suffix": "suffix"
+ }`,
+ req: api.GenerateRequest{
+ Model: "test-model",
+ Prompt: "Hello",
+ Options: map[string]any{
+ "frequency_penalty": 0.0,
+ "presence_penalty": 0.0,
+ "temperature": 0.8,
+ "top_p": 1.0,
+ "stop": []any{"\n", "stop"},
+ },
+ Suffix: "suffix",
+ Stream: &False,
+ },
+ },
+ {
+ name: "completions handler error forwarding",
+ body: `{
+ "model": "test-model",
+ "prompt": "Hello",
+ "temperature": null,
+ "stop": [1, 2],
+ "suffix": "suffix"
+ }`,
+ err: ErrorResponse{
+ Error: Error{
+ Message: "invalid type for 'stop' field: float64",
+ Type: "invalid_request_error",
+ },
+ },
+ },
+ }
+
+ endpoint := func(c *gin.Context) {
+ c.Status(http.StatusOK)
+ }
+
+ gin.SetMode(gin.TestMode)
+ router := gin.New()
+ router.Use(CompletionsMiddleware(), captureRequestMiddleware(&capturedRequest))
+ router.Handle(http.MethodPost, "/api/generate", endpoint)
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ req, _ := http.NewRequest(http.MethodPost, "/api/generate", strings.NewReader(tc.body))
+ req.Header.Set("Content-Type", "application/json")
+
+ resp := httptest.NewRecorder()
+ router.ServeHTTP(resp, req)
+
+ var errResp ErrorResponse
+ if resp.Code != http.StatusOK {
+ if err := json.Unmarshal(resp.Body.Bytes(), &errResp); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if capturedRequest != nil && !reflect.DeepEqual(tc.req, *capturedRequest) {
+ t.Fatal("requests did not match")
+ }
+
+ if !reflect.DeepEqual(tc.err, errResp) {
+ t.Fatal("errors did not match")
+ }
+
+ capturedRequest = nil
+ })
+ }
+}
+
+func TestEmbeddingsMiddleware(t *testing.T) {
+ type testCase struct {
+ name string
+ body string
+ req api.EmbedRequest
+ err ErrorResponse
+ }
+
+ var capturedRequest *api.EmbedRequest
+
+ testCases := []testCase{
+ {
+ name: "embed handler single input",
+ body: `{
+ "input": "Hello",
+ "model": "test-model"
+ }`,
+ req: api.EmbedRequest{
+ Input: "Hello",
+ Model: "test-model",
+ },
+ },
+ {
+ name: "embed handler batch input",
+ body: `{
+ "input": ["Hello", "World"],
+ "model": "test-model"
+ }`,
+ req: api.EmbedRequest{
+ Input: []any{"Hello", "World"},
+ Model: "test-model",
+ },
+ },
+ {
+ name: "embed handler error forwarding",
+ body: `{
+ "model": "test-model"
+ }`,
+ err: ErrorResponse{
+ Error: Error{
+ Message: "invalid input",
+ Type: "invalid_request_error",
+ },
+ },
+ },
+ }
+
+ endpoint := func(c *gin.Context) {
+ c.Status(http.StatusOK)
+ }
+
+ gin.SetMode(gin.TestMode)
+ router := gin.New()
+ router.Use(EmbeddingsMiddleware(), captureRequestMiddleware(&capturedRequest))
+ router.Handle(http.MethodPost, "/api/embed", endpoint)
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ req, _ := http.NewRequest(http.MethodPost, "/api/embed", strings.NewReader(tc.body))
+ req.Header.Set("Content-Type", "application/json")
+
+ resp := httptest.NewRecorder()
+ router.ServeHTTP(resp, req)
+
+ var errResp ErrorResponse
+ if resp.Code != http.StatusOK {
+ if err := json.Unmarshal(resp.Body.Bytes(), &errResp); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if capturedRequest != nil && !reflect.DeepEqual(tc.req, *capturedRequest) {
+ t.Fatal("requests did not match")
+ }
+
+ if !reflect.DeepEqual(tc.err, errResp) {
+ t.Fatal("errors did not match")
+ }
+
+ capturedRequest = nil
+ })
+ }
+}
+
+func TestListMiddleware(t *testing.T) {
+ type testCase struct {
+ name string
+ endpoint func(c *gin.Context)
+ resp string
+ }
+
+ testCases := []testCase{
+ {
+ name: "list handler",
+ endpoint: func(c *gin.Context) {
+ c.JSON(http.StatusOK, api.ListResponse{
+ Models: []api.ListModelResponse{
+ {
+ Name: "test-model",
+ ModifiedAt: time.Unix(int64(1686935002), 0).UTC(),
+ },
+ },
+ })
+ },
+ resp: `{
+ "object": "list",
+ "data": [
+ {
+ "id": "test-model",
+ "object": "model",
+ "created": 1686935002,
+ "owned_by": "library"
+ }
+ ]
+ }`,
+ },
+ {
+ name: "list handler empty output",
+ endpoint: func(c *gin.Context) {
+ c.JSON(http.StatusOK, api.ListResponse{})
+ },
+ resp: `{
+ "object": "list",
+ "data": null
+ }`,
+ },
+ }
+
+ gin.SetMode(gin.TestMode)
+
+ for _, tc := range testCases {
+ router := gin.New()
+ router.Use(ListMiddleware())
+ router.Handle(http.MethodGet, "/api/tags", tc.endpoint)
+ req, _ := http.NewRequest(http.MethodGet, "/api/tags", nil)
+
+ resp := httptest.NewRecorder()
+ router.ServeHTTP(resp, req)
+
+ var expected, actual map[string]any
+ err := json.Unmarshal([]byte(tc.resp), &expected)
+ if err != nil {
+ t.Fatalf("failed to unmarshal expected response: %v", err)
+ }
+
+ err = json.Unmarshal(resp.Body.Bytes(), &actual)
+ if err != nil {
+ t.Fatalf("failed to unmarshal actual response: %v", err)
+ }
+
+ if !reflect.DeepEqual(expected, actual) {
+ t.Errorf("responses did not match\nExpected: %+v\nActual: %+v", expected, actual)
+ }
+ }
+}
+
+func TestRetrieveMiddleware(t *testing.T) {
+ type testCase struct {
+ name string
+ endpoint func(c *gin.Context)
+ resp string
+ }
+
+ testCases := []testCase{
+ {
+ name: "retrieve handler",
+ endpoint: func(c *gin.Context) {
+ c.JSON(http.StatusOK, api.ShowResponse{
+ ModifiedAt: time.Unix(int64(1686935002), 0).UTC(),
+ })
+ },
+ resp: `{
+ "id":"test-model",
+ "object":"model",
+ "created":1686935002,
+ "owned_by":"library"}
+ `,
+ },
+ {
+ name: "retrieve handler error forwarding",
+ endpoint: func(c *gin.Context) {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "model not found"})
+ },
+ resp: `{
+ "error": {
+ "code": null,
+ "message": "model not found",
+ "param": null,
+ "type": "api_error"
+ }
+ }`,
+ },
+ }
+
+ gin.SetMode(gin.TestMode)
+
+ for _, tc := range testCases {
+ router := gin.New()
+ router.Use(RetrieveMiddleware())
+ router.Handle(http.MethodGet, "/api/show/:model", tc.endpoint)
+ req, _ := http.NewRequest(http.MethodGet, "/api/show/test-model", nil)
+
+ resp := httptest.NewRecorder()
+ router.ServeHTTP(resp, req)
+
+ var expected, actual map[string]any
+ err := json.Unmarshal([]byte(tc.resp), &expected)
+ if err != nil {
+ t.Fatalf("failed to unmarshal expected response: %v", err)
+ }
+
+ err = json.Unmarshal(resp.Body.Bytes(), &actual)
+ if err != nil {
+ t.Fatalf("failed to unmarshal actual response: %v", err)
+ }
+
+ if !reflect.DeepEqual(expected, actual) {
+ t.Errorf("responses did not match\nExpected: %+v\nActual: %+v", expected, actual)
+ }
+ }
+}
diff --git a/parser/parser.go b/parser/parser.go
index 19d32c29..7f566da4 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -6,116 +6,299 @@ import (
"errors"
"fmt"
"io"
- "log"
+ "strconv"
+ "strings"
+
+ "golang.org/x/text/encoding/unicode"
+ "golang.org/x/text/transform"
)
+type File struct {
+ Commands []Command
+}
+
+func (f File) String() string {
+ var sb strings.Builder
+ for _, cmd := range f.Commands {
+ fmt.Fprintln(&sb, cmd.String())
+ }
+
+ return sb.String()
+}
+
type Command struct {
Name string
Args string
}
-func (c *Command) Reset() {
- c.Name = ""
- c.Args = ""
+func (c Command) String() string {
+ var sb strings.Builder
+ switch c.Name {
+ case "model":
+ fmt.Fprintf(&sb, "FROM %s", c.Args)
+ case "license", "template", "system", "adapter":
+ fmt.Fprintf(&sb, "%s %s", strings.ToUpper(c.Name), quote(c.Args))
+ case "message":
+ role, message, _ := strings.Cut(c.Args, ": ")
+ fmt.Fprintf(&sb, "MESSAGE %s %s", role, quote(message))
+ default:
+ fmt.Fprintf(&sb, "PARAMETER %s %s", c.Name, quote(c.Args))
+ }
+
+ return sb.String()
}
-func Parse(reader io.Reader) ([]Command, error) {
- var commands []Command
- var command, modelCommand Command
+type state int
- scanner := bufio.NewScanner(reader)
- scanner.Buffer(make([]byte, 0, bufio.MaxScanTokenSize), bufio.MaxScanTokenSize)
- scanner.Split(scanModelfile)
- for scanner.Scan() {
- line := scanner.Bytes()
+const (
+ stateNil state = iota
+ stateName
+ stateValue
+ stateParameter
+ stateMessage
+ stateComment
+)
- fields := bytes.SplitN(line, []byte(" "), 2)
- if len(fields) == 0 || len(fields[0]) == 0 {
- continue
+var (
+ errMissingFrom = errors.New("no FROM line")
+ errInvalidMessageRole = errors.New("message role must be one of \"system\", \"user\", or \"assistant\"")
+ errInvalidCommand = errors.New("command must be one of \"from\", \"license\", \"template\", \"system\", \"adapter\", \"parameter\", or \"message\"")
+)
+
+func ParseFile(r io.Reader) (*File, error) {
+ var cmd Command
+ var curr state
+ var b bytes.Buffer
+ var role string
+
+ var f File
+
+ tr := unicode.BOMOverride(unicode.UTF8.NewDecoder())
+ br := bufio.NewReader(transform.NewReader(r, tr))
+
+ for {
+ r, _, err := br.ReadRune()
+ if errors.Is(err, io.EOF) {
+ break
+ } else if err != nil {
+ return nil, err
}
- switch string(bytes.ToUpper(fields[0])) {
- case "FROM":
- command.Name = "model"
- command.Args = string(bytes.TrimSpace(fields[1]))
- // copy command for validation
- modelCommand = command
- case "ADAPTER":
- command.Name = string(bytes.ToLower(fields[0]))
- command.Args = string(bytes.TrimSpace(fields[1]))
- case "LICENSE", "TEMPLATE", "SYSTEM", "PROMPT":
- command.Name = string(bytes.ToLower(fields[0]))
- command.Args = string(fields[1])
- case "PARAMETER":
- fields = bytes.SplitN(fields[1], []byte(" "), 2)
- if len(fields) < 2 {
- return nil, fmt.Errorf("missing value for %s", fields)
+ next, r, err := parseRuneForState(r, curr)
+ if errors.Is(err, io.ErrUnexpectedEOF) {
+ return nil, fmt.Errorf("%w: %s", err, b.String())
+ } else if err != nil {
+ return nil, err
+ }
+
+ // process the state transition, some transitions need to be intercepted and redirected
+ if next != curr {
+ switch curr {
+ case stateName:
+ if !isValidCommand(b.String()) {
+ return nil, errInvalidCommand
+ }
+
+ // next state sometimes depends on the current buffer value
+ switch s := strings.ToLower(b.String()); s {
+ case "from":
+ cmd.Name = "model"
+ case "parameter":
+ // transition to stateParameter which sets command name
+ next = stateParameter
+ case "message":
+ // transition to stateMessage which validates the message role
+ next = stateMessage
+ fallthrough
+ default:
+ cmd.Name = s
+ }
+ case stateParameter:
+ cmd.Name = b.String()
+ case stateMessage:
+ if !isValidMessageRole(b.String()) {
+ return nil, errInvalidMessageRole
+ }
+
+ role = b.String()
+ case stateComment, stateNil:
+ // pass
+ case stateValue:
+ s, ok := unquote(strings.TrimSpace(b.String()))
+ if !ok || isSpace(r) {
+ if _, err := b.WriteRune(r); err != nil {
+ return nil, err
+ }
+
+ continue
+ }
+
+ if role != "" {
+ s = role + ": " + s
+ role = ""
+ }
+
+ cmd.Args = s
+ f.Commands = append(f.Commands, cmd)
}
- command.Name = string(fields[0])
- command.Args = string(bytes.TrimSpace(fields[1]))
- case "EMBED":
- return nil, fmt.Errorf("deprecated command: EMBED is no longer supported, use the /embed API endpoint instead")
+ b.Reset()
+ curr = next
+ }
+
+ if strconv.IsPrint(r) {
+ if _, err := b.WriteRune(r); err != nil {
+ return nil, err
+ }
+ }
+ }
+
+ // flush the buffer
+ switch curr {
+ case stateComment, stateNil:
+ // pass; nothing to flush
+ case stateValue:
+ s, ok := unquote(strings.TrimSpace(b.String()))
+ if !ok {
+ return nil, io.ErrUnexpectedEOF
+ }
+
+ if role != "" {
+ s = role + ": " + s
+ }
+
+ cmd.Args = s
+ f.Commands = append(f.Commands, cmd)
+ default:
+ return nil, io.ErrUnexpectedEOF
+ }
+
+ for _, cmd := range f.Commands {
+ if cmd.Name == "model" {
+ return &f, nil
+ }
+ }
+
+ return nil, errMissingFrom
+}
+
+func parseRuneForState(r rune, cs state) (state, rune, error) {
+ switch cs {
+ case stateNil:
+ switch {
+ case r == '#':
+ return stateComment, 0, nil
+ case isSpace(r), isNewline(r):
+ return stateNil, 0, nil
default:
- if !bytes.HasPrefix(fields[0], []byte("#")) {
- // log a warning for unknown commands
- log.Printf("WARNING: Unknown command: %s", fields[0])
- }
- continue
+ return stateName, r, nil
+ }
+ case stateName:
+ switch {
+ case isAlpha(r):
+ return stateName, r, nil
+ case isSpace(r):
+ return stateValue, 0, nil
+ default:
+ return stateNil, 0, errInvalidCommand
+ }
+ case stateValue:
+ switch {
+ case isNewline(r):
+ return stateNil, r, nil
+ case isSpace(r):
+ return stateNil, r, nil
+ default:
+ return stateValue, r, nil
+ }
+ case stateParameter:
+ switch {
+ case isAlpha(r), isNumber(r), r == '_':
+ return stateParameter, r, nil
+ case isSpace(r):
+ return stateValue, 0, nil
+ default:
+ return stateNil, 0, io.ErrUnexpectedEOF
+ }
+ case stateMessage:
+ switch {
+ case isAlpha(r):
+ return stateMessage, r, nil
+ case isSpace(r):
+ return stateValue, 0, nil
+ default:
+ return stateNil, 0, io.ErrUnexpectedEOF
+ }
+ case stateComment:
+ switch {
+ case isNewline(r):
+ return stateNil, 0, nil
+ default:
+ return stateComment, 0, nil
+ }
+ default:
+ return stateNil, 0, errors.New("")
+ }
+}
+
+func quote(s string) string {
+ if strings.Contains(s, "\n") || strings.HasPrefix(s, " ") || strings.HasSuffix(s, " ") {
+ if strings.Contains(s, "\"") {
+ return `"""` + s + `"""`
}
- commands = append(commands, command)
- command.Reset()
+ return `"` + s + `"`
}
- if modelCommand.Args == "" {
- return nil, errors.New("no FROM line for the model was specified")
- }
-
- return commands, scanner.Err()
+ return s
}
-func scanModelfile(data []byte, atEOF bool) (advance int, token []byte, err error) {
- advance, token, err = scan([]byte(`"""`), []byte(`"""`), data, atEOF)
- if err != nil {
- return 0, nil, err
- }
-
- if advance > 0 && token != nil {
- return advance, token, nil
- }
-
- advance, token, err = scan([]byte(`"`), []byte(`"`), data, atEOF)
- if err != nil {
- return 0, nil, err
- }
-
- if advance > 0 && token != nil {
- return advance, token, nil
- }
-
- return bufio.ScanLines(data, atEOF)
-}
-
-func scan(openBytes, closeBytes, data []byte, atEOF bool) (advance int, token []byte, err error) {
- newline := bytes.IndexByte(data, '\n')
-
- if start := bytes.Index(data, openBytes); start >= 0 && start < newline {
- end := bytes.Index(data[start+len(openBytes):], closeBytes)
- if end < 0 {
- if atEOF {
- return 0, nil, fmt.Errorf("unterminated %s: expecting %s", openBytes, closeBytes)
- } else {
- return 0, nil, nil
- }
+func unquote(s string) (string, bool) {
+ // TODO: single quotes
+ if len(s) >= 3 && s[:3] == `"""` {
+ if len(s) >= 6 && s[len(s)-3:] == `"""` {
+ return s[3 : len(s)-3], true
}
- n := start + len(openBytes) + end + len(closeBytes)
-
- newData := data[:start]
- newData = append(newData, data[start+len(openBytes):n-len(closeBytes)]...)
- return n, newData, nil
+ return "", false
}
- return 0, nil, nil
+ if len(s) >= 1 && s[0] == '"' {
+ if len(s) >= 2 && s[len(s)-1] == '"' {
+ return s[1 : len(s)-1], true
+ }
+
+ return "", false
+ }
+
+ return s, true
+}
+
+func isAlpha(r rune) bool {
+ return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z'
+}
+
+func isNumber(r rune) bool {
+ return r >= '0' && r <= '9'
+}
+
+func isSpace(r rune) bool {
+ return r == ' ' || r == '\t'
+}
+
+func isNewline(r rune) bool {
+ return r == '\r' || r == '\n'
+}
+
+func isValidMessageRole(role string) bool {
+ return role == "system" || role == "user" || role == "assistant"
+}
+
+func isValidCommand(cmd string) bool {
+ switch strings.ToLower(cmd) {
+ case "from", "license", "template", "system", "adapter", "parameter", "message":
+ return true
+ default:
+ return false
+ }
}
diff --git a/parser/parser_test.go b/parser/parser_test.go
new file mode 100644
index 00000000..ebd8a7ff
--- /dev/null
+++ b/parser/parser_test.go
@@ -0,0 +1,643 @@
+package parser
+
+import (
+ "bytes"
+ "encoding/binary"
+ "fmt"
+ "io"
+ "strings"
+ "testing"
+ "unicode/utf16"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/unicode"
+)
+
+func TestParseFileFile(t *testing.T) {
+ input := `
+FROM model1
+ADAPTER adapter1
+LICENSE MIT
+PARAMETER param1 value1
+PARAMETER param2 value2
+TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|>
+
+{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>
+
+{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>
+
+{{ .Response }}<|eot_id|>"""
+`
+
+ reader := strings.NewReader(input)
+
+ modelfile, err := ParseFile(reader)
+ require.NoError(t, err)
+
+ expectedCommands := []Command{
+ {Name: "model", Args: "model1"},
+ {Name: "adapter", Args: "adapter1"},
+ {Name: "license", Args: "MIT"},
+ {Name: "param1", Args: "value1"},
+ {Name: "param2", Args: "value2"},
+ {Name: "template", Args: "{{ if .System }}<|start_header_id|>system<|end_header_id|>\n\n{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>\n\n{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>\n\n{{ .Response }}<|eot_id|>"},
+ }
+
+ assert.Equal(t, expectedCommands, modelfile.Commands)
+}
+
+func TestParseFileTrimSpace(t *testing.T) {
+ input := `
+FROM " model 1"
+ADAPTER adapter3
+LICENSE "MIT "
+PARAMETER param1 value1
+PARAMETER param2 value2
+TEMPLATE """ {{ if .System }}<|start_header_id|>system<|end_header_id|>
+
+{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>
+
+{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>
+
+{{ .Response }}<|eot_id|> """
+`
+
+ reader := strings.NewReader(input)
+
+ modelfile, err := ParseFile(reader)
+ require.NoError(t, err)
+
+ expectedCommands := []Command{
+ {Name: "model", Args: " model 1"},
+ {Name: "adapter", Args: "adapter3"},
+ {Name: "license", Args: "MIT "},
+ {Name: "param1", Args: "value1"},
+ {Name: "param2", Args: "value2"},
+ {Name: "template", Args: " {{ if .System }}<|start_header_id|>system<|end_header_id|>\n\n{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>\n\n{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>\n\n{{ .Response }}<|eot_id|> "},
+ }
+
+ assert.Equal(t, expectedCommands, modelfile.Commands)
+}
+
+func TestParseFileFrom(t *testing.T) {
+ cases := []struct {
+ input string
+ expected []Command
+ err error
+ }{
+ {
+ "FROM \"FOO BAR \"",
+ []Command{{Name: "model", Args: "FOO BAR "}},
+ nil,
+ },
+ {
+ "FROM \"FOO BAR\"\nPARAMETER param1 value1",
+ []Command{{Name: "model", Args: "FOO BAR"}, {Name: "param1", Args: "value1"}},
+ nil,
+ },
+ {
+ "FROM FOOO BAR ",
+ []Command{{Name: "model", Args: "FOOO BAR"}},
+ nil,
+ },
+ {
+ "FROM /what/is/the path ",
+ []Command{{Name: "model", Args: "/what/is/the path"}},
+ nil,
+ },
+ {
+ "FROM foo",
+ []Command{{Name: "model", Args: "foo"}},
+ nil,
+ },
+ {
+ "FROM /path/to/model",
+ []Command{{Name: "model", Args: "/path/to/model"}},
+ nil,
+ },
+ {
+ "FROM /path/to/model/fp16.bin",
+ []Command{{Name: "model", Args: "/path/to/model/fp16.bin"}},
+ nil,
+ },
+ {
+ "FROM llama3:latest",
+ []Command{{Name: "model", Args: "llama3:latest"}},
+ nil,
+ },
+ {
+ "FROM llama3:7b-instruct-q4_K_M",
+ []Command{{Name: "model", Args: "llama3:7b-instruct-q4_K_M"}},
+ nil,
+ },
+ {
+ "", nil, errMissingFrom,
+ },
+ {
+ "PARAMETER param1 value1",
+ nil,
+ errMissingFrom,
+ },
+ {
+ "PARAMETER param1 value1\nFROM foo",
+ []Command{{Name: "param1", Args: "value1"}, {Name: "model", Args: "foo"}},
+ nil,
+ },
+ {
+ "PARAMETER what the \nFROM lemons make lemonade ",
+ []Command{{Name: "what", Args: "the"}, {Name: "model", Args: "lemons make lemonade"}},
+ nil,
+ },
+ }
+
+ for _, c := range cases {
+ t.Run("", func(t *testing.T) {
+ modelfile, err := ParseFile(strings.NewReader(c.input))
+ require.ErrorIs(t, err, c.err)
+ if modelfile != nil {
+ assert.Equal(t, c.expected, modelfile.Commands)
+ }
+ })
+ }
+}
+
+func TestParseFileParametersMissingValue(t *testing.T) {
+ input := `
+FROM foo
+PARAMETER param1
+`
+
+ reader := strings.NewReader(input)
+
+ _, err := ParseFile(reader)
+ require.ErrorIs(t, err, io.ErrUnexpectedEOF)
+}
+
+func TestParseFileBadCommand(t *testing.T) {
+ input := `
+FROM foo
+BADCOMMAND param1 value1
+`
+ _, err := ParseFile(strings.NewReader(input))
+ require.ErrorIs(t, err, errInvalidCommand)
+}
+
+func TestParseFileMessages(t *testing.T) {
+ cases := []struct {
+ input string
+ expected []Command
+ err error
+ }{
+ {
+ `
+FROM foo
+MESSAGE system You are a file parser. Always parse things.
+`,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "message", Args: "system: You are a file parser. Always parse things."},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+MESSAGE system You are a file parser. Always parse things.`,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "message", Args: "system: You are a file parser. Always parse things."},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+MESSAGE system You are a file parser. Always parse things.
+MESSAGE user Hey there!
+MESSAGE assistant Hello, I want to parse all the things!
+`,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "message", Args: "system: You are a file parser. Always parse things."},
+ {Name: "message", Args: "user: Hey there!"},
+ {Name: "message", Args: "assistant: Hello, I want to parse all the things!"},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+MESSAGE system """
+You are a multiline file parser. Always parse things.
+"""
+ `,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "message", Args: "system: \nYou are a multiline file parser. Always parse things.\n"},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+MESSAGE badguy I'm a bad guy!
+`,
+ nil,
+ errInvalidMessageRole,
+ },
+ {
+ `
+FROM foo
+MESSAGE system
+`,
+ nil,
+ io.ErrUnexpectedEOF,
+ },
+ {
+ `
+FROM foo
+MESSAGE system`,
+ nil,
+ io.ErrUnexpectedEOF,
+ },
+ }
+
+ for _, c := range cases {
+ t.Run("", func(t *testing.T) {
+ modelfile, err := ParseFile(strings.NewReader(c.input))
+ require.ErrorIs(t, err, c.err)
+ if modelfile != nil {
+ assert.Equal(t, c.expected, modelfile.Commands)
+ }
+ })
+ }
+}
+
+func TestParseFileQuoted(t *testing.T) {
+ cases := []struct {
+ multiline string
+ expected []Command
+ err error
+ }{
+ {
+ `
+FROM foo
+SYSTEM """
+This is a
+multiline system.
+"""
+ `,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "system", Args: "\nThis is a\nmultiline system.\n"},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+SYSTEM """
+This is a
+multiline system."""
+ `,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "system", Args: "\nThis is a\nmultiline system."},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+SYSTEM """This is a
+multiline system."""
+ `,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "system", Args: "This is a\nmultiline system."},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+SYSTEM """This is a multiline system."""
+ `,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "system", Args: "This is a multiline system."},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+SYSTEM """This is a multiline system.""
+ `,
+ nil,
+ io.ErrUnexpectedEOF,
+ },
+ {
+ `
+FROM foo
+SYSTEM "
+ `,
+ nil,
+ io.ErrUnexpectedEOF,
+ },
+ {
+ `
+FROM foo
+SYSTEM """
+This is a multiline system with "quotes".
+"""
+`,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "system", Args: "\nThis is a multiline system with \"quotes\".\n"},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+SYSTEM """"""
+`,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "system", Args: ""},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+SYSTEM ""
+`,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "system", Args: ""},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+SYSTEM "'"
+`,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "system", Args: "'"},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+SYSTEM """''"'""'""'"'''''""'""'"""
+`,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "system", Args: `''"'""'""'"'''''""'""'`},
+ },
+ nil,
+ },
+ {
+ `
+FROM foo
+TEMPLATE """
+{{ .Prompt }}
+"""`,
+ []Command{
+ {Name: "model", Args: "foo"},
+ {Name: "template", Args: "\n{{ .Prompt }}\n"},
+ },
+ nil,
+ },
+ }
+
+ for _, c := range cases {
+ t.Run("", func(t *testing.T) {
+ modelfile, err := ParseFile(strings.NewReader(c.multiline))
+ require.ErrorIs(t, err, c.err)
+ if modelfile != nil {
+ assert.Equal(t, c.expected, modelfile.Commands)
+ }
+ })
+ }
+}
+
+func TestParseFileParameters(t *testing.T) {
+ cases := map[string]struct {
+ name, value string
+ }{
+ "numa true": {"numa", "true"},
+ "num_ctx 1": {"num_ctx", "1"},
+ "num_batch 1": {"num_batch", "1"},
+ "num_gqa 1": {"num_gqa", "1"},
+ "num_gpu 1": {"num_gpu", "1"},
+ "main_gpu 1": {"main_gpu", "1"},
+ "low_vram true": {"low_vram", "true"},
+ "f16_kv true": {"f16_kv", "true"},
+ "logits_all true": {"logits_all", "true"},
+ "vocab_only true": {"vocab_only", "true"},
+ "use_mmap true": {"use_mmap", "true"},
+ "use_mlock true": {"use_mlock", "true"},
+ "num_thread 1": {"num_thread", "1"},
+ "num_keep 1": {"num_keep", "1"},
+ "seed 1": {"seed", "1"},
+ "num_predict 1": {"num_predict", "1"},
+ "top_k 1": {"top_k", "1"},
+ "top_p 1.0": {"top_p", "1.0"},
+ "min_p 0.05": {"min_p", "0.05"},
+ "tfs_z 1.0": {"tfs_z", "1.0"},
+ "typical_p 1.0": {"typical_p", "1.0"},
+ "repeat_last_n 1": {"repeat_last_n", "1"},
+ "temperature 1.0": {"temperature", "1.0"},
+ "repeat_penalty 1.0": {"repeat_penalty", "1.0"},
+ "presence_penalty 1.0": {"presence_penalty", "1.0"},
+ "frequency_penalty 1.0": {"frequency_penalty", "1.0"},
+ "mirostat 1": {"mirostat", "1"},
+ "mirostat_tau 1.0": {"mirostat_tau", "1.0"},
+ "mirostat_eta 1.0": {"mirostat_eta", "1.0"},
+ "penalize_newline true": {"penalize_newline", "true"},
+ "stop ### User:": {"stop", "### User:"},
+ "stop ### User: ": {"stop", "### User:"},
+ "stop \"### User:\"": {"stop", "### User:"},
+ "stop \"### User: \"": {"stop", "### User: "},
+ "stop \"\"\"### User:\"\"\"": {"stop", "### User:"},
+ "stop \"\"\"### User:\n\"\"\"": {"stop", "### User:\n"},
+ "stop <|endoftext|>": {"stop", "<|endoftext|>"},
+ "stop <|eot_id|>": {"stop", "<|eot_id|>"},
+ "stop </s>": {"stop", "</s>"},
+ }
+
+ for k, v := range cases {
+ t.Run(k, func(t *testing.T) {
+ var b bytes.Buffer
+ fmt.Fprintln(&b, "FROM foo")
+ fmt.Fprintln(&b, "PARAMETER", k)
+ modelfile, err := ParseFile(&b)
+ require.NoError(t, err)
+
+ assert.Equal(t, []Command{
+ {Name: "model", Args: "foo"},
+ {Name: v.name, Args: v.value},
+ }, modelfile.Commands)
+ })
+ }
+}
+
+func TestParseFileComments(t *testing.T) {
+ cases := []struct {
+ input string
+ expected []Command
+ }{
+ {
+ `
+# comment
+FROM foo
+ `,
+ []Command{
+ {Name: "model", Args: "foo"},
+ },
+ },
+ }
+
+ for _, c := range cases {
+ t.Run("", func(t *testing.T) {
+ modelfile, err := ParseFile(strings.NewReader(c.input))
+ require.NoError(t, err)
+ assert.Equal(t, c.expected, modelfile.Commands)
+ })
+ }
+}
+
+func TestParseFileFormatParseFile(t *testing.T) {
+ cases := []string{
+ `
+FROM foo
+ADAPTER adapter1
+LICENSE MIT
+PARAMETER param1 value1
+PARAMETER param2 value2
+TEMPLATE template1
+MESSAGE system You are a file parser. Always parse things.
+MESSAGE user Hey there!
+MESSAGE assistant Hello, I want to parse all the things!
+`,
+ `
+FROM foo
+ADAPTER adapter1
+LICENSE MIT
+PARAMETER param1 value1
+PARAMETER param2 value2
+TEMPLATE template1
+MESSAGE system """
+You are a store greeter. Always responsed with "Hello!".
+"""
+MESSAGE user Hey there!
+MESSAGE assistant Hello, I want to parse all the things!
+`,
+ `
+FROM foo
+ADAPTER adapter1
+LICENSE """
+Very long and boring legal text.
+Blah blah blah.
+"Oh look, a quote!"
+"""
+
+PARAMETER param1 value1
+PARAMETER param2 value2
+TEMPLATE template1
+MESSAGE system """
+You are a store greeter. Always responsed with "Hello!".
+"""
+MESSAGE user Hey there!
+MESSAGE assistant Hello, I want to parse all the things!
+`,
+ `
+FROM foo
+SYSTEM ""
+`,
+ }
+
+ for _, c := range cases {
+ t.Run("", func(t *testing.T) {
+ modelfile, err := ParseFile(strings.NewReader(c))
+ require.NoError(t, err)
+
+ modelfile2, err := ParseFile(strings.NewReader(modelfile.String()))
+ require.NoError(t, err)
+
+ assert.Equal(t, modelfile, modelfile2)
+ })
+ }
+}
+
+func TestParseFileUTF16ParseFile(t *testing.T) {
+ data := `FROM bob
+PARAMETER param1 1
+PARAMETER param2 4096
+SYSTEM You are a utf16 file.
+`
+
+ expected := []Command{
+ {Name: "model", Args: "bob"},
+ {Name: "param1", Args: "1"},
+ {Name: "param2", Args: "4096"},
+ {Name: "system", Args: "You are a utf16 file."},
+ }
+
+ t.Run("le", func(t *testing.T) {
+ var b bytes.Buffer
+ require.NoError(t, binary.Write(&b, binary.LittleEndian, []byte{0xff, 0xfe}))
+ require.NoError(t, binary.Write(&b, binary.LittleEndian, utf16.Encode([]rune(data))))
+
+ actual, err := ParseFile(&b)
+ require.NoError(t, err)
+
+ assert.Equal(t, expected, actual.Commands)
+ })
+
+ t.Run("be", func(t *testing.T) {
+ var b bytes.Buffer
+ require.NoError(t, binary.Write(&b, binary.BigEndian, []byte{0xfe, 0xff}))
+ require.NoError(t, binary.Write(&b, binary.BigEndian, utf16.Encode([]rune(data))))
+
+ actual, err := ParseFile(&b)
+ require.NoError(t, err)
+ assert.Equal(t, expected, actual.Commands)
+ })
+}
+
+func TestParseMultiByte(t *testing.T) {
+ input := `FROM test
+ SYSTEM 你好👋`
+
+ expect := []Command{
+ {Name: "model", Args: "test"},
+ {Name: "system", Args: "你好👋"},
+ }
+
+ encodings := []encoding.Encoding{
+ unicode.UTF8,
+ unicode.UTF16(unicode.LittleEndian, unicode.UseBOM),
+ unicode.UTF16(unicode.BigEndian, unicode.UseBOM),
+ }
+
+ for _, encoding := range encodings {
+ t.Run(fmt.Sprintf("%s", encoding), func(t *testing.T) {
+ s, err := encoding.NewEncoder().String(input)
+ require.NoError(t, err)
+
+ actual, err := ParseFile(strings.NewReader(s))
+ require.NoError(t, err)
+
+ assert.Equal(t, expect, actual.Commands)
+ })
+ }
+}
diff --git a/progress/bar.go b/progress/bar.go
index 9a0da825..410b6e23 100644
--- a/progress/bar.go
+++ b/progress/bar.go
@@ -6,8 +6,9 @@ import (
"strings"
"time"
- "github.com/jmorganca/ollama/format"
"golang.org/x/term"
+
+ "github.com/ollama/ollama/format"
)
type Bar struct {
diff --git a/progress/progress.go b/progress/progress.go
index 78917e9c..102830a8 100644
--- a/progress/progress.go
+++ b/progress/progress.go
@@ -59,7 +59,7 @@ func (p *Progress) StopAndClear() bool {
stopped := p.stop()
if stopped {
// clear all progress lines
- for i := 0; i < p.pos; i++ {
+ for i := range p.pos {
if i > 0 {
fmt.Fprint(p.w, "\033[A")
}
@@ -77,7 +77,7 @@ func (p *Progress) Add(key string, state State) {
p.states = append(p.states, state)
}
-func (p *Progress) render() error {
+func (p *Progress) render() {
p.mu.Lock()
defer p.mu.Unlock()
@@ -85,7 +85,7 @@ func (p *Progress) render() error {
defer fmt.Fprint(p.w, "\033[?25h")
// clear already rendered progress lines
- for i := 0; i < p.pos; i++ {
+ for i := range p.pos {
if i > 0 {
fmt.Fprint(p.w, "\033[A")
}
@@ -101,8 +101,6 @@ func (p *Progress) render() error {
}
p.pos = len(p.states)
-
- return nil
}
func (p *Progress) start() {
diff --git a/progress/spinner.go b/progress/spinner.go
index 02f3f9fb..e39a45ee 100644
--- a/progress/spinner.go
+++ b/progress/spinner.go
@@ -3,11 +3,12 @@ package progress
import (
"fmt"
"strings"
+ "sync/atomic"
"time"
)
type Spinner struct {
- message string
+ message atomic.Value
messageWidth int
parts []string
@@ -21,20 +22,25 @@ type Spinner struct {
func NewSpinner(message string) *Spinner {
s := &Spinner{
- message: message,
parts: []string{
"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏",
},
started: time.Now(),
}
+ s.SetMessage(message)
go s.start()
return s
}
+func (s *Spinner) SetMessage(message string) {
+ s.message.Store(message)
+}
+
func (s *Spinner) String() string {
var sb strings.Builder
- if len(s.message) > 0 {
- message := strings.TrimSpace(s.message)
+
+ if message, ok := s.message.Load().(string); ok && len(message) > 0 {
+ message := strings.TrimSpace(message)
if s.messageWidth > 0 && len(message) > s.messageWidth {
message = message[:s.messageWidth]
}
diff --git a/readline/buffer.go b/readline/buffer.go
index d66b512a..d91fe0a9 100644
--- a/readline/buffer.go
+++ b/readline/buffer.go
@@ -5,51 +5,82 @@ import (
"os"
"github.com/emirpasic/gods/lists/arraylist"
+ "github.com/mattn/go-runewidth"
"golang.org/x/term"
)
type Buffer struct {
- Pos int
- Buf *arraylist.List
- Prompt *Prompt
- LineWidth int
- Width int
- Height int
+ DisplayPos int
+ Pos int
+ Buf *arraylist.List
+ // LineHasSpace is an arraylist of bools to keep track of whether a line has a space at the end
+ LineHasSpace *arraylist.List
+ Prompt *Prompt
+ LineWidth int
+ Width int
+ Height int
}
func NewBuffer(prompt *Prompt) (*Buffer, error) {
fd := int(os.Stdout.Fd())
- width, height, err := term.GetSize(fd)
- if err != nil {
- fmt.Println("Error getting size:", err)
- return nil, err
+ width, height := 80, 24
+ if termWidth, termHeight, err := term.GetSize(fd); err == nil {
+ width, height = termWidth, termHeight
}
- lwidth := width - len(prompt.Prompt)
- if prompt.UseAlt {
- lwidth = width - len(prompt.AltPrompt)
- }
+ lwidth := width - len(prompt.prompt())
b := &Buffer{
- Pos: 0,
- Buf: arraylist.New(),
- Prompt: prompt,
- Width: width,
- Height: height,
- LineWidth: lwidth,
+ DisplayPos: 0,
+ Pos: 0,
+ Buf: arraylist.New(),
+ LineHasSpace: arraylist.New(),
+ Prompt: prompt,
+ Width: width,
+ Height: height,
+ LineWidth: lwidth,
}
return b, nil
}
+func (b *Buffer) GetLineSpacing(line int) bool {
+ hasSpace, _ := b.LineHasSpace.Get(line)
+
+ if hasSpace == nil {
+ return false
+ }
+
+ return hasSpace.(bool)
+}
+
func (b *Buffer) MoveLeft() {
if b.Pos > 0 {
- if b.Pos%b.LineWidth == 0 {
- fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width))
- } else {
- fmt.Print(CursorLeft)
+ // asserts that we retrieve a rune
+ if e, ok := b.Buf.Get(b.Pos - 1); ok {
+ if r, ok := e.(rune); ok {
+ rLength := runewidth.RuneWidth(r)
+
+ if b.DisplayPos%b.LineWidth == 0 {
+ fmt.Print(CursorUp + CursorBOL + CursorRightN(b.Width))
+ if rLength == 2 {
+ fmt.Print(CursorLeft)
+ }
+
+ line := b.DisplayPos/b.LineWidth - 1
+ hasSpace := b.GetLineSpacing(line)
+ if hasSpace {
+ b.DisplayPos -= 1
+ fmt.Print(CursorLeft)
+ }
+ } else {
+ fmt.Print(CursorLeftN(rLength))
+ }
+
+ b.Pos -= 1
+ b.DisplayPos -= rLength
+ }
}
- b.Pos -= 1
}
}
@@ -75,18 +106,32 @@ func (b *Buffer) MoveLeftWord() {
}
func (b *Buffer) MoveRight() {
- if b.Pos < b.Size() {
- b.Pos += 1
- if b.Pos%b.LineWidth == 0 {
- fmt.Printf(CursorDown + CursorBOL + cursorRightN(b.PromptSize()))
- } else {
- fmt.Print(CursorRight)
+ if b.Pos < b.Buf.Size() {
+ if e, ok := b.Buf.Get(b.Pos); ok {
+ if r, ok := e.(rune); ok {
+ rLength := runewidth.RuneWidth(r)
+ b.Pos += 1
+ hasSpace := b.GetLineSpacing(b.DisplayPos / b.LineWidth)
+ b.DisplayPos += rLength
+
+ if b.DisplayPos%b.LineWidth == 0 {
+ fmt.Print(CursorDown + CursorBOL + CursorRightN(len(b.Prompt.prompt())))
+ } else if (b.DisplayPos-rLength)%b.LineWidth == b.LineWidth-1 && hasSpace {
+ fmt.Print(CursorDown + CursorBOL + CursorRightN(len(b.Prompt.prompt())+rLength))
+ b.DisplayPos += 1
+ } else if b.LineHasSpace.Size() > 0 && b.DisplayPos%b.LineWidth == b.LineWidth-1 && hasSpace {
+ fmt.Print(CursorDown + CursorBOL + CursorRightN(len(b.Prompt.prompt())))
+ b.DisplayPos += 1
+ } else {
+ fmt.Print(CursorRightN(rLength))
+ }
+ }
}
}
}
func (b *Buffer) MoveRightWord() {
- if b.Pos < b.Size() {
+ if b.Pos < b.Buf.Size() {
for {
b.MoveRight()
v, _ := b.Buf.Get(b.Pos)
@@ -94,7 +139,7 @@ func (b *Buffer) MoveRightWord() {
break
}
- if b.Pos == b.Size() {
+ if b.Pos == b.Buf.Size() {
break
}
}
@@ -103,103 +148,197 @@ func (b *Buffer) MoveRightWord() {
func (b *Buffer) MoveToStart() {
if b.Pos > 0 {
- currLine := b.Pos / b.LineWidth
+ currLine := b.DisplayPos / b.LineWidth
if currLine > 0 {
- for cnt := 0; cnt < currLine; cnt++ {
+ for range currLine {
fmt.Print(CursorUp)
}
}
- fmt.Printf(CursorBOL + cursorRightN(b.PromptSize()))
+ fmt.Print(CursorBOL + CursorRightN(len(b.Prompt.prompt())))
b.Pos = 0
+ b.DisplayPos = 0
}
}
func (b *Buffer) MoveToEnd() {
- if b.Pos < b.Size() {
- currLine := b.Pos / b.LineWidth
- totalLines := b.Size() / b.LineWidth
+ if b.Pos < b.Buf.Size() {
+ currLine := b.DisplayPos / b.LineWidth
+ totalLines := b.DisplaySize() / b.LineWidth
if currLine < totalLines {
- for cnt := 0; cnt < totalLines-currLine; cnt++ {
+ for range totalLines - currLine {
fmt.Print(CursorDown)
}
- remainder := b.Size() % b.LineWidth
- fmt.Printf(CursorBOL + cursorRightN(b.PromptSize()+remainder))
+ remainder := b.DisplaySize() % b.LineWidth
+ fmt.Print(CursorBOL + CursorRightN(len(b.Prompt.prompt())+remainder))
} else {
- fmt.Print(cursorRightN(b.Size() - b.Pos))
+ fmt.Print(CursorRightN(b.DisplaySize() - b.DisplayPos))
}
- b.Pos = b.Size()
+ b.Pos = b.Buf.Size()
+ b.DisplayPos = b.DisplaySize()
}
}
-func (b *Buffer) Size() int {
- return b.Buf.Size()
-}
-
-func min(n, m int) int {
- if n > m {
- return m
+func (b *Buffer) DisplaySize() int {
+ sum := 0
+ for i := range b.Buf.Size() {
+ if e, ok := b.Buf.Get(i); ok {
+ if r, ok := e.(rune); ok {
+ sum += runewidth.RuneWidth(r)
+ }
+ }
}
- return n
-}
-func (b *Buffer) PromptSize() int {
- if b.Prompt.UseAlt {
- return len(b.Prompt.AltPrompt)
- }
- return len(b.Prompt.Prompt)
+ return sum
}
func (b *Buffer) Add(r rune) {
if b.Pos == b.Buf.Size() {
- fmt.Printf("%c", r)
- b.Buf.Add(r)
- b.Pos += 1
- if b.Pos > 0 && b.Pos%b.LineWidth == 0 {
+ b.AddChar(r, false)
+ } else {
+ b.AddChar(r, true)
+ }
+}
+
+func (b *Buffer) AddChar(r rune, insert bool) {
+ rLength := runewidth.RuneWidth(r)
+ b.DisplayPos += rLength
+
+ if b.Pos > 0 {
+ if b.DisplayPos%b.LineWidth == 0 {
+ fmt.Printf("%c", r)
fmt.Printf("\n%s", b.Prompt.AltPrompt)
+
+ if insert {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth-1, false)
+ } else {
+ b.LineHasSpace.Add(false)
+ }
+
+ // this case occurs when a double-width rune crosses the line boundary
+ } else if b.DisplayPos%b.LineWidth < (b.DisplayPos-rLength)%b.LineWidth {
+ if insert {
+ fmt.Print(ClearToEOL)
+ }
+ fmt.Printf("\n%s", b.Prompt.AltPrompt)
+ b.DisplayPos += 1
+ fmt.Printf("%c", r)
+
+ if insert {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth-1, true)
+ } else {
+ b.LineHasSpace.Add(true)
+ }
+ } else {
+ fmt.Printf("%c", r)
}
} else {
fmt.Printf("%c", r)
+ }
+
+ if insert {
b.Buf.Insert(b.Pos, r)
- b.Pos += 1
- if b.Pos > 0 && b.Pos%b.LineWidth == 0 {
- fmt.Printf("\n%s", b.Prompt.AltPrompt)
- }
+ } else {
+ b.Buf.Add(r)
+ }
+
+ b.Pos += 1
+
+ if insert {
b.drawRemaining()
}
}
+func (b *Buffer) countRemainingLineWidth(place int) int {
+ var sum int
+ counter := -1
+ var prevLen int
+
+ for place <= b.LineWidth {
+ counter += 1
+ sum += prevLen
+ if e, ok := b.Buf.Get(b.Pos + counter); ok {
+ if r, ok := e.(rune); ok {
+ place += runewidth.RuneWidth(r)
+ prevLen = len(string(r))
+ }
+ } else {
+ break
+ }
+ }
+
+ return sum
+}
+
func (b *Buffer) drawRemaining() {
var place int
remainingText := b.StringN(b.Pos)
if b.Pos > 0 {
- place = b.Pos % b.LineWidth
+ place = b.DisplayPos % b.LineWidth
}
fmt.Print(CursorHide)
// render the rest of the current line
- currLine := remainingText[:min(b.LineWidth-place, len(remainingText))]
+ currLineLength := b.countRemainingLineWidth(place)
+
+ currLine := remainingText[:min(currLineLength, len(remainingText))]
+ currLineSpace := runewidth.StringWidth(currLine)
+ remLength := runewidth.StringWidth(remainingText)
+
if len(currLine) > 0 {
- fmt.Printf(ClearToEOL + currLine)
- fmt.Print(cursorLeftN(len(currLine)))
+ fmt.Print(ClearToEOL + currLine + CursorLeftN(currLineSpace))
} else {
fmt.Print(ClearToEOL)
}
+ if currLineSpace != b.LineWidth-place && currLineSpace != remLength {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth, true)
+ } else if currLineSpace != b.LineWidth-place {
+ b.LineHasSpace.Remove(b.DisplayPos / b.LineWidth)
+ } else {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth, false)
+ }
+
+ if (b.DisplayPos+currLineSpace)%b.LineWidth == 0 && currLine == remainingText {
+ fmt.Print(CursorRightN(currLineSpace))
+ fmt.Printf("\n%s", b.Prompt.AltPrompt)
+ fmt.Print(CursorUp + CursorBOL + CursorRightN(b.Width-currLineSpace))
+ }
+
// render the other lines
- if len(remainingText) > len(currLine) {
- remaining := []rune(remainingText[len(currLine):])
+ if remLength > currLineSpace {
+ remaining := (remainingText[len(currLine):])
var totalLines int
- for i, c := range remaining {
- if i%b.LineWidth == 0 {
+ var displayLength int
+ var lineLength int = currLineSpace
+
+ for _, c := range remaining {
+ if displayLength == 0 || (displayLength+runewidth.RuneWidth(c))%b.LineWidth < displayLength%b.LineWidth {
fmt.Printf("\n%s", b.Prompt.AltPrompt)
totalLines += 1
+
+ if displayLength != 0 {
+ if lineLength == b.LineWidth {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth+totalLines-1, false)
+ } else {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth+totalLines-1, true)
+ }
+ }
+
+ lineLength = 0
}
+
+ displayLength += runewidth.RuneWidth(c)
+ lineLength += runewidth.RuneWidth(c)
fmt.Printf("%c", c)
}
- fmt.Print(ClearToEOL)
- fmt.Print(cursorUpN(totalLines))
- fmt.Printf(CursorBOL + cursorRightN(b.Width-len(currLine)))
+ fmt.Print(ClearToEOL + CursorUpN(totalLines) + CursorBOL + CursorRightN(b.Width-currLineSpace))
+
+ hasSpace := b.GetLineSpacing(b.DisplayPos / b.LineWidth)
+
+ if hasSpace && b.DisplayPos%b.LineWidth != b.LineWidth-1 {
+ fmt.Print(CursorLeft)
+ }
}
fmt.Print(CursorShow)
@@ -207,47 +346,80 @@ func (b *Buffer) drawRemaining() {
func (b *Buffer) Remove() {
if b.Buf.Size() > 0 && b.Pos > 0 {
- if b.Pos%b.LineWidth == 0 {
- // if the user backspaces over the word boundary, do this magic to clear the line
- // and move to the end of the previous line
- fmt.Printf(CursorBOL + ClearToEOL)
- fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width) + " " + CursorLeft)
- } else {
- fmt.Printf(CursorLeft + " " + CursorLeft)
- }
+ if e, ok := b.Buf.Get(b.Pos - 1); ok {
+ if r, ok := e.(rune); ok {
+ rLength := runewidth.RuneWidth(r)
+ hasSpace := b.GetLineSpacing(b.DisplayPos/b.LineWidth - 1)
- var eraseExtraLine bool
- if (b.Size()-1)%b.LineWidth == 0 {
- eraseExtraLine = true
- }
+ if b.DisplayPos%b.LineWidth == 0 {
+ // if the user backspaces over the word boundary, do this magic to clear the line
+ // and move to the end of the previous line
+ fmt.Print(CursorBOL + ClearToEOL + CursorUp + CursorBOL + CursorRightN(b.Width))
- b.Pos -= 1
- b.Buf.Remove(b.Pos)
+ if b.DisplaySize()%b.LineWidth < (b.DisplaySize()-rLength)%b.LineWidth {
+ b.LineHasSpace.Remove(b.DisplayPos/b.LineWidth - 1)
+ }
- if b.Pos < b.Size() {
- b.drawRemaining()
- // this erases a line which is left over when backspacing in the middle of a line and there
- // are trailing characters which go over the line width boundary
- if eraseExtraLine {
- remainingLines := (b.Size() - b.Pos) / b.LineWidth
- fmt.Printf(cursorDownN(remainingLines+1) + CursorBOL + ClearToEOL)
- place := b.Pos % b.LineWidth
- fmt.Printf(cursorUpN(remainingLines+1) + cursorRightN(place+len(b.Prompt.Prompt)))
+ if hasSpace {
+ b.DisplayPos -= 1
+ fmt.Print(CursorLeft)
+ }
+
+ if rLength == 2 {
+ fmt.Print(CursorLeft + " " + CursorLeftN(2))
+ } else {
+ fmt.Print(" " + CursorLeft)
+ }
+ } else if (b.DisplayPos-rLength)%b.LineWidth == 0 && hasSpace {
+ fmt.Print(CursorBOL + ClearToEOL + CursorUp + CursorBOL + CursorRightN(b.Width))
+
+ if b.Pos == b.Buf.Size() {
+ b.LineHasSpace.Remove(b.DisplayPos/b.LineWidth - 1)
+ }
+ b.DisplayPos -= 1
+ } else {
+ fmt.Print(CursorLeftN(rLength))
+ for range rLength {
+ fmt.Print(" ")
+ }
+ fmt.Print(CursorLeftN(rLength))
+ }
+
+ var eraseExtraLine bool
+ if (b.DisplaySize()-1)%b.LineWidth == 0 || (rLength == 2 && ((b.DisplaySize()-2)%b.LineWidth == 0)) || b.DisplaySize()%b.LineWidth == 0 {
+ eraseExtraLine = true
+ }
+
+ b.Pos -= 1
+ b.DisplayPos -= rLength
+ b.Buf.Remove(b.Pos)
+
+ if b.Pos < b.Buf.Size() {
+ b.drawRemaining()
+ // this erases a line which is left over when backspacing in the middle of a line and there
+ // are trailing characters which go over the line width boundary
+ if eraseExtraLine {
+ remainingLines := (b.DisplaySize() - b.DisplayPos) / b.LineWidth
+ fmt.Print(CursorDownN(remainingLines+1) + CursorBOL + ClearToEOL)
+ place := b.DisplayPos % b.LineWidth
+ fmt.Print(CursorUpN(remainingLines+1) + CursorRightN(place+len(b.Prompt.prompt())))
+ }
+ }
}
}
}
}
func (b *Buffer) Delete() {
- if b.Size() > 0 && b.Pos < b.Size() {
+ if b.Buf.Size() > 0 && b.Pos < b.Buf.Size() {
b.Buf.Remove(b.Pos)
b.drawRemaining()
- if b.Size()%b.LineWidth == 0 {
- if b.Pos != b.Size() {
- remainingLines := (b.Size() - b.Pos) / b.LineWidth
- fmt.Printf(cursorDownN(remainingLines) + CursorBOL + ClearToEOL)
- place := b.Pos % b.LineWidth
- fmt.Printf(cursorUpN(remainingLines) + cursorRightN(place+len(b.Prompt.Prompt)))
+ if b.DisplaySize()%b.LineWidth == 0 {
+ if b.DisplayPos != b.DisplaySize() {
+ remainingLines := (b.DisplaySize() - b.DisplayPos) / b.LineWidth
+ fmt.Print(CursorDownN(remainingLines) + CursorBOL + ClearToEOL)
+ place := b.DisplayPos % b.LineWidth
+ fmt.Print(CursorUpN(remainingLines) + CursorRightN(place+len(b.Prompt.prompt())))
}
}
}
@@ -262,9 +434,9 @@ func (b *Buffer) DeleteBefore() {
}
func (b *Buffer) DeleteRemaining() {
- if b.Size() > 0 && b.Pos < b.Size() {
- charsToDel := b.Size() - b.Pos
- for cnt := 0; cnt < charsToDel; cnt++ {
+ if b.DisplaySize() > 0 && b.Pos < b.DisplaySize() {
+ charsToDel := b.Buf.Size() - b.Pos
+ for range charsToDel {
b.Delete()
}
}
@@ -294,31 +466,34 @@ func (b *Buffer) DeleteWord() {
}
func (b *Buffer) ClearScreen() {
- fmt.Printf(ClearScreen + CursorReset + b.Prompt.Prompt)
+ fmt.Print(ClearScreen + CursorReset + b.Prompt.prompt())
if b.IsEmpty() {
- ph := b.Prompt.Placeholder
- fmt.Printf(ColorGrey + ph + cursorLeftN(len(ph)) + ColorDefault)
+ ph := b.Prompt.placeholder()
+ fmt.Print(ColorGrey + ph + CursorLeftN(len(ph)) + ColorDefault)
} else {
- currPos := b.Pos
+ currPos := b.DisplayPos
+ currIndex := b.Pos
b.Pos = 0
+ b.DisplayPos = 0
b.drawRemaining()
- fmt.Printf(CursorReset + cursorRightN(len(b.Prompt.Prompt)))
+ fmt.Print(CursorReset + CursorRightN(len(b.Prompt.prompt())))
if currPos > 0 {
targetLine := currPos / b.LineWidth
if targetLine > 0 {
- for cnt := 0; cnt < targetLine; cnt++ {
+ for range targetLine {
fmt.Print(CursorDown)
}
}
remainder := currPos % b.LineWidth
if remainder > 0 {
- fmt.Print(cursorRightN(remainder))
+ fmt.Print(CursorRightN(remainder))
}
if currPos%b.LineWidth == 0 {
- fmt.Printf(CursorBOL + b.Prompt.AltPrompt)
+ fmt.Print(CursorBOL + b.Prompt.AltPrompt)
}
}
- b.Pos = currPos
+ b.Pos = currIndex
+ b.DisplayPos = currPos
}
}
@@ -327,9 +502,20 @@ func (b *Buffer) IsEmpty() bool {
}
func (b *Buffer) Replace(r []rune) {
+ b.DisplayPos = 0
b.Pos = 0
+ lineNums := b.DisplaySize() / b.LineWidth
+
b.Buf.Clear()
- fmt.Printf(ClearLine + CursorBOL + b.Prompt.Prompt)
+
+ fmt.Print(CursorBOL + ClearToEOL)
+
+ for range lineNums {
+ fmt.Print(CursorUp + CursorBOL + ClearToEOL)
+ }
+
+ fmt.Print(CursorBOL + b.Prompt.prompt())
+
for _, c := range r {
b.Add(c)
}
@@ -346,7 +532,7 @@ func (b *Buffer) StringN(n int) string {
func (b *Buffer) StringNM(n, m int) string {
var s string
if m == 0 {
- m = b.Size()
+ m = b.Buf.Size()
}
for cnt := n; cnt < m; cnt++ {
c, _ := b.Buf.Get(cnt)
@@ -354,19 +540,3 @@ func (b *Buffer) StringNM(n, m int) string {
}
return s
}
-
-func cursorLeftN(n int) string {
- return fmt.Sprintf(CursorLeftN, n)
-}
-
-func cursorRightN(n int) string {
- return fmt.Sprintf(CursorRightN, n)
-}
-
-func cursorUpN(n int) string {
- return fmt.Sprintf(CursorUpN, n)
-}
-
-func cursorDownN(n int) string {
- return fmt.Sprintf(CursorDownN, n)
-}
diff --git a/readline/errors.go b/readline/errors.go
index 40e40cb7..bb3fbd47 100644
--- a/readline/errors.go
+++ b/readline/errors.go
@@ -4,9 +4,7 @@ import (
"errors"
)
-var (
- ErrInterrupt = errors.New("Interrupt")
-)
+var ErrInterrupt = errors.New("Interrupt")
type InterruptError struct {
Line []rune
diff --git a/readline/history.go b/readline/history.go
index 57fccb7c..9c6b930b 100644
--- a/readline/history.go
+++ b/readline/history.go
@@ -23,7 +23,7 @@ type History struct {
func NewHistory() (*History, error) {
h := &History{
Buf: arraylist.New(),
- Limit: 100, //resizeme
+ Limit: 100, // resizeme
Autosave: true,
Enabled: true,
}
@@ -49,7 +49,7 @@ func (h *History) Init() error {
h.Filename = path
- f, err := os.OpenFile(path, os.O_CREATE|os.O_RDONLY, 0600)
+ f, err := os.OpenFile(path, os.O_CREATE|os.O_RDONLY, 0o600)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
@@ -62,7 +62,7 @@ func (h *History) Init() error {
for {
line, err := r.ReadString('\n')
if err != nil {
- if err == io.EOF {
+ if errors.Is(err, io.EOF) {
break
}
return err
@@ -84,14 +84,14 @@ func (h *History) Add(l []rune) {
h.Compact()
h.Pos = h.Size()
if h.Autosave {
- h.Save()
+ _ = h.Save()
}
}
func (h *History) Compact() {
s := h.Buf.Size()
if s > h.Limit {
- for cnt := 0; cnt < s-h.Limit; cnt++ {
+ for range s - h.Limit {
h.Buf.Remove(0)
}
}
@@ -132,17 +132,19 @@ func (h *History) Save() error {
tmpFile := h.Filename + ".tmp"
- f, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0666)
+ f, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0o600)
if err != nil {
return err
}
defer f.Close()
buf := bufio.NewWriter(f)
- for cnt := 0; cnt < h.Size(); cnt++ {
+ for cnt := range h.Size() {
v, _ := h.Buf.Get(cnt)
line, _ := v.([]rune)
- buf.WriteString(string(line) + "\n")
+ if _, err := buf.WriteString(string(line) + "\n"); err != nil {
+ return err
+ }
}
buf.Flush()
f.Close()
diff --git a/readline/readline.go b/readline/readline.go
index c4c73520..1c14fe10 100644
--- a/readline/readline.go
+++ b/readline/readline.go
@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"os"
- "syscall"
)
type Prompt struct {
@@ -16,8 +15,24 @@ type Prompt struct {
UseAlt bool
}
+func (p *Prompt) prompt() string {
+ if p.UseAlt {
+ return p.AltPrompt
+ }
+ return p.Prompt
+}
+
+func (p *Prompt) placeholder() string {
+ if p.UseAlt {
+ return p.AltPlaceholder
+ }
+ return p.Placeholder
+}
+
type Terminal struct {
outchan chan rune
+ rawmode bool
+ termios any
}
type Instance struct {
@@ -46,18 +61,29 @@ func New(prompt Prompt) (*Instance, error) {
}
func (i *Instance) Readline() (string, error) {
- prompt := i.Prompt.Prompt
- if i.Prompt.UseAlt || i.Pasting {
+ if !i.Terminal.rawmode {
+ fd := os.Stdin.Fd()
+ termios, err := SetRawMode(fd)
+ if err != nil {
+ return "", err
+ }
+ i.Terminal.rawmode = true
+ i.Terminal.termios = termios
+ }
+
+ prompt := i.Prompt.prompt()
+ if i.Pasting {
+ // force alt prompt when pasting
prompt = i.Prompt.AltPrompt
}
fmt.Print(prompt)
- fd := int(syscall.Stdin)
- termios, err := SetRawMode(fd)
- if err != nil {
- return "", err
- }
- defer UnsetRawMode(fd, termios)
+ defer func() {
+ fd := os.Stdin.Fd()
+ //nolint:errcheck
+ UnsetRawMode(fd, i.Terminal.termios)
+ i.Terminal.rawmode = false
+ }()
buf, _ := NewBuffer(i.Prompt)
@@ -71,11 +97,8 @@ func (i *Instance) Readline() (string, error) {
// don't show placeholder when pasting unless we're in multiline mode
showPlaceholder := !i.Pasting || i.Prompt.UseAlt
if buf.IsEmpty() && showPlaceholder {
- ph := i.Prompt.Placeholder
- if i.Prompt.UseAlt {
- ph = i.Prompt.AltPlaceholder
- }
- fmt.Printf(ColorGrey + ph + fmt.Sprintf(CursorLeftN, len(ph)) + ColorDefault)
+ ph := i.Prompt.placeholder()
+ fmt.Print(ColorGrey + ph + CursorLeftN(len(ph)) + ColorDefault)
}
r, err := i.Terminal.Read()
@@ -112,7 +135,7 @@ func (i *Instance) Readline() (string, error) {
buf.MoveRight()
case CharBracketedPaste:
var code string
- for cnt := 0; cnt < 3; cnt++ {
+ for range 3 {
r, err = i.Terminal.Read()
if err != nil {
return "", io.EOF
@@ -126,7 +149,7 @@ func (i *Instance) Readline() (string, error) {
i.Pasting = false
}
case KeyDel:
- if buf.Size() > 0 {
+ if buf.DisplaySize() > 0 {
buf.Delete()
}
metaDel = true
@@ -174,11 +197,11 @@ func (i *Instance) Readline() (string, error) {
buf.Remove()
case CharTab:
// todo: convert back to real tabs
- for cnt := 0; cnt < 8; cnt++ {
+ for range 8 {
buf.Add(' ')
}
case CharDelete:
- if buf.Size() > 0 {
+ if buf.DisplaySize() > 0 {
buf.Delete()
} else {
return "", io.EOF
@@ -192,8 +215,9 @@ func (i *Instance) Readline() (string, error) {
case CharCtrlW:
buf.DeleteWord()
case CharCtrlZ:
- return handleCharCtrlZ(fd, termios)
- case CharEnter:
+ fd := os.Stdin.Fd()
+ return handleCharCtrlZ(fd, i.Terminal.termios)
+ case CharEnter, CharCtrlJ:
output := buf.String()
if output != "" {
i.History.Add([]rune(output))
@@ -207,7 +231,7 @@ func (i *Instance) Readline() (string, error) {
metaDel = false
continue
}
- if r >= CharSpace || r == CharEnter {
+ if r >= CharSpace || r == CharEnter || r == CharCtrlJ {
buf.Add(r)
}
}
@@ -223,8 +247,16 @@ func (i *Instance) HistoryDisable() {
}
func NewTerminal() (*Terminal, error) {
+ fd := os.Stdin.Fd()
+ termios, err := SetRawMode(fd)
+ if err != nil {
+ return nil, err
+ }
+
t := &Terminal{
outchan: make(chan rune),
+ rawmode: true,
+ termios: termios,
}
go t.ioloop()
diff --git a/readline/readline_unix.go b/readline/readline_unix.go
index a1e34d27..d48b9176 100644
--- a/readline/readline_unix.go
+++ b/readline/readline_unix.go
@@ -6,12 +6,13 @@ import (
"syscall"
)
-func handleCharCtrlZ(fd int, termios *Termios) (string, error) {
- if err := UnsetRawMode(fd, termios); err != nil {
+func handleCharCtrlZ(fd uintptr, termios any) (string, error) {
+ t := termios.(*Termios)
+ if err := UnsetRawMode(fd, t); err != nil {
return "", err
}
- syscall.Kill(0, syscall.SIGSTOP)
+ _ = syscall.Kill(0, syscall.SIGSTOP)
// on resume...
return "", nil
diff --git a/readline/readline_windows.go b/readline/readline_windows.go
index c8178903..a131d0ef 100644
--- a/readline/readline_windows.go
+++ b/readline/readline_windows.go
@@ -1,6 +1,6 @@
package readline
-func handleCharCtrlZ(fd int, state *State) (string, error) {
+func handleCharCtrlZ(fd uintptr, state any) (string, error) {
// not supported
return "", nil
}
diff --git a/readline/term.go b/readline/term.go
index 45757e6a..5584cd25 100644
--- a/readline/term.go
+++ b/readline/term.go
@@ -8,7 +8,7 @@ import (
type Termios syscall.Termios
-func SetRawMode(fd int) (*Termios, error) {
+func SetRawMode(fd uintptr) (*Termios, error) {
termios, err := getTermios(fd)
if err != nil {
return nil, err
@@ -25,12 +25,13 @@ func SetRawMode(fd int) (*Termios, error) {
return termios, setTermios(fd, &newTermios)
}
-func UnsetRawMode(fd int, termios *Termios) error {
- return setTermios(fd, termios)
+func UnsetRawMode(fd uintptr, termios any) error {
+ t := termios.(*Termios)
+ return setTermios(fd, t)
}
// IsTerminal returns true if the given file descriptor is a terminal.
-func IsTerminal(fd int) bool {
+func IsTerminal(fd uintptr) bool {
_, err := getTermios(fd)
return err == nil
}
diff --git a/readline/term_bsd.go b/readline/term_bsd.go
index 04b912f0..80bee6b3 100644
--- a/readline/term_bsd.go
+++ b/readline/term_bsd.go
@@ -7,17 +7,17 @@ import (
"unsafe"
)
-func getTermios(fd int) (*Termios, error) {
+func getTermios(fd uintptr) (*Termios, error) {
termios := new(Termios)
- _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TIOCGETA, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
+ _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, syscall.TIOCGETA, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
if err != 0 {
return nil, err
}
return termios, nil
}
-func setTermios(fd int, termios *Termios) error {
- _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TIOCSETA, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
+func setTermios(fd uintptr, termios *Termios) error {
+ _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, syscall.TIOCSETA, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
if err != 0 {
return err
}
diff --git a/readline/term_linux.go b/readline/term_linux.go
index 2d6211dd..e9e36da9 100644
--- a/readline/term_linux.go
+++ b/readline/term_linux.go
@@ -7,20 +7,22 @@ import (
"unsafe"
)
-const tcgets = 0x5401
-const tcsets = 0x5402
+const (
+ tcgets = 0x5401
+ tcsets = 0x5402
+)
-func getTermios(fd int) (*Termios, error) {
+func getTermios(fd uintptr) (*Termios, error) {
termios := new(Termios)
- _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), tcgets, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
+ _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, tcgets, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
if err != 0 {
return nil, err
}
return termios, nil
}
-func setTermios(fd int, termios *Termios) error {
- _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), tcsets, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
+func setTermios(fd uintptr, termios *Termios) error {
+ _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, tcsets, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
if err != 0 {
return err
}
diff --git a/readline/term_windows.go b/readline/term_windows.go
index 3d1c80e1..3b35149b 100644
--- a/readline/term_windows.go
+++ b/readline/term_windows.go
@@ -1,29 +1,7 @@
package readline
import (
- "syscall"
- "unsafe"
-)
-
-const (
- enableLineInput = 2
- enableWindowInput = 8
- enableMouseInput = 16
- enableInsertMode = 32
- enableQuickEditMode = 64
- enableExtendedFlags = 128
- enableProcessedOutput = 1
- enableWrapAtEolOutput = 2
- enableAutoPosition = 256 // Cursor position is not affected by writing data to the console.
- enableEchoInput = 4 // Characters are written to the console as they're read.
- enableProcessedInput = 1 // Enables input processing (like recognizing Ctrl+C).
-)
-
-var kernel32 = syscall.NewLazyDLL("kernel32.dll")
-
-var (
- procGetConsoleMode = kernel32.NewProc("GetConsoleMode")
- procSetConsoleMode = kernel32.NewProc("SetConsoleMode")
+ "golang.org/x/sys/windows"
)
type State struct {
@@ -31,32 +9,30 @@ type State struct {
}
// IsTerminal checks if the given file descriptor is associated with a terminal
-func IsTerminal(fd int) bool {
+func IsTerminal(fd uintptr) bool {
var st uint32
- r, _, e := syscall.SyscallN(procGetConsoleMode.Addr(), uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
- // if the call succeeds and doesn't produce an error, it's a terminal
- return r != 0 && e == 0
+ err := windows.GetConsoleMode(windows.Handle(fd), &st)
+ return err == nil
}
-func SetRawMode(fd int) (*State, error) {
+func SetRawMode(fd uintptr) (*State, error) {
var st uint32
- // retrieve the current mode of the terminal
- _, _, e := syscall.SyscallN(procGetConsoleMode.Addr(), uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
- if e != 0 {
- return nil, error(e)
+ if err := windows.GetConsoleMode(windows.Handle(fd), &st); err != nil {
+ return nil, err
}
- // modify the mode to set it to raw
- raw := st &^ (enableEchoInput | enableProcessedInput | enableLineInput | enableProcessedOutput)
- // apply the new mode to the terminal
- _, _, e = syscall.SyscallN(procSetConsoleMode.Addr(), uintptr(fd), uintptr(raw), 0)
- if e != 0 {
- return nil, error(e)
+
+ // this enables raw mode by turning off various flags in the console mode: https://pkg.go.dev/golang.org/x/sys/windows#pkg-constants
+ raw := st &^ (windows.ENABLE_ECHO_INPUT | windows.ENABLE_PROCESSED_INPUT | windows.ENABLE_LINE_INPUT | windows.ENABLE_PROCESSED_OUTPUT)
+
+ // turn on ENABLE_VIRTUAL_TERMINAL_INPUT to enable escape sequences
+ raw |= windows.ENABLE_VIRTUAL_TERMINAL_INPUT
+ if err := windows.SetConsoleMode(windows.Handle(fd), raw); err != nil {
+ return nil, err
}
- // return the original state so that it can be restored later
return &State{st}, nil
}
-func UnsetRawMode(fd int, state *State) error {
- _, _, err := syscall.SyscallN(procSetConsoleMode.Addr(), uintptr(fd), uintptr(state.mode), 0)
- return err
+func UnsetRawMode(fd uintptr, state any) error {
+ s := state.(*State)
+ return windows.SetConsoleMode(windows.Handle(fd), s.mode)
}
diff --git a/readline/types.go b/readline/types.go
index 3b88588f..e136d996 100644
--- a/readline/types.go
+++ b/readline/types.go
@@ -1,5 +1,7 @@
package readline
+import "strconv"
+
const (
CharNull = 0
CharLineStart = 1
@@ -41,34 +43,49 @@ const (
)
const (
- CursorUp = "\033[1A"
- CursorDown = "\033[1B"
- CursorRight = "\033[1C"
- CursorLeft = "\033[1D"
+ Esc = "\x1b"
- CursorSave = "\033[s"
- CursorRestore = "\033[u"
+ CursorSave = Esc + "[s"
+ CursorRestore = Esc + "[u"
- CursorUpN = "\033[%dA"
- CursorDownN = "\033[%dB"
- CursorRightN = "\033[%dC"
- CursorLeftN = "\033[%dD"
+ CursorEOL = Esc + "[E"
+ CursorBOL = Esc + "[1G"
+ CursorHide = Esc + "[?25l"
+ CursorShow = Esc + "[?25h"
- CursorEOL = "\033[E"
- CursorBOL = "\033[1G"
- CursorHide = "\033[?25l"
- CursorShow = "\033[?25h"
+ ClearToEOL = Esc + "[K"
+ ClearLine = Esc + "[2K"
+ ClearScreen = Esc + "[2J"
+ CursorReset = Esc + "[0;0f"
- ClearToEOL = "\033[K"
- ClearLine = "\033[2K"
- ClearScreen = "\033[2J"
- CursorReset = "\033[0;0f"
+ ColorGrey = Esc + "[38;5;245m"
+ ColorDefault = Esc + "[0m"
- ColorGrey = "\033[38;5;245m"
- ColorDefault = "\033[0m"
+ StartBracketedPaste = Esc + "[?2004h"
+ EndBracketedPaste = Esc + "[?2004l"
+)
- StartBracketedPaste = "\033[?2004h"
- EndBracketedPaste = "\033[?2004l"
+func CursorUpN(n int) string {
+ return Esc + "[" + strconv.Itoa(n) + "A"
+}
+
+func CursorDownN(n int) string {
+ return Esc + "[" + strconv.Itoa(n) + "B"
+}
+
+func CursorRightN(n int) string {
+ return Esc + "[" + strconv.Itoa(n) + "C"
+}
+
+func CursorLeftN(n int) string {
+ return Esc + "[" + strconv.Itoa(n) + "D"
+}
+
+var (
+ CursorUp = CursorUpN(1)
+ CursorDown = CursorDownN(1)
+ CursorRight = CursorRightN(1)
+ CursorLeft = CursorLeftN(1)
)
const (
diff --git a/runners/common.go b/runners/common.go
new file mode 100644
index 00000000..681c397b
--- /dev/null
+++ b/runners/common.go
@@ -0,0 +1,384 @@
+package runners
+
+import (
+ "compress/gzip"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "runtime"
+ "slices"
+ "strconv"
+ "strings"
+ "sync"
+ "syscall"
+
+ "golang.org/x/sync/errgroup"
+
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/gpu"
+)
+
+const (
+ binGlob = "*/*/*/*"
+)
+
+var (
+ lock sync.Mutex
+ runnersDir = ""
+)
+
+// Return the location where runners are stored
+// If runners are payloads, this will either extract them
+// or refresh them if any have disappeared due to tmp cleaners
+func Refresh(payloadFS fs.FS) (string, error) {
+ lock.Lock()
+ defer lock.Unlock()
+ var err error
+
+ // Wire up extra logging on our first load
+ if runnersDir == "" {
+ defer func() {
+ var runners []string
+ for v := range GetAvailableServers(runnersDir) {
+ runners = append(runners, v)
+ }
+ slog.Info("Dynamic LLM libraries", "runners", runners)
+ slog.Debug("Override detection logic by setting OLLAMA_LLM_LIBRARY")
+ }()
+ }
+
+ if hasPayloads(payloadFS) {
+ if runnersDir == "" {
+ runnersDir, err = extractRunners(payloadFS)
+ } else {
+ err = refreshRunners(payloadFS, runnersDir)
+ }
+ } else if runnersDir == "" {
+ runnersDir, err = locateRunners()
+ }
+
+ return runnersDir, err
+}
+
+func Cleanup(payloadFS fs.FS) {
+ lock.Lock()
+ defer lock.Unlock()
+ if hasPayloads(payloadFS) && runnersDir != "" {
+ // We want to fully clean up the tmpdir parent of the payloads dir
+ tmpDir := filepath.Clean(filepath.Join(runnersDir, ".."))
+ slog.Debug("cleaning up", "dir", tmpDir)
+ err := os.RemoveAll(tmpDir)
+ if err != nil {
+ slog.Warn("failed to clean up", "dir", tmpDir, "err", err)
+ }
+ }
+}
+
+func locateRunners() (string, error) {
+ exe, err := os.Executable()
+ if err != nil {
+ return "", err
+ }
+
+ cwd, err := os.Getwd()
+ if err != nil {
+ return "", err
+ }
+
+ var paths []string
+ for _, root := range []string{filepath.Dir(exe), filepath.Join(filepath.Dir(exe), envconfig.LibRelativeToExe()), cwd} {
+ paths = append(paths,
+ root,
+ filepath.Join(root, runtime.GOOS+"-"+runtime.GOARCH),
+ filepath.Join(root, "dist", runtime.GOOS+"-"+runtime.GOARCH),
+ )
+ }
+
+ // Try a few variations to improve developer experience when building from source in the local tree
+ for _, path := range paths {
+ candidate := filepath.Join(path, "lib", "ollama", "runners")
+ if _, err := os.Stat(candidate); err == nil {
+ return candidate, nil
+ }
+ }
+ return "", fmt.Errorf("unable to locate runners in any search path %v", paths)
+}
+
+// Return true if we're carying nested payloads for the runners
+func hasPayloads(payloadFS fs.FS) bool {
+ files, err := fs.Glob(payloadFS, binGlob)
+ if err != nil || len(files) == 0 || (len(files) == 1 && strings.Contains(files[0], "placeholder")) {
+ return false
+ }
+ return true
+}
+
+func extractRunners(payloadFS fs.FS) (string, error) {
+ cleanupTmpDirs()
+ tmpDir, err := os.MkdirTemp(envconfig.TmpDir(), "ollama")
+ if err != nil {
+ return "", fmt.Errorf("failed to generate tmp dir: %w", err)
+ }
+ // Track our pid so we can clean up orphaned tmpdirs
+ n := filepath.Join(tmpDir, "ollama.pid")
+ if err := os.WriteFile(n, []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil {
+ slog.Warn("failed to write pid file", "file", n, "error", err)
+ }
+ // We create a distinct subdirectory for payloads within the tmpdir
+ // This will typically look like /tmp/ollama3208993108/runners on linux
+ rDir := filepath.Join(tmpDir, "runners")
+
+ slog.Info("extracting embedded files", "dir", rDir)
+ return rDir, refreshRunners(payloadFS, rDir)
+}
+
+func refreshRunners(payloadFS fs.FS, rDir string) error {
+ // extract or refresh server libraries
+ err := extractFiles(payloadFS, rDir, binGlob)
+ if err != nil {
+ return fmt.Errorf("extract binaries: %v", err)
+ }
+ return nil
+}
+
+// extract extracts the embedded files to the target directory
+func extractFiles(payloadFS fs.FS, targetDir string, glob string) error {
+ files, err := fs.Glob(payloadFS, glob)
+ if err != nil || len(files) == 0 {
+ // Should not happen
+ return fmt.Errorf("extractFiles called without payload present")
+ }
+
+ if err := os.MkdirAll(targetDir, 0o755); err != nil {
+ return fmt.Errorf("extractFiles could not mkdir %s: %v", targetDir, err)
+ }
+
+ g := new(errgroup.Group)
+
+ // $OS/$GOARCH/$RUNNER/$FILE
+ for _, file := range files {
+ filename := file
+
+ runner := filepath.Base(filepath.Dir(filename))
+
+ slog.Debug("extracting", "runner", runner, "payload", filename)
+
+ g.Go(func() error {
+ srcf, err := payloadFS.Open(filename)
+ if err != nil {
+ return err
+ }
+ defer srcf.Close()
+
+ src := io.Reader(srcf)
+ if strings.HasSuffix(filename, ".gz") {
+ src, err = gzip.NewReader(src)
+ if err != nil {
+ return fmt.Errorf("decompress payload %s: %v", filename, err)
+ }
+ filename = strings.TrimSuffix(filename, ".gz")
+ }
+
+ runnerDir := filepath.Join(targetDir, runner)
+ if err := os.MkdirAll(runnerDir, 0o755); err != nil {
+ return fmt.Errorf("extractFiles could not mkdir %s: %v", runnerDir, err)
+ }
+
+ base := filepath.Base(filename)
+ destFilename := filepath.Join(runnerDir, base)
+
+ _, err = os.Stat(destFilename)
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ destFile, err := os.OpenFile(destFilename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
+ if err != nil {
+ return fmt.Errorf("write payload %s: %v", filename, err)
+ }
+ defer destFile.Close()
+ if _, err := io.Copy(destFile, src); err != nil {
+ return fmt.Errorf("copy payload %s: %v", filename, err)
+ }
+ case err != nil:
+ return fmt.Errorf("stat payload %s: %v", filename, err)
+ }
+ return nil
+ })
+ }
+
+ err = g.Wait()
+ if err != nil {
+ slog.Error("failed to extract files", "error", err)
+ // If we fail to extract, the payload dir is most likely unusable, so cleanup whatever we extracted
+ err := os.RemoveAll(targetDir)
+ if err != nil {
+ slog.Warn("failed to cleanup incomplete payload dir", "dir", targetDir, "error", err)
+ }
+ return err
+ }
+ return nil
+}
+
+// Best effort to clean up prior tmpdirs
+func cleanupTmpDirs() {
+ tmpDir := envconfig.TmpDir()
+ if tmpDir == "" {
+ tmpDir = os.TempDir()
+ }
+ matches, err := filepath.Glob(filepath.Join(tmpDir, "ollama*", "ollama.pid"))
+ if err != nil {
+ return
+ }
+
+ for _, match := range matches {
+ raw, err := os.ReadFile(match)
+ if errors.Is(err, os.ErrNotExist) {
+ slog.Debug("not a ollama runtime directory, skipping", "path", match)
+ continue
+ } else if err != nil {
+ slog.Warn("could not read ollama.pid, skipping", "path", match, "error", err)
+ continue
+ }
+
+ pid, err := strconv.Atoi(string(raw))
+ if err != nil {
+ slog.Warn("invalid pid, skipping", "path", match, "error", err)
+ continue
+ }
+
+ p, err := os.FindProcess(pid)
+ if err == nil && !errors.Is(p.Signal(syscall.Signal(0)), os.ErrProcessDone) {
+ slog.Warn("process still running, skipping", "pid", pid, "path", match)
+ continue
+ }
+
+ if err := os.Remove(match); err != nil {
+ slog.Warn("could not cleanup stale pidfile", "path", match, "error", err)
+ }
+
+ runners := filepath.Join(filepath.Dir(match), "runners")
+ if err := os.RemoveAll(runners); err != nil {
+ slog.Warn("could not cleanup stale runners", "path", runners, "error", err)
+ }
+
+ if err := os.Remove(filepath.Dir(match)); err != nil {
+ slog.Warn("could not cleanup stale tmpdir", "path", filepath.Dir(match), "error", err)
+ }
+ }
+}
+
+// directory names are the name of the runner and may contain an optional
+// variant prefixed with '_' as the separator. For example, "cuda_v11" and
+// "cuda_v12" or "cpu" and "cpu_avx2". Any library without a variant is the
+// lowest common denominator
+func GetAvailableServers(payloadsDir string) map[string]string {
+ if payloadsDir == "" {
+ slog.Error("empty runner dir")
+ return nil
+ }
+
+ // glob payloadsDir for files that start with ollama_
+ pattern := filepath.Join(payloadsDir, "*", "ollama_*")
+
+ files, err := filepath.Glob(pattern)
+ if err != nil {
+ slog.Debug("could not glob", "pattern", pattern, "error", err)
+ return nil
+ }
+
+ servers := make(map[string]string)
+ for _, file := range files {
+ slog.Debug("availableServers : found", "file", file)
+ servers[filepath.Base(filepath.Dir(file))] = filepath.Dir(file)
+ }
+
+ return servers
+}
+
+// serversForGpu returns a list of compatible servers give the provided GPU
+// info, ordered by performance. assumes Init() has been called
+// TODO - switch to metadata based mapping
+func ServersForGpu(info gpu.GpuInfo) []string {
+ // glob workDir for files that start with ollama_
+ availableServers := GetAvailableServers(runnersDir)
+ requested := info.Library
+ if info.Variant != gpu.CPUCapabilityNone.String() {
+ requested += "_" + info.Variant
+ }
+
+ servers := []string{}
+
+ // exact match first
+ for a := range availableServers {
+ if a == requested {
+ servers = []string{a}
+
+ if a == "metal" {
+ return servers
+ }
+
+ break
+ }
+ }
+
+ alt := []string{}
+
+ // Then for GPUs load alternates and sort the list for consistent load ordering
+ if info.Library != "cpu" {
+ for a := range availableServers {
+ if info.Library == strings.Split(a, "_")[0] && a != requested {
+ alt = append(alt, a)
+ }
+ }
+
+ slices.Sort(alt)
+ servers = append(servers, alt...)
+ }
+
+ if !(runtime.GOOS == "darwin" && runtime.GOARCH == "arm64") {
+ // Load up the best CPU variant if not primary requested
+ if info.Library != "cpu" {
+ variant := gpu.GetCPUCapability()
+ // If no variant, then we fall back to default
+ // If we have a variant, try that if we find an exact match
+ // Attempting to run the wrong CPU instructions will panic the
+ // process
+ if variant != gpu.CPUCapabilityNone {
+ for cmp := range availableServers {
+ if cmp == "cpu_"+variant.String() {
+ servers = append(servers, cmp)
+ break
+ }
+ }
+ } else {
+ servers = append(servers, "cpu")
+ }
+ }
+
+ if len(servers) == 0 {
+ servers = []string{"cpu"}
+ }
+ }
+
+ return servers
+}
+
+// Return the optimal server for this CPU architecture
+func ServerForCpu() string {
+ if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
+ return "metal"
+ }
+ variant := gpu.GetCPUCapability()
+ availableServers := GetAvailableServers(runnersDir)
+ if variant != gpu.CPUCapabilityNone {
+ for cmp := range availableServers {
+ if cmp == "cpu_"+variant.String() {
+ return cmp
+ }
+ }
+ }
+ return "cpu"
+}
diff --git a/runners/runners_test.go b/runners/runners_test.go
new file mode 100644
index 00000000..e6439448
--- /dev/null
+++ b/runners/runners_test.go
@@ -0,0 +1,50 @@
+package runners
+
+import (
+ "log/slog"
+ "os"
+ "path"
+ "runtime"
+ "strings"
+ "testing"
+ "testing/fstest"
+)
+
+func TestRefreshRunners(t *testing.T) {
+ slog.SetLogLoggerLevel(slog.LevelDebug)
+
+ payloadFS := fstest.MapFS{
+ path.Join(runtime.GOOS, runtime.GOARCH, "foo", "ollama_llama_server"): {Data: []byte("hello, world\n")},
+ }
+ tmpDir, err := os.MkdirTemp("", "testing")
+ if err != nil {
+ t.Fatalf("failed to make tmp dir %s", err)
+ }
+ t.Setenv("OLLAMA_TMPDIR", tmpDir)
+ rDir, err := Refresh(payloadFS)
+ if err != nil {
+ t.Fatalf("failed to extract to %s %s", tmpDir, err)
+ }
+ if !strings.Contains(rDir, tmpDir) {
+ t.Fatalf("runner dir %s was not in tmp dir %s", rDir, tmpDir)
+ }
+
+ // spot check results
+ servers := GetAvailableServers(rDir)
+ if len(servers) < 1 {
+ t.Fatalf("expected at least 1 server")
+ }
+
+ // Refresh contents
+ rDir, err = extractRunners(payloadFS)
+ if err != nil {
+ t.Fatalf("failed to extract to %s %s", tmpDir, err)
+ }
+ if !strings.Contains(rDir, tmpDir) {
+ t.Fatalf("runner dir %s was not in tmp dir %s", rDir, tmpDir)
+ }
+
+ cleanupTmpDirs()
+
+ Cleanup(payloadFS)
+}
diff --git a/scripts/build_darwin.sh b/scripts/build_darwin.sh
index b37f6d0e..981f1067 100755
--- a/scripts/build_darwin.sh
+++ b/scripts/build_darwin.sh
@@ -1,31 +1,52 @@
#!/bin/sh
-set -eu
+set -e
-export VERSION=${VERSION:-0.0.0}
-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/jmorganca/ollama/version.Version=$VERSION\" \"-X=github.com/jmorganca/ollama/server.mode=release\"'"
+. $(dirname $0)/env.sh
mkdir -p dist
for TARGETARCH in arm64 amd64; do
- GOOS=darwin GOARCH=$TARGETARCH go generate ./...
- CGO_ENABLED=1 GOOS=darwin GOARCH=$TARGETARCH go build -o dist/ollama-darwin-$TARGETARCH
- rm -rf llm/llama.cpp/*/build
+ if [ -n "${OLLAMA_NEW_RUNNERS}" ]; then
+ echo "Building Go runner darwin $TARGETARCH"
+ rm -rf llama/build
+ GOOS=darwin ARCH=$TARGETARCH GOARCH=$TARGETARCH make -C llama -j 8
+ else
+ echo "Building C++ runner darwin $TARGETARCH"
+ rm -rf llm/build
+ GOOS=darwin GOARCH=$TARGETARCH go generate ./...
+ fi
+ CGO_ENABLED=1 GOOS=darwin GOARCH=$TARGETARCH go build -trimpath -o dist/ollama-darwin-$TARGETARCH
+ CGO_ENABLED=1 GOOS=darwin GOARCH=$TARGETARCH go build -trimpath -cover -o dist/ollama-darwin-$TARGETARCH-cov
done
-lipo -create -output dist/ollama dist/ollama-darwin-*
-rm -f dist/ollama-darwin-*
-codesign --deep --force --options=runtime --sign "$APPLE_IDENTITY" --timestamp dist/ollama
+lipo -create -output dist/ollama dist/ollama-darwin-arm64 dist/ollama-darwin-amd64
+rm -f dist/ollama-darwin-arm64 dist/ollama-darwin-amd64
+if [ -n "$APPLE_IDENTITY" ]; then
+ codesign --deep --force --options=runtime --sign "$APPLE_IDENTITY" --timestamp dist/ollama
+else
+ echo "Skipping code signing - set APPLE_IDENTITY"
+fi
chmod +x dist/ollama
-# build and sign the mac app
-npm install --prefix app
-npm run --prefix app make:sign
-cp app/out/make/zip/darwin/universal/Ollama-darwin-universal-$VERSION.zip dist/Ollama-darwin.zip
+# build and optionally sign the mac app
+npm install --prefix macapp
+if [ -n "$APPLE_IDENTITY" ]; then
+ npm run --prefix macapp make:sign
+else
+ npm run --prefix macapp make
+fi
+cp macapp/out/make/zip/darwin/universal/Ollama-darwin-universal-$VERSION.zip dist/Ollama-darwin.zip
# sign the binary and rename it
-codesign -f --timestamp -s "$APPLE_IDENTITY" --identifier ai.ollama.ollama --options=runtime dist/ollama
+if [ -n "$APPLE_IDENTITY" ]; then
+ codesign -f --timestamp -s "$APPLE_IDENTITY" --identifier ai.ollama.ollama --options=runtime dist/ollama
+else
+ echo "WARNING: Skipping code signing - set APPLE_IDENTITY"
+fi
ditto -c -k --keepParent dist/ollama dist/temp.zip
-xcrun notarytool submit dist/temp.zip --wait --timeout 10m --apple-id $APPLE_ID --password $APPLE_PASSWORD --team-id $APPLE_TEAM_ID
+if [ -n "$APPLE_IDENTITY" ]; then
+ xcrun notarytool submit dist/temp.zip --wait --timeout 10m --apple-id $APPLE_ID --password $APPLE_PASSWORD --team-id $APPLE_TEAM_ID
+fi
mv dist/ollama dist/ollama-darwin
rm -f dist/temp.zip
diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh
index 0c795fb4..ea747a3c 100755
--- a/scripts/build_docker.sh
+++ b/scripts/build_docker.sh
@@ -2,16 +2,34 @@
set -eu
-export VERSION=${VERSION:-0.0.0}
-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/jmorganca/ollama/version.Version=$VERSION\" \"-X=github.com/jmorganca/ollama/server.mode=release\"'"
+. $(dirname $0)/env.sh
+
+# Set PUSH to a non-empty string to trigger push instead of load
+PUSH=${PUSH:-""}
+
+if [ -z "${PUSH}" ] ; then
+ echo "Building ${FINAL_IMAGE_REPO}:$VERSION locally. set PUSH=1 to push"
+ LOAD_OR_PUSH="--load"
+else
+ echo "Will be pushing ${FINAL_IMAGE_REPO}:$VERSION"
+ LOAD_OR_PUSH="--push"
+fi
docker buildx build \
- --load \
- --platform=linux/arm64,linux/amd64 \
- --build-arg=VERSION \
- --build-arg=GOFLAGS \
- --cache-from type=local,src=.cache \
- --cache-to type=local,dest=.cache \
- -f Dockerfile \
- -t ollama/ollama:$VERSION \
+ ${LOAD_OR_PUSH} \
+ --platform=${PLATFORM} \
+ ${OLLAMA_COMMON_BUILD_ARGS} \
+ -f ${DOCKERFILE_DIR}Dockerfile \
+ -t ${FINAL_IMAGE_REPO}:$VERSION \
.
+
+if echo $PLATFORM | grep "amd64" > /dev/null; then
+ docker buildx build \
+ ${LOAD_OR_PUSH} \
+ --platform=linux/amd64 \
+ ${OLLAMA_COMMON_BUILD_ARGS} \
+ --target runtime-rocm \
+ -f ${DOCKERFILE_DIR}Dockerfile \
+ -t ${FINAL_IMAGE_REPO}:$VERSION-rocm \
+ .
+fi
\ No newline at end of file
diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh
index 602fd755..48fdd37b 100755
--- a/scripts/build_linux.sh
+++ b/scripts/build_linux.sh
@@ -1,15 +1,29 @@
#!/bin/sh
+#
+# Mac ARM users, rosetta can be flaky, so to use a remote x86 builder
+#
+# docker context create amd64 --docker host=ssh://mybuildhost
+# docker buildx create --name mybuilder amd64 --platform linux/amd64
+# docker buildx create --name mybuilder --append desktop-linux --platform linux/arm64
+# docker buildx use mybuilder
+
set -eu
-export VERSION=${VERSION:-0.0.0}
-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/jmorganca/ollama/version.Version=$VERSION\" \"-X=github.com/jmorganca/ollama/server.mode=release\"'"
+. $(dirname $0)/env.sh
mkdir -p dist
-for TARGETARCH in amd64 arm64; do
- docker buildx build --load --platform=linux/$TARGETARCH --build-arg=VERSION --build-arg=GOFLAGS --build-arg=CGO_CFLAGS --cache-from type=local,src=.cache --cache-to type=local,dest=.cache -f Dockerfile.build -t builder:$TARGETARCH .
- docker create --platform linux/$TARGETARCH --name builder-$TARGETARCH builder:$TARGETARCH
- docker cp builder-$TARGETARCH:/go/src/github.com/jmorganca/ollama/ollama ./dist/ollama-linux-$TARGETARCH
- docker rm builder-$TARGETARCH
-done
+docker buildx build \
+ --output type=local,dest=./dist/ \
+ --platform=${PLATFORM} \
+ ${OLLAMA_COMMON_BUILD_ARGS} \
+ --target dist \
+ -f ${DOCKERFILE_DIR}Dockerfile \
+ .
+
+# buildx behavior changes for single vs. multiplatform
+if echo $PLATFORM | grep "," > /dev/null ; then
+ mv -f ./dist/linux_*64/ollama* ./dist/
+ rmdir ./dist/linux_*64
+fi
\ No newline at end of file
diff --git a/scripts/build_remote.py b/scripts/build_remote.py
index db824e4b..2ab58ad7 100755
--- a/scripts/build_remote.py
+++ b/scripts/build_remote.py
@@ -60,9 +60,17 @@ subprocess.check_call(['ssh', netloc, 'cd', path, ';', 'git', 'checkout', branch
# subprocess.check_call(['ssh', netloc, 'cd', path, ';', 'env'])
# TODO - or consider paramiko maybe
-print("Performing generate")
-subprocess.check_call(['ssh', netloc, 'cd', path, ';', GoCmd, 'generate', './...'])
+print("Running Windows Build Script")
+subprocess.check_call(['ssh', netloc, 'cd', path, ';', "powershell", "-ExecutionPolicy", "Bypass", "-File", "./scripts/build_windows.ps1"])
+
+# print("Building")
+# subprocess.check_call(['ssh', netloc, 'cd', path, ';', GoCmd, 'build', '.'])
+
+print("Copying built result")
+subprocess.check_call(['scp', netloc +":"+ path + "/ollama.exe", './dist/'])
+
+print("Copying installer")
+subprocess.check_call(['scp', netloc +":"+ path + "/dist/Ollama Setup.exe", './dist/'])
+
-print("Building")
-subprocess.check_call(['ssh', netloc, 'cd', path, ';', GoCmd, 'build', '.'])
diff --git a/scripts/build_windows.ps1 b/scripts/build_windows.ps1
new file mode 100644
index 00000000..c29ba20e
--- /dev/null
+++ b/scripts/build_windows.ps1
@@ -0,0 +1,243 @@
+#!powershell
+#
+# powershell -ExecutionPolicy Bypass -File .\scripts\build_windows.ps1
+#
+# gcloud auth application-default login
+
+$ErrorActionPreference = "Stop"
+
+function checkEnv() {
+ if ($null -ne $env:ARCH ) {
+ $script:ARCH = $env:ARCH
+ } else {
+ $arch=([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)
+ if ($null -ne $arch) {
+ $script:ARCH = ($arch.ToString().ToLower()).Replace("x64", "amd64")
+ } else {
+ write-host "WARNING: old powershell detected, assuming amd64 architecture - set `$env:ARCH to override"
+ $script:ARCH="amd64"
+ }
+ }
+ $script:TARGET_ARCH=$script:ARCH
+ Write-host "Building for ${script:TARGET_ARCH}"
+ write-host "Locating required tools and paths"
+ $script:SRC_DIR=$PWD
+ if ($null -eq $env:VCToolsRedistDir) {
+ $MSVC_INSTALL=(Get-CimInstance MSFT_VSInstance -Namespace root/cimv2/vs)[0].InstallLocation
+ $env:VCToolsRedistDir=(get-item "${MSVC_INSTALL}\VC\Redist\MSVC\*")[0]
+ }
+ # Locate CUDA versions
+ # Note: this assumes every version found will be built
+ $cudaList=(get-item "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v*\bin\" -ea 'silentlycontinue')
+ if ($cudaList.length -eq 0) {
+ $d=(get-command -ea 'silentlycontinue' nvcc).path
+ if ($null -ne $d) {
+ $script:CUDA_DIRS=@($d| split-path -parent)
+ }
+ } else {
+ $script:CUDA_DIRS=$cudaList
+ }
+
+ $inoSetup=(get-item "C:\Program Files*\Inno Setup*\")
+ if ($inoSetup.length -gt 0) {
+ $script:INNO_SETUP_DIR=$inoSetup[0]
+ }
+
+ $script:DIST_DIR="${script:SRC_DIR}\dist\windows-${script:TARGET_ARCH}"
+ $env:CGO_ENABLED="1"
+ Write-Output "Checking version"
+ if (!$env:VERSION) {
+ $data=(git describe --tags --first-parent --abbrev=7 --long --dirty --always)
+ $pattern="v(.+)"
+ if ($data -match $pattern) {
+ $script:VERSION=$matches[1]
+ }
+ } else {
+ $script:VERSION=$env:VERSION
+ }
+ $pattern = "(\d+[.]\d+[.]\d+).*"
+ if ($script:VERSION -match $pattern) {
+ $script:PKG_VERSION=$matches[1]
+ } else {
+ $script:PKG_VERSION="0.0.0"
+ }
+ write-host "Building Ollama $script:VERSION with package version $script:PKG_VERSION"
+
+ # Note: Windows Kits 10 signtool crashes with GCP's plugin
+ if ($null -eq $env:SIGN_TOOL) {
+ ${script:SignTool}="C:\Program Files (x86)\Windows Kits\8.1\bin\x64\signtool.exe"
+ } else {
+ ${script:SignTool}=${env:SIGN_TOOL}
+ }
+ if ("${env:KEY_CONTAINER}") {
+ ${script:OLLAMA_CERT}=$(resolve-path "${script:SRC_DIR}\ollama_inc.crt")
+ Write-host "Code signing enabled"
+ } else {
+ write-host "Code signing disabled - please set KEY_CONTAINERS to sign and copy ollama_inc.crt to the top of the source tree"
+ }
+
+}
+
+
+function buildOllama() {
+ if ($null -eq ${env:OLLAMA_SKIP_GENERATE}) {
+ write-host "Building ollama runners"
+ Remove-Item -ea 0 -recurse -force -path "${script:SRC_DIR}\dist\windows-${script:ARCH}"
+ if ($null -eq ${env:OLLAMA_NEW_RUNNERS}) {
+ # Start by skipping CUDA to build everything else
+ write-host "Building ollama runners"
+ powershell -Command { $env:OLLAMA_SKIP_CUDA_GENERATE="1"; & go generate ./... }
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+
+ # Then skip everyhting else and build all the CUDA variants
+ foreach ($env:CUDA_LIB_DIR in $script:CUDA_DIRS) {
+ write-host "Building CUDA ${env:CUDA_LIB_DIR} runner"
+
+ if ($env:CUDA_LIB_DIR.Contains("v12")) {
+ powershell -Command {
+ $env:OLLAMA_SKIP_CUDA_GENERATE=""
+ $env:OLLAMA_SKIP_STATIC_GENERATE="1"
+ $env:OLLAMA_SKIP_CPU_GENERATE="1"
+ $env:OLLAMA_SKIP_ONEAPI_GENERATE="1"
+ $env:OLLAMA_SKIP_ROCM_GENERATE="1"
+ $env:CMAKE_CUDA_ARCHITECTURES="60;61;62;70;72;75;80;86;87;89;90;90a"
+ $env:OLLAMA_CUSTOM_CUDA_DEFS="-DGGML_CUDA_USE_GRAPHS=on"
+ $env:CUDA_PATH=split-path -path $env:CUDA_LIB_DIR -parent
+ $env:PATH="$envs:CUDA_LIB_DIR;$env:PATH"
+ & go generate ./...
+ }
+ } else {
+ powershell -Command {
+ $env:OLLAMA_SKIP_CUDA_GENERATE=""
+ $env:OLLAMA_SKIP_STATIC_GENERATE="1"
+ $env:OLLAMA_SKIP_CPU_GENERATE="1"
+ $env:OLLAMA_SKIP_ONEAPI_GENERATE="1"
+ $env:OLLAMA_SKIP_ROCM_GENERATE="1"
+ $env:CMAKE_CUDA_ARCHITECTURES="50;52;53;60;61;62;70;72;75;80;86"
+ $env:OLLAMA_CUSTOM_CUDA_DEFS=""
+ $env:CUDA_PATH=split-path -path $env:CUDA_LIB_DIR -parent
+ $env:PATH="$envs:CUDA_LIB_DIR;$env:PATH"
+ & go generate ./...
+ }
+ }
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ }
+ } else {
+ & make -C llama -j 12
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ }
+
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ } else {
+ write-host "Skipping generate step with OLLAMA_SKIP_GENERATE set"
+ }
+ write-host "Building ollama CLI"
+ & go build -trimpath -ldflags "-s -w -X=github.com/ollama/ollama/version.Version=$script:VERSION -X=github.com/ollama/ollama/server.mode=release" .
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ if ("${env:KEY_CONTAINER}") {
+ & "${script:SignTool}" sign /v /fd sha256 /t http://timestamp.digicert.com /f "${script:OLLAMA_CERT}" `
+ /csp "Google Cloud KMS Provider" /kc ${env:KEY_CONTAINER} ollama.exe
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ }
+ New-Item -ItemType Directory -Path .\dist\windows-${script:TARGET_ARCH}\ -Force
+ cp .\ollama.exe .\dist\windows-${script:TARGET_ARCH}\
+}
+
+function buildApp() {
+ write-host "Building Ollama App"
+ cd "${script:SRC_DIR}\app"
+ & windres -l 0 -o ollama.syso ollama.rc
+ & go build -trimpath -ldflags "-s -w -H windowsgui -X=github.com/ollama/ollama/version.Version=$script:VERSION -X=github.com/ollama/ollama/server.mode=release" -o "${script:SRC_DIR}\dist\windows-${script:TARGET_ARCH}-app.exe" .
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ if ("${env:KEY_CONTAINER}") {
+ & "${script:SignTool}" sign /v /fd sha256 /t http://timestamp.digicert.com /f "${script:OLLAMA_CERT}" `
+ /csp "Google Cloud KMS Provider" /kc ${env:KEY_CONTAINER} "${script:SRC_DIR}\dist\windows-${script:TARGET_ARCH}-app.exe"
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ }
+}
+
+function gatherDependencies() {
+ if ($null -eq $env:VCToolsRedistDir) {
+ write-error "Unable to locate VC Install location - please use a Developer shell"
+ exit 1
+ }
+ write-host "Gathering runtime dependencies from $env:VCToolsRedistDir"
+ cd "${script:SRC_DIR}"
+ md "${script:DIST_DIR}\lib\ollama" -ea 0 > $null
+
+ # TODO - this varies based on host build system and MSVC version - drive from dumpbin output
+ # currently works for Win11 + MSVC 2019 + Cuda V11
+ if ($script:TARGET_ARCH -eq "amd64") {
+ $depArch="x64"
+ } else {
+ $depArch=$script:TARGET_ARCH
+ }
+ if ($depArch -eq "amd64") {
+ cp "${env:VCToolsRedistDir}\${depArch}\Microsoft.VC*.CRT\msvcp140*.dll" "${script:DIST_DIR}\lib\ollama\"
+ cp "${env:VCToolsRedistDir}\${depArch}\Microsoft.VC*.CRT\vcruntime140.dll" "${script:DIST_DIR}\lib\ollama\"
+ cp "${env:VCToolsRedistDir}\${depArch}\Microsoft.VC*.CRT\vcruntime140_1.dll" "${script:DIST_DIR}\lib\ollama\"
+ $llvmCrtDir="$env:VCToolsRedistDir\..\..\..\Tools\Llvm\${depArch}\bin"
+ foreach ($part in $("runtime", "stdio", "filesystem", "math", "convert", "heap", "string", "time", "locale", "environment")) {
+ write-host "cp ${llvmCrtDir}\api-ms-win-crt-${part}*.dll ${script:DIST_DIR}\lib\ollama\"
+ cp "${llvmCrtDir}\api-ms-win-crt-${part}*.dll" "${script:DIST_DIR}\lib\ollama\"
+ }
+ } else {
+ # Carying the dll's doesn't seem to work, so use the redist installer
+ copy-item -path "${env:VCToolsRedistDir}\vc_redist.arm64.exe" -destination "${script:DIST_DIR}" -verbose
+ }
+
+
+ cp "${script:SRC_DIR}\app\ollama_welcome.ps1" "${script:SRC_DIR}\dist\"
+ if ("${env:KEY_CONTAINER}") {
+ write-host "about to sign"
+ foreach ($file in (get-childitem "${script:DIST_DIR}\lib\ollama\cu*.dll") + @("${script:SRC_DIR}\dist\ollama_welcome.ps1")){
+ write-host "signing $file"
+ & "${script:SignTool}" sign /v /fd sha256 /t http://timestamp.digicert.com /f "${script:OLLAMA_CERT}" `
+ /csp "Google Cloud KMS Provider" /kc ${env:KEY_CONTAINER} $file
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+ }
+ }
+}
+
+function buildInstaller() {
+ if ($null -eq ${script:INNO_SETUP_DIR}) {
+ write-host "Inno Setup not present, skipping installer build"
+ return
+ }
+ write-host "Building Ollama Installer"
+ cd "${script:SRC_DIR}\app"
+ $env:PKG_VERSION=$script:PKG_VERSION
+ if ("${env:KEY_CONTAINER}") {
+ & "${script:INNO_SETUP_DIR}\ISCC.exe" /DARCH=$script:TARGET_ARCH /SMySignTool="${script:SignTool} sign /fd sha256 /t http://timestamp.digicert.com /f ${script:OLLAMA_CERT} /csp `$qGoogle Cloud KMS Provider`$q /kc ${env:KEY_CONTAINER} `$f" .\ollama.iss
+ } else {
+ & "${script:INNO_SETUP_DIR}\ISCC.exe" /DARCH=$script:TARGET_ARCH .\ollama.iss
+ }
+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+}
+
+function distZip() {
+ write-host "Generating stand-alone distribution zip file ${script:SRC_DIR}\dist\ollama-windows-${script:TARGET_ARCH}.zip"
+ Compress-Archive -Path "${script:SRC_DIR}\dist\windows-${script:TARGET_ARCH}\*" -DestinationPath "${script:SRC_DIR}\dist\ollama-windows-${script:TARGET_ARCH}.zip" -Force
+}
+
+checkEnv
+try {
+ if ($($args.count) -eq 0) {
+ buildOllama
+ buildApp
+ gatherDependencies
+ buildInstaller
+ distZip
+ } else {
+ for ( $i = 0; $i -lt $args.count; $i++ ) {
+ write-host "performing $($args[$i])"
+ & $($args[$i])
+ }
+ }
+} catch {
+ write-host "Build Failed"
+ write-host $_
+} finally {
+ set-location $script:SRC_DIR
+ $env:PKG_VERSION=""
+}
diff --git a/scripts/env.sh b/scripts/env.sh
new file mode 100644
index 00000000..537d3eee
--- /dev/null
+++ b/scripts/env.sh
@@ -0,0 +1,30 @@
+# Common environment setup across build*.sh scripts
+
+export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")}
+export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
+# TODO - consider `docker buildx ls --format=json` to autodiscover platform capability
+PLATFORM=${PLATFORM:-"linux/arm64,linux/amd64"}
+DOCKER_ORG=${DOCKER_ORG:-"ollama"}
+RELEASE_IMAGE_REPO=${RELEASE_IMAGE_REPO:-"${DOCKER_ORG}/release"}
+FINAL_IMAGE_REPO=${FINAL_IMAGE_REPO:-"${DOCKER_ORG}/ollama"}
+OLLAMA_COMMON_BUILD_ARGS="--build-arg=VERSION \
+ --build-arg=GOFLAGS \
+ --build-arg=OLLAMA_CUSTOM_CPU_DEFS \
+ --build-arg=OLLAMA_SKIP_CUDA_GENERATE \
+ --build-arg=OLLAMA_SKIP_CUDA_11_GENERATE \
+ --build-arg=OLLAMA_SKIP_CUDA_12_GENERATE \
+ --build-arg=CUDA_V11_ARCHITECTURES \
+ --build-arg=CUDA_V12_ARCHITECTURES \
+ --build-arg=OLLAMA_SKIP_ROCM_GENERATE \
+ --build-arg=OLLAMA_FAST_BUILD \
+ --build-arg=AMDGPU_TARGETS"
+OLLAMA_NEW_RUNNERS=${OLLAMA_NEW_RUNNERS:-""}
+if [ -n "${OLLAMA_NEW_RUNNERS}" ]; then
+ DOCKERFILE_DIR="./llama/"
+else
+ DOCKERFILE_DIR="./"
+fi
+
+echo "Building Ollama"
+echo "VERSION=$VERSION"
+echo "PLATFORM=$PLATFORM"
\ No newline at end of file
diff --git a/scripts/fast.sh b/scripts/fast.sh
new file mode 100755
index 00000000..8fd1e908
--- /dev/null
+++ b/scripts/fast.sh
@@ -0,0 +1,20 @@
+#/bin/sh
+
+# Wrapper script to speed up builds by disabling some permutations and reduce compatibility matrix
+# Don't use for release builds, but suitable for local developer iteration
+
+# Only build cuda v12
+export OLLAMA_SKIP_CUDA_11_GENERATE=1
+# Major versions only
+export CUDA_V12_ARCHITECTURES="60;70;80;90"
+# Skip ROCm
+export OLLAMA_SKIP_ROCM_GENERATE=1
+# Disable various less common quants and fattn
+export OLLAMA_FAST_BUILD=1
+
+if [ $# -ne 1 ] ; then
+ echo "Usage: ./scripts/fast.sh <build_script>"
+ exit 1
+fi
+
+exec $1
\ No newline at end of file
diff --git a/scripts/install.sh b/scripts/install.sh
index 30004584..79a7b564 100644
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -33,6 +33,17 @@ case "$ARCH" in
*) error "Unsupported architecture: $ARCH" ;;
esac
+IS_WSL2=false
+
+KERN=$(uname -r)
+case "$KERN" in
+ *icrosoft*WSL2 | *icrosoft*wsl2) IS_WSL2=true;;
+ *icrosoft) error "Microsoft WSL1 is not currently supported. Please use WSL2 with 'wsl --set-version <distro> 2'" ;;
+ *) ;;
+esac
+
+VER_PARAM="${OLLAMA_VERSION:+?version=$OLLAMA_VERSION}"
+
SUDO=
if [ "$(id -u)" -ne 0 ]; then
# Running as root, no need for sudo
@@ -52,19 +63,39 @@ if [ -n "$NEEDS" ]; then
exit 1
fi
-status "Downloading ollama..."
-curl --fail --show-error --location --progress-bar -o $TEMP_DIR/ollama "https://ollama.ai/download/ollama-linux-$ARCH"
-
for BINDIR in /usr/local/bin /usr/bin /bin; do
echo $PATH | grep -q $BINDIR && break || continue
done
+OLLAMA_INSTALL_DIR=$(dirname ${BINDIR})
-status "Installing ollama to $BINDIR..."
+status "Installing ollama to $OLLAMA_INSTALL_DIR"
$SUDO install -o0 -g0 -m755 -d $BINDIR
-$SUDO install -o0 -g0 -m755 $TEMP_DIR/ollama $BINDIR/ollama
+$SUDO install -o0 -g0 -m755 -d "$OLLAMA_INSTALL_DIR"
+if curl -I --silent --fail --location "https://ollama.com/download/ollama-linux-${ARCH}.tgz${VER_PARAM}" >/dev/null ; then
+ status "Downloading Linux ${ARCH} bundle"
+ curl --fail --show-error --location --progress-bar \
+ "https://ollama.com/download/ollama-linux-${ARCH}.tgz${VER_PARAM}" | \
+ $SUDO tar -xzf - -C "$OLLAMA_INSTALL_DIR"
+ BUNDLE=1
+ if [ "$OLLAMA_INSTALL_DIR/bin/ollama" != "$BINDIR/ollama" ] ; then
+ status "Making ollama accessible in the PATH in $BINDIR"
+ $SUDO ln -sf "$OLLAMA_INSTALL_DIR/ollama" "$BINDIR/ollama"
+ fi
+else
+ status "Downloading Linux ${ARCH} CLI"
+ curl --fail --show-error --location --progress-bar -o "$TEMP_DIR/ollama"\
+ "https://ollama.com/download/ollama-linux-${ARCH}${VER_PARAM}"
+ $SUDO install -o0 -g0 -m755 $TEMP_DIR/ollama $OLLAMA_INSTALL_DIR/ollama
+ BUNDLE=0
+ if [ "$OLLAMA_INSTALL_DIR/ollama" != "$BINDIR/ollama" ] ; then
+ status "Making ollama accessible in the PATH in $BINDIR"
+ $SUDO ln -sf "$OLLAMA_INSTALL_DIR/ollama" "$BINDIR/ollama"
+ fi
+fi
-install_success() {
- status 'The Ollama API is now available at 0.0.0.0:11434.'
+
+install_success() {
+ status 'The Ollama API is now available at 127.0.0.1:11434.'
status 'Install complete. Run "ollama" from the command line.'
}
trap install_success EXIT
@@ -74,7 +105,15 @@ trap install_success EXIT
configure_systemd() {
if ! id ollama >/dev/null 2>&1; then
status "Creating ollama user..."
- $SUDO useradd -r -s /bin/false -m -d /usr/share/ollama ollama
+ $SUDO useradd -r -s /bin/false -U -m -d /usr/share/ollama ollama
+ fi
+ if getent group render >/dev/null 2>&1; then
+ status "Adding ollama user to render group..."
+ $SUDO usermod -a -G render ollama
+ fi
+ if getent group video >/dev/null 2>&1; then
+ status "Adding ollama user to video group..."
+ $SUDO usermod -a -G video ollama
fi
status "Adding current user to ollama group..."
@@ -114,15 +153,35 @@ if available systemctl; then
configure_systemd
fi
+# WSL2 only supports GPUs via nvidia passthrough
+# so check for nvidia-smi to determine if GPU is available
+if [ "$IS_WSL2" = true ]; then
+ if available nvidia-smi && [ -n "$(nvidia-smi | grep -o "CUDA Version: [0-9]*\.[0-9]*")" ]; then
+ status "Nvidia GPU detected."
+ fi
+ install_success
+ exit 0
+fi
+
+# Install GPU dependencies on Linux
if ! available lspci && ! available lshw; then
- warning "Unable to detect NVIDIA GPU. Install lspci or lshw to automatically detect and install NVIDIA CUDA drivers."
+ warning "Unable to detect NVIDIA/AMD GPU. Install lspci or lshw to automatically detect and install GPU dependencies."
exit 0
fi
check_gpu() {
+ # Look for devices based on vendor ID for NVIDIA and AMD
case $1 in
- lspci) available lspci && lspci -d '10de:' | grep -q 'NVIDIA' || return 1 ;;
- lshw) available lshw && $SUDO lshw -c display -numeric | grep -q 'vendor: .* \[10DE\]' || return 1 ;;
+ lspci)
+ case $2 in
+ nvidia) available lspci && lspci -d '10de:' | grep -q 'NVIDIA' || return 1 ;;
+ amdgpu) available lspci && lspci -d '1002:' | grep -q 'AMD' || return 1 ;;
+ esac ;;
+ lshw)
+ case $2 in
+ nvidia) available lshw && $SUDO lshw -c display -numeric -disable network | grep -q 'vendor: .* \[10DE\]' || return 1 ;;
+ amdgpu) available lshw && $SUDO lshw -c display -numeric -disable network | grep -q 'vendor: .* \[1002\]' || return 1 ;;
+ esac ;;
nvidia-smi) available nvidia-smi || return 1 ;;
esac
}
@@ -132,25 +191,66 @@ if check_gpu nvidia-smi; then
exit 0
fi
-if ! check_gpu lspci && ! check_gpu lshw; then
+if ! check_gpu lspci nvidia && ! check_gpu lshw nvidia && ! check_gpu lspci amdgpu && ! check_gpu lshw amdgpu; then
install_success
- warning "No NVIDIA GPU detected. Ollama will run in CPU-only mode."
+ warning "No NVIDIA/AMD GPU detected. Ollama will run in CPU-only mode."
exit 0
fi
+if check_gpu lspci amdgpu || check_gpu lshw amdgpu; then
+ if [ $BUNDLE -ne 0 ]; then
+ status "Downloading Linux ROCm ${ARCH} bundle"
+ curl --fail --show-error --location --progress-bar \
+ "https://ollama.com/download/ollama-linux-${ARCH}-rocm.tgz${VER_PARAM}" | \
+ $SUDO tar -xzf - -C "$OLLAMA_INSTALL_DIR"
+
+ install_success
+ status "AMD GPU ready."
+ exit 0
+ fi
+ # Look for pre-existing ROCm v6 before downloading the dependencies
+ for search in "${HIP_PATH:-''}" "${ROCM_PATH:-''}" "/opt/rocm" "/usr/lib64"; do
+ if [ -n "${search}" ] && [ -e "${search}/libhipblas.so.2" -o -e "${search}/lib/libhipblas.so.2" ]; then
+ status "Compatible AMD GPU ROCm library detected at ${search}"
+ install_success
+ exit 0
+ fi
+ done
+
+ status "Downloading AMD GPU dependencies..."
+ $SUDO rm -rf /usr/share/ollama/lib
+ $SUDO chmod o+x /usr/share/ollama
+ $SUDO install -o ollama -g ollama -m 755 -d /usr/share/ollama/lib/rocm
+ curl --fail --show-error --location --progress-bar "https://ollama.com/download/ollama-linux-amd64-rocm.tgz${VER_PARAM}" \
+ | $SUDO tar zx --owner ollama --group ollama -C /usr/share/ollama/lib/rocm .
+ install_success
+ status "AMD GPU ready."
+ exit 0
+fi
+
+CUDA_REPO_ERR_MSG="NVIDIA GPU detected, but your OS and Architecture are not supported by NVIDIA. Please install the CUDA driver manually https://docs.nvidia.com/cuda/cuda-installation-guide-linux/"
# ref: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#rhel-7-centos-7
# ref: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#rhel-8-rocky-8
# ref: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#rhel-9-rocky-9
# ref: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#fedora
install_cuda_driver_yum() {
status 'Installing NVIDIA repository...'
+
case $PACKAGE_MANAGER in
yum)
$SUDO $PACKAGE_MANAGER -y install yum-utils
- $SUDO $PACKAGE_MANAGER-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/$1$2/$(uname -m)/cuda-$1$2.repo
+ if curl -I --silent --fail --location "https://developer.download.nvidia.com/compute/cuda/repos/$1$2/$(uname -m | sed -e 's/aarch64/sbsa/')/cuda-$1$2.repo" >/dev/null ; then
+ $SUDO $PACKAGE_MANAGER-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/$1$2/$(uname -m | sed -e 's/aarch64/sbsa/')/cuda-$1$2.repo
+ else
+ error $CUDA_REPO_ERR_MSG
+ fi
;;
dnf)
- $SUDO $PACKAGE_MANAGER config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/$1$2/$(uname -m)/cuda-$1$2.repo
+ if curl -I --silent --fail --location "https://developer.download.nvidia.com/compute/cuda/repos/$1$2/$(uname -m | sed -e 's/aarch64/sbsa/')/cuda-$1$2.repo" >/dev/null ; then
+ $SUDO $PACKAGE_MANAGER config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/$1$2/$(uname -m | sed -e 's/aarch64/sbsa/')/cuda-$1$2.repo
+ else
+ error $CUDA_REPO_ERR_MSG
+ fi
;;
esac
@@ -175,7 +275,11 @@ install_cuda_driver_yum() {
# ref: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#debian
install_cuda_driver_apt() {
status 'Installing NVIDIA repository...'
- curl -fsSL -o $TEMP_DIR/cuda-keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/$1$2/$(uname -m)/cuda-keyring_1.1-1_all.deb
+ if curl -I --silent --fail --location "https://developer.download.nvidia.com/compute/cuda/repos/$1$2/$(uname -m | sed -e 's/aarch64/sbsa/')/cuda-keyring_1.1-1_all.deb" >/dev/null ; then
+ curl -fsSL -o $TEMP_DIR/cuda-keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/$1$2/$(uname -m | sed -e 's/aarch64/sbsa/')/cuda-keyring_1.1-1_all.deb
+ else
+ error $CUDA_REPO_ERR_MSG
+ fi
case $1 in
debian)
@@ -219,15 +323,15 @@ if ! check_gpu nvidia-smi || [ -z "$(nvidia-smi | grep -o "CUDA Version: [0-9]*\
case $OS_NAME in
centos|rhel) install_cuda_driver_yum 'rhel' $(echo $OS_VERSION | cut -d '.' -f 1) ;;
rocky) install_cuda_driver_yum 'rhel' $(echo $OS_VERSION | cut -c1) ;;
- fedora) install_cuda_driver_yum $OS_NAME $OS_VERSION ;;
- amzn) install_cuda_driver_yum 'fedora' '35' ;;
+ fedora) [ $OS_VERSION -lt '39' ] && install_cuda_driver_yum $OS_NAME $OS_VERSION || install_cuda_driver_yum $OS_NAME '39';;
+ amzn) install_cuda_driver_yum 'fedora' '37' ;;
debian) install_cuda_driver_apt $OS_NAME $OS_VERSION ;;
ubuntu) install_cuda_driver_apt $OS_NAME $(echo $OS_VERSION | sed 's/\.//') ;;
*) exit ;;
esac
fi
-if ! lsmod | grep -q nvidia; then
+if ! lsmod | grep -q nvidia || ! lsmod | grep -q nvidia_uvm; then
KERNEL_RELEASE="$(uname -r)"
case $OS_NAME in
rocky) $SUDO $PACKAGE_MANAGER -y install kernel-devel kernel-headers ;;
@@ -248,7 +352,19 @@ if ! lsmod | grep -q nvidia; then
fi
$SUDO modprobe nvidia
+ $SUDO modprobe nvidia_uvm
fi
+# make sure the NVIDIA modules are loaded on boot with nvidia-persistenced
+if available nvidia-persistenced; then
+ $SUDO touch /etc/modules-load.d/nvidia.conf
+ MODULES="nvidia nvidia-uvm"
+ for MODULE in $MODULES; do
+ if ! grep -qxF "$MODULE" /etc/modules-load.d/nvidia.conf; then
+ echo "$MODULE" | $SUDO tee -a /etc/modules-load.d/nvidia.conf > /dev/null
+ fi
+ done
+fi
-status "NVIDIA CUDA drivers installed."
+status "NVIDIA GPU ready."
+install_success
diff --git a/scripts/push_docker.sh b/scripts/push_docker.sh
index 31865c2b..c140c27c 100755
--- a/scripts/push_docker.sh
+++ b/scripts/push_docker.sh
@@ -3,14 +3,13 @@
set -eu
export VERSION=${VERSION:-0.0.0}
-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/jmorganca/ollama/version.Version=$VERSION\" \"-X=github.com/jmorganca/ollama/server.mode=release\"'"
+export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
-docker buildx build \
+docker build \
--push \
--platform=linux/arm64,linux/amd64 \
--build-arg=VERSION \
--build-arg=GOFLAGS \
- --cache-from type=local,src=.cache \
-f Dockerfile \
-t ollama/ollama -t ollama/ollama:$VERSION \
.
diff --git a/scripts/rh_linux_deps.sh b/scripts/rh_linux_deps.sh
new file mode 100644
index 00000000..d0cadd45
--- /dev/null
+++ b/scripts/rh_linux_deps.sh
@@ -0,0 +1,78 @@
+#!/bin/sh
+
+# Script for common Dockerfile dependency installation in redhat linux based images
+
+set -ex
+set -o pipefail
+MACHINE=$(uname -m)
+
+if grep -i "centos" /etc/system-release >/dev/null; then
+ # As of 7/1/2024 mirrorlist.centos.org has been taken offline, so adjust accordingly
+ sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
+ sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
+ sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
+
+ # Centos 7 derivatives have too old of a git version to run our generate script
+ # uninstall and ignore failures
+ yum remove -y git
+ yum -y install epel-release centos-release-scl
+
+ # The release packages reinstate the mirrors, undo that again
+ sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
+ sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
+ sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
+
+ yum -y install dnf
+ if [ "${MACHINE}" = "x86_64" ]; then
+ yum -y install https://repo.ius.io/ius-release-el7.rpm
+ dnf install -y git236
+ else
+ dnf install -y rh-git227-git
+ ln -s /opt/rh/rh-git227/root/usr/bin/git /usr/local/bin/git
+ fi
+ dnf install -y devtoolset-10-gcc devtoolset-10-gcc-c++ pigz findutils
+elif grep -i "rocky" /etc/system-release >/dev/null; then
+ # Temporary workaround until rocky 8 AppStream ships GCC 10.4 (10.3 is incompatible with NVCC)
+ cat << EOF > /etc/yum.repos.d/Rocky-Vault.repo
+[vault]
+name=Rocky Vault
+baseurl=https://dl.rockylinux.org/vault/rocky/8.5/AppStream/\$basearch/os/
+gpgcheck=1
+enabled=1
+countme=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial
+EOF
+ dnf install -y git \
+ gcc-toolset-10-gcc-10.2.1-8.2.el8 \
+ gcc-toolset-10-gcc-c++-10.2.1-8.2.el8 \
+ findutils \
+ yum-utils \
+ pigz
+else
+ echo "ERROR Unexpected distro"
+ exit 1
+fi
+
+if [ "${MACHINE}" = "x86_64" ] ; then
+ curl -s -L https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz | tar -Jx -C /tmp --strip-components 1 && \
+ mv /tmp/ccache /usr/local/bin/
+else
+ yum -y install epel-release
+ yum install -y ccache
+fi
+
+if [ -n "${CMAKE_VERSION}" ]; then
+ curl -s -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz | tar -zx -C /usr --strip-components 1
+fi
+
+if [ -n "${GOLANG_VERSION}" ]; then
+ if [ "${MACHINE}" = "x86_64" ]; then
+ GO_ARCH="amd64"
+ else
+ GO_ARCH="arm64"
+ fi
+ mkdir -p /usr/local
+ curl -s -L https://dl.google.com/go/go${GOLANG_VERSION}.linux-${GO_ARCH}.tar.gz | tar xz -C /usr/local
+ ln -s /usr/local/go/bin/go /usr/local/bin/go
+ ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
+fi
diff --git a/scripts/setup_integration_tests.sh b/scripts/setup_integration_tests.sh
deleted file mode 100755
index 851ce49d..00000000
--- a/scripts/setup_integration_tests.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-
-# This script sets up integration tests which run the full stack to verify
-# inference locally
-#
-# To run the relevant tests use
-# go test -tags=integration ./server
-set -e
-set -o pipefail
-
-REPO=$(dirname $0)/../
-export OLLAMA_MODELS=${REPO}/test_data/models
-REGISTRY_SCHEME=https
-REGISTRY=registry.ollama.ai
-TEST_MODELS=("library/orca-mini:latest" "library/llava:7b")
-ACCEPT_HEADER="Accept: application/vnd.docker.distribution.manifest.v2+json"
-
-for model in ${TEST_MODELS[@]}; do
- TEST_MODEL=$(echo ${model} | cut -f1 -d:)
- TEST_MODEL_TAG=$(echo ${model} | cut -f2 -d:)
- mkdir -p ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/
- mkdir -p ${OLLAMA_MODELS}/blobs/
-
- echo "Pulling manifest for ${TEST_MODEL}:${TEST_MODEL_TAG}"
- curl -s --header "${ACCEPT_HEADER}" \
- -o ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} \
- ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/manifests/${TEST_MODEL_TAG}
-
- CFG_HASH=$(cat ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} | jq -r ".config.digest")
- echo "Pulling config blob ${CFG_HASH}"
- curl -L -C - --header "${ACCEPT_HEADER}" \
- -o ${OLLAMA_MODELS}/blobs/${CFG_HASH} \
- ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/blobs/${CFG_HASH}
-
- for LAYER in $(cat ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} | jq -r ".layers[].digest"); do
- echo "Pulling blob ${LAYER}"
- curl -L -C - --header "${ACCEPT_HEADER}" \
- -o ${OLLAMA_MODELS}/blobs/${LAYER} \
- ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/blobs/${LAYER}
- done
-done
diff --git a/scripts/tag_latest.sh b/scripts/tag_latest.sh
new file mode 100755
index 00000000..1f56f036
--- /dev/null
+++ b/scripts/tag_latest.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -eu
+
+# For developers, you can override the DOCKER_ORG to generate multiarch manifests
+# DOCKER_ORG=jdoe VERSION=0.1.30 ./scripts/tag_latest.sh
+DOCKER_ORG=${DOCKER_ORG:-"ollama"}
+FINAL_IMAGE_REPO=${FINAL_IMAGE_REPO:-"${DOCKER_ORG}/ollama"}
+
+echo "Updating ${FINAL_IMAGE_REPO}:latest -> ${FINAL_IMAGE_REPO}:${VERSION}"
+docker buildx imagetools create -t ${FINAL_IMAGE_REPO}:latest ${FINAL_IMAGE_REPO}:${VERSION}
+echo "Updating ${FINAL_IMAGE_REPO}:rocm -> ${FINAL_IMAGE_REPO}:${VERSION}-rocm"
+docker buildx imagetools create -t ${FINAL_IMAGE_REPO}:rocm ${FINAL_IMAGE_REPO}:${VERSION}-rocm
diff --git a/server/auth.go b/server/auth.go
index 6c151ac3..dcef5bf9 100644
--- a/server/auth.go
+++ b/server/auth.go
@@ -1,7 +1,6 @@
package server
import (
- "bytes"
"context"
"crypto/rand"
"crypto/sha256"
@@ -10,166 +9,87 @@ import (
"encoding/json"
"fmt"
"io"
- "log"
"net/http"
"net/url"
- "os"
- "path/filepath"
"strconv"
"strings"
"time"
- "golang.org/x/crypto/ssh"
-
- "github.com/jmorganca/ollama/api"
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/auth"
)
-type AuthRedirect struct {
+type registryChallenge struct {
Realm string
Service string
Scope string
}
-type SignatureData struct {
- Method string
- Path string
- Data []byte
-}
-
-func generateNonce(length int) (string, error) {
- nonce := make([]byte, length)
- _, err := rand.Read(nonce)
- if err != nil {
- return "", err
- }
- return base64.RawURLEncoding.EncodeToString(nonce), nil
-}
-
-func (r AuthRedirect) URL() (*url.URL, error) {
+func (r registryChallenge) URL() (*url.URL, error) {
redirectURL, err := url.Parse(r.Realm)
if err != nil {
return nil, err
}
values := redirectURL.Query()
-
values.Add("service", r.Service)
-
for _, s := range strings.Split(r.Scope, " ") {
values.Add("scope", s)
}
values.Add("ts", strconv.FormatInt(time.Now().Unix(), 10))
- nonce, err := generateNonce(16)
+ nonce, err := auth.NewNonce(rand.Reader, 16)
if err != nil {
return nil, err
}
+
values.Add("nonce", nonce)
redirectURL.RawQuery = values.Encode()
return redirectURL, nil
}
-func getAuthToken(ctx context.Context, redirData AuthRedirect) (string, error) {
- redirectURL, err := redirData.URL()
+func getAuthorizationToken(ctx context.Context, challenge registryChallenge) (string, error) {
+ redirectURL, err := challenge.URL()
if err != nil {
return "", err
}
- home, err := os.UserHomeDir()
- if err != nil {
- return "", err
- }
-
- keyPath := filepath.Join(home, ".ollama", "id_ed25519")
-
- rawKey, err := os.ReadFile(keyPath)
- if err != nil {
- log.Printf("Failed to load private key: %v", err)
- return "", err
- }
-
- s := SignatureData{
- Method: http.MethodGet,
- Path: redirectURL.String(),
- Data: nil,
- }
-
- sig, err := s.Sign(rawKey)
- if err != nil {
- return "", err
- }
+ sha256sum := sha256.Sum256(nil)
+ data := []byte(fmt.Sprintf("%s,%s,%s", http.MethodGet, redirectURL.String(), base64.StdEncoding.EncodeToString([]byte(hex.EncodeToString(sha256sum[:])))))
headers := make(http.Header)
- headers.Set("Authorization", sig)
- resp, err := makeRequest(ctx, http.MethodGet, redirectURL, headers, nil, nil)
- if err != nil {
- log.Printf("couldn't get token: %q", err)
- return "", err
- }
- defer resp.Body.Close()
-
- if resp.StatusCode >= http.StatusBadRequest {
- body, _ := io.ReadAll(resp.Body)
- return "", fmt.Errorf("on pull registry responded with code %d: %s", resp.StatusCode, body)
- }
-
- respBody, err := io.ReadAll(resp.Body)
+ signature, err := auth.Sign(ctx, data)
if err != nil {
return "", err
}
- var tok api.TokenResponse
- if err := json.Unmarshal(respBody, &tok); err != nil {
+ headers.Add("Authorization", signature)
+
+ response, err := makeRequest(ctx, http.MethodGet, redirectURL, headers, nil, ®istryOptions{})
+ if err != nil {
+ return "", err
+ }
+ defer response.Body.Close()
+
+ body, err := io.ReadAll(response.Body)
+ if err != nil {
+ return "", fmt.Errorf("%d: %v", response.StatusCode, err)
+ }
+
+ if response.StatusCode >= http.StatusBadRequest {
+ if len(body) > 0 {
+ return "", fmt.Errorf("%d: %s", response.StatusCode, body)
+ } else {
+ return "", fmt.Errorf("%d", response.StatusCode)
+ }
+ }
+
+ var token api.TokenResponse
+ if err := json.Unmarshal(body, &token); err != nil {
return "", err
}
- return tok.Token, nil
-}
-
-// Bytes returns a byte slice of the data to sign for the request
-func (s SignatureData) Bytes() []byte {
- // We first derive the content hash of the request body using:
- // base64(hex(sha256(request body)))
-
- hash := sha256.Sum256(s.Data)
- hashHex := make([]byte, hex.EncodedLen(len(hash)))
- hex.Encode(hashHex, hash[:])
- contentHash := base64.StdEncoding.EncodeToString(hashHex)
-
- // We then put the entire request together in a serialize string using:
- // "<method>,<uri>,<content hash>"
- // e.g. "GET,http://localhost,OTdkZjM1O..."
-
- return []byte(strings.Join([]string{s.Method, s.Path, contentHash}, ","))
-}
-
-// SignData takes a SignatureData object and signs it with a raw private key
-func (s SignatureData) Sign(rawKey []byte) (string, error) {
- privateKey, err := ssh.ParseRawPrivateKey(rawKey)
- if err != nil {
- return "", err
- }
-
- signer, err := ssh.NewSignerFromKey(privateKey)
- if err != nil {
- return "", err
- }
-
- // get the pubkey, but remove the type
- pubKey := ssh.MarshalAuthorizedKey(signer.PublicKey())
- parts := bytes.Split(pubKey, []byte(" "))
- if len(parts) < 2 {
- return "", fmt.Errorf("malformed public key")
- }
-
- signedData, err := signer.Sign(nil, s.Bytes())
- if err != nil {
- return "", err
- }
-
- // signature is <pubkey>:<signature>
- sig := fmt.Sprintf("%s:%s", bytes.TrimSpace(parts[1]), base64.StdEncoding.EncodeToString(signedData.Blob))
- return sig, nil
+ return token.Token, nil
}
diff --git a/server/download.go b/server/download.go
index 3c823f32..a3b53189 100644
--- a/server/download.go
+++ b/server/download.go
@@ -6,8 +6,9 @@ import (
"errors"
"fmt"
"io"
- "log"
+ "log/slog"
"math"
+ "math/rand/v2"
"net/http"
"net/url"
"os"
@@ -21,8 +22,15 @@ import (
"golang.org/x/sync/errgroup"
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/format"
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/format"
+)
+
+const maxRetries = 6
+
+var (
+ errMaxRetriesExceeded = errors.New("max retries exceeded")
+ errPartStalled = errors.New("part stalled")
)
var blobDownloadManager sync.Map
@@ -38,7 +46,7 @@ type blobDownload struct {
context.CancelFunc
- done bool
+ done chan struct{}
err error
references atomic.Int32
}
@@ -47,13 +55,46 @@ type blobDownloadPart struct {
N int
Offset int64
Size int64
- Completed int64
+ Completed atomic.Int64
+
+ lastUpdatedMu sync.Mutex
+ lastUpdated time.Time
*blobDownload `json:"-"`
}
+type jsonBlobDownloadPart struct {
+ N int
+ Offset int64
+ Size int64
+ Completed int64
+}
+
+func (p *blobDownloadPart) MarshalJSON() ([]byte, error) {
+ return json.Marshal(jsonBlobDownloadPart{
+ N: p.N,
+ Offset: p.Offset,
+ Size: p.Size,
+ Completed: p.Completed.Load(),
+ })
+}
+
+func (p *blobDownloadPart) UnmarshalJSON(b []byte) error {
+ var j jsonBlobDownloadPart
+ if err := json.Unmarshal(b, &j); err != nil {
+ return err
+ }
+ *p = blobDownloadPart{
+ N: j.N,
+ Offset: j.Offset,
+ Size: j.Size,
+ }
+ p.Completed.Store(j.Completed)
+ return nil
+}
+
const (
- numDownloadParts = 64
+ numDownloadParts = 16
minDownloadPartSize int64 = 100 * format.MegaByte
maxDownloadPartSize int64 = 1000 * format.MegaByte
)
@@ -65,19 +106,30 @@ func (p *blobDownloadPart) Name() string {
}
func (p *blobDownloadPart) StartsAt() int64 {
- return p.Offset + p.Completed
+ return p.Offset + p.Completed.Load()
}
func (p *blobDownloadPart) StopsAt() int64 {
return p.Offset + p.Size
}
-func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *RegistryOptions) error {
+func (p *blobDownloadPart) Write(b []byte) (n int, err error) {
+ n = len(b)
+ p.blobDownload.Completed.Add(int64(n))
+ p.lastUpdatedMu.Lock()
+ p.lastUpdated = time.Now()
+ p.lastUpdatedMu.Unlock()
+ return n, nil
+}
+
+func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *registryOptions) error {
partFilePaths, err := filepath.Glob(b.Name + "-partial-*")
if err != nil {
return err
}
+ b.done = make(chan struct{})
+
for _, partFilePath := range partFilePaths {
part, err := b.readPart(partFilePath)
if err != nil {
@@ -85,7 +137,7 @@ func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *R
}
b.Total += part.Size
- b.Completed.Add(part.Completed)
+ b.Completed.Add(part.Completed.Load())
b.Parts = append(b.Parts, part)
}
@@ -98,7 +150,7 @@ func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *R
b.Total, _ = strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
- var size = b.Total / numDownloadParts
+ size := b.Total / numDownloadParts
switch {
case size < minDownloadPartSize:
size = minDownloadPartSize
@@ -120,31 +172,105 @@ func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *R
}
}
- log.Printf("downloading %s in %d %s part(s)", b.Digest[7:19], len(b.Parts), format.HumanBytes(b.Parts[0].Size))
+ slog.Info(fmt.Sprintf("downloading %s in %d %s part(s)", b.Digest[7:19], len(b.Parts), format.HumanBytes(b.Parts[0].Size)))
return nil
}
-func (b *blobDownload) Run(ctx context.Context, requestURL *url.URL, opts *RegistryOptions) {
+func (b *blobDownload) Run(ctx context.Context, requestURL *url.URL, opts *registryOptions) {
+ defer close(b.done)
b.err = b.run(ctx, requestURL, opts)
}
-func (b *blobDownload) run(ctx context.Context, requestURL *url.URL, opts *RegistryOptions) error {
+func newBackoff(maxBackoff time.Duration) func(ctx context.Context) error {
+ var n int
+ return func(ctx context.Context) error {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+
+ n++
+
+ // n^2 backoff timer is a little smoother than the
+ // common choice of 2^n.
+ d := min(time.Duration(n*n)*10*time.Millisecond, maxBackoff)
+ // Randomize the delay between 0.5-1.5 x msec, in order
+ // to prevent accidental "thundering herd" problems.
+ d = time.Duration(float64(d) * (rand.Float64() + 0.5))
+ t := time.NewTimer(d)
+ defer t.Stop()
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-t.C:
+ return nil
+ }
+ }
+}
+
+func (b *blobDownload) run(ctx context.Context, requestURL *url.URL, opts *registryOptions) error {
defer blobDownloadManager.Delete(b.Digest)
ctx, b.CancelFunc = context.WithCancel(ctx)
- file, err := os.OpenFile(b.Name+"-partial", os.O_CREATE|os.O_RDWR, 0644)
+ file, err := os.OpenFile(b.Name+"-partial", os.O_CREATE|os.O_RDWR, 0o644)
if err != nil {
return err
}
defer file.Close()
+ setSparse(file)
- file.Truncate(b.Total)
+ _ = file.Truncate(b.Total)
+
+ directURL, err := func() (*url.URL, error) {
+ ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ defer cancel()
+
+ backoff := newBackoff(10 * time.Second)
+ for {
+ // shallow clone opts to be used in the closure
+ // without affecting the outer opts.
+ newOpts := new(registryOptions)
+ *newOpts = *opts
+
+ newOpts.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+ if len(via) > 10 {
+ return errors.New("maximum redirects exceeded (10) for directURL")
+ }
+
+ // if the hostname is the same, allow the redirect
+ if req.URL.Hostname() == requestURL.Hostname() {
+ return nil
+ }
+
+ // stop at the first redirect that is not
+ // the same hostname as the original
+ // request.
+ return http.ErrUseLastResponse
+ }
+
+ resp, err := makeRequestWithRetry(ctx, http.MethodGet, requestURL, nil, nil, newOpts)
+ if err != nil {
+ slog.Warn("failed to get direct URL; backing off and retrying", "err", err)
+ if err := backoff(ctx); err != nil {
+ return nil, err
+ }
+ continue
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusTemporaryRedirect && resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("unexpected status code %d", resp.StatusCode)
+ }
+ return resp.Location()
+ }
+ }()
+ if err != nil {
+ return err
+ }
g, inner := errgroup.WithContext(ctx)
g.SetLimit(numDownloadParts)
for i := range b.Parts {
part := b.Parts[i]
- if part.Completed == part.Size {
+ if part.Completed.Load() == part.Size {
continue
}
@@ -152,14 +278,17 @@ func (b *blobDownload) run(ctx context.Context, requestURL *url.URL, opts *Regis
var err error
for try := 0; try < maxRetries; try++ {
w := io.NewOffsetWriter(file, part.StartsAt())
- err = b.downloadChunk(inner, requestURL, w, part, opts)
+ err = b.downloadChunk(inner, directURL, w, part)
switch {
case errors.Is(err, context.Canceled), errors.Is(err, syscall.ENOSPC):
// return immediately if the context is canceled or the device is out of space
return err
+ case errors.Is(err, errPartStalled):
+ try--
+ continue
case err != nil:
sleep := time.Second * time.Duration(math.Pow(2, float64(try)))
- log.Printf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep)
+ slog.Info(fmt.Sprintf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep))
time.Sleep(sleep)
continue
default:
@@ -190,33 +319,68 @@ func (b *blobDownload) run(ctx context.Context, requestURL *url.URL, opts *Regis
return err
}
- b.done = true
return nil
}
-func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, w io.Writer, part *blobDownloadPart, opts *RegistryOptions) error {
- headers := make(http.Header)
- headers.Set("Range", fmt.Sprintf("bytes=%d-%d", part.StartsAt(), part.StopsAt()-1))
- resp, err := makeRequestWithRetry(ctx, http.MethodGet, requestURL, headers, nil, opts)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
+func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, w io.Writer, part *blobDownloadPart) error {
+ g, ctx := errgroup.WithContext(ctx)
+ g.Go(func() error {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL.String(), nil)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", part.StartsAt(), part.StopsAt()-1))
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
- n, err := io.Copy(w, io.TeeReader(resp.Body, b))
- if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, io.ErrUnexpectedEOF) {
- // rollback progress
- b.Completed.Add(-n)
- return err
- }
+ n, err := io.CopyN(w, io.TeeReader(resp.Body, part), part.Size-part.Completed.Load())
+ if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, io.ErrUnexpectedEOF) {
+ // rollback progress
+ b.Completed.Add(-n)
+ return err
+ }
- part.Completed += n
- if err := b.writePart(part.Name(), part); err != nil {
- return err
- }
+ part.Completed.Add(n)
+ if err := b.writePart(part.Name(), part); err != nil {
+ return err
+ }
- // return nil or context.Canceled or UnexpectedEOF (resumable)
- return err
+ // return nil or context.Canceled or UnexpectedEOF (resumable)
+ return err
+ })
+
+ g.Go(func() error {
+ ticker := time.NewTicker(time.Second)
+ for {
+ select {
+ case <-ticker.C:
+ if part.Completed.Load() >= part.Size {
+ return nil
+ }
+
+ part.lastUpdatedMu.Lock()
+ lastUpdated := part.lastUpdated
+ part.lastUpdatedMu.Unlock()
+
+ if !lastUpdated.IsZero() && time.Since(lastUpdated) > 5*time.Second {
+ const msg = "%s part %d stalled; retrying. If this persists, press ctrl-c to exit, then 'ollama pull' to find a faster connection."
+ slog.Info(fmt.Sprintf(msg, b.Digest[7:19], part.N))
+ // reset last updated
+ part.lastUpdatedMu.Lock()
+ part.lastUpdated = time.Time{}
+ part.lastUpdatedMu.Unlock()
+ return errPartStalled
+ }
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+ })
+
+ return g.Wait()
}
func (b *blobDownload) newPart(offset, size int64) error {
@@ -246,7 +410,7 @@ func (b *blobDownload) readPart(partName string) (*blobDownloadPart, error) {
}
func (b *blobDownload) writePart(partName string, part *blobDownloadPart) error {
- partFile, err := os.OpenFile(partName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
+ partFile, err := os.OpenFile(partName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o644)
if err != nil {
return err
}
@@ -255,12 +419,6 @@ func (b *blobDownload) writePart(partName string, part *blobDownloadPart) error
return json.NewEncoder(partFile).Encode(part)
}
-func (b *blobDownload) Write(p []byte) (n int, err error) {
- n = len(p)
- b.Completed.Add(int64(n))
- return n, nil
-}
-
func (b *blobDownload) acquire() {
b.references.Add(1)
}
@@ -278,47 +436,40 @@ func (b *blobDownload) Wait(ctx context.Context, fn func(api.ProgressResponse))
ticker := time.NewTicker(60 * time.Millisecond)
for {
select {
+ case <-b.done:
+ return b.err
case <-ticker.C:
+ fn(api.ProgressResponse{
+ Status: fmt.Sprintf("pulling %s", b.Digest[7:19]),
+ Digest: b.Digest,
+ Total: b.Total,
+ Completed: b.Completed.Load(),
+ })
case <-ctx.Done():
return ctx.Err()
}
-
- fn(api.ProgressResponse{
- Status: fmt.Sprintf("pulling %s", b.Digest[7:19]),
- Digest: b.Digest,
- Total: b.Total,
- Completed: b.Completed.Load(),
- })
-
- if b.done || b.err != nil {
- return b.err
- }
}
}
type downloadOpts struct {
mp ModelPath
digest string
- regOpts *RegistryOptions
+ regOpts *registryOptions
fn func(api.ProgressResponse)
}
-const maxRetries = 6
-
-var errMaxRetriesExceeded = errors.New("max retries exceeded")
-
// downloadBlob downloads a blob from the registry and stores it in the blobs directory
-func downloadBlob(ctx context.Context, opts downloadOpts) error {
+func downloadBlob(ctx context.Context, opts downloadOpts) (cacheHit bool, _ error) {
fp, err := GetBlobsPath(opts.digest)
if err != nil {
- return err
+ return false, err
}
fi, err := os.Stat(fp)
switch {
case errors.Is(err, os.ErrNotExist):
case err != nil:
- return err
+ return false, err
default:
opts.fn(api.ProgressResponse{
Status: fmt.Sprintf("pulling %s", opts.digest[7:19]),
@@ -327,7 +478,7 @@ func downloadBlob(ctx context.Context, opts downloadOpts) error {
Completed: fi.Size(),
})
- return nil
+ return true, nil
}
data, ok := blobDownloadManager.LoadOrStore(opts.digest, &blobDownload{Name: fp, Digest: opts.digest})
@@ -337,11 +488,12 @@ func downloadBlob(ctx context.Context, opts downloadOpts) error {
requestURL = requestURL.JoinPath("v2", opts.mp.GetNamespaceRepository(), "blobs", opts.digest)
if err := download.Prepare(ctx, requestURL, opts.regOpts); err != nil {
blobDownloadManager.Delete(opts.digest)
- return err
+ return false, err
}
+ //nolint:contextcheck
go download.Run(context.Background(), requestURL, opts.regOpts)
}
- return download.Wait(ctx, opts.fn)
+ return false, download.Wait(ctx, opts.fn)
}
diff --git a/server/fixblobs.go b/server/fixblobs.go
new file mode 100644
index 00000000..6e6b3114
--- /dev/null
+++ b/server/fixblobs.go
@@ -0,0 +1,26 @@
+package server
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// fixBlobs walks the provided dir and replaces (":") to ("-") in the file
+// prefix. (e.g. sha256:1234 -> sha256-1234)
+func fixBlobs(dir string) error {
+ return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ baseName := filepath.Base(path)
+ typ, sha, ok := strings.Cut(baseName, ":")
+ if ok && typ == "sha256" {
+ newPath := filepath.Join(filepath.Dir(path), typ+"-"+sha)
+ if err := os.Rename(path, newPath); err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+}
diff --git a/server/fixblobs_test.go b/server/fixblobs_test.go
new file mode 100644
index 00000000..c2d4851a
--- /dev/null
+++ b/server/fixblobs_test.go
@@ -0,0 +1,83 @@
+package server
+
+import (
+ "io/fs"
+ "os"
+ "path/filepath"
+ "runtime"
+ "slices"
+ "strings"
+ "testing"
+)
+
+func TestFixBlobs(t *testing.T) {
+ cases := []struct {
+ path []string
+ want []string
+ }{
+ {path: []string{"sha256-1234"}, want: []string{"sha256-1234"}},
+ {path: []string{"sha256:1234"}, want: []string{"sha256-1234"}},
+ {path: []string{"sha259:5678"}, want: []string{"sha259:5678"}},
+ {path: []string{"sha256:abcd"}, want: []string{"sha256-abcd"}},
+ {path: []string{"x/y/sha256:abcd"}, want: []string{"x/y/sha256-abcd"}},
+ {path: []string{"x:y/sha256:abcd"}, want: []string{"x:y/sha256-abcd"}},
+ {path: []string{"x:y/sha256:abcd"}, want: []string{"x:y/sha256-abcd"}},
+ {path: []string{"x:y/sha256:abcd", "sha256:1234"}, want: []string{"x:y/sha256-abcd", "sha256-1234"}},
+ {path: []string{"x:y/sha256:abcd", "sha256-1234"}, want: []string{"x:y/sha256-abcd", "sha256-1234"}},
+ }
+
+ for _, tt := range cases {
+ t.Run(strings.Join(tt.path, "|"), func(t *testing.T) {
+ hasColon := slices.ContainsFunc(tt.path, func(s string) bool { return strings.Contains(s, ":") })
+ if hasColon && runtime.GOOS == "windows" {
+ t.Skip("skipping test on windows")
+ }
+
+ rootDir := t.TempDir()
+ for _, path := range tt.path {
+ fullPath := filepath.Join(rootDir, path)
+ fullDir, _ := filepath.Split(fullPath)
+
+ t.Logf("creating dir %s", fullDir)
+ if err := os.MkdirAll(fullDir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+
+ t.Logf("writing file %s", fullPath)
+ if err := os.WriteFile(fullPath, nil, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if err := fixBlobs(rootDir); err != nil {
+ t.Fatal(err)
+ }
+
+ got := slurpFiles(os.DirFS(rootDir))
+
+ slices.Sort(tt.want)
+ slices.Sort(got)
+ if !slices.Equal(got, tt.want) {
+ t.Fatalf("got = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func slurpFiles(fsys fs.FS) []string {
+ var sfs []string
+ fn := func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ return nil
+ }
+ sfs = append(sfs, path)
+ return nil
+ }
+ if err := fs.WalkDir(fsys, ".", fn); err != nil {
+ panic(err)
+ }
+ return sfs
+}
diff --git a/server/images.go b/server/images.go
index 85b96395..9d2e1959 100644
--- a/server/images.go
+++ b/server/images.go
@@ -2,37 +2,61 @@ package server
import (
"bytes"
+ "cmp"
"context"
"crypto/sha256"
+ "encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
+ "log/slog"
"net/http"
"net/url"
"os"
"path/filepath"
"runtime"
+ "slices"
"strconv"
"strings"
- "text/template"
- "text/template/parse"
- "golang.org/x/exp/slices"
-
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/llm"
- "github.com/jmorganca/ollama/parser"
- "github.com/jmorganca/ollama/version"
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/auth"
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/format"
+ "github.com/ollama/ollama/llama"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/parser"
+ "github.com/ollama/ollama/template"
+ "github.com/ollama/ollama/types/errtypes"
+ "github.com/ollama/ollama/types/model"
+ "github.com/ollama/ollama/version"
)
-type RegistryOptions struct {
+var (
+ errCapabilities = errors.New("does not support")
+ errCapabilityCompletion = errors.New("completion")
+ errCapabilityTools = errors.New("tools")
+ errCapabilityInsert = errors.New("insert")
+)
+
+type Capability string
+
+const (
+ CapabilityCompletion = Capability("completion")
+ CapabilityTools = Capability("tools")
+ CapabilityInsert = Capability("insert")
+)
+
+type registryOptions struct {
Insecure bool
Username string
Password string
Token string
+
+ CheckRedirect func(req *http.Request, via []*http.Request) error
}
type Model struct {
@@ -40,172 +64,132 @@ type Model struct {
Config ConfigV2
ShortName string
ModelPath string
- OriginalModel string
+ ParentModel string
AdapterPaths []string
ProjectorPaths []string
- Template string
System string
License []string
Digest string
- Size int64
Options map[string]interface{}
+ Messages []api.Message
+
+ Template *template.Template
}
-type PromptVars struct {
- System string
- Prompt string
- Response string
- First bool
-}
-
-// extractParts extracts the parts of the template before and after the {{.Response}} node.
-func extractParts(tmplStr string) (pre string, post string, err error) {
- tmpl, err := template.New("").Parse(tmplStr)
- if err != nil {
- return "", "", err
- }
-
- var foundResponse bool
-
- for _, node := range tmpl.Tree.Root.Nodes {
- if node.Type() == parse.NodeAction && node.String() == "{{.Response}}" {
- foundResponse = true
- }
- if !foundResponse {
- pre += node.String()
- } else {
- post += node.String()
- }
- }
-
- return pre, post, nil
-}
-
-func Prompt(promptTemplate string, p PromptVars) (string, error) {
- var prompt strings.Builder
- // Use the "missingkey=zero" option to handle missing variables without panicking
- tmpl, err := template.New("").Option("missingkey=zero").Parse(promptTemplate)
- if err != nil {
- return "", err
- }
-
- vars := map[string]any{
- "System": p.System,
- "Prompt": p.Prompt,
- "Response": p.Response,
- "First": p.First,
- }
-
- var sb strings.Builder
- if err := tmpl.Execute(&sb, vars); err != nil {
- return "", err
- }
- prompt.WriteString(sb.String())
-
- if !strings.Contains(prompt.String(), p.Response) {
- // if the response is not in the prompt template, append it to the end
- prompt.WriteString(p.Response)
- }
-
- return prompt.String(), nil
-}
-
-// PreResponsePrompt returns the prompt before the response tag
-func (m *Model) PreResponsePrompt(p PromptVars) (string, error) {
- if p.System == "" {
- // use the default system prompt for this model if one is not specified
- p.System = m.System
- }
- pre, _, err := extractParts(m.Template)
- if err != nil {
- return "", err
- }
-
- return Prompt(pre, p)
-}
-
-// PostResponseTemplate returns the template after the response tag
-func (m *Model) PostResponseTemplate(p PromptVars) (string, error) {
- if p.System == "" {
- // use the default system prompt for this model if one is not specified
- p.System = m.System
- }
- _, post, err := extractParts(m.Template)
- if err != nil {
- return "", err
- }
-
- if post == "" {
- // if there is no post-response template, return the provided response
- return p.Response, nil
- }
-
- return Prompt(post, p)
-}
-
-func (m *Model) ChatPrompt(msgs []api.Message) (string, []api.ImageData, error) {
- // build the prompt from the list of messages
- var prompt strings.Builder
- var currentImages []api.ImageData
- currentVars := PromptVars{
- First: true,
- System: m.System,
- }
-
- writePrompt := func() error {
- p, err := Prompt(m.Template, currentVars)
- if err != nil {
- return err
- }
- prompt.WriteString(p)
- currentVars = PromptVars{}
- return nil
- }
-
- for _, msg := range msgs {
- switch strings.ToLower(msg.Role) {
- case "system":
- if currentVars.System != "" {
- if err := writePrompt(); err != nil {
- return "", nil, err
- }
+// CheckCapabilities checks if the model has the specified capabilities returning an error describing
+// any missing or unknown capabilities
+func (m *Model) CheckCapabilities(caps ...Capability) error {
+ var errs []error
+ for _, cap := range caps {
+ switch cap {
+ case CapabilityCompletion:
+ f, err := os.Open(m.ModelPath)
+ if err != nil {
+ slog.Error("couldn't open model file", "error", err)
+ continue
}
- currentVars.System = msg.Content
- case "user":
- if currentVars.Prompt != "" {
- if err := writePrompt(); err != nil {
- return "", nil, err
- }
+ defer f.Close()
+
+ // TODO(mxyng): decode the GGML into model to avoid doing this multiple times
+ ggml, _, err := llm.DecodeGGML(f, 0)
+ if err != nil {
+ slog.Error("couldn't decode ggml", "error", err)
+ continue
}
- currentVars.Prompt = msg.Content
- currentImages = msg.Images
- case "assistant":
- currentVars.Response = msg.Content
- if err := writePrompt(); err != nil {
- return "", nil, err
+
+ if _, ok := ggml.KV()[fmt.Sprintf("%s.pooling_type", ggml.KV().Architecture())]; ok {
+ errs = append(errs, errCapabilityCompletion)
+ }
+ case CapabilityTools:
+ if !slices.Contains(m.Template.Vars(), "tools") {
+ errs = append(errs, errCapabilityTools)
+ }
+ case CapabilityInsert:
+ vars := m.Template.Vars()
+ if !slices.Contains(vars, "suffix") {
+ errs = append(errs, errCapabilityInsert)
}
default:
- return "", nil, fmt.Errorf("invalid role: %s, role must be one of [system, user, assistant]", msg.Role)
+ slog.Error("unknown capability", "capability", cap)
+ return fmt.Errorf("unknown capability: %s", cap)
}
}
- // Append the last set of vars if they are non-empty
- if currentVars.Prompt != "" || currentVars.System != "" {
- p, err := m.PreResponsePrompt(currentVars)
- if err != nil {
- return "", nil, fmt.Errorf("pre-response template: %w", err)
- }
- prompt.WriteString(p)
+ if err := errors.Join(errs...); err != nil {
+ return fmt.Errorf("%w %w", errCapabilities, errors.Join(errs...))
}
- return prompt.String(), currentImages, nil
+ return nil
}
-type ManifestV2 struct {
- SchemaVersion int `json:"schemaVersion"`
- MediaType string `json:"mediaType"`
- Config *Layer `json:"config"`
- Layers []*Layer `json:"layers"`
+func (m *Model) String() string {
+ var modelfile parser.File
+
+ modelfile.Commands = append(modelfile.Commands, parser.Command{
+ Name: "model",
+ Args: m.ModelPath,
+ })
+
+ for _, adapter := range m.AdapterPaths {
+ modelfile.Commands = append(modelfile.Commands, parser.Command{
+ Name: "adapter",
+ Args: adapter,
+ })
+ }
+
+ for _, projector := range m.ProjectorPaths {
+ modelfile.Commands = append(modelfile.Commands, parser.Command{
+ Name: "model",
+ Args: projector,
+ })
+ }
+
+ if m.Template != nil {
+ modelfile.Commands = append(modelfile.Commands, parser.Command{
+ Name: "template",
+ Args: m.Template.String(),
+ })
+ }
+
+ if m.System != "" {
+ modelfile.Commands = append(modelfile.Commands, parser.Command{
+ Name: "system",
+ Args: m.System,
+ })
+ }
+
+ for k, v := range m.Options {
+ switch v := v.(type) {
+ case []any:
+ for _, s := range v {
+ modelfile.Commands = append(modelfile.Commands, parser.Command{
+ Name: k,
+ Args: fmt.Sprintf("%v", s),
+ })
+ }
+ default:
+ modelfile.Commands = append(modelfile.Commands, parser.Command{
+ Name: k,
+ Args: fmt.Sprintf("%v", v),
+ })
+ }
+ }
+
+ for _, license := range m.License {
+ modelfile.Commands = append(modelfile.Commands, parser.Command{
+ Name: "license",
+ Args: license,
+ })
+ }
+
+ for _, msg := range m.Messages {
+ modelfile.Commands = append(modelfile.Commands, parser.Command{
+ Name: "message",
+ Args: fmt.Sprintf("%s: %s", msg.Role, msg.Content),
+ })
+ }
+
+ return modelfile.String()
}
type ConfigV2 struct {
@@ -221,75 +205,31 @@ type ConfigV2 struct {
RootFS RootFS `json:"rootfs"`
}
-func (c *ConfigV2) SetModelFormat(format string) {
- if c.ModelFormat == "" {
- c.ModelFormat = format
- }
-}
-
-func (c *ConfigV2) SetModelFamily(families ...string) {
- for _, family := range families {
- if c.ModelFamily == "" {
- c.ModelFamily = family
- }
-
- if !slices.Contains(c.ModelFamilies, family) {
- c.ModelFamilies = append(c.ModelFamilies, family)
- }
- }
-}
-
-func (c *ConfigV2) SetModelType(modelType string) {
- if c.ModelType == "" {
- c.ModelType = modelType
- }
-}
-
-func (c *ConfigV2) SetFileType(fileType string) {
- if c.FileType == "" {
- c.FileType = fileType
- }
-}
-
type RootFS struct {
Type string `json:"type"`
DiffIDs []string `json:"diff_ids"`
}
-func (m *ManifestV2) GetTotalSize() (total int64) {
- for _, layer := range m.Layers {
- total += layer.Size
- }
-
- total += m.Config.Size
- return total
-}
-
-func GetManifest(mp ModelPath) (*ManifestV2, string, error) {
+func GetManifest(mp ModelPath) (*Manifest, string, error) {
fp, err := mp.GetManifestPath()
if err != nil {
return nil, "", err
}
- if _, err = os.Stat(fp); err != nil {
- return nil, "", err
- }
-
- var manifest *ManifestV2
-
- bts, err := os.ReadFile(fp)
+ f, err := os.Open(fp)
if err != nil {
- return nil, "", fmt.Errorf("couldn't open file '%s'", fp)
+ return nil, "", err
}
+ defer f.Close()
- shaSum := sha256.Sum256(bts)
- shaStr := hex.EncodeToString(shaSum[:])
+ sha256sum := sha256.New()
- if err := json.Unmarshal(bts, &manifest); err != nil {
+ var manifest Manifest
+ if err := json.NewDecoder(io.TeeReader(f, sha256sum)).Decode(&manifest); err != nil {
return nil, "", err
}
- return manifest, shaStr, nil
+ return &manifest, hex.EncodeToString(sha256sum.Sum(nil)), nil
}
func GetModel(name string) (*Model, error) {
@@ -303,24 +243,24 @@ func GetModel(name string) (*Model, error) {
Name: mp.GetFullTagname(),
ShortName: mp.GetShortTagname(),
Digest: digest,
- Template: "{{ .Prompt }}",
- License: []string{},
- Size: manifest.GetTotalSize(),
+ Template: template.DefaultTemplate,
}
- filename, err := GetBlobsPath(manifest.Config.Digest)
- if err != nil {
- return nil, err
- }
+ if manifest.Config.Digest != "" {
+ filename, err := GetBlobsPath(manifest.Config.Digest)
+ if err != nil {
+ return nil, err
+ }
- configFile, err := os.Open(filename)
- if err != nil {
- return nil, err
- }
- defer configFile.Close()
+ configFile, err := os.Open(filename)
+ if err != nil {
+ return nil, err
+ }
+ defer configFile.Close()
- if err := json.NewDecoder(configFile).Decode(&model.Config); err != nil {
- return nil, err
+ if err := json.NewDecoder(configFile).Decode(&model.Config); err != nil {
+ return nil, err
+ }
}
for _, layer := range manifest.Layers {
@@ -332,22 +272,26 @@ func GetModel(name string) (*Model, error) {
switch layer.MediaType {
case "application/vnd.ollama.image.model":
model.ModelPath = filename
- model.OriginalModel = layer.From
+ model.ParentModel = layer.From
case "application/vnd.ollama.image.embed":
// Deprecated in versions > 0.1.2
// TODO: remove this warning in a future version
- log.Print("WARNING: model contains embeddings, but embeddings in modelfiles have been deprecated and will be ignored.")
+ slog.Info("WARNING: model contains embeddings, but embeddings in modelfiles have been deprecated and will be ignored.")
case "application/vnd.ollama.image.adapter":
model.AdapterPaths = append(model.AdapterPaths, filename)
case "application/vnd.ollama.image.projector":
model.ProjectorPaths = append(model.ProjectorPaths, filename)
- case "application/vnd.ollama.image.template":
+ case "application/vnd.ollama.image.prompt",
+ "application/vnd.ollama.image.template":
bts, err := os.ReadFile(filename)
if err != nil {
return nil, err
}
- model.Template = string(bts)
+ model.Template, err = template.Parse(string(bts))
+ if err != nil {
+ return nil, err
+ }
case "application/vnd.ollama.image.system":
bts, err := os.ReadFile(filename)
if err != nil {
@@ -355,13 +299,6 @@ func GetModel(name string) (*Model, error) {
}
model.System = string(bts)
- case "application/vnd.ollama.image.prompt":
- bts, err := os.ReadFile(filename)
- if err != nil {
- return nil, err
- }
-
- model.Template = string(bts)
case "application/vnd.ollama.image.params":
params, err := os.Open(filename)
if err != nil {
@@ -373,6 +310,16 @@ func GetModel(name string) (*Model, error) {
if err = json.NewDecoder(params).Decode(&model.Options); err != nil {
return nil, err
}
+ case "application/vnd.ollama.image.messages":
+ msgs, err := os.Open(filename)
+ if err != nil {
+ return nil, err
+ }
+ defer msgs.Close()
+
+ if err = json.NewDecoder(msgs).Decode(&model.Messages); err != nil {
+ return nil, err
+ }
case "application/vnd.ollama.image.license":
bts, err := os.ReadFile(filename)
if err != nil {
@@ -385,7 +332,7 @@ func GetModel(name string) (*Model, error) {
return model, nil
}
-func realpath(mfDir, from string) string {
+func realpath(rel, from string) string {
abspath, err := filepath.Abs(from)
if err != nil {
return from
@@ -402,15 +349,15 @@ func realpath(mfDir, from string) string {
return filepath.Join(home, from[2:])
}
- if _, err := os.Stat(filepath.Join(mfDir, from)); err == nil {
+ if _, err := os.Stat(filepath.Join(rel, from)); err == nil {
// this is a file relative to the Modelfile
- return filepath.Join(mfDir, from)
+ return filepath.Join(rel, from)
}
return abspath
}
-func CreateModel(ctx context.Context, name, modelFileDir string, commands []parser.Command, fn func(resp api.ProgressResponse)) error {
+func CreateModel(ctx context.Context, name model.Name, modelFileDir, quantization string, modelfile *parser.File, fn func(resp api.ProgressResponse)) (err error) {
config := ConfigV2{
OS: "linux",
Architecture: "amd64",
@@ -419,250 +366,252 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
},
}
- deleteMap := make(map[string]struct{})
+ var messages []*api.Message
+ parameters := make(map[string]any)
- var layers Layers
-
- params := make(map[string][]string)
- fromParams := make(map[string]any)
-
- for _, c := range commands {
- log.Printf("[%s] - %s", c.Name, c.Args)
+ var layers []Layer
+ var baseLayers []*layerGGML
+ for _, c := range modelfile.Commands {
mediatype := fmt.Sprintf("application/vnd.ollama.image.%s", c.Name)
+ command := c.Name
- switch c.Name {
- case "model":
- if strings.HasPrefix(c.Args, "@") {
- blobPath, err := GetBlobsPath(strings.TrimPrefix(c.Args, "@"))
+ switch command {
+ case "model", "adapter":
+ if name := model.ParseName(c.Args); name.IsValid() && command == "model" {
+ baseLayers, err = parseFromModel(ctx, name, fn)
+ if err != nil {
+ return err
+ }
+ } else if strings.HasPrefix(c.Args, "@") {
+ digest := strings.TrimPrefix(c.Args, "@")
+ if ib, ok := intermediateBlobs[digest]; ok {
+ p, err := GetBlobsPath(ib)
+ if err != nil {
+ return err
+ }
+
+ if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
+ // pass
+ } else if err != nil {
+ return err
+ } else {
+ fn(api.ProgressResponse{Status: fmt.Sprintf("using cached layer %s", ib)})
+ digest = ib
+ }
+ }
+
+ blobpath, err := GetBlobsPath(digest)
if err != nil {
return err
}
- c.Args = blobPath
+ blob, err := os.Open(blobpath)
+ if err != nil {
+ return err
+ }
+ defer blob.Close()
+
+ baseLayers, err = parseFromFile(ctx, command, baseLayers, blob, digest, fn)
+ if err != nil {
+ return err
+ }
+ } else if file, err := os.Open(realpath(modelFileDir, c.Args)); err == nil {
+ defer file.Close()
+
+ baseLayers, err = parseFromFile(ctx, command, baseLayers, file, "", fn)
+ if err != nil {
+ return err
+ }
+ } else {
+ return fmt.Errorf("invalid model reference: %s", c.Args)
}
- bin, err := os.Open(realpath(modelFileDir, c.Args))
- if err != nil {
- // not a file on disk so must be a model reference
- modelpath := ParseModelPath(c.Args)
- manifest, _, err := GetManifest(modelpath)
- switch {
- case errors.Is(err, os.ErrNotExist):
- fn(api.ProgressResponse{Status: "pulling model"})
- if err := PullModel(ctx, c.Args, &RegistryOptions{}, fn); err != nil {
- return err
- }
-
- manifest, _, err = GetManifest(modelpath)
+ for _, baseLayer := range baseLayers {
+ if quantization != "" &&
+ baseLayer.MediaType == "application/vnd.ollama.image.model" &&
+ baseLayer.GGML != nil &&
+ baseLayer.GGML.Name() == "gguf" {
+ want, err := llm.ParseFileType(quantization)
if err != nil {
return err
}
- case err != nil:
- return err
- }
- fn(api.ProgressResponse{Status: "reading model metadata"})
- fromConfigPath, err := GetBlobsPath(manifest.Config.Digest)
- if err != nil {
- return err
- }
+ ft := baseLayer.GGML.KV().FileType()
+ if !slices.Contains([]string{"F16", "F32"}, ft.String()) {
+ return errors.New("quantization is only supported for F16 and F32 models")
+ } else if want != ft {
+ fn(api.ProgressResponse{Status: fmt.Sprintf("quantizing %s model to %s", ft, quantization)})
- fromConfigFile, err := os.Open(fromConfigPath)
- if err != nil {
- return err
- }
- defer fromConfigFile.Close()
-
- var fromConfig ConfigV2
- if err := json.NewDecoder(fromConfigFile).Decode(&fromConfig); err != nil {
- return err
- }
-
- // if the model is not in gguf format, pull the base model to try and get it in gguf format
- if fromConfig.ModelFormat != "gguf" {
- fn(api.ProgressResponse{Status: "updating base model"})
- parent, err := GetModel(c.Args)
- if err != nil {
- return err
- }
- if err := PullModel(ctx, parent.OriginalModel, &RegistryOptions{}, fn); err != nil {
- log.Printf("error pulling model: %v", err)
- }
- // Reset the file pointer to the beginning of the file
- _, err = fromConfigFile.Seek(0, 0)
- if err != nil {
- return fmt.Errorf("update from config after pull: %w", err)
- }
- if err := json.NewDecoder(fromConfigFile).Decode(&fromConfig); err != nil {
- return err
- }
- }
-
- // if the model is still not in gguf format, error out
- if fromConfig.ModelFormat != "gguf" {
- return fmt.Errorf("%s is not in gguf format, this base model is not compatible with this version of ollama", c.Args)
- }
-
- config.SetModelFormat(fromConfig.ModelFormat)
- config.SetModelFamily(append(fromConfig.ModelFamilies, fromConfig.ModelFamily)...)
- config.SetModelType(fromConfig.ModelType)
- config.SetFileType(fromConfig.FileType)
-
- for _, layer := range manifest.Layers {
- deleteMap[layer.Digest] = struct{}{}
- if layer.MediaType == "application/vnd.ollama.image.params" {
- fromParamsPath, err := GetBlobsPath(layer.Digest)
+ blob, err := GetBlobsPath(baseLayer.Digest)
if err != nil {
return err
}
- fromParamsFile, err := os.Open(fromParamsPath)
+ temp, err := os.CreateTemp(filepath.Dir(blob), quantization)
if err != nil {
return err
}
- defer fromParamsFile.Close()
+ defer temp.Close()
+ defer os.Remove(temp.Name())
- if err := json.NewDecoder(fromParamsFile).Decode(&fromParams); err != nil {
+ if err := llama.Quantize(blob, temp.Name(), uint32(want)); err != nil {
return err
}
- }
- layer, err := NewLayerFromLayer(layer.Digest, layer.MediaType, modelpath.GetShortTagname())
- if err != nil {
- return err
- }
+ layer, err := NewLayer(temp, baseLayer.MediaType)
+ if err != nil {
+ return err
+ }
- layers.Add(layer)
- }
+ if _, err := temp.Seek(0, io.SeekStart); err != nil {
+ return err
+ }
- deleteMap[manifest.Config.Digest] = struct{}{}
- continue
- }
- defer bin.Close()
+ ggml, _, err := llm.DecodeGGML(temp, 0)
+ if err != nil {
+ return err
+ }
- var offset int64
- CREATE:
- for {
- fn(api.ProgressResponse{Status: "creating model layer"})
-
- bin.Seek(offset, io.SeekStart)
- ggml, err := llm.DecodeGGML(bin)
- if err != nil {
- switch {
- case errors.Is(err, io.EOF):
- break CREATE
- case errors.Is(err, llm.ErrUnsupportedFormat):
- return fmt.Errorf("model binary specified in FROM field is not a valid gguf format model, %w", err)
- default:
- return err
+ baseLayer.Layer = layer
+ baseLayer.GGML = ggml
}
}
- config.SetModelFormat(ggml.Name())
- config.SetModelFamily(ggml.ModelFamily())
- config.SetModelType(ggml.ModelType())
- config.SetFileType(ggml.FileType())
-
- mediatype := mediatype
- if ggml.ModelFamily() == "clip" {
- mediatype = "application/vnd.ollama.image.projector"
+ if baseLayer.GGML != nil {
+ config.ModelFormat = cmp.Or(config.ModelFormat, baseLayer.GGML.Name())
+ config.ModelFamily = cmp.Or(config.ModelFamily, baseLayer.GGML.KV().Architecture())
+ config.ModelType = cmp.Or(config.ModelType, format.HumanNumber(baseLayer.GGML.KV().ParameterCount()))
+ config.FileType = cmp.Or(config.FileType, baseLayer.GGML.KV().FileType().String())
+ config.ModelFamilies = append(config.ModelFamilies, baseLayer.GGML.KV().Architecture())
}
- sr := io.NewSectionReader(bin, offset, ggml.Size)
- layer, err := NewLayer(sr, mediatype)
- if err != nil {
- return err
+ layers = append(layers, baseLayer.Layer)
+ }
+ case "license", "template", "system":
+ if c.Name == "template" {
+ if _, err := template.Parse(c.Args); err != nil {
+ return fmt.Errorf("%w: %s", errBadTemplate, err)
}
-
- layers.Add(layer)
-
- offset += ggml.Size
- }
- case "adapter":
- if strings.HasPrefix(c.Args, "@") {
- blobPath, err := GetBlobsPath(strings.TrimPrefix(c.Args, "@"))
- if err != nil {
- return err
- }
-
- c.Args = blobPath
}
- fn(api.ProgressResponse{Status: "creating adapter layer"})
- bin, err := os.Open(realpath(modelFileDir, c.Args))
- if err != nil {
- return err
- }
- defer bin.Close()
+ if c.Name != "license" {
+ // replace
+ layers = slices.DeleteFunc(layers, func(layer Layer) bool {
+ if layer.MediaType != mediatype {
+ return false
+ }
- layer, err := NewLayer(bin, mediatype)
+ if err := layer.Remove(); err != nil {
+ return false
+ }
+
+ return true
+ })
+ }
+
+ blob := strings.NewReader(c.Args)
+ layer, err := NewLayer(blob, mediatype)
if err != nil {
return err
}
- layers.Add(layer)
- case "license":
- fn(api.ProgressResponse{Status: "creating license layer"})
-
- bin := strings.NewReader(c.Args)
- layer, err := NewLayer(bin, mediatype)
- if err != nil {
- return err
+ layers = append(layers, layer)
+ case "message":
+ role, content, ok := strings.Cut(c.Args, ": ")
+ if !ok {
+ return fmt.Errorf("invalid message: %s", c.Args)
}
- layers.Add(layer)
- case "template", "system":
- fn(api.ProgressResponse{Status: fmt.Sprintf("creating %s layer", c.Name)})
-
- bin := strings.NewReader(c.Args)
- layer, err := NewLayer(bin, mediatype)
- if err != nil {
- return err
- }
-
- layers.Replace(layer)
+ messages = append(messages, &api.Message{Role: role, Content: content})
default:
- params[c.Name] = append(params[c.Name], c.Args)
+ ps, err := api.FormatParams(map[string][]string{c.Name: {c.Args}})
+ if err != nil {
+ return err
+ }
+
+ for k, v := range ps {
+ if ks, ok := parameters[k].([]string); ok {
+ parameters[k] = append(ks, v.([]string)...)
+ } else if vs, ok := v.([]string); ok {
+ parameters[k] = vs
+ } else {
+ parameters[k] = v
+ }
+ }
}
}
- if len(params) > 0 {
- fn(api.ProgressResponse{Status: "creating parameters layer"})
+ var err2 error
+ layers = slices.DeleteFunc(layers, func(layer Layer) bool {
+ switch layer.MediaType {
+ case "application/vnd.ollama.image.message":
+ // if there are new messages, remove the inherited ones
+ if len(messages) > 0 {
+ return true
+ }
- formattedParams, err := api.FormatParams(params)
+ return false
+ case "application/vnd.ollama.image.params":
+ // merge inherited parameters with new ones
+ r, err := layer.Open()
+ if err != nil {
+ err2 = err
+ return false
+ }
+ defer r.Close()
+
+ var ps map[string]any
+ if err := json.NewDecoder(r).Decode(&ps); err != nil {
+ err2 = err
+ return false
+ }
+
+ for k, v := range ps {
+ if _, ok := parameters[k]; !ok {
+ parameters[k] = v
+ }
+ }
+
+ return true
+ default:
+ return false
+ }
+ })
+
+ if err2 != nil {
+ return err2
+ }
+
+ if len(messages) > 0 {
+ var b bytes.Buffer
+ if err := json.NewEncoder(&b).Encode(messages); err != nil {
+ return err
+ }
+
+ layer, err := NewLayer(&b, "application/vnd.ollama.image.messages")
if err != nil {
return err
}
- for k, v := range fromParams {
- if _, ok := formattedParams[k]; !ok {
- formattedParams[k] = v
- }
- }
-
- // xxx - can this be removed?
- if config.ModelType == "65B" {
- if gqa, ok := formattedParams["gqa"].(int); ok && gqa == 8 {
- config.ModelType = "70B"
- }
- }
+ layers = append(layers, layer)
+ }
+ if len(parameters) > 0 {
var b bytes.Buffer
- if err := json.NewEncoder(&b).Encode(formattedParams); err != nil {
+ if err := json.NewEncoder(&b).Encode(parameters); err != nil {
return err
}
- fn(api.ProgressResponse{Status: "creating config layer"})
layer, err := NewLayer(&b, "application/vnd.ollama.image.params")
if err != nil {
return err
}
- layers.Replace(layer)
+ layers = append(layers, layer)
}
- digests := make([]string, len(layers.items))
- for i, layer := range layers.items {
+ digests := make([]string, len(layers))
+ for i, layer := range layers {
digests[i] = layer.Digest
}
@@ -678,31 +627,21 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
return err
}
- delete(deleteMap, configLayer.Digest)
-
- for _, layer := range append(layers.items, configLayer) {
- committed, err := layer.Commit()
- if err != nil {
- return err
+ for _, layer := range append(layers, configLayer) {
+ if layer.status != "" {
+ fn(api.ProgressResponse{Status: layer.status})
}
-
- status := "writing layer"
- if !committed {
- status = "using already created layer"
- }
-
- fn(api.ProgressResponse{Status: fmt.Sprintf("%s %s", status, layer.Digest)})
-
- delete(deleteMap, layer.Digest)
}
+ old, _ := ParseNamedManifest(name)
+
fn(api.ProgressResponse{Status: "writing manifest"})
- if err := WriteManifest(name, configLayer, layers.items); err != nil {
+ if err := WriteManifest(name, configLayer, layers); err != nil {
return err
}
- if noprune := os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
- if err := deleteUnusedLayers(nil, deleteMap, false); err != nil {
+ if !envconfig.NoPrune() && old != nil {
+ if err := old.RemoveLayers(); err != nil {
return err
}
}
@@ -711,91 +650,69 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
return nil
}
-func CopyModel(src, dest string) error {
- srcModelPath := ParseModelPath(src)
- srcPath, err := srcModelPath.GetManifestPath()
+func CopyModel(src, dst model.Name) error {
+ if !dst.IsFullyQualified() {
+ return model.Unqualified(dst)
+ }
+ if !src.IsFullyQualified() {
+ return model.Unqualified(src)
+ }
+
+ if src.Filepath() == dst.Filepath() {
+ return nil
+ }
+
+ manifests, err := GetManifestPath()
if err != nil {
return err
}
- destModelPath := ParseModelPath(dest)
- destPath, err := destModelPath.GetManifestPath()
- if err != nil {
- return err
- }
- if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
+ dstpath := filepath.Join(manifests, dst.Filepath())
+ if err := os.MkdirAll(filepath.Dir(dstpath), 0o755); err != nil {
return err
}
- // copy the file
- input, err := os.ReadFile(srcPath)
+ srcpath := filepath.Join(manifests, src.Filepath())
+ srcfile, err := os.Open(srcpath)
if err != nil {
- fmt.Println("Error reading file:", err)
return err
}
+ defer srcfile.Close()
- err = os.WriteFile(destPath, input, 0o644)
+ dstfile, err := os.Create(dstpath)
if err != nil {
- fmt.Println("Error reading file:", err)
return err
}
+ defer dstfile.Close()
- return nil
+ _, err = io.Copy(dstfile, srcfile)
+ return err
}
-func deleteUnusedLayers(skipModelPath *ModelPath, deleteMap map[string]struct{}, dryRun bool) error {
- fp, err := GetManifestPath()
+func deleteUnusedLayers(deleteMap map[string]struct{}) error {
+ manifests, err := Manifests()
if err != nil {
return err
}
- walkFunc := func(path string, info os.FileInfo, _ error) error {
- if info.IsDir() {
- return nil
- }
-
- dir, file := filepath.Split(path)
- dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
- tag := strings.Join([]string{dir, file}, ":")
- fmp := ParseModelPath(tag)
-
- // skip the manifest we're trying to delete
- if skipModelPath != nil && skipModelPath.GetFullTagname() == fmp.GetFullTagname() {
- return nil
- }
-
- // save (i.e. delete from the deleteMap) any files used in other manifests
- manifest, _, err := GetManifest(fmp)
- if err != nil {
- return nil
- }
-
+ for _, manifest := range manifests {
for _, layer := range manifest.Layers {
delete(deleteMap, layer.Digest)
}
delete(deleteMap, manifest.Config.Digest)
- return nil
- }
-
- if err := filepath.Walk(fp, walkFunc); err != nil {
- return err
}
// only delete the files which are still in the deleteMap
for k := range deleteMap {
fp, err := GetBlobsPath(k)
if err != nil {
- log.Printf("couldn't get file path for '%s': %v", k, err)
+ slog.Info(fmt.Sprintf("couldn't get file path for '%s': %v", k, err))
continue
}
- if !dryRun {
- if err := os.Remove(fp); err != nil {
- log.Printf("couldn't remove file '%s': %v", fp, err)
- continue
- }
- } else {
- log.Printf("wanted to remove: %s", fp)
+ if err := os.Remove(fp); err != nil {
+ slog.Info(fmt.Sprintf("couldn't remove file '%s': %v", fp, err))
+ continue
}
}
@@ -811,28 +728,37 @@ func PruneLayers() error {
blobs, err := os.ReadDir(p)
if err != nil {
- log.Printf("couldn't read dir '%s': %v", p, err)
+ slog.Info(fmt.Sprintf("couldn't read dir '%s': %v", p, err))
return err
}
for _, blob := range blobs {
name := blob.Name()
- if runtime.GOOS == "windows" {
- name = strings.ReplaceAll(name, "-", ":")
- }
- if strings.HasPrefix(name, "sha256:") {
- deleteMap[name] = struct{}{}
+ name = strings.ReplaceAll(name, "-", ":")
+
+ _, err := GetBlobsPath(name)
+ if err != nil {
+ if errors.Is(err, ErrInvalidDigestFormat) {
+ // remove invalid blobs (e.g. partial downloads)
+ if err := os.Remove(filepath.Join(p, blob.Name())); err != nil {
+ slog.Error("couldn't remove blob", "blob", blob.Name(), "error", err)
+ }
+ }
+
+ continue
}
+
+ deleteMap[name] = struct{}{}
}
- log.Printf("total blobs: %d", len(deleteMap))
+ slog.Info(fmt.Sprintf("total blobs: %d", len(deleteMap)))
- err = deleteUnusedLayers(nil, deleteMap, false)
- if err != nil {
- return err
+ if err := deleteUnusedLayers(deleteMap); err != nil {
+ slog.Error(fmt.Sprintf("couldn't remove unused layers: %v", err))
+ return nil
}
- log.Printf("total unused blobs removed: %d", len(deleteMap))
+ slog.Info(fmt.Sprintf("total unused blobs removed: %d", len(deleteMap)))
return nil
}
@@ -870,104 +796,12 @@ func PruneDirectory(path string) error {
return nil
}
-func DeleteModel(name string) error {
- mp := ParseModelPath(name)
- manifest, _, err := GetManifest(mp)
- if err != nil {
- return err
- }
-
- deleteMap := make(map[string]struct{})
- for _, layer := range manifest.Layers {
- deleteMap[layer.Digest] = struct{}{}
- }
- deleteMap[manifest.Config.Digest] = struct{}{}
-
- err = deleteUnusedLayers(&mp, deleteMap, false)
- if err != nil {
- return err
- }
-
- fp, err := mp.GetManifestPath()
- if err != nil {
- return err
- }
- err = os.Remove(fp)
- if err != nil {
- log.Printf("couldn't remove manifest file '%s': %v", fp, err)
- return err
- }
-
- return nil
-}
-
-func ShowModelfile(model *Model) (string, error) {
- var mt struct {
- *Model
- From string
- Parameters map[string][]any
- }
-
- mt.Parameters = make(map[string][]any)
- for k, v := range model.Options {
- if s, ok := v.([]any); ok {
- mt.Parameters[k] = s
- continue
- }
-
- mt.Parameters[k] = []any{v}
- }
-
- mt.Model = model
- mt.From = model.ModelPath
-
- if model.OriginalModel != "" {
- mt.From = model.OriginalModel
- }
-
- modelFile := `# Modelfile generated by "ollama show"
-# To build a new Modelfile based on this one, replace the FROM line with:
-# FROM {{ .ShortName }}
-
-FROM {{ .From }}
-TEMPLATE """{{ .Template }}"""
-
-{{- if .System }}
-SYSTEM """{{ .System }}"""
-{{- end }}
-
-{{- range $adapter := .AdapterPaths }}
-ADAPTER {{ $adapter }}
-{{- end }}
-
-{{- range $k, $v := .Parameters }}
-{{- range $parameter := $v }}
-PARAMETER {{ $k }} {{ printf "%#v" $parameter }}
-{{- end }}
-{{- end }}`
-
- tmpl, err := template.New("").Parse(modelFile)
- if err != nil {
- log.Printf("error parsing template: %q", err)
- return "", err
- }
-
- var buf bytes.Buffer
-
- if err = tmpl.Execute(&buf, mt); err != nil {
- log.Printf("error executing template: %q", err)
- return "", err
- }
-
- return buf.String(), nil
-}
-
-func PushModel(ctx context.Context, name string, regOpts *RegistryOptions, fn func(api.ProgressResponse)) error {
+func PushModel(ctx context.Context, name string, regOpts *registryOptions, fn func(api.ProgressResponse)) error {
mp := ParseModelPath(name)
fn(api.ProgressResponse{Status: "retrieving manifest"})
if mp.ProtocolScheme == "http" && !regOpts.Insecure {
- return fmt.Errorf("insecure protocol http")
+ return errors.New("insecure protocol http")
}
manifest, _, err := GetManifest(mp)
@@ -976,16 +810,15 @@ func PushModel(ctx context.Context, name string, regOpts *RegistryOptions, fn fu
return err
}
- var layers []*Layer
+ var layers []Layer
layers = append(layers, manifest.Layers...)
- layers = append(layers, manifest.Config)
+ if manifest.Config.Digest != "" {
+ layers = append(layers, manifest.Config)
+ }
for _, layer := range layers {
if err := uploadBlob(ctx, mp, layer, regOpts, fn); err != nil {
- log.Printf("error uploading blob: %v", err)
- if errors.Is(err, errUnauthorized) {
- return fmt.Errorf("unable to push %s, make sure this namespace exists and you are authorized to push to it", ParseModelPath(name).GetNamespaceRepository())
- }
+ slog.Info(fmt.Sprintf("error uploading blob: %v", err))
return err
}
}
@@ -1012,32 +845,27 @@ func PushModel(ctx context.Context, name string, regOpts *RegistryOptions, fn fu
return nil
}
-func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn func(api.ProgressResponse)) error {
+func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn func(api.ProgressResponse)) error {
mp := ParseModelPath(name)
- var manifest *ManifestV2
- var err error
- var noprune string
-
// build deleteMap to prune unused layers
deleteMap := make(map[string]struct{})
-
- if noprune = os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
- manifest, _, err = GetManifest(mp)
- if err != nil && !errors.Is(err, os.ErrNotExist) {
- return err
+ manifest, _, err := GetManifest(mp)
+ if errors.Is(err, os.ErrNotExist) {
+ // noop
+ } else if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ } else {
+ for _, l := range manifest.Layers {
+ deleteMap[l.Digest] = struct{}{}
}
-
- if manifest != nil {
- for _, l := range manifest.Layers {
- deleteMap[l.Digest] = struct{}{}
- }
+ if manifest.Config.Digest != "" {
deleteMap[manifest.Config.Digest] = struct{}{}
}
}
if mp.ProtocolScheme == "http" && !regOpts.Insecure {
- return fmt.Errorf("insecure protocol http")
+ return errors.New("insecure protocol http")
}
fn(api.ProgressResponse{Status: "pulling manifest"})
@@ -1047,27 +875,33 @@ func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn fu
return fmt.Errorf("pull model manifest: %s", err)
}
- var layers []*Layer
+ var layers []Layer
layers = append(layers, manifest.Layers...)
- layers = append(layers, manifest.Config)
+ if manifest.Config.Digest != "" {
+ layers = append(layers, manifest.Config)
+ }
+ skipVerify := make(map[string]bool)
for _, layer := range layers {
- if err := downloadBlob(
- ctx,
- downloadOpts{
- mp: mp,
- digest: layer.Digest,
- regOpts: regOpts,
- fn: fn,
- }); err != nil {
+ cacheHit, err := downloadBlob(ctx, downloadOpts{
+ mp: mp,
+ digest: layer.Digest,
+ regOpts: regOpts,
+ fn: fn,
+ })
+ if err != nil {
return err
}
+ skipVerify[layer.Digest] = cacheHit
delete(deleteMap, layer.Digest)
}
delete(deleteMap, manifest.Config.Digest)
fn(api.ProgressResponse{Status: "verifying sha256 digest"})
for _, layer := range layers {
+ if skipVerify[layer.Digest] {
+ continue
+ }
if err := verifyBlob(layer.Digest); err != nil {
if errors.Is(err, errDigestMismatch) {
// something went wrong, delete the blob
@@ -1077,7 +911,7 @@ func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn fu
}
if err := os.Remove(fp); err != nil {
// log this, but return the original error
- log.Printf("couldn't remove file with digest mismatch '%s': %v", fp, err)
+ slog.Info(fmt.Sprintf("couldn't remove file with digest mismatch '%s': %v", fp, err))
}
}
return err
@@ -1101,15 +935,14 @@ func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn fu
err = os.WriteFile(fp, manifestJSON, 0o644)
if err != nil {
- log.Printf("couldn't write to %s", fp)
+ slog.Info(fmt.Sprintf("couldn't write to %s", fp))
return err
}
- if noprune == "" {
- fn(api.ProgressResponse{Status: "removing any unused layers"})
- err = deleteUnusedLayers(nil, deleteMap, false)
- if err != nil {
- return err
+ if !envconfig.NoPrune() && len(deleteMap) > 0 {
+ fn(api.ProgressResponse{Status: "removing unused layers"})
+ if err := deleteUnusedLayers(deleteMap); err != nil {
+ fn(api.ProgressResponse{Status: fmt.Sprintf("couldn't remove unused layers: %v", err)})
}
}
@@ -1118,7 +951,7 @@ func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn fu
return nil
}
-func pullModelManifest(ctx context.Context, mp ModelPath, regOpts *RegistryOptions) (*ManifestV2, error) {
+func pullModelManifest(ctx context.Context, mp ModelPath, regOpts *registryOptions) (*Manifest, error) {
requestURL := mp.BaseURL().JoinPath("v2", mp.GetNamespaceRepository(), "manifests", mp.Tag)
headers := make(http.Header)
@@ -1129,12 +962,12 @@ func pullModelManifest(ctx context.Context, mp ModelPath, regOpts *RegistryOptio
}
defer resp.Body.Close()
- var m *ManifestV2
+ var m Manifest
if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
return nil, err
}
- return m, err
+ return &m, err
}
// GetSHA256Digest returns the SHA256 hash of a given buffer and returns it, and the size of buffer
@@ -1148,55 +981,96 @@ func GetSHA256Digest(r io.Reader) (string, int64) {
return fmt.Sprintf("sha256:%x", h.Sum(nil)), n
}
-var errUnauthorized = fmt.Errorf("unauthorized")
+var errUnauthorized = errors.New("unauthorized: access denied")
-func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.ReadSeeker, regOpts *RegistryOptions) (*http.Response, error) {
- resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts)
- if err != nil {
- if !errors.Is(err, context.Canceled) {
- log.Printf("request failed: %v", err)
- }
-
- return nil, err
+// getTokenSubject returns the subject of a JWT token, it does not validate the token
+func getTokenSubject(token string) string {
+ parts := strings.Split(token, ".")
+ if len(parts) != 3 {
+ return ""
}
- switch {
- case resp.StatusCode == http.StatusUnauthorized:
- // Handle authentication error with one retry
- auth := resp.Header.Get("www-authenticate")
- authRedir := ParseAuthRedirectString(auth)
- token, err := getAuthToken(ctx, authRedir)
+ payload := parts[1]
+ payloadBytes, err := base64.RawURLEncoding.DecodeString(payload)
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to decode jwt payload: %v", err))
+ return ""
+ }
+
+ var payloadMap map[string]interface{}
+ if err := json.Unmarshal(payloadBytes, &payloadMap); err != nil {
+ slog.Error(fmt.Sprintf("failed to unmarshal payload JSON: %v", err))
+ return ""
+ }
+
+ sub, ok := payloadMap["sub"]
+ if !ok {
+ slog.Error("jwt does not contain 'sub' field")
+ return ""
+ }
+
+ return fmt.Sprintf("%s", sub)
+}
+
+func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.ReadSeeker, regOpts *registryOptions) (*http.Response, error) {
+ anonymous := true // access will default to anonymous if no user is found associated with the public key
+ for range 2 {
+ resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts)
if err != nil {
+ if !errors.Is(err, context.Canceled) {
+ slog.Info(fmt.Sprintf("request failed: %v", err))
+ }
+
return nil, err
}
- regOpts.Token = token
- if body != nil {
- _, err = body.Seek(0, io.SeekStart)
+
+ switch {
+ case resp.StatusCode == http.StatusUnauthorized:
+ resp.Body.Close()
+
+ // Handle authentication error with one retry
+ challenge := parseRegistryChallenge(resp.Header.Get("www-authenticate"))
+ token, err := getAuthorizationToken(ctx, challenge)
if err != nil {
return nil, err
}
+ anonymous = getTokenSubject(token) == "anonymous"
+ regOpts.Token = token
+ if body != nil {
+ _, err = body.Seek(0, io.SeekStart)
+ if err != nil {
+ return nil, err
+ }
+ }
+ case resp.StatusCode == http.StatusNotFound:
+ resp.Body.Close()
+ return nil, os.ErrNotExist
+ case resp.StatusCode >= http.StatusBadRequest:
+ defer resp.Body.Close()
+ responseBody, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("%d: %s", resp.StatusCode, err)
+ }
+ return nil, fmt.Errorf("%d: %s", resp.StatusCode, responseBody)
+ default:
+ return resp, nil
}
-
- resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts)
- if resp.StatusCode == http.StatusUnauthorized {
- return nil, errUnauthorized
- }
-
- return resp, err
- case resp.StatusCode == http.StatusNotFound:
- return nil, os.ErrNotExist
- case resp.StatusCode >= http.StatusBadRequest:
- responseBody, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("%d: %s", resp.StatusCode, err)
- }
- return nil, fmt.Errorf("%d: %s", resp.StatusCode, responseBody)
}
- return resp, nil
+ if anonymous {
+ // no user is associated with the public key, and the request requires non-anonymous access
+ pubKey, nestedErr := auth.GetPublicKey()
+ if nestedErr != nil {
+ slog.Error(fmt.Sprintf("couldn't get public key: %v", nestedErr))
+ return nil, errUnauthorized
+ }
+ return nil, &errtypes.UnknownOllamaKey{Key: pubKey}
+ }
+ // user is associated with the public key, but is not authorized to make the request
+ return nil, errUnauthorized
}
-func makeRequest(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.Reader, regOpts *RegistryOptions) (*http.Response, error) {
+func makeRequest(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.Reader, regOpts *registryOptions) (*http.Response, error) {
if requestURL.Scheme != "http" && regOpts != nil && regOpts.Insecure {
requestURL.Scheme = "http"
}
@@ -1229,18 +1103,9 @@ func makeRequest(ctx context.Context, method string, requestURL *url.URL, header
req.ContentLength = contentLength
}
- proxyURL, err := http.ProxyFromEnvironment(req)
- if err != nil {
- return nil, err
- }
-
- client := http.Client{
- Transport: &http.Transport{
- Proxy: http.ProxyURL(proxyURL),
- },
- }
-
- resp, err := client.Do(req)
+ resp, err := (&http.Client{
+ CheckRedirect: regOpts.CheckRedirect,
+ }).Do(req)
if err != nil {
return nil, err
}
@@ -1271,17 +1136,17 @@ func getValue(header, key string) string {
return header[startIdx:endIdx]
}
-func ParseAuthRedirectString(authStr string) AuthRedirect {
+func parseRegistryChallenge(authStr string) registryChallenge {
authStr = strings.TrimPrefix(authStr, "Bearer ")
- return AuthRedirect{
+ return registryChallenge{
Realm: getValue(authStr, "realm"),
Service: getValue(authStr, "service"),
Scope: getValue(authStr, "scope"),
}
}
-var errDigestMismatch = fmt.Errorf("digest mismatch, file must be downloaded again")
+var errDigestMismatch = errors.New("digest mismatch, file must be downloaded again")
func verifyBlob(digest string) error {
fp, err := GetBlobsPath(digest)
diff --git a/server/images_test.go b/server/images_test.go
deleted file mode 100644
index 55408e7a..00000000
--- a/server/images_test.go
+++ /dev/null
@@ -1,347 +0,0 @@
-package server
-
-import (
- "strings"
- "testing"
-
- "github.com/jmorganca/ollama/api"
-)
-
-func TestPrompt(t *testing.T) {
- tests := []struct {
- name string
- template string
- vars PromptVars
- want string
- wantErr bool
- }{
- {
- name: "System Prompt",
- template: "[INST] {{ .System }} {{ .Prompt }} [/INST]",
- vars: PromptVars{
- System: "You are a Wizard.",
- Prompt: "What are the potion ingredients?",
- },
- want: "[INST] You are a Wizard. What are the potion ingredients? [/INST]",
- },
- {
- name: "System Prompt with Response",
- template: "[INST] {{ .System }} {{ .Prompt }} [/INST] {{ .Response }}",
- vars: PromptVars{
- System: "You are a Wizard.",
- Prompt: "What are the potion ingredients?",
- Response: "I don't know.",
- },
- want: "[INST] You are a Wizard. What are the potion ingredients? [/INST] I don't know.",
- },
- {
- name: "Conditional Logic Nodes",
- template: "[INST] {{if .First}}Hello!{{end}} {{ .System }} {{ .Prompt }} [/INST] {{ .Response }}",
- vars: PromptVars{
- First: true,
- System: "You are a Wizard.",
- Prompt: "What are the potion ingredients?",
- Response: "I don't know.",
- },
- want: "[INST] Hello! You are a Wizard. What are the potion ingredients? [/INST] I don't know.",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := Prompt(tt.template, tt.vars)
- if (err != nil) != tt.wantErr {
- t.Errorf("Prompt() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- if got != tt.want {
- t.Errorf("Prompt() got = %v, want %v", got, tt.want)
- }
- })
- }
-}
-
-func TestModel_PreResponsePrompt(t *testing.T) {
- tests := []struct {
- name string
- template string
- vars PromptVars
- want string
- wantErr bool
- }{
- {
- name: "No Response in Template",
- template: "[INST] {{ .System }} {{ .Prompt }} [/INST]",
- vars: PromptVars{
- System: "You are a Wizard.",
- Prompt: "What are the potion ingredients?",
- },
- want: "[INST] You are a Wizard. What are the potion ingredients? [/INST]",
- },
- {
- name: "Response in Template",
- template: "[INST] {{ .System }} {{ .Prompt }} [/INST] {{ .Response }}",
- vars: PromptVars{
- System: "You are a Wizard.",
- Prompt: "What are the potion ingredients?",
- },
- want: "[INST] You are a Wizard. What are the potion ingredients? [/INST] ",
- },
- {
- name: "Response in Template with Trailing Formatting",
- template: "<|im_start|>user\n{{ .Prompt }}<|im_end|><|im_start|>assistant\n{{ .Response }}<|im_end|>",
- vars: PromptVars{
- Prompt: "What are the potion ingredients?",
- },
- want: "<|im_start|>user\nWhat are the potion ingredients?<|im_end|><|im_start|>assistant\n",
- },
- {
- name: "Response in Template with Alternative Formatting",
- template: "<|im_start|>user\n{{.Prompt}}<|im_end|><|im_start|>assistant\n{{.Response}}<|im_end|>",
- vars: PromptVars{
- Prompt: "What are the potion ingredients?",
- },
- want: "<|im_start|>user\nWhat are the potion ingredients?<|im_end|><|im_start|>assistant\n",
- },
- }
-
- for _, tt := range tests {
- m := Model{Template: tt.template}
- t.Run(tt.name, func(t *testing.T) {
- got, err := m.PreResponsePrompt(tt.vars)
- if (err != nil) != tt.wantErr {
- t.Errorf("PreResponsePrompt() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- if got != tt.want {
- t.Errorf("PreResponsePrompt() got = %v, want %v", got, tt.want)
- }
- })
- }
-}
-
-func TestModel_PostResponsePrompt(t *testing.T) {
- tests := []struct {
- name string
- template string
- vars PromptVars
- want string
- wantErr bool
- }{
- {
- name: "No Response in Template",
- template: "[INST] {{ .System }} {{ .Prompt }} [/INST]",
- vars: PromptVars{
- Response: "I don't know.",
- },
- want: "I don't know.",
- },
- {
- name: "Response in Template",
- template: "[INST] {{ .System }} {{ .Prompt }} [/INST] {{ .Response }}",
- vars: PromptVars{
- Response: "I don't know.",
- },
- want: "I don't know.",
- },
- {
- name: "Response in Template with Trailing Formatting",
- template: "<|im_start|>user\n{{ .Prompt }}<|im_end|><|im_start|>assistant\n{{ .Response }}<|im_end|>",
- vars: PromptVars{
- Response: "I don't know.",
- },
- want: "I don't know.<|im_end|>",
- },
- {
- name: "Response in Template with Alternative Formatting",
- template: "<|im_start|>user\n{{.Prompt}}<|im_end|><|im_start|>assistant\n{{.Response}}<|im_end|>",
- vars: PromptVars{
- Response: "I don't know.",
- },
- want: "I don't know.<|im_end|>",
- },
- }
-
- for _, tt := range tests {
- m := Model{Template: tt.template}
- t.Run(tt.name, func(t *testing.T) {
- got, err := m.PostResponseTemplate(tt.vars)
- if (err != nil) != tt.wantErr {
- t.Errorf("PostResponseTemplate() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- if got != tt.want {
- t.Errorf("PostResponseTemplate() got = %v, want %v", got, tt.want)
- }
- })
- }
-}
-
-func TestModel_PreResponsePrompt_PostResponsePrompt(t *testing.T) {
- tests := []struct {
- name string
- template string
- preVars PromptVars
- postVars PromptVars
- want string
- wantErr bool
- }{
- {
- name: "Response in Template",
- template: "<|im_start|>user\n{{.Prompt}}<|im_end|><|im_start|>assistant\n{{.Response}}<|im_end|>",
- preVars: PromptVars{
- Prompt: "What are the potion ingredients?",
- },
- postVars: PromptVars{
- Prompt: "What are the potion ingredients?",
- Response: "Sugar.",
- },
- want: "<|im_start|>user\nWhat are the potion ingredients?<|im_end|><|im_start|>assistant\nSugar.<|im_end|>",
- },
- {
- name: "No Response in Template",
- template: "<|im_start|>user\n{{.Prompt}}<|im_end|><|im_start|>assistant\n",
- preVars: PromptVars{
- Prompt: "What are the potion ingredients?",
- },
- postVars: PromptVars{
- Prompt: "What are the potion ingredients?",
- Response: "Spice.",
- },
- want: "<|im_start|>user\nWhat are the potion ingredients?<|im_end|><|im_start|>assistant\nSpice.",
- },
- }
-
- for _, tt := range tests {
- m := Model{Template: tt.template}
- t.Run(tt.name, func(t *testing.T) {
- pre, err := m.PreResponsePrompt(tt.preVars)
- if (err != nil) != tt.wantErr {
- t.Errorf("PreResponsePrompt() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- post, err := m.PostResponseTemplate(tt.postVars)
- if err != nil {
- t.Errorf("PostResponseTemplate() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- result := pre + post
- if result != tt.want {
- t.Errorf("Prompt() got = %v, want %v", result, tt.want)
- }
- })
- }
-}
-
-func TestChat(t *testing.T) {
- tests := []struct {
- name string
- template string
- msgs []api.Message
- want string
- wantErr string
- }{
- {
- name: "Single Message",
- template: "[INST] {{ .System }} {{ .Prompt }} [/INST]",
- msgs: []api.Message{
- {
- Role: "system",
- Content: "You are a Wizard.",
- },
- {
- Role: "user",
- Content: "What are the potion ingredients?",
- },
- },
- want: "[INST] You are a Wizard. What are the potion ingredients? [/INST]",
- },
- {
- name: "First Message",
- template: "[INST] {{if .First}}Hello!{{end}} {{ .System }} {{ .Prompt }} [/INST]",
- msgs: []api.Message{
- {
- Role: "system",
- Content: "You are a Wizard.",
- },
- {
- Role: "user",
- Content: "What are the potion ingredients?",
- },
- {
- Role: "assistant",
- Content: "eye of newt",
- },
- {
- Role: "user",
- Content: "Anything else?",
- },
- },
- want: "[INST] Hello! You are a Wizard. What are the potion ingredients? [/INST]eye of newt[INST] Anything else? [/INST]",
- },
- {
- name: "Message History",
- template: "[INST] {{ .System }} {{ .Prompt }} [/INST]",
- msgs: []api.Message{
- {
- Role: "system",
- Content: "You are a Wizard.",
- },
- {
- Role: "user",
- Content: "What are the potion ingredients?",
- },
- {
- Role: "assistant",
- Content: "sugar",
- },
- {
- Role: "user",
- Content: "Anything else?",
- },
- },
- want: "[INST] You are a Wizard. What are the potion ingredients? [/INST]sugar[INST] Anything else? [/INST]",
- },
- {
- name: "Assistant Only",
- template: "[INST] {{ .System }} {{ .Prompt }} [/INST]",
- msgs: []api.Message{
- {
- Role: "assistant",
- Content: "everything nice",
- },
- },
- want: "[INST] [/INST]everything nice",
- },
- {
- name: "Invalid Role",
- msgs: []api.Message{
- {
- Role: "not-a-role",
- Content: "howdy",
- },
- },
- wantErr: "invalid role: not-a-role",
- },
- }
-
- for _, tt := range tests {
- m := Model{
- Template: tt.template,
- }
- t.Run(tt.name, func(t *testing.T) {
- got, _, err := m.ChatPrompt(tt.msgs)
- if tt.wantErr != "" {
- if err == nil {
- t.Errorf("ChatPrompt() expected error, got nil")
- }
- if !strings.Contains(err.Error(), tt.wantErr) {
- t.Errorf("ChatPrompt() error = %v, wantErr %v", err, tt.wantErr)
- }
- }
- if got != tt.want {
- t.Errorf("ChatPrompt() got = %v, want %v", got, tt.want)
- }
- })
- }
-}
diff --git a/server/layer.go b/server/layer.go
new file mode 100644
index 00000000..0bdee72b
--- /dev/null
+++ b/server/layer.go
@@ -0,0 +1,129 @@
+package server
+
+import (
+ "crypto/sha256"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+)
+
+type Layer struct {
+ MediaType string `json:"mediaType"`
+ Digest string `json:"digest"`
+ Size int64 `json:"size"`
+ From string `json:"from,omitempty"`
+ status string
+}
+
+func NewLayer(r io.Reader, mediatype string) (Layer, error) {
+ blobs, err := GetBlobsPath("")
+ if err != nil {
+ return Layer{}, err
+ }
+
+ temp, err := os.CreateTemp(blobs, "sha256-")
+ if err != nil {
+ return Layer{}, err
+ }
+ defer temp.Close()
+ defer os.Remove(temp.Name())
+
+ sha256sum := sha256.New()
+ n, err := io.Copy(io.MultiWriter(temp, sha256sum), r)
+ if err != nil {
+ return Layer{}, err
+ }
+
+ if err := temp.Close(); err != nil {
+ return Layer{}, err
+ }
+
+ digest := fmt.Sprintf("sha256:%x", sha256sum.Sum(nil))
+ blob, err := GetBlobsPath(digest)
+ if err != nil {
+ return Layer{}, err
+ }
+
+ status := "using existing layer"
+ if _, err := os.Stat(blob); err != nil {
+ status = "creating new layer"
+ if err := os.Rename(temp.Name(), blob); err != nil {
+ return Layer{}, err
+ }
+ if err := os.Chmod(blob, 0o644); err != nil {
+ return Layer{}, err
+ }
+ }
+
+ return Layer{
+ MediaType: mediatype,
+ Digest: digest,
+ Size: n,
+ status: fmt.Sprintf("%s %s", status, digest),
+ }, nil
+}
+
+func NewLayerFromLayer(digest, mediatype, from string) (Layer, error) {
+ if digest == "" {
+ return Layer{}, errors.New("creating new layer from layer with empty digest")
+ }
+
+ blob, err := GetBlobsPath(digest)
+ if err != nil {
+ return Layer{}, err
+ }
+
+ fi, err := os.Stat(blob)
+ if err != nil {
+ return Layer{}, err
+ }
+
+ return Layer{
+ MediaType: mediatype,
+ Digest: digest,
+ Size: fi.Size(),
+ From: from,
+ status: fmt.Sprintf("using existing layer %s", digest),
+ }, nil
+}
+
+func (l *Layer) Open() (io.ReadSeekCloser, error) {
+ if l.Digest == "" {
+ return nil, errors.New("opening layer with empty digest")
+ }
+
+ blob, err := GetBlobsPath(l.Digest)
+ if err != nil {
+ return nil, err
+ }
+
+ return os.Open(blob)
+}
+
+func (l *Layer) Remove() error {
+ if l.Digest == "" {
+ return nil
+ }
+
+ ms, err := Manifests()
+ if err != nil {
+ return err
+ }
+
+ for _, m := range ms {
+ for _, layer := range append(m.Layers, m.Config) {
+ if layer.Digest == l.Digest {
+ // something is using this layer
+ return nil
+ }
+ }
+ }
+
+ blob, err := GetBlobsPath(l.Digest)
+ if err != nil {
+ return err
+ }
+
+ return os.Remove(blob)
+}
diff --git a/server/layers.go b/server/layers.go
deleted file mode 100644
index adbf7175..00000000
--- a/server/layers.go
+++ /dev/null
@@ -1,109 +0,0 @@
-package server
-
-import (
- "crypto/sha256"
- "fmt"
- "io"
- "os"
- "runtime"
- "strings"
-
- "golang.org/x/exp/slices"
-)
-
-type Layers struct {
- items []*Layer
-}
-
-func (ls *Layers) Add(layer *Layer) {
- if layer.Size > 0 {
- ls.items = append(ls.items, layer)
- }
-}
-
-func (ls *Layers) Replace(layer *Layer) {
- if layer.Size > 0 {
- mediatype := layer.MediaType
- layers := slices.DeleteFunc(ls.items, func(l *Layer) bool {
- return l.MediaType == mediatype
- })
-
- ls.items = append(layers, layer)
- }
-}
-
-type Layer struct {
- MediaType string `json:"mediaType"`
- Digest string `json:"digest"`
- Size int64 `json:"size"`
- From string `json:"from,omitempty"`
-
- tempFileName string
-}
-
-func NewLayer(r io.Reader, mediatype string) (*Layer, error) {
- blobs, err := GetBlobsPath("")
- if err != nil {
- return nil, err
- }
-
- delimiter := ":"
- if runtime.GOOS == "windows" {
- delimiter = "-"
- }
-
- pattern := strings.Join([]string{"sha256", "*-partial"}, delimiter)
- temp, err := os.CreateTemp(blobs, pattern)
- if err != nil {
- return nil, err
- }
- defer temp.Close()
-
- sha256sum := sha256.New()
- n, err := io.Copy(io.MultiWriter(temp, sha256sum), r)
- if err != nil {
- return nil, err
- }
-
- return &Layer{
- MediaType: mediatype,
- Digest: fmt.Sprintf("sha256:%x", sha256sum.Sum(nil)),
- Size: n,
- tempFileName: temp.Name(),
- }, nil
-}
-
-func NewLayerFromLayer(digest, mediatype, from string) (*Layer, error) {
- blob, err := GetBlobsPath(digest)
- if err != nil {
- return nil, err
- }
-
- fi, err := os.Stat(blob)
- if err != nil {
- return nil, err
- }
-
- return &Layer{
- MediaType: mediatype,
- Digest: digest,
- Size: fi.Size(),
- From: from,
- }, nil
-}
-
-func (l *Layer) Commit() (bool, error) {
- // always remove temp
- defer os.Remove(l.tempFileName)
-
- blob, err := GetBlobsPath(l.Digest)
- if err != nil {
- return false, err
- }
-
- if _, err := os.Stat(blob); err != nil {
- return true, os.Rename(l.tempFileName, blob)
- }
-
- return false, nil
-}
diff --git a/server/llm_test.go b/server/llm_test.go
deleted file mode 100644
index 7f63a64d..00000000
--- a/server/llm_test.go
+++ /dev/null
@@ -1,123 +0,0 @@
-//go:build integration
-
-package server
-
-import (
- "context"
- "os"
- "strings"
- "sync"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/llm"
-)
-
-// TODO - this would ideally be in the llm package, but that would require some refactoring of interfaces in the server
-// package to avoid circular dependencies
-
-// WARNING - these tests will fail on mac if you don't manually copy ggml-metal.metal to this dir (./server)
-//
-// TODO - Fix this ^^
-
-var (
- req = [2]api.GenerateRequest{
- {
- Model: "orca-mini",
- Prompt: "tell me a short story about agi?",
- Options: map[string]interface{}{},
- }, {
- Model: "orca-mini",
- Prompt: "what is the origin of the us thanksgiving holiday?",
- Options: map[string]interface{}{},
- },
- }
- resp = [2]string{
- "once upon a time",
- "united states thanksgiving",
- }
-)
-
-func TestIntegrationSimpleOrcaMini(t *testing.T) {
- SkipIFNoTestData(t)
- workDir, err := os.MkdirTemp("", "ollama")
- require.NoError(t, err)
- defer os.RemoveAll(workDir)
- require.NoError(t, llm.Init(workDir))
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
- defer cancel()
- opts := api.DefaultOptions()
- opts.Seed = 42
- opts.Temperature = 0.0
- model, llmRunner := PrepareModelForPrompts(t, req[0].Model, opts)
- defer llmRunner.Close()
- response := OneShotPromptResponse(t, ctx, req[0], model, llmRunner)
- assert.Contains(t, strings.ToLower(response), resp[0])
-}
-
-// TODO
-// The server always loads a new runner and closes the old one, which forces serial execution
-// At present this test case fails with concurrency problems. Eventually we should try to
-// get true concurrency working with n_parallel support in the backend
-func TestIntegrationConcurrentPredictOrcaMini(t *testing.T) {
- SkipIFNoTestData(t)
-
- t.Skip("concurrent prediction on single runner not currently supported")
-
- workDir, err := os.MkdirTemp("", "ollama")
- require.NoError(t, err)
- defer os.RemoveAll(workDir)
- require.NoError(t, llm.Init(workDir))
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
- defer cancel()
- opts := api.DefaultOptions()
- opts.Seed = 42
- opts.Temperature = 0.0
- var wg sync.WaitGroup
- wg.Add(len(req))
- model, llmRunner := PrepareModelForPrompts(t, req[0].Model, opts)
- defer llmRunner.Close()
- for i := 0; i < len(req); i++ {
- go func(i int) {
- defer wg.Done()
- response := OneShotPromptResponse(t, ctx, req[i], model, llmRunner)
- t.Logf("Prompt: %s\nResponse: %s", req[0].Prompt, response)
- assert.Contains(t, strings.ToLower(response), resp[i], "error in thread %d (%s)", i, req[i].Prompt)
- }(i)
- }
- wg.Wait()
-}
-
-func TestIntegrationConcurrentRunnersOrcaMini(t *testing.T) {
- SkipIFNoTestData(t)
- workDir, err := os.MkdirTemp("", "ollama")
- require.NoError(t, err)
- defer os.RemoveAll(workDir)
- require.NoError(t, llm.Init(workDir))
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
- defer cancel()
- opts := api.DefaultOptions()
- opts.Seed = 42
- opts.Temperature = 0.0
- var wg sync.WaitGroup
- wg.Add(len(req))
-
- t.Logf("Running %d concurrently", len(req))
- for i := 0; i < len(req); i++ {
- go func(i int) {
- defer wg.Done()
- model, llmRunner := PrepareModelForPrompts(t, req[0].Model, opts)
- defer llmRunner.Close()
- response := OneShotPromptResponse(t, ctx, req[i], model, llmRunner)
- t.Logf("Prompt: %s\nResponse: %s", req[0].Prompt, response)
- assert.Contains(t, strings.ToLower(response), resp[i], "error in thread %d (%s)", i, req[i].Prompt)
- }(i)
- }
- wg.Wait()
-}
-
-// TODO - create a parallel test with 2 different models once we support concurrency
diff --git a/server/llm_utils_test.go b/server/llm_utils_test.go
deleted file mode 100644
index 92ee1039..00000000
--- a/server/llm_utils_test.go
+++ /dev/null
@@ -1,75 +0,0 @@
-//go:build integration
-
-package server
-
-import (
- "context"
- "errors"
- "os"
- "path"
- "runtime"
- "testing"
-
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/llm"
- "github.com/stretchr/testify/require"
-)
-
-func SkipIFNoTestData(t *testing.T) {
- modelDir := getModelDir()
- if _, err := os.Stat(modelDir); errors.Is(err, os.ErrNotExist) {
- t.Skipf("%s does not exist - skipping integration tests", modelDir)
- }
-}
-
-func getModelDir() string {
- _, filename, _, _ := runtime.Caller(0)
- return path.Dir(path.Dir(filename) + "/../test_data/models/.")
-}
-
-func PrepareModelForPrompts(t *testing.T, modelName string, opts api.Options) (*Model, llm.LLM) {
- modelDir := getModelDir()
- os.Setenv("OLLAMA_MODELS", modelDir)
- model, err := GetModel(modelName)
- require.NoError(t, err, "GetModel ")
- err = opts.FromMap(model.Options)
- require.NoError(t, err, "opts from model ")
- runner, err := llm.New("unused", model.ModelPath, model.AdapterPaths, model.ProjectorPaths, opts)
- require.NoError(t, err, "llm.New failed")
- return model, runner
-}
-
-func OneShotPromptResponse(t *testing.T, ctx context.Context, req api.GenerateRequest, model *Model, runner llm.LLM) string {
- prompt, err := model.PreResponsePrompt(PromptVars{
- System: req.System,
- Prompt: req.Prompt,
- First: len(req.Context) == 0,
- })
- require.NoError(t, err, "prompt generation failed")
- success := make(chan bool, 1)
- response := ""
- cb := func(r llm.PredictResult) {
-
- if !r.Done {
- response += r.Content
- } else {
- success <- true
- }
- }
-
- predictReq := llm.PredictOpts{
- Prompt: prompt,
- Format: req.Format,
- Images: req.Images,
- }
- err = runner.Predict(ctx, predictReq, cb)
- require.NoError(t, err, "predict call failed")
-
- select {
- case <-ctx.Done():
- t.Errorf("failed to complete before timeout: \n%s", response)
- return ""
- case <-success:
- return response
- }
-}
diff --git a/server/manifest.go b/server/manifest.go
new file mode 100644
index 00000000..6b04753f
--- /dev/null
+++ b/server/manifest.go
@@ -0,0 +1,171 @@
+package server
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "os"
+ "path/filepath"
+
+ "github.com/ollama/ollama/types/model"
+)
+
+type Manifest struct {
+ SchemaVersion int `json:"schemaVersion"`
+ MediaType string `json:"mediaType"`
+ Config Layer `json:"config"`
+ Layers []Layer `json:"layers"`
+
+ filepath string
+ fi os.FileInfo
+ digest string
+}
+
+func (m *Manifest) Size() (size int64) {
+ for _, layer := range append(m.Layers, m.Config) {
+ size += layer.Size
+ }
+
+ return
+}
+
+func (m *Manifest) Remove() error {
+ if err := os.Remove(m.filepath); err != nil {
+ return err
+ }
+
+ manifests, err := GetManifestPath()
+ if err != nil {
+ return err
+ }
+
+ return PruneDirectory(manifests)
+}
+
+func (m *Manifest) RemoveLayers() error {
+ for _, layer := range append(m.Layers, m.Config) {
+ if layer.Digest != "" {
+ if err := layer.Remove(); errors.Is(err, os.ErrNotExist) {
+ slog.Debug("layer does not exist", "digest", layer.Digest)
+ } else if err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func ParseNamedManifest(n model.Name) (*Manifest, error) {
+ if !n.IsFullyQualified() {
+ return nil, model.Unqualified(n)
+ }
+
+ manifests, err := GetManifestPath()
+ if err != nil {
+ return nil, err
+ }
+
+ p := filepath.Join(manifests, n.Filepath())
+
+ var m Manifest
+ f, err := os.Open(p)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ fi, err := f.Stat()
+ if err != nil {
+ return nil, err
+ }
+
+ sha256sum := sha256.New()
+ if err := json.NewDecoder(io.TeeReader(f, sha256sum)).Decode(&m); err != nil {
+ return nil, err
+ }
+
+ m.filepath = p
+ m.fi = fi
+ m.digest = hex.EncodeToString(sha256sum.Sum(nil))
+
+ return &m, nil
+}
+
+func WriteManifest(name model.Name, config Layer, layers []Layer) error {
+ manifests, err := GetManifestPath()
+ if err != nil {
+ return err
+ }
+
+ p := filepath.Join(manifests, name.Filepath())
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ return err
+ }
+
+ f, err := os.Create(p)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ m := Manifest{
+ SchemaVersion: 2,
+ MediaType: "application/vnd.docker.distribution.manifest.v2+json",
+ Config: config,
+ Layers: layers,
+ }
+
+ return json.NewEncoder(f).Encode(m)
+}
+
+func Manifests() (map[model.Name]*Manifest, error) {
+ manifests, err := GetManifestPath()
+ if err != nil {
+ return nil, err
+ }
+
+ // TODO(mxyng): use something less brittle
+ matches, err := filepath.Glob(filepath.Join(manifests, "*", "*", "*", "*"))
+ if err != nil {
+ return nil, err
+ }
+
+ ms := make(map[model.Name]*Manifest)
+ for _, match := range matches {
+ fi, err := os.Stat(match)
+ if err != nil {
+ return nil, err
+ }
+
+ if !fi.IsDir() {
+ rel, err := filepath.Rel(manifests, match)
+ if err != nil {
+ slog.Warn("bad filepath", "path", match, "error", err)
+ continue
+ }
+
+ n := model.ParseNameFromFilepath(rel)
+ if !n.IsValid() {
+ slog.Warn("bad manifest name", "path", rel)
+ continue
+ }
+
+ m, err := ParseNamedManifest(n)
+ if syntax := &(json.SyntaxError{}); errors.As(err, &syntax) {
+ slog.Warn("bad manifest", "name", n, "error", err)
+ continue
+ } else if err != nil {
+ return nil, fmt.Errorf("%s: %w", n, err)
+ }
+
+ ms[n] = m
+ }
+ }
+
+ return ms, nil
+}
diff --git a/server/manifest_test.go b/server/manifest_test.go
new file mode 100644
index 00000000..70ab7fa2
--- /dev/null
+++ b/server/manifest_test.go
@@ -0,0 +1,150 @@
+package server
+
+import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "slices"
+ "testing"
+
+ "github.com/ollama/ollama/types/model"
+)
+
+func createManifest(t *testing.T, path, name string) {
+ t.Helper()
+
+ p := filepath.Join(path, "manifests", name)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+
+ f, err := os.Create(p)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+
+ if err := json.NewEncoder(f).Encode(Manifest{}); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestManifests(t *testing.T) {
+ cases := map[string]struct {
+ ps []string
+ wantValidCount int
+ wantInvalidCount int
+ }{
+ "empty": {},
+ "single": {
+ ps: []string{
+ filepath.Join("host", "namespace", "model", "tag"),
+ },
+ wantValidCount: 1,
+ },
+ "multiple": {
+ ps: []string{
+ filepath.Join("registry.ollama.ai", "library", "llama3", "latest"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q4_0"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q4_1"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q8_0"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q5_0"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q5_1"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q2_K"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q3_K_S"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q3_K_M"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q3_K_L"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q4_K_S"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q4_K_M"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q5_K_S"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q5_K_M"),
+ filepath.Join("registry.ollama.ai", "library", "llama3", "q6_K"),
+ },
+ wantValidCount: 15,
+ },
+ "hidden": {
+ ps: []string{
+ filepath.Join("host", "namespace", "model", "tag"),
+ filepath.Join("host", "namespace", "model", ".hidden"),
+ },
+ wantValidCount: 1,
+ wantInvalidCount: 1,
+ },
+ "subdir": {
+ ps: []string{
+ filepath.Join("host", "namespace", "model", "tag", "one"),
+ filepath.Join("host", "namespace", "model", "tag", "another", "one"),
+ },
+ wantInvalidCount: 2,
+ },
+ "upper tag": {
+ ps: []string{
+ filepath.Join("host", "namespace", "model", "TAG"),
+ },
+ wantValidCount: 1,
+ },
+ "upper model": {
+ ps: []string{
+ filepath.Join("host", "namespace", "MODEL", "tag"),
+ },
+ wantValidCount: 1,
+ },
+ "upper namespace": {
+ ps: []string{
+ filepath.Join("host", "NAMESPACE", "model", "tag"),
+ },
+ wantValidCount: 1,
+ },
+ "upper host": {
+ ps: []string{
+ filepath.Join("HOST", "namespace", "model", "tag"),
+ },
+ wantValidCount: 1,
+ },
+ }
+
+ for n, wants := range cases {
+ t.Run(n, func(t *testing.T) {
+ d := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", d)
+
+ for _, p := range wants.ps {
+ createManifest(t, d, p)
+ }
+
+ ms, err := Manifests()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var ns []model.Name
+ for k := range ms {
+ ns = append(ns, k)
+ }
+
+ var gotValidCount, gotInvalidCount int
+ for _, p := range wants.ps {
+ n := model.ParseNameFromFilepath(p)
+ if n.IsValid() {
+ gotValidCount++
+ } else {
+ gotInvalidCount++
+ }
+
+ if !n.IsValid() && slices.Contains(ns, n) {
+ t.Errorf("unexpected invalid name: %s", p)
+ } else if n.IsValid() && !slices.Contains(ns, n) {
+ t.Errorf("missing valid name: %s", p)
+ }
+ }
+
+ if gotValidCount != wants.wantValidCount {
+ t.Errorf("got valid count %d, want %d", gotValidCount, wants.wantValidCount)
+ }
+
+ if gotInvalidCount != wants.wantInvalidCount {
+ t.Errorf("got invalid count %d, want %d", gotInvalidCount, wants.wantInvalidCount)
+ }
+ })
+ }
+}
diff --git a/server/manifests.go b/server/manifests.go
deleted file mode 100644
index 4e42b2ca..00000000
--- a/server/manifests.go
+++ /dev/null
@@ -1,34 +0,0 @@
-package server
-
-import (
- "bytes"
- "encoding/json"
- "os"
- "path/filepath"
-)
-
-func WriteManifest(name string, config *Layer, layers []*Layer) error {
- manifest := ManifestV2{
- SchemaVersion: 2,
- MediaType: "application/vnd.docker.distribution.manifest.v2+json",
- Config: config,
- Layers: layers,
- }
-
- var b bytes.Buffer
- if err := json.NewEncoder(&b).Encode(manifest); err != nil {
- return err
- }
-
- modelpath := ParseModelPath(name)
- manifestPath, err := modelpath.GetManifestPath()
- if err != nil {
- return err
- }
-
- if err := os.MkdirAll(filepath.Dir(manifestPath), 0755); err != nil {
- return err
- }
-
- return os.WriteFile(manifestPath, b.Bytes(), 0644)
-}
diff --git a/server/model.go b/server/model.go
new file mode 100644
index 00000000..124693d3
--- /dev/null
+++ b/server/model.go
@@ -0,0 +1,394 @@
+package server
+
+import (
+ "archive/zip"
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "text/template/parse"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/convert"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/template"
+ "github.com/ollama/ollama/types/model"
+)
+
+var intermediateBlobs map[string]string = make(map[string]string)
+
+type layerGGML struct {
+ Layer
+ *llm.GGML
+}
+
+func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
+ m, err := ParseNamedManifest(name)
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ if err := PullModel(ctx, name.String(), ®istryOptions{}, fn); err != nil {
+ return nil, err
+ }
+
+ m, err = ParseNamedManifest(name)
+ if err != nil {
+ return nil, err
+ }
+ case err != nil:
+ return nil, err
+ }
+
+ for _, layer := range m.Layers {
+ layer, err := NewLayerFromLayer(layer.Digest, layer.MediaType, name.DisplayShortest())
+ if err != nil {
+ return nil, err
+ }
+
+ switch layer.MediaType {
+ case "application/vnd.ollama.image.model",
+ "application/vnd.ollama.image.projector",
+ "application/vnd.ollama.image.adapter":
+ blobpath, err := GetBlobsPath(layer.Digest)
+ if err != nil {
+ return nil, err
+ }
+
+ blob, err := os.Open(blobpath)
+ if err != nil {
+ return nil, err
+ }
+ defer blob.Close()
+
+ ggml, _, err := llm.DecodeGGML(blob, 0)
+ if err != nil {
+ return nil, err
+ }
+
+ layers = append(layers, &layerGGML{layer, ggml})
+ default:
+ layers = append(layers, &layerGGML{layer, nil})
+ }
+ }
+
+ return layers, nil
+}
+
+func parseFromZipFile(_ context.Context, command string, baseLayers []*layerGGML, f *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
+ fi, err := f.Stat()
+ if err != nil {
+ return nil, err
+ }
+
+ r, err := zip.NewReader(f, fi.Size())
+ if err != nil {
+ return nil, err
+ }
+
+ p, err := os.MkdirTemp(filepath.Dir(f.Name()), "")
+ if err != nil {
+ return nil, err
+ }
+ defer os.RemoveAll(p)
+
+ fn(api.ProgressResponse{Status: "converting model"})
+ // TODO(mxyng): this should write directly into a layer
+ // e.g. NewLayer(arch.Reader(), "application/vnd.ollama.image.model")
+ t, err := os.CreateTemp(p, "fp16")
+ if err != nil {
+ return nil, err
+ }
+ defer t.Close()
+ defer os.Remove(t.Name())
+
+ var layerType string
+
+ switch command {
+ case "adapter":
+ var baseModel *llm.GGML
+ for _, l := range baseLayers {
+ if l.GGML != nil {
+ baseModel = l.GGML
+ break
+ }
+ }
+
+ if baseModel == nil {
+ return nil, fmt.Errorf("no base model specified for the adapter")
+ }
+
+ if err := convert.ConvertAdapter(convert.NewZipReader(r, p, 32<<20), t, baseModel.KV()); err != nil {
+ return nil, err
+ }
+ layerType = "application/vnd.ollama.image.adapter"
+ case "model":
+ if err := convert.ConvertModel(convert.NewZipReader(r, p, 32<<20), t); err != nil {
+ return nil, err
+ }
+ layerType = "application/vnd.ollama.image.model"
+ }
+
+ if _, err := t.Seek(0, io.SeekStart); err != nil {
+ return nil, err
+ }
+
+ layer, err := NewLayer(t, layerType)
+ if err != nil {
+ return nil, err
+ }
+
+ bin, err := layer.Open()
+ if err != nil {
+ return nil, err
+ }
+ defer bin.Close()
+
+ ggml, _, err := llm.DecodeGGML(bin, 0)
+ if err != nil {
+ return nil, err
+ }
+
+ layers = append(layers, &layerGGML{layer, ggml})
+
+ intermediateBlobs[digest] = layer.Digest
+ return detectChatTemplate(layers)
+}
+
+func parseFromFile(ctx context.Context, command string, baseLayers []*layerGGML, file *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
+ sr := io.NewSectionReader(file, 0, 512)
+ contentType, err := detectContentType(sr)
+ if err != nil {
+ return nil, err
+ }
+
+ switch contentType {
+ case "gguf", "ggla":
+ // noop
+ case "application/zip":
+ return parseFromZipFile(ctx, command, baseLayers, file, digest, fn)
+ default:
+ return nil, fmt.Errorf("unsupported content type: %s", contentType)
+ }
+
+ stat, err := file.Stat()
+ if err != nil {
+ return nil, err
+ }
+
+ var offset int64
+ for offset < stat.Size() {
+ ggml, n, err := llm.DecodeGGML(file, 0)
+ if errors.Is(err, io.EOF) {
+ break
+ } else if err != nil {
+ return nil, err
+ }
+
+ mediatype := "application/vnd.ollama.image.model"
+ if ggml.Name() == "ggla" || ggml.KV().Kind() == "adapter" {
+ mediatype = "application/vnd.ollama.image.adapter"
+ } else if ggml.KV().Architecture() == "clip" {
+ mediatype = "application/vnd.ollama.image.projector"
+ }
+
+ var layer Layer
+ if digest != "" && n == stat.Size() && offset == 0 {
+ layer, err = NewLayerFromLayer(digest, mediatype, file.Name())
+ if err != nil {
+ slog.Debug("could not create new layer from layer", "error", err)
+ }
+ }
+
+ // Fallback to creating layer from file copy (either NewLayerFromLayer failed, or digest empty/n != stat.Size())
+ if layer.Digest == "" {
+ layer, err = NewLayer(io.NewSectionReader(file, offset, n), mediatype)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ layers = append(layers, &layerGGML{layer, ggml})
+ offset = n
+ }
+
+ return detectChatTemplate(layers)
+}
+
+func detectChatTemplate(layers []*layerGGML) ([]*layerGGML, error) {
+ for _, layer := range layers {
+ if s := layer.GGML.KV().ChatTemplate(); s != "" {
+ if t, err := template.Named(s); err != nil {
+ slog.Debug("template detection", "error", err)
+ } else {
+ layer, err := NewLayer(t.Reader(), "application/vnd.ollama.image.template")
+ if err != nil {
+ return nil, err
+ }
+
+ layer.status = fmt.Sprintf("using autodetected template %s", t.Name)
+ layers = append(layers, &layerGGML{layer, nil})
+
+ if t.Parameters != nil {
+ var b bytes.Buffer
+ if err := json.NewEncoder(&b).Encode(t.Parameters); err != nil {
+ return nil, err
+ }
+
+ layer, err := NewLayer(&b, "application/vnd.ollama.image.params")
+ if err != nil {
+ return nil, err
+ }
+
+ layers = append(layers, &layerGGML{layer, nil})
+ }
+ }
+ }
+ }
+
+ return layers, nil
+}
+
+func detectContentType(r io.Reader) (string, error) {
+ var b bytes.Buffer
+ if _, err := io.Copy(&b, r); err != nil {
+ return "", err
+ }
+
+ if contentType := llm.DetectGGMLType(b.Bytes()); contentType != "" {
+ return contentType, nil
+ }
+
+ if contentType := http.DetectContentType(b.Bytes()); contentType != "application/octet-stream" {
+ return contentType, nil
+ }
+
+ return "unknown", nil
+}
+
+func parseObjects(s string) []map[string]any {
+ var objs []map[string]any
+ for offset := 0; offset < len(s); {
+ var obj map[string]any
+ decoder := json.NewDecoder(strings.NewReader(s[offset:]))
+ if err := decoder.Decode(&obj); errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+ break
+ } else if syntax := &(json.SyntaxError{}); errors.As(err, &syntax) {
+ // skip over any syntax errors
+ offset += int(syntax.Offset)
+ } else if unmarshalType := &(json.UnmarshalTypeError{}); errors.As(err, &unmarshalType) {
+ // skip over any unmarshalable types
+ offset += int(unmarshalType.Offset)
+ } else if err != nil {
+ return nil
+ } else {
+ offset += int(decoder.InputOffset())
+ objs = append(objs, obj)
+ }
+ }
+
+ return objs
+}
+
+// parseToolCalls attempts to parse a JSON string into a slice of ToolCalls.
+// mxyng: this only really works if the input contains tool calls in some JSON format
+func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) {
+ // create a subtree from the node that ranges over .ToolCalls
+ tmpl := m.Template.Subtree(func(n parse.Node) bool {
+ if t, ok := n.(*parse.RangeNode); ok {
+ return slices.Contains(template.Identifiers(t.Pipe), "ToolCalls")
+ }
+
+ return false
+ })
+
+ if tmpl == nil {
+ return nil, false
+ }
+
+ var b bytes.Buffer
+ if err := tmpl.Execute(&b, map[string][]api.ToolCall{
+ "ToolCalls": {
+ {
+ Function: api.ToolCallFunction{
+ Name: "@@name@@",
+ Arguments: api.ToolCallFunctionArguments{
+ "@@argument@@": 1,
+ },
+ },
+ },
+ },
+ }); err != nil {
+ return nil, false
+ }
+
+ templateObjects := parseObjects(b.String())
+ if len(templateObjects) == 0 {
+ return nil, false
+ }
+
+ // find the keys that correspond to the name and arguments fields
+ var name, arguments string
+ for k, v := range templateObjects[0] {
+ switch v.(type) {
+ case string:
+ name = k
+ case map[string]any:
+ arguments = k
+ }
+ }
+
+ if name == "" || arguments == "" {
+ return nil, false
+ }
+
+ responseObjects := parseObjects(s)
+ if len(responseObjects) == 0 {
+ return nil, false
+ }
+
+ // collect all nested objects
+ var collect func(any) []map[string]any
+ collect = func(obj any) (all []map[string]any) {
+ switch o := obj.(type) {
+ case map[string]any:
+ all = append(all, o)
+ for _, v := range o {
+ all = append(all, collect(v)...)
+ }
+ case []any:
+ for _, v := range o {
+ all = append(all, collect(v)...)
+ }
+ }
+
+ return all
+ }
+
+ var objs []map[string]any
+ for _, p := range responseObjects {
+ objs = append(objs, collect(p)...)
+ }
+
+ var toolCalls []api.ToolCall
+ for _, kv := range objs {
+ n, nok := kv[name].(string)
+ a, aok := kv[arguments].(map[string]any)
+ if nok && aok {
+ toolCalls = append(toolCalls, api.ToolCall{
+ Function: api.ToolCallFunction{
+ Name: n,
+ Arguments: a,
+ },
+ })
+ }
+ }
+
+ return toolCalls, len(toolCalls) > 0
+}
diff --git a/server/model_test.go b/server/model_test.go
new file mode 100644
index 00000000..304d4655
--- /dev/null
+++ b/server/model_test.go
@@ -0,0 +1,262 @@
+package server
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/template"
+)
+
+func readFile(t *testing.T, base, name string) *bytes.Buffer {
+ t.Helper()
+
+ bts, err := os.ReadFile(filepath.Join(base, name))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return bytes.NewBuffer(bts)
+}
+
+func TestExecuteWithTools(t *testing.T) {
+ p := filepath.Join("testdata", "tools")
+ cases := []struct {
+ model string
+ output string
+ ok bool
+ }{
+ {"mistral", `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, true},
+ {"mistral", `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]
+
+The temperature in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.`, true},
+ {"mistral", `I'm not aware of that information. However, I can suggest searching for the weather using the "get_current_weather" function:
+
+ [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, true},
+ {"mistral", " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", false},
+ {"command-r-plus", "Action: ```json" + `
+[
+ {
+ "tool_name": "get_current_weather",
+ "parameters": {
+ "format": "fahrenheit",
+ "location": "San Francisco, CA"
+ }
+ },
+ {
+ "tool_name": "get_current_weather",
+ "parameters": {
+ "format": "celsius",
+ "location": "Toronto, Canada"
+ }
+ }
+]
+` + "```", true},
+ {"command-r-plus", " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", false},
+ {"firefunction", ` functools[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, true},
+ {"firefunction", " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", false},
+ {"llama3-groq-tool-use", `<tool_call>
+{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}
+{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}
+</tool_call>`, true},
+ {"xlam", `{"tool_calls": [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]}`, true},
+ {"nemotron", `<toolcall>{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]} </toolcall>`, true},
+ }
+
+ var tools []api.Tool
+ if err := json.Unmarshal(readFile(t, p, "tools.json").Bytes(), &tools); err != nil {
+ t.Fatal(err)
+ }
+
+ var messages []api.Message
+ if err := json.Unmarshal(readFile(t, p, "messages.json").Bytes(), &messages); err != nil {
+ t.Fatal(err)
+ }
+
+ calls := []api.ToolCall{
+ {
+ Function: api.ToolCallFunction{
+ Name: "get_current_weather",
+ Arguments: api.ToolCallFunctionArguments{
+ "format": "fahrenheit",
+ "location": "San Francisco, CA",
+ },
+ },
+ },
+ {
+ Function: api.ToolCallFunction{
+ Name: "get_current_weather",
+ Arguments: api.ToolCallFunctionArguments{
+ "format": "celsius",
+ "location": "Toronto, Canada",
+ },
+ },
+ },
+ }
+
+ for _, tt := range cases {
+ t.Run(tt.model, func(t *testing.T) {
+ tmpl, err := template.Parse(readFile(t, p, fmt.Sprintf("%s.gotmpl", tt.model)).String())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ t.Run("template", func(t *testing.T) {
+ var actual bytes.Buffer
+ if err := tmpl.Execute(&actual, template.Values{Tools: tools, Messages: messages}); err != nil {
+ t.Fatal(err)
+ }
+
+ if diff := cmp.Diff(actual.String(), readFile(t, p, fmt.Sprintf("%s.out", tt.model)).String()); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+
+ t.Run("parse", func(t *testing.T) {
+ m := &Model{Template: tmpl}
+ actual, ok := m.parseToolCalls(tt.output)
+ if ok != tt.ok {
+ t.Fatalf("expected %t, got %t", tt.ok, ok)
+ }
+
+ if tt.ok {
+ if diff := cmp.Diff(actual, calls); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ }
+ })
+ })
+ }
+}
+
+func TestParseFromFileFromLayer(t *testing.T) {
+ tempModels := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", tempModels)
+
+ file, err := os.CreateTemp(tempModels, "")
+ if err != nil {
+ t.Fatalf("failed to open file: %v", err)
+ }
+ defer file.Close()
+ if err := llm.WriteGGUF(file, llm.KV{"general.architecture": "gemma"}, []llm.Tensor{}); err != nil {
+ t.Fatalf("failed to write gguf: %v", err)
+ }
+
+ if _, err := file.Seek(0, io.SeekStart); err != nil {
+ t.Fatalf("failed to seek to start: %v", err)
+ }
+
+ layers, err := parseFromFile(context.Background(), "model", []*layerGGML{}, file, "", func(api.ProgressResponse) {})
+ if err != nil {
+ t.Fatalf("failed to parse from file: %v", err)
+ }
+
+ if len(layers) != 1 {
+ t.Fatalf("got %d != want 1", len(layers))
+ }
+
+ if _, err := file.Seek(0, io.SeekStart); err != nil {
+ t.Fatalf("failed to seek to start: %v", err)
+ }
+
+ layers2, err := parseFromFile(context.Background(), "model", []*layerGGML{}, file, layers[0].Digest, func(api.ProgressResponse) {})
+ if err != nil {
+ t.Fatalf("failed to parse from file: %v", err)
+ }
+ if len(layers2) != 1 {
+ t.Fatalf("got %d != want 1", len(layers2))
+ }
+
+ if layers[0].Digest != layers2[0].Digest {
+ t.Fatalf("got %s != want %s", layers[0].Digest, layers2[0].Digest)
+ }
+
+ if layers[0].Size != layers2[0].Size {
+ t.Fatalf("got %d != want %d", layers[0].Size, layers2[0].Size)
+ }
+
+ if layers[0].MediaType != layers2[0].MediaType {
+ t.Fatalf("got %v != want %v", layers[0].MediaType, layers2[0].MediaType)
+ }
+}
+
+func TestParseLayerFromCopy(t *testing.T) {
+ tempModels := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", tempModels)
+
+ file2, err := os.CreateTemp(tempModels, "")
+ if err != nil {
+ t.Fatalf("failed to open file: %v", err)
+ }
+ defer file2.Close()
+
+ for range 5 {
+ if err := llm.WriteGGUF(file2, llm.KV{"general.architecture": "gemma"}, []llm.Tensor{}); err != nil {
+ t.Fatalf("failed to write gguf: %v", err)
+ }
+ }
+
+ if _, err := file2.Seek(0, io.SeekStart); err != nil {
+ t.Fatalf("failed to seek to start: %v", err)
+ }
+
+ layers, err := parseFromFile(context.Background(), "model", []*layerGGML{}, file2, "", func(api.ProgressResponse) {})
+ if err != nil {
+ t.Fatalf("failed to parse from file: %v", err)
+ }
+
+ if len(layers) != 5 {
+ t.Fatalf("got %d != want 5", len(layers))
+ }
+}
+
+func TestParseObjects(t *testing.T) {
+ tests := []struct {
+ input string
+ want []map[string]any
+ }{
+ {
+ input: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`,
+ want: []map[string]any{
+ {"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}},
+ {"name": "get_current_weather", "arguments": map[string]any{"format": "celsius", "location": "Toronto, Canada"}},
+ },
+ },
+ {
+ input: `<toolcall>{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} </toolcall>`,
+ want: []map[string]any{
+ {"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}},
+ },
+ },
+ {
+ input: `<toolcall>{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} </toolcall> <toolcall>{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, ON"}} </toolcall>`,
+ want: []map[string]any{
+ {"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}},
+ {"name": "get_current_weather", "arguments": map[string]any{"format": "celsius", "location": "Toronto, ON"}},
+ },
+ },
+ {
+ input: `{"name": "get_current_weather", "arguments": `,
+ want: nil,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.input, func(t *testing.T) {
+ got := parseObjects(tc.input)
+
+ if diff := cmp.Diff(got, tc.want); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+ }
+}
diff --git a/server/modelpath.go b/server/modelpath.go
index 651ee1be..d498c467 100644
--- a/server/modelpath.go
+++ b/server/modelpath.go
@@ -6,8 +6,10 @@ import (
"net/url"
"os"
"path/filepath"
- "runtime"
+ "regexp"
"strings"
+
+ "github.com/ollama/ollama/envconfig"
)
type ModelPath struct {
@@ -26,9 +28,10 @@ const (
)
var (
- ErrInvalidImageFormat = errors.New("invalid image format")
- ErrInvalidProtocol = errors.New("invalid protocol scheme")
- ErrInsecureProtocol = errors.New("insecure protocol http")
+ ErrInvalidImageFormat = errors.New("invalid image format")
+ ErrInvalidProtocol = errors.New("invalid protocol scheme")
+ ErrInsecureProtocol = errors.New("insecure protocol http")
+ ErrInvalidDigestFormat = errors.New("invalid digest format")
)
func ParseModelPath(name string) ModelPath {
@@ -46,7 +49,8 @@ func ParseModelPath(name string) ModelPath {
name = after
}
- parts := strings.Split(name, string(os.PathSeparator))
+ name = strings.ReplaceAll(name, string(os.PathSeparator), "/")
+ parts := strings.Split(name, "/")
switch len(parts) {
case 3:
mp.Registry = parts[0]
@@ -69,18 +73,6 @@ func ParseModelPath(name string) ModelPath {
var errModelPathInvalid = errors.New("invalid model path")
-func (mp ModelPath) Validate() error {
- if mp.Repository == "" {
- return fmt.Errorf("%w: model repository name is required", errModelPathInvalid)
- }
-
- if strings.Contains(mp.Tag, ":") {
- return fmt.Errorf("%w: ':' (colon) is not allowed in tag names", errModelPathInvalid)
- }
-
- return nil
-}
-
func (mp ModelPath) GetNamespaceRepository() string {
return fmt.Sprintf("%s/%s", mp.Namespace, mp.Repository)
}
@@ -99,27 +91,13 @@ func (mp ModelPath) GetShortTagname() string {
return fmt.Sprintf("%s/%s/%s:%s", mp.Registry, mp.Namespace, mp.Repository, mp.Tag)
}
-// modelsDir returns the value of the OLLAMA_MODELS environment variable or the user's home directory if OLLAMA_MODELS is not set.
-// The models directory is where Ollama stores its model files and manifests.
-func modelsDir() (string, error) {
- if models, exists := os.LookupEnv("OLLAMA_MODELS"); exists {
- return models, nil
- }
- home, err := os.UserHomeDir()
- if err != nil {
- return "", err
- }
- return filepath.Join(home, ".ollama", "models"), nil
-}
-
// GetManifestPath returns the path to the manifest file for the given model path, it is up to the caller to create the directory if it does not exist.
func (mp ModelPath) GetManifestPath() (string, error) {
- dir, err := modelsDir()
- if err != nil {
- return "", err
+ if p := filepath.Join(mp.Registry, mp.Namespace, mp.Repository, mp.Tag); filepath.IsLocal(p) {
+ return filepath.Join(envconfig.Models(), "manifests", p), nil
}
- return filepath.Join(dir, "manifests", mp.Registry, mp.Namespace, mp.Repository, mp.Tag), nil
+ return "", errModelPathInvalid
}
func (mp ModelPath) BaseURL() *url.URL {
@@ -130,12 +108,7 @@ func (mp ModelPath) BaseURL() *url.URL {
}
func GetManifestPath() (string, error) {
- dir, err := modelsDir()
- if err != nil {
- return "", err
- }
-
- path := filepath.Join(dir, "manifests")
+ path := filepath.Join(envconfig.Models(), "manifests")
if err := os.MkdirAll(path, 0o755); err != nil {
return "", err
}
@@ -144,16 +117,16 @@ func GetManifestPath() (string, error) {
}
func GetBlobsPath(digest string) (string, error) {
- dir, err := modelsDir()
- if err != nil {
- return "", err
+ // only accept actual sha256 digests
+ pattern := "^sha256[:-][0-9a-fA-F]{64}$"
+ re := regexp.MustCompile(pattern)
+
+ if digest != "" && !re.MatchString(digest) {
+ return "", ErrInvalidDigestFormat
}
- if runtime.GOOS == "windows" {
- digest = strings.ReplaceAll(digest, ":", "-")
- }
-
- path := filepath.Join(dir, "blobs", digest)
+ digest = strings.ReplaceAll(digest, ":", "-")
+ path := filepath.Join(envconfig.Models(), "blobs", digest)
dirPath := filepath.Dir(path)
if digest == "" {
dirPath = path
diff --git a/server/modelpath_test.go b/server/modelpath_test.go
index 8b26d52c..ef26266b 100644
--- a/server/modelpath_test.go
+++ b/server/modelpath_test.go
@@ -1,6 +1,75 @@
package server
-import "testing"
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetBlobsPath(t *testing.T) {
+ // GetBlobsPath expects an actual directory to exist
+ dir, err := os.MkdirTemp("", "ollama-test")
+ require.NoError(t, err)
+ defer os.RemoveAll(dir)
+
+ tests := []struct {
+ name string
+ digest string
+ expected string
+ err error
+ }{
+ {
+ "empty digest",
+ "",
+ filepath.Join(dir, "blobs"),
+ nil,
+ },
+ {
+ "valid with colon",
+ "sha256:456402914e838a953e0cf80caa6adbe75383d9e63584a964f504a7bbb8f7aad9",
+ filepath.Join(dir, "blobs", "sha256-456402914e838a953e0cf80caa6adbe75383d9e63584a964f504a7bbb8f7aad9"),
+ nil,
+ },
+ {
+ "valid with dash",
+ "sha256-456402914e838a953e0cf80caa6adbe75383d9e63584a964f504a7bbb8f7aad9",
+ filepath.Join(dir, "blobs", "sha256-456402914e838a953e0cf80caa6adbe75383d9e63584a964f504a7bbb8f7aad9"),
+ nil,
+ },
+ {
+ "digest too short",
+ "sha256-45640291",
+ "",
+ ErrInvalidDigestFormat,
+ },
+ {
+ "digest too long",
+ "sha256-456402914e838a953e0cf80caa6adbe75383d9e63584a964f504a7bbb8f7aad9aaaaaaaaaa",
+ "",
+ ErrInvalidDigestFormat,
+ },
+ {
+ "digest invalid chars",
+ "../sha256-456402914e838a953e0cf80caa6adbe75383d9e63584a964f504a7bbb8f7a",
+ "",
+ ErrInvalidDigestFormat,
+ },
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", dir)
+
+ got, err := GetBlobsPath(tc.digest)
+
+ require.ErrorIs(t, tc.err, err, tc.name)
+ assert.Equal(t, tc.expected, got, tc.name)
+ })
+ }
+}
func TestParseModelPath(t *testing.T) {
tests := []struct {
@@ -86,3 +155,10 @@ func TestParseModelPath(t *testing.T) {
})
}
}
+
+func TestInsecureModelpath(t *testing.T) {
+ mp := ParseModelPath("../../..:something")
+ if _, err := mp.GetManifestPath(); !errors.Is(err, errModelPathInvalid) {
+ t.Errorf("expected error: %v", err)
+ }
+}
diff --git a/server/prompt.go b/server/prompt.go
new file mode 100644
index 00000000..be0d4969
--- /dev/null
+++ b/server/prompt.go
@@ -0,0 +1,74 @@
+package server
+
+import (
+ "bytes"
+ "context"
+ "log/slog"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/template"
+)
+
+type tokenizeFunc func(context.Context, string) ([]int, error)
+
+// chatPrompt accepts a list of messages and returns the prompt and images that should be used for the next chat turn.
+// chatPrompt truncates any messages that exceed the context window of the model, making sure to always include 1) the
+// latest message and 2) system messages
+func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.Options, msgs []api.Message, tools []api.Tool) (prompt string, images []llm.ImageData, _ error) {
+ var system []api.Message
+ // always include the last message
+ n := len(msgs) - 1
+ // in reverse, find all messages that fit into context window
+ for i := n - 1; i >= 0; i-- {
+ system = make([]api.Message, 0)
+ for j := range i {
+ if msgs[j].Role == "system" {
+ system = append(system, msgs[j])
+ }
+ }
+
+ var b bytes.Buffer
+ if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[i:]...), Tools: tools}); err != nil {
+ return "", nil, err
+ }
+
+ s, err := tokenize(ctx, b.String())
+ if err != nil {
+ return "", nil, err
+ }
+
+ c := len(s)
+ if m.ProjectorPaths != nil {
+ for _, m := range msgs[i:] {
+ // images are represented as 768 sized embeddings
+ // TODO: get embedding length from project metadata
+ c += 768 * len(m.Images)
+ }
+ }
+
+ if c > opts.NumCtx {
+ slog.Debug("truncating input messages which exceed context length", "truncated", len(msgs[i:]))
+ break
+ } else {
+ n = i
+ }
+ }
+
+ // truncate any messages that do not fit into the context window
+ var b bytes.Buffer
+ if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[n:]...), Tools: tools}); err != nil {
+ return "", nil, err
+ }
+
+ for _, m := range msgs[n:] {
+ for _, i := range m.Images {
+ images = append(images, llm.ImageData{
+ ID: len(images),
+ Data: i,
+ })
+ }
+ }
+
+ return b.String(), images, nil
+}
diff --git a/server/prompt_test.go b/server/prompt_test.go
new file mode 100644
index 00000000..5fe3d4c5
--- /dev/null
+++ b/server/prompt_test.go
@@ -0,0 +1,211 @@
+package server
+
+import (
+ "bytes"
+ "context"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/template"
+)
+
+func TestChatPrompt(t *testing.T) {
+ type expect struct {
+ prompt string
+ images [][]byte
+ }
+
+ cases := []struct {
+ name string
+ limit int
+ msgs []api.Message
+ expect
+ }{
+ {
+ name: "messages",
+ limit: 64,
+ msgs: []api.Message{
+ {Role: "user", Content: "You're a test, Harry!"},
+ {Role: "assistant", Content: "I-I'm a what?"},
+ {Role: "user", Content: "A test. And a thumping good one at that, I'd wager."},
+ },
+ expect: expect{
+ prompt: "You're a test, Harry! I-I'm a what? A test. And a thumping good one at that, I'd wager. ",
+ },
+ },
+ {
+ name: "truncate messages",
+ limit: 1,
+ msgs: []api.Message{
+ {Role: "user", Content: "You're a test, Harry!"},
+ {Role: "assistant", Content: "I-I'm a what?"},
+ {Role: "user", Content: "A test. And a thumping good one at that, I'd wager."},
+ },
+ expect: expect{
+ prompt: "A test. And a thumping good one at that, I'd wager. ",
+ },
+ },
+ {
+ name: "truncate messages with image",
+ limit: 64,
+ msgs: []api.Message{
+ {Role: "user", Content: "You're a test, Harry!"},
+ {Role: "assistant", Content: "I-I'm a what?"},
+ {Role: "user", Content: "A test. And a thumping good one at that, I'd wager.", Images: []api.ImageData{[]byte("something")}},
+ },
+ expect: expect{
+ prompt: "[img-0] A test. And a thumping good one at that, I'd wager. ",
+ images: [][]byte{
+ []byte("something"),
+ },
+ },
+ },
+ {
+ name: "truncate messages with images",
+ limit: 64,
+ msgs: []api.Message{
+ {Role: "user", Content: "You're a test, Harry!", Images: []api.ImageData{[]byte("something")}},
+ {Role: "assistant", Content: "I-I'm a what?"},
+ {Role: "user", Content: "A test. And a thumping good one at that, I'd wager.", Images: []api.ImageData{[]byte("somethingelse")}},
+ },
+ expect: expect{
+ prompt: "[img-0] A test. And a thumping good one at that, I'd wager. ",
+ images: [][]byte{
+ []byte("somethingelse"),
+ },
+ },
+ },
+ {
+ name: "messages with images",
+ limit: 2048,
+ msgs: []api.Message{
+ {Role: "user", Content: "You're a test, Harry!", Images: []api.ImageData{[]byte("something")}},
+ {Role: "assistant", Content: "I-I'm a what?"},
+ {Role: "user", Content: "A test. And a thumping good one at that, I'd wager.", Images: []api.ImageData{[]byte("somethingelse")}},
+ },
+ expect: expect{
+ prompt: "[img-0] You're a test, Harry! I-I'm a what? [img-1] A test. And a thumping good one at that, I'd wager. ",
+ images: [][]byte{
+ []byte("something"),
+ []byte("somethingelse"),
+ },
+ },
+ },
+ {
+ name: "message with image tag",
+ limit: 2048,
+ msgs: []api.Message{
+ {Role: "user", Content: "You're a test, Harry! [img]", Images: []api.ImageData{[]byte("something")}},
+ {Role: "assistant", Content: "I-I'm a what?"},
+ {Role: "user", Content: "A test. And a thumping good one at that, I'd wager.", Images: []api.ImageData{[]byte("somethingelse")}},
+ },
+ expect: expect{
+ prompt: "You're a test, Harry! [img-0] I-I'm a what? [img-1] A test. And a thumping good one at that, I'd wager. ",
+ images: [][]byte{
+ []byte("something"),
+ []byte("somethingelse"),
+ },
+ },
+ },
+ {
+ name: "messages with interleaved images",
+ limit: 2048,
+ msgs: []api.Message{
+ {Role: "user", Content: "You're a test, Harry!"},
+ {Role: "user", Images: []api.ImageData{[]byte("something")}},
+ {Role: "user", Images: []api.ImageData{[]byte("somethingelse")}},
+ {Role: "assistant", Content: "I-I'm a what?"},
+ {Role: "user", Content: "A test. And a thumping good one at that, I'd wager."},
+ },
+ expect: expect{
+ prompt: "You're a test, Harry!\n\n[img-0]\n\n[img-1] I-I'm a what? A test. And a thumping good one at that, I'd wager. ",
+ images: [][]byte{
+ []byte("something"),
+ []byte("somethingelse"),
+ },
+ },
+ },
+ {
+ name: "truncate message with interleaved images",
+ limit: 1024,
+ msgs: []api.Message{
+ {Role: "user", Content: "You're a test, Harry!"},
+ {Role: "user", Images: []api.ImageData{[]byte("something")}},
+ {Role: "user", Images: []api.ImageData{[]byte("somethingelse")}},
+ {Role: "assistant", Content: "I-I'm a what?"},
+ {Role: "user", Content: "A test. And a thumping good one at that, I'd wager."},
+ },
+ expect: expect{
+ prompt: "[img-0] I-I'm a what? A test. And a thumping good one at that, I'd wager. ",
+ images: [][]byte{
+ []byte("somethingelse"),
+ },
+ },
+ },
+ {
+ name: "message with system prompt",
+ limit: 2048,
+ msgs: []api.Message{
+ {Role: "system", Content: "You are the Test Who Lived."},
+ {Role: "user", Content: "You're a test, Harry!"},
+ {Role: "assistant", Content: "I-I'm a what?"},
+ {Role: "user", Content: "A test. And a thumping good one at that, I'd wager."},
+ },
+ expect: expect{
+ prompt: "You are the Test Who Lived. You're a test, Harry! I-I'm a what? A test. And a thumping good one at that, I'd wager. ",
+ },
+ },
+ {
+ name: "out of order system",
+ limit: 2048,
+ msgs: []api.Message{
+ {Role: "user", Content: "You're a test, Harry!"},
+ {Role: "assistant", Content: "I-I'm a what?"},
+ {Role: "system", Content: "You are the Test Who Lived."},
+ {Role: "user", Content: "A test. And a thumping good one at that, I'd wager."},
+ },
+ expect: expect{
+ prompt: "You're a test, Harry! I-I'm a what? You are the Test Who Lived. A test. And a thumping good one at that, I'd wager. ",
+ },
+ },
+ }
+
+ tmpl, err := template.Parse(`
+{{- if .System }}{{ .System }} {{ end }}
+{{- if .Prompt }}{{ .Prompt }} {{ end }}
+{{- if .Response }}{{ .Response }} {{ end }}`)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for _, tt := range cases {
+ t.Run(tt.name, func(t *testing.T) {
+ model := Model{Template: tmpl, ProjectorPaths: []string{"vision"}}
+ opts := api.Options{Runner: api.Runner{NumCtx: tt.limit}}
+ prompt, images, err := chatPrompt(context.TODO(), &model, mockRunner{}.Tokenize, &opts, tt.msgs, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if diff := cmp.Diff(prompt, tt.prompt); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+
+ if len(images) != len(tt.images) {
+ t.Fatalf("expected %d images, got %d", len(tt.images), len(images))
+ }
+
+ for i := range images {
+ if images[i].ID != i {
+ t.Errorf("expected ID %d, got %d", i, images[i].ID)
+ }
+
+ if !bytes.Equal(images[i].Data, tt.images[i]) {
+ t.Errorf("expected %q, got %q", tt.images[i], images[i])
+ }
+ }
+ })
+ }
+}
diff --git a/server/routes.go b/server/routes.go
index 123f964b..23f9dbfd 100644
--- a/server/routes.go
+++ b/server/routes.go
@@ -1,40 +1,49 @@
package server
import (
+ "bytes"
+ "cmp"
"context"
"encoding/json"
"errors"
"fmt"
"io"
- "io/fs"
- "log"
+ "log/slog"
+ "math"
"net"
"net/http"
+ "net/netip"
"os"
"os/signal"
"path/filepath"
- "reflect"
- "runtime"
- "strconv"
+ "slices"
"strings"
- "sync"
"syscall"
"time"
"github.com/gin-contrib/cors"
"github.com/gin-gonic/gin"
+ "golang.org/x/sync/errgroup"
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/gpu"
- "github.com/jmorganca/ollama/llm"
- "github.com/jmorganca/ollama/parser"
- "github.com/jmorganca/ollama/version"
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/build"
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/gpu"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/openai"
+ "github.com/ollama/ollama/parser"
+ "github.com/ollama/ollama/runners"
+ "github.com/ollama/ollama/template"
+ "github.com/ollama/ollama/types/errtypes"
+ "github.com/ollama/ollama/types/model"
+ "github.com/ollama/ollama/version"
)
var mode string = gin.DebugMode
type Server struct {
- WorkDir string
+ addr net.Addr
+ sched *Scheduler
}
func init() {
@@ -49,289 +58,268 @@ func init() {
gin.SetMode(mode)
}
-var loaded struct {
- mu sync.Mutex
-
- runner llm.LLM
-
- expireAt time.Time
- expireTimer *time.Timer
-
- *Model
- *api.Options
-}
-
-var defaultSessionDuration = 5 * time.Minute
-
-// load a model into memory if it is not already loaded, it is up to the caller to lock loaded.mu before calling this function
-func load(c *gin.Context, modelName string, reqOpts map[string]interface{}, sessionDuration time.Duration) (*Model, error) {
- model, err := GetModel(modelName)
- if err != nil {
- return nil, err
- }
-
- workDir := c.GetString("workDir")
+var (
+ errRequired = errors.New("is required")
+ errBadTemplate = errors.New("template error")
+)
+func modelOptions(model *Model, requestOpts map[string]interface{}) (api.Options, error) {
opts := api.DefaultOptions()
if err := opts.FromMap(model.Options); err != nil {
- log.Printf("could not load model options: %v", err)
- return nil, err
+ return api.Options{}, err
}
- if err := opts.FromMap(reqOpts); err != nil {
- return nil, err
+ if err := opts.FromMap(requestOpts); err != nil {
+ return api.Options{}, err
}
- needLoad := loaded.runner == nil || // is there a model loaded?
- loaded.ModelPath != model.ModelPath || // has the base model changed?
- !reflect.DeepEqual(loaded.AdapterPaths, model.AdapterPaths) || // have the adapters changed?
- !reflect.DeepEqual(loaded.Options.Runner, opts.Runner) // have the runner options changed?
-
- if needLoad {
- if loaded.runner != nil {
- log.Println("changing loaded model")
- loaded.runner.Close()
- loaded.runner = nil
- loaded.Model = nil
- loaded.Options = nil
- }
-
- llmRunner, err := llm.New(workDir, model.ModelPath, model.AdapterPaths, model.ProjectorPaths, opts)
- if err != nil {
- // some older models are not compatible with newer versions of llama.cpp
- // show a generalized compatibility error until there is a better way to
- // check for model compatibility
- if errors.Is(llm.ErrUnsupportedFormat, err) || strings.Contains(err.Error(), "failed to load model") {
- err = fmt.Errorf("%v: this model may be incompatible with your version of Ollama. If you previously pulled this model, try updating it by running `ollama pull %s`", err, model.ShortName)
- }
-
- return nil, err
- }
-
- loaded.Model = model
- loaded.runner = llmRunner
- loaded.Options = &opts
- }
-
- loaded.expireAt = time.Now().Add(sessionDuration)
-
- if loaded.expireTimer == nil {
- loaded.expireTimer = time.AfterFunc(sessionDuration, func() {
- loaded.mu.Lock()
- defer loaded.mu.Unlock()
-
- if time.Now().Before(loaded.expireAt) {
- return
- }
-
- if loaded.runner != nil {
- loaded.runner.Close()
- }
-
- loaded.runner = nil
- loaded.Model = nil
- loaded.Options = nil
- })
- }
-
- loaded.expireTimer.Reset(sessionDuration)
- return model, nil
+ return opts, nil
}
-func GenerateHandler(c *gin.Context) {
- loaded.mu.Lock()
- defer loaded.mu.Unlock()
+// scheduleRunner schedules a runner after validating inputs such as capabilities and model options.
+// It returns the allocated runner, model instance, and consolidated options if successful and error otherwise.
+func (s *Server) scheduleRunner(ctx context.Context, name string, caps []Capability, requestOpts map[string]any, keepAlive *api.Duration) (llm.LlamaServer, *Model, *api.Options, error) {
+ if name == "" {
+ return nil, nil, nil, fmt.Errorf("model %w", errRequired)
+ }
+ model, err := GetModel(name)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+
+ if err := model.CheckCapabilities(caps...); err != nil {
+ return nil, nil, nil, fmt.Errorf("%s %w", name, err)
+ }
+
+ opts, err := modelOptions(model, requestOpts)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+
+ runnerCh, errCh := s.sched.GetRunner(ctx, model, opts, keepAlive)
+ var runner *runnerRef
+ select {
+ case runner = <-runnerCh:
+ case err = <-errCh:
+ return nil, nil, nil, err
+ }
+
+ return runner.llama, model, &opts, nil
+}
+
+func (s *Server) GenerateHandler(c *gin.Context) {
checkpointStart := time.Now()
var req api.GenerateRequest
- err := c.ShouldBindJSON(&req)
-
- switch {
- case errors.Is(err, io.EOF):
+ if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
return
- case err != nil:
+ } else if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
- // validate the request
- switch {
- case req.Model == "":
- c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
+ // expire the runner
+ if req.Prompt == "" && req.KeepAlive != nil && int(req.KeepAlive.Seconds()) == 0 {
+ model, err := GetModel(req.Model)
+ if err != nil {
+ switch {
+ case os.IsNotExist(err):
+ c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
+ case err.Error() == "invalid model name":
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ default:
+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ }
+ return
+ }
+ s.sched.expireRunner(model)
+
+ c.JSON(http.StatusOK, api.GenerateResponse{
+ Model: req.Model,
+ CreatedAt: time.Now().UTC(),
+ Response: "",
+ Done: true,
+ DoneReason: "unload",
+ })
return
- case len(req.Format) > 0 && req.Format != "json":
- c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be json"})
+ }
+
+ if req.Format != "" && req.Format != "json" {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be empty or \"json\""})
return
- case req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0):
+ } else if req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0) {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
return
}
- sessionDuration := defaultSessionDuration
- model, err := load(c, req.Model, req.Options, sessionDuration)
- if err != nil {
- var pErr *fs.PathError
- switch {
- case errors.As(err, &pErr):
- c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
- case errors.Is(err, api.ErrInvalidOpts):
- c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
- default:
- c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
- }
- return
+ caps := []Capability{CapabilityCompletion}
+ if req.Suffix != "" {
+ caps = append(caps, CapabilityInsert)
}
- // an empty request loads the model
- if req.Prompt == "" && req.Template == "" && req.System == "" {
- c.JSON(http.StatusOK, api.GenerateResponse{
- CreatedAt: time.Now().UTC(),
- Model: req.Model,
- Done: true})
+ r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, caps, req.Options, req.KeepAlive)
+ if errors.Is(err, errCapabilityCompletion) {
+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support generate", req.Model)})
+ return
+ } else if err != nil {
+ handleScheduleError(c, req.Model, err)
return
}
checkpointLoaded := time.Now()
- var prompt string
- var promptVars PromptVars
- switch {
- case req.Raw:
- prompt = req.Prompt
- case req.Prompt != "":
- if req.Template != "" {
- // override the default model template
- model.Template = req.Template
- }
+ if req.Prompt == "" {
+ c.JSON(http.StatusOK, api.GenerateResponse{
+ Model: req.Model,
+ CreatedAt: time.Now().UTC(),
+ Done: true,
+ DoneReason: "load",
+ })
+ return
+ }
- var rebuild strings.Builder
- if req.Context != nil {
- // TODO: context is deprecated, at some point the context logic within this conditional should be removed
- prevCtx, err := loaded.runner.Decode(c.Request.Context(), req.Context)
+ images := make([]llm.ImageData, len(req.Images))
+ for i := range req.Images {
+ images[i] = llm.ImageData{ID: i, Data: req.Images[i]}
+ }
+
+ prompt := req.Prompt
+ if !req.Raw {
+ tmpl := m.Template
+ if req.Template != "" {
+ tmpl, err = template.Parse(req.Template)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
+ }
- // Remove leading spaces from prevCtx if present
- prevCtx = strings.TrimPrefix(prevCtx, " ")
- rebuild.WriteString(prevCtx)
+ var values template.Values
+ if req.Suffix != "" {
+ values.Prompt = prompt
+ values.Suffix = req.Suffix
+ } else {
+ var msgs []api.Message
+ if req.System != "" {
+ msgs = append(msgs, api.Message{Role: "system", Content: req.System})
+ } else if m.System != "" {
+ msgs = append(msgs, api.Message{Role: "system", Content: m.System})
+ }
+
+ if req.Context == nil {
+ msgs = append(msgs, m.Messages...)
+ }
+
+ for _, i := range images {
+ msgs = append(msgs, api.Message{Role: "user", Content: fmt.Sprintf("[img-%d]", i.ID)})
+ }
+
+ values.Messages = append(msgs, api.Message{Role: "user", Content: req.Prompt})
}
- promptVars = PromptVars{
- System: req.System,
- Prompt: req.Prompt,
- First: len(req.Context) == 0,
+
+ var b bytes.Buffer
+ if req.Context != nil {
+ s, err := r.Detokenize(c.Request.Context(), req.Context)
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ }
+ b.WriteString(s)
}
- p, err := model.PreResponsePrompt(promptVars)
- if err != nil {
+
+ if err := tmpl.Execute(&b, values); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
- rebuild.WriteString(p)
- prompt = rebuild.String()
+
+ prompt = b.String()
}
+ slog.Debug("generate request", "prompt", prompt, "images", images)
+
ch := make(chan any)
- var generated strings.Builder
go func() {
+ // TODO (jmorganca): avoid building the response twice both here and below
+ var sb strings.Builder
defer close(ch)
-
- fn := func(r llm.PredictResult) {
- // Update model expiration
- loaded.expireAt = time.Now().Add(sessionDuration)
- loaded.expireTimer.Reset(sessionDuration)
-
- // Build up the full response
- if _, err := generated.WriteString(r.Content); err != nil {
- ch <- gin.H{"error": err.Error()}
- return
- }
-
- resp := api.GenerateResponse{
- Model: req.Model,
- CreatedAt: time.Now().UTC(),
- Done: r.Done,
- Response: r.Content,
+ if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
+ Prompt: prompt,
+ Images: images,
+ Format: req.Format,
+ Options: opts,
+ }, func(cr llm.CompletionResponse) {
+ res := api.GenerateResponse{
+ Model: req.Model,
+ CreatedAt: time.Now().UTC(),
+ Response: cr.Content,
+ Done: cr.Done,
+ DoneReason: cr.DoneReason,
Metrics: api.Metrics{
- PromptEvalCount: r.PromptEvalCount,
- PromptEvalDuration: r.PromptEvalDuration,
- EvalCount: r.EvalCount,
- EvalDuration: r.EvalDuration,
+ PromptEvalCount: cr.PromptEvalCount,
+ PromptEvalDuration: cr.PromptEvalDuration,
+ EvalCount: cr.EvalCount,
+ EvalDuration: cr.EvalDuration,
},
}
- if r.Done {
- resp.TotalDuration = time.Since(checkpointStart)
- resp.LoadDuration = checkpointLoaded.Sub(checkpointStart)
+ if _, err := sb.WriteString(cr.Content); err != nil {
+ ch <- gin.H{"error": err.Error()}
+ }
+
+ if cr.Done {
+ res.TotalDuration = time.Since(checkpointStart)
+ res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
if !req.Raw {
- // append the generated text to the history and template it if needed
- promptVars.Response = generated.String()
- result, err := model.PostResponseTemplate(promptVars)
+ tokens, err := r.Tokenize(c.Request.Context(), prompt+sb.String())
if err != nil {
ch <- gin.H{"error": err.Error()}
return
}
- embd, err := loaded.runner.Encode(c.Request.Context(), prompt+result)
- if err != nil {
- ch <- gin.H{"error": err.Error()}
- return
- }
- resp.Context = embd
+ res.Context = tokens
}
}
- ch <- resp
- }
-
- // Start prediction
- predictReq := llm.PredictOpts{
- Prompt: prompt,
- Format: req.Format,
- Images: req.Images,
- }
- if err := loaded.runner.Predict(c.Request.Context(), predictReq, fn); err != nil {
+ ch <- res
+ }); err != nil {
ch <- gin.H{"error": err.Error()}
}
}()
if req.Stream != nil && !*req.Stream {
- // Accumulate responses into the final response
- var final api.GenerateResponse
+ var r api.GenerateResponse
var sb strings.Builder
- for resp := range ch {
- switch r := resp.(type) {
+ for rr := range ch {
+ switch t := rr.(type) {
case api.GenerateResponse:
- sb.WriteString(r.Response)
- final = r
+ sb.WriteString(t.Response)
+ r = t
case gin.H:
- if errorMsg, ok := r["error"].(string); ok {
- c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
- return
- } else {
- c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in response"})
- return
+ msg, ok := t["error"].(string)
+ if !ok {
+ msg = "unexpected error format in response"
}
+
+ c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
+ return
default:
- c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error"})
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
return
}
}
- final.Response = sb.String()
- c.JSON(http.StatusOK, final)
+ r.Response = sb.String()
+ c.JSON(http.StatusOK, r)
return
}
streamResponse(c, ch)
}
-func EmbeddingHandler(c *gin.Context) {
- loaded.mu.Lock()
- defer loaded.mu.Unlock()
-
- var req api.EmbeddingRequest
+func (s *Server) EmbedHandler(c *gin.Context) {
+ checkpointStart := time.Now()
+ var req api.EmbedRequest
err := c.ShouldBindJSON(&req)
switch {
case errors.Is(err, io.EOF):
@@ -342,45 +330,168 @@ func EmbeddingHandler(c *gin.Context) {
return
}
- if req.Model == "" {
- c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
- return
+ truncate := true
+
+ if req.Truncate != nil && !*req.Truncate {
+ truncate = false
}
- sessionDuration := defaultSessionDuration
- _, err = load(c, req.Model, req.Options, sessionDuration)
- if err != nil {
- var pErr *fs.PathError
- switch {
- case errors.As(err, &pErr):
- c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
- case errors.Is(err, api.ErrInvalidOpts):
- c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
- default:
- c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ var input []string
+
+ switch i := req.Input.(type) {
+ case string:
+ if len(i) > 0 {
+ input = append(input, i)
+ }
+ case []any:
+ for _, v := range i {
+ if _, ok := v.(string); !ok {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
+ return
+ }
+ input = append(input, v.(string))
+ }
+ default:
+ if req.Input != nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
+ return
}
- return
}
- if !loaded.Options.EmbeddingOnly {
- c.JSON(http.StatusBadRequest, gin.H{"error": "embedding option must be set to true"})
- return
- }
-
- embedding, err := loaded.runner.Embedding(c.Request.Context(), req.Prompt)
+ r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
if err != nil {
- log.Printf("embedding generation failed: %v", err)
- c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
+ handleScheduleError(c, req.Model, err)
return
}
- resp := api.EmbeddingResponse{
- Embedding: embedding,
+ checkpointLoaded := time.Now()
+
+ if len(input) == 0 {
+ c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}})
+ return
+ }
+
+ kvData, err := getKVData(m.ModelPath, false)
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ }
+
+ var count int
+ for i, s := range input {
+ tokens, err := r.Tokenize(c.Request.Context(), s)
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ }
+
+ ctxLen := min(opts.NumCtx, int(kvData.ContextLength()))
+ if len(tokens) > ctxLen {
+ if !truncate {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "input length exceeds maximum context length"})
+ return
+ }
+
+ tokens = tokens[:ctxLen]
+ s, err = r.Detokenize(c.Request.Context(), tokens)
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ }
+ }
+
+ count += len(tokens)
+
+ input[i] = s
+ }
+
+ var g errgroup.Group
+ embeddings := make([][]float32, len(input))
+ for i, text := range input {
+ g.Go(func() error {
+ embedding, err := r.Embedding(c.Request.Context(), text)
+ if err != nil {
+ return err
+ }
+ embeddings[i] = normalize(embedding)
+ return nil
+ })
+ }
+
+ if err := g.Wait(); err != nil {
+ slog.Error("embedding generation failed", "error", err)
+ c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Errorf("failed to generate embeddings: %v", err)})
+ return
+ }
+
+ resp := api.EmbedResponse{
+ Model: req.Model,
+ Embeddings: embeddings,
+ TotalDuration: time.Since(checkpointStart),
+ LoadDuration: checkpointLoaded.Sub(checkpointStart),
+ PromptEvalCount: count,
}
c.JSON(http.StatusOK, resp)
}
-func PullModelHandler(c *gin.Context) {
+func normalize(vec []float32) []float32 {
+ var sum float32
+ for _, v := range vec {
+ sum += v * v
+ }
+
+ norm := float32(0.0)
+ if sum > 0 {
+ norm = float32(1.0 / math.Sqrt(float64(sum)))
+ }
+
+ for i := range vec {
+ vec[i] *= norm
+ }
+ return vec
+}
+
+func (s *Server) EmbeddingsHandler(c *gin.Context) {
+ var req api.EmbeddingRequest
+ if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
+ return
+ } else if err != nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+
+ r, _, _, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
+ if err != nil {
+ handleScheduleError(c, req.Model, err)
+ return
+ }
+
+ // an empty request loads the model
+ if req.Prompt == "" {
+ c.JSON(http.StatusOK, api.EmbeddingResponse{Embedding: []float64{}})
+ return
+ }
+
+ embedding, err := r.Embedding(c.Request.Context(), req.Prompt)
+ if err != nil {
+ slog.Info(fmt.Sprintf("embedding generation failed: %v", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
+ return
+ }
+
+ var e []float64
+ for _, v := range embedding {
+ e = append(e, float64(v))
+ }
+
+ resp := api.EmbeddingResponse{
+ Embedding: e,
+ }
+ c.JSON(http.StatusOK, resp)
+}
+
+func (s *Server) PullHandler(c *gin.Context) {
var req api.PullRequest
err := c.ShouldBindJSON(&req)
switch {
@@ -392,8 +503,14 @@ func PullModelHandler(c *gin.Context) {
return
}
- if req.Name == "" {
- c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
+ name := model.ParseName(cmp.Or(req.Model, req.Name))
+ if !name.IsValid() {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid model name"})
+ return
+ }
+
+ if err := checkNameExists(name); err != nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
@@ -404,14 +521,14 @@ func PullModelHandler(c *gin.Context) {
ch <- r
}
- regOpts := &RegistryOptions{
+ regOpts := ®istryOptions{
Insecure: req.Insecure,
}
ctx, cancel := context.WithCancel(c.Request.Context())
defer cancel()
- if err := PullModel(ctx, req.Name, regOpts, fn); err != nil {
+ if err := PullModel(ctx, name.DisplayShortest(), regOpts, fn); err != nil {
ch <- gin.H{"error": err.Error()}
}
}()
@@ -424,7 +541,7 @@ func PullModelHandler(c *gin.Context) {
streamResponse(c, ch)
}
-func PushModelHandler(c *gin.Context) {
+func (s *Server) PushHandler(c *gin.Context) {
var req api.PushRequest
err := c.ShouldBindJSON(&req)
switch {
@@ -436,8 +553,13 @@ func PushModelHandler(c *gin.Context) {
return
}
- if req.Name == "" {
- c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
+ var model string
+ if req.Model != "" {
+ model = req.Model
+ } else if req.Name != "" {
+ model = req.Name
+ } else {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
return
}
@@ -448,14 +570,14 @@ func PushModelHandler(c *gin.Context) {
ch <- r
}
- regOpts := &RegistryOptions{
+ regOpts := ®istryOptions{
Insecure: req.Insecure,
}
ctx, cancel := context.WithCancel(c.Request.Context())
defer cancel()
- if err := PushModel(ctx, req.Name, regOpts, fn); err != nil {
+ if err := PushModel(ctx, model, regOpts, fn); err != nil {
ch <- gin.H{"error": err.Error()}
}
}()
@@ -468,46 +590,60 @@ func PushModelHandler(c *gin.Context) {
streamResponse(c, ch)
}
-func CreateModelHandler(c *gin.Context) {
- var req api.CreateRequest
- err := c.ShouldBindJSON(&req)
- switch {
- case errors.Is(err, io.EOF):
+func checkNameExists(name model.Name) error {
+ names, err := Manifests()
+ if err != nil {
+ return err
+ }
+
+ for n := range names {
+ if strings.EqualFold(n.Filepath(), name.Filepath()) && n != name {
+ return errors.New("a model with that name already exists")
+ }
+ }
+
+ return nil
+}
+
+func (s *Server) CreateHandler(c *gin.Context) {
+ var r api.CreateRequest
+ if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
return
- case err != nil:
+ } else if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
- if req.Name == "" {
- c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
+ name := model.ParseName(cmp.Or(r.Model, r.Name))
+ if !name.IsValid() {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": errtypes.InvalidModelNameErrMsg})
return
}
- if err := ParseModelPath(req.Name).Validate(); err != nil {
+ if err := checkNameExists(name); err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
- if req.Path == "" && req.Modelfile == "" {
+ if r.Path == "" && r.Modelfile == "" {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "path or modelfile are required"})
return
}
- var modelfile io.Reader = strings.NewReader(req.Modelfile)
- if req.Path != "" && req.Modelfile == "" {
- mf, err := os.Open(req.Path)
+ var sr io.Reader = strings.NewReader(r.Modelfile)
+ if r.Path != "" && r.Modelfile == "" {
+ f, err := os.Open(r.Path)
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("error reading modelfile: %s", err)})
return
}
- defer mf.Close()
+ defer f.Close()
- modelfile = mf
+ sr = f
}
- commands, err := parser.Parse(modelfile)
+ f, err := parser.ParseFile(sr)
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
@@ -523,12 +659,15 @@ func CreateModelHandler(c *gin.Context) {
ctx, cancel := context.WithCancel(c.Request.Context())
defer cancel()
- if err := CreateModel(ctx, req.Name, filepath.Dir(req.Path), commands, fn); err != nil {
+ quantization := cmp.Or(r.Quantize, r.Quantization)
+ if err := CreateModel(ctx, name, filepath.Dir(r.Path), strings.ToUpper(quantization), f, fn); errors.Is(err, errBadTemplate) {
+ ch <- gin.H{"error": err.Error(), "status": http.StatusBadRequest}
+ } else if err != nil {
ch <- gin.H{"error": err.Error()}
}
}()
- if req.Stream != nil && !*req.Stream {
+ if r.Stream != nil && !*r.Stream {
waitForStream(c, ch)
return
}
@@ -536,47 +675,45 @@ func CreateModelHandler(c *gin.Context) {
streamResponse(c, ch)
}
-func DeleteModelHandler(c *gin.Context) {
- var req api.DeleteRequest
- err := c.ShouldBindJSON(&req)
- switch {
- case errors.Is(err, io.EOF):
+func (s *Server) DeleteHandler(c *gin.Context) {
+ var r api.DeleteRequest
+ if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
return
- case err != nil:
+ } else if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
- if req.Name == "" {
- c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
+ n := model.ParseName(cmp.Or(r.Model, r.Name))
+ if !n.IsValid() {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("name %q is invalid", cmp.Or(r.Model, r.Name))})
return
}
- if err := DeleteModel(req.Name); err != nil {
- if os.IsNotExist(err) {
- c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
- } else {
+ m, err := ParseNamedManifest(n)
+ if err != nil {
+ switch {
+ case os.IsNotExist(err):
+ c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", cmp.Or(r.Model, r.Name))})
+ default:
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
}
return
}
- manifestsPath, err := GetManifestPath()
- if err != nil {
+ if err := m.Remove(); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
- if err := PruneDirectory(manifestsPath); err != nil {
+ if err := m.RemoveLayers(); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
-
- c.JSON(http.StatusOK, nil)
}
-func ShowModelHandler(c *gin.Context) {
+func (s *Server) ShowHandler(c *gin.Context) {
var req api.ShowRequest
err := c.ShouldBindJSON(&req)
switch {
@@ -588,16 +725,23 @@ func ShowModelHandler(c *gin.Context) {
return
}
- if req.Name == "" {
- c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
+ if req.Model != "" {
+ // noop
+ } else if req.Name != "" {
+ req.Model = req.Name
+ } else {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
return
}
- resp, err := GetModelInfo(req.Name)
+ resp, err := GetModelInfo(req)
if err != nil {
- if os.IsNotExist(err) {
- c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
- } else {
+ switch {
+ case os.IsNotExist(err):
+ c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
+ case err.Error() == "invalid model name":
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ default:
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
}
return
@@ -606,156 +750,203 @@ func ShowModelHandler(c *gin.Context) {
c.JSON(http.StatusOK, resp)
}
-func GetModelInfo(name string) (*api.ShowResponse, error) {
- model, err := GetModel(name)
+func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
+ m, err := GetModel(req.Model)
if err != nil {
return nil, err
}
modelDetails := api.ModelDetails{
- Format: model.Config.ModelFormat,
- Family: model.Config.ModelFamily,
- Families: model.Config.ModelFamilies,
- ParameterSize: model.Config.ModelType,
- QuantizationLevel: model.Config.FileType,
+ ParentModel: m.ParentModel,
+ Format: m.Config.ModelFormat,
+ Family: m.Config.ModelFamily,
+ Families: m.Config.ModelFamilies,
+ ParameterSize: m.Config.ModelType,
+ QuantizationLevel: m.Config.FileType,
}
- resp := &api.ShowResponse{
- License: strings.Join(model.License, "\n"),
- System: model.System,
- Template: model.Template,
- Details: modelDetails,
+ if req.System != "" {
+ m.System = req.System
}
- mf, err := ShowModelfile(model)
+ msgs := make([]api.Message, len(m.Messages))
+ for i, msg := range m.Messages {
+ msgs[i] = api.Message{Role: msg.Role, Content: msg.Content}
+ }
+
+ n := model.ParseName(req.Model)
+ if !n.IsValid() {
+ return nil, errors.New("invalid model name")
+ }
+
+ manifest, err := ParseNamedManifest(n)
if err != nil {
return nil, err
}
- resp.Modelfile = mf
+ resp := &api.ShowResponse{
+ License: strings.Join(m.License, "\n"),
+ System: m.System,
+ Template: m.Template.String(),
+ Details: modelDetails,
+ Messages: msgs,
+ ModifiedAt: manifest.fi.ModTime(),
+ }
var params []string
cs := 30
- for k, v := range model.Options {
+ for k, v := range m.Options {
switch val := v.(type) {
- case string:
- params = append(params, fmt.Sprintf("%-*s %s", cs, k, val))
- case int:
- params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(val)))
- case float64:
- params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(val, 'f', 0, 64)))
- case bool:
- params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(val)))
case []interface{}:
for _, nv := range val {
- switch nval := nv.(type) {
- case string:
- params = append(params, fmt.Sprintf("%-*s %s", cs, k, nval))
- case int:
- params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(nval)))
- case float64:
- params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(nval, 'f', 0, 64)))
- case bool:
- params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(nval)))
- }
+ params = append(params, fmt.Sprintf("%-*s %#v", cs, k, nv))
}
+ default:
+ params = append(params, fmt.Sprintf("%-*s %#v", cs, k, v))
}
}
resp.Parameters = strings.Join(params, "\n")
+ for k, v := range req.Options {
+ if _, ok := req.Options[k]; ok {
+ m.Options[k] = v
+ }
+ }
+
+ var sb strings.Builder
+ fmt.Fprintln(&sb, "# Modelfile generated by \"ollama show\"")
+ fmt.Fprintln(&sb, "# To build a new Modelfile based on this, replace FROM with:")
+ fmt.Fprintf(&sb, "# FROM %s\n\n", m.ShortName)
+ fmt.Fprint(&sb, m.String())
+ resp.Modelfile = sb.String()
+
+ kvData, err := getKVData(m.ModelPath, req.Verbose)
+ if err != nil {
+ return nil, err
+ }
+ delete(kvData, "general.name")
+ delete(kvData, "tokenizer.chat_template")
+ resp.ModelInfo = kvData
+
+ if len(m.ProjectorPaths) > 0 {
+ projectorData, err := getKVData(m.ProjectorPaths[0], req.Verbose)
+ if err != nil {
+ return nil, err
+ }
+ resp.ProjectorInfo = projectorData
+ }
+
return resp, nil
}
-func ListModelsHandler(c *gin.Context) {
- models := make([]api.ModelResponse, 0)
- fp, err := GetManifestPath()
+func getKVData(digest string, verbose bool) (llm.KV, error) {
+ maxArraySize := 0
+ if verbose {
+ maxArraySize = -1
+ }
+ kvData, err := llm.LoadModel(digest, maxArraySize)
+ if err != nil {
+ return nil, err
+ }
+
+ kv := kvData.KV()
+
+ if !verbose {
+ for k := range kv {
+ if t, ok := kv[k].([]any); len(t) > 5 && ok {
+ kv[k] = []any{}
+ }
+ }
+ }
+
+ return kv, nil
+}
+
+func (s *Server) ListHandler(c *gin.Context) {
+ ms, err := Manifests()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
- modelResponse := func(modelName string) (api.ModelResponse, error) {
- model, err := GetModel(modelName)
- if err != nil {
- return api.ModelResponse{}, err
- }
+ models := []api.ListModelResponse{}
+ for n, m := range ms {
+ var cf ConfigV2
- modelDetails := api.ModelDetails{
- Format: model.Config.ModelFormat,
- Family: model.Config.ModelFamily,
- Families: model.Config.ModelFamilies,
- ParameterSize: model.Config.ModelType,
- QuantizationLevel: model.Config.FileType,
- }
-
- return api.ModelResponse{
- Name: model.ShortName,
- Size: model.Size,
- Digest: model.Digest,
- Details: modelDetails,
- }, nil
- }
-
- walkFunc := func(path string, info os.FileInfo, _ error) error {
- if !info.IsDir() {
- dir, file := filepath.Split(path)
- dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
- tag := strings.Join([]string{dir, file}, ":")
-
- resp, err := modelResponse(tag)
+ if m.Config.Digest != "" {
+ f, err := m.Config.Open()
if err != nil {
- log.Printf("skipping file: %s", fp)
- return nil
+ slog.Warn("bad manifest filepath", "name", n, "error", err)
+ continue
}
+ defer f.Close()
- resp.ModifiedAt = info.ModTime()
- models = append(models, resp)
+ if err := json.NewDecoder(f).Decode(&cf); err != nil {
+ slog.Warn("bad manifest config", "name", n, "error", err)
+ continue
+ }
}
- return nil
+ // tag should never be masked
+ models = append(models, api.ListModelResponse{
+ Model: n.DisplayShortest(),
+ Name: n.DisplayShortest(),
+ Size: m.Size(),
+ Digest: m.digest,
+ ModifiedAt: m.fi.ModTime(),
+ Details: api.ModelDetails{
+ Format: cf.ModelFormat,
+ Family: cf.ModelFamily,
+ Families: cf.ModelFamilies,
+ ParameterSize: cf.ModelType,
+ QuantizationLevel: cf.FileType,
+ },
+ })
}
- if err := filepath.Walk(fp, walkFunc); err != nil {
- c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
- return
- }
+ slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
+ // most recently modified first
+ return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
+ })
c.JSON(http.StatusOK, api.ListResponse{Models: models})
}
-func CopyModelHandler(c *gin.Context) {
- var req api.CopyRequest
- err := c.ShouldBindJSON(&req)
- switch {
- case errors.Is(err, io.EOF):
+func (s *Server) CopyHandler(c *gin.Context) {
+ var r api.CopyRequest
+ if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
return
- case err != nil:
+ } else if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
- if req.Source == "" || req.Destination == "" {
- c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "source add destination are required"})
+ src := model.ParseName(r.Source)
+ if !src.IsValid() {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("source %q is invalid", r.Source)})
return
}
- if err := ParseModelPath(req.Destination).Validate(); err != nil {
+ dst := model.ParseName(r.Destination)
+ if !dst.IsValid() {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("destination %q is invalid", r.Destination)})
+ return
+ }
+
+ if err := checkNameExists(dst); err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
- if err := CopyModel(req.Source, req.Destination); err != nil {
- if os.IsNotExist(err) {
- c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Source)})
- } else {
- c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
- }
- return
+ if err := CopyModel(src, dst); errors.Is(err, os.ErrNotExist) {
+ c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)})
+ } else if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
}
}
-func HeadBlobHandler(c *gin.Context) {
+func (s *Server) HeadBlobHandler(c *gin.Context) {
path, err := GetBlobsPath(c.Param("digest"))
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
@@ -770,7 +961,44 @@ func HeadBlobHandler(c *gin.Context) {
c.Status(http.StatusOK)
}
-func CreateBlobHandler(c *gin.Context) {
+func (s *Server) CreateBlobHandler(c *gin.Context) {
+ if ib, ok := intermediateBlobs[c.Param("digest")]; ok {
+ p, err := GetBlobsPath(ib)
+ if err != nil {
+ c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ }
+
+ if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
+ slog.Info("evicting intermediate blob which no longer exists", "digest", ib)
+ delete(intermediateBlobs, c.Param("digest"))
+ } else if err != nil {
+ c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ } else {
+ c.Status(http.StatusOK)
+ return
+ }
+ }
+
+ path, err := GetBlobsPath(c.Param("digest"))
+ if err != nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+
+ _, err = os.Stat(path)
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ // noop
+ case err != nil:
+ c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ default:
+ c.Status(http.StatusOK)
+ return
+ }
+
layer, err := NewLayer(c.Request.Body, "")
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
@@ -782,77 +1010,137 @@ func CreateBlobHandler(c *gin.Context) {
return
}
- if _, err := layer.Commit(); err != nil {
- c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
- return
- }
-
c.Status(http.StatusCreated)
}
-var defaultAllowOrigins = []string{
- "localhost",
- "127.0.0.1",
- "0.0.0.0",
-}
+func isLocalIP(ip netip.Addr) bool {
+ if interfaces, err := net.Interfaces(); err == nil {
+ for _, iface := range interfaces {
+ addrs, err := iface.Addrs()
+ if err != nil {
+ continue
+ }
-func NewServer() (*Server, error) {
- workDir, err := os.MkdirTemp("", "ollama")
- if err != nil {
- return nil, err
+ for _, a := range addrs {
+ if parsed, _, err := net.ParseCIDR(a.String()); err == nil {
+ if parsed.String() == ip.String() {
+ return true
+ }
+ }
+ }
+ }
}
- return &Server{
- WorkDir: workDir,
- }, nil
+ return false
+}
+
+func allowedHost(host string) bool {
+ if host == "" || host == "localhost" {
+ return true
+ }
+
+ if hostname, err := os.Hostname(); err == nil && host == hostname {
+ return true
+ }
+
+ tlds := []string{
+ "localhost",
+ "local",
+ "internal",
+ }
+
+ // check if the host is a local TLD
+ for _, tld := range tlds {
+ if strings.HasSuffix(host, "."+tld) {
+ return true
+ }
+ }
+
+ return false
+}
+
+func allowedHostsMiddleware(addr net.Addr) gin.HandlerFunc {
+ return func(c *gin.Context) {
+ if addr == nil {
+ c.Next()
+ return
+ }
+
+ if addr, err := netip.ParseAddrPort(addr.String()); err == nil && !addr.Addr().IsLoopback() {
+ c.Next()
+ return
+ }
+
+ host, _, err := net.SplitHostPort(c.Request.Host)
+ if err != nil {
+ host = c.Request.Host
+ }
+
+ if addr, err := netip.ParseAddr(host); err == nil {
+ if addr.IsLoopback() || addr.IsPrivate() || addr.IsUnspecified() || isLocalIP(addr) {
+ c.Next()
+ return
+ }
+ }
+
+ if allowedHost(host) {
+ if c.Request.Method == http.MethodOptions {
+ c.AbortWithStatus(http.StatusNoContent)
+ return
+ }
+
+ c.Next()
+ return
+ }
+
+ c.AbortWithStatus(http.StatusForbidden)
+ }
}
func (s *Server) GenerateRoutes() http.Handler {
- var origins []string
- if o := os.Getenv("OLLAMA_ORIGINS"); o != "" {
- origins = strings.Split(o, ",")
- }
-
config := cors.DefaultConfig()
config.AllowWildcard = true
-
- config.AllowOrigins = origins
- for _, allowOrigin := range defaultAllowOrigins {
- config.AllowOrigins = append(config.AllowOrigins,
- fmt.Sprintf("http://%s", allowOrigin),
- fmt.Sprintf("https://%s", allowOrigin),
- fmt.Sprintf("http://%s:*", allowOrigin),
- fmt.Sprintf("https://%s:*", allowOrigin),
- )
+ config.AllowBrowserExtensions = true
+ config.AllowHeaders = []string{"Authorization", "Content-Type", "User-Agent", "Accept", "X-Requested-With"}
+ openAIProperties := []string{"lang", "package-version", "os", "arch", "runtime", "runtime-version", "async"}
+ for _, prop := range openAIProperties {
+ config.AllowHeaders = append(config.AllowHeaders, "x-stainless-"+prop)
}
+ config.AllowOrigins = envconfig.Origins()
r := gin.Default()
r.Use(
cors.New(config),
- func(c *gin.Context) {
- c.Set("workDir", s.WorkDir)
- c.Next()
- },
+ allowedHostsMiddleware(s.addr),
)
- r.POST("/api/pull", PullModelHandler)
- r.POST("/api/generate", GenerateHandler)
- r.POST("/api/chat", ChatHandler)
- r.POST("/api/embeddings", EmbeddingHandler)
- r.POST("/api/create", CreateModelHandler)
- r.POST("/api/push", PushModelHandler)
- r.POST("/api/copy", CopyModelHandler)
- r.DELETE("/api/delete", DeleteModelHandler)
- r.POST("/api/show", ShowModelHandler)
- r.POST("/api/blobs/:digest", CreateBlobHandler)
- r.HEAD("/api/blobs/:digest", HeadBlobHandler)
+ r.POST("/api/pull", s.PullHandler)
+ r.POST("/api/generate", s.GenerateHandler)
+ r.POST("/api/chat", s.ChatHandler)
+ r.POST("/api/embed", s.EmbedHandler)
+ r.POST("/api/embeddings", s.EmbeddingsHandler)
+ r.POST("/api/create", s.CreateHandler)
+ r.POST("/api/push", s.PushHandler)
+ r.POST("/api/copy", s.CopyHandler)
+ r.DELETE("/api/delete", s.DeleteHandler)
+ r.POST("/api/show", s.ShowHandler)
+ r.POST("/api/blobs/:digest", s.CreateBlobHandler)
+ r.HEAD("/api/blobs/:digest", s.HeadBlobHandler)
+ r.GET("/api/ps", s.PsHandler)
+
+ // Compatibility endpoints
+ r.POST("/v1/chat/completions", openai.ChatMiddleware(), s.ChatHandler)
+ r.POST("/v1/completions", openai.CompletionsMiddleware(), s.GenerateHandler)
+ r.POST("/v1/embeddings", openai.EmbeddingsMiddleware(), s.EmbedHandler)
+ r.GET("/v1/models", openai.ListMiddleware(), s.ListHandler)
+ r.GET("/v1/models/:model", openai.RetrieveMiddleware(), s.ShowHandler)
for _, method := range []string{http.MethodGet, http.MethodHead} {
r.Handle(method, "/", func(c *gin.Context) {
c.String(http.StatusOK, "Ollama is running")
})
- r.Handle(method, "/api/tags", ListModelsHandler)
+ r.Handle(method, "/api/tags", s.ListHandler)
r.Handle(method, "/api/version", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"version": version.Version})
})
@@ -862,7 +1150,36 @@ func (s *Server) GenerateRoutes() http.Handler {
}
func Serve(ln net.Listener) error {
- if noprune := os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
+ level := slog.LevelInfo
+ if envconfig.Debug() {
+ level = slog.LevelDebug
+ }
+
+ slog.Info("server config", "env", envconfig.Values())
+ handler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+ Level: level,
+ AddSource: true,
+ ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr {
+ if attr.Key == slog.SourceKey {
+ source := attr.Value.Any().(*slog.Source)
+ source.File = filepath.Base(source.File)
+ }
+
+ return attr
+ },
+ })
+
+ slog.SetDefault(slog.New(handler))
+
+ blobsDir, err := GetBlobsPath("")
+ if err != nil {
+ return err
+ }
+ if err := fixBlobs(blobsDir); err != nil {
+ return err
+ }
+
+ if !envconfig.NoPrune() {
// clean up unused layers and manifests
if err := PruneLayers(); err != nil {
return err
@@ -878,15 +1195,24 @@ func Serve(ln net.Listener) error {
}
}
- s, err := NewServer()
- if err != nil {
- return err
- }
- r := s.GenerateRoutes()
+ ctx, done := context.WithCancel(context.Background())
+ schedCtx, schedDone := context.WithCancel(ctx)
+ sched := InitScheduler(schedCtx)
+ s := &Server{addr: ln.Addr(), sched: sched}
- log.Printf("Listening on %s (version %s)", ln.Addr(), version.Version)
+ http.Handle("/", s.GenerateRoutes())
+
+ slog.Info(fmt.Sprintf("Listening on %s (version %s)", ln.Addr(), version.Version))
srvr := &http.Server{
- Handler: r,
+ // Use http.DefaultServeMux so we get net/http/pprof for
+ // free.
+ //
+ // TODO(bmizerany): Decide if we want to make this
+ // configurable so it is not exposed by default, or allow
+ // users to bind it to a different port. This was a quick
+ // and easy way to get pprof, but it may not be the best
+ // way.
+ Handler: nil,
}
// listen for a ctrl+c and stop any loaded llm
@@ -894,24 +1220,32 @@ func Serve(ln net.Listener) error {
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-signals
- if loaded.runner != nil {
- loaded.runner.Close()
- }
- os.RemoveAll(s.WorkDir)
- os.Exit(0)
+ srvr.Close()
+ schedDone()
+ sched.unloadAllRunners()
+ runners.Cleanup(build.EmbedFS)
+ done()
}()
- if err := llm.Init(s.WorkDir); err != nil {
- return fmt.Errorf("unable to initialize llm library %w", err)
- }
- if runtime.GOOS == "linux" { // TODO - windows too
- // check compatibility to log warnings
- if _, err := gpu.CheckVRAM(); err != nil {
- log.Print(err.Error())
- }
+ if _, err := runners.Refresh(build.EmbedFS); err != nil {
+ return fmt.Errorf("unable to initialize llm runners %w", err)
}
- return srvr.Serve(ln)
+ s.sched.Run(schedCtx)
+
+ // At startup we retrieve GPU information so we can get log messages before loading a model
+ // This will log warnings to the log in case we have problems with detected GPUs
+ gpus := gpu.GetGPUInfo()
+ gpus.LogDetails()
+
+ err = srvr.Serve(ln)
+ // If server is closed from the signal handler, wait for the ctx to be done
+ // otherwise error out quickly
+ if !errors.Is(err, http.ErrServerClosed) {
+ return err
+ }
+ <-ctx.Done()
+ return nil
}
func waitForStream(c *gin.Context, ch chan interface{}) {
@@ -924,11 +1258,15 @@ func waitForStream(c *gin.Context, ch chan interface{}) {
return
}
case gin.H:
+ status, ok := r["status"].(int)
+ if !ok {
+ status = http.StatusInternalServerError
+ }
if errorMsg, ok := r["error"].(string); ok {
- c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
+ c.JSON(status, gin.H{"error": errorMsg})
return
} else {
- c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in progress response"})
+ c.JSON(status, gin.H{"error": "unexpected error format in progress response"})
return
}
default:
@@ -949,14 +1287,14 @@ func streamResponse(c *gin.Context, ch chan any) {
bts, err := json.Marshal(val)
if err != nil {
- log.Printf("streamResponse: json.Marshal failed with %s", err)
+ slog.Info(fmt.Sprintf("streamResponse: json.Marshal failed with %s", err))
return false
}
// Delineate chunks with new-line delimiter
bts = append(bts, '\n')
if _, err := w.Write(bts); err != nil {
- log.Printf("streamResponse: w.Write failed with %s", err)
+ slog.Info(fmt.Sprintf("streamResponse: w.Write failed with %s", err))
return false
}
@@ -964,77 +1302,140 @@ func streamResponse(c *gin.Context, ch chan any) {
})
}
-func ChatHandler(c *gin.Context) {
- loaded.mu.Lock()
- defer loaded.mu.Unlock()
+func (s *Server) PsHandler(c *gin.Context) {
+ models := []api.ProcessModelResponse{}
+ for _, v := range s.sched.loaded {
+ model := v.model
+ modelDetails := api.ModelDetails{
+ Format: model.Config.ModelFormat,
+ Family: model.Config.ModelFamily,
+ Families: model.Config.ModelFamilies,
+ ParameterSize: model.Config.ModelType,
+ QuantizationLevel: model.Config.FileType,
+ }
+
+ mr := api.ProcessModelResponse{
+ Model: model.ShortName,
+ Name: model.ShortName,
+ Size: int64(v.estimatedTotal),
+ SizeVRAM: int64(v.estimatedVRAM),
+ Digest: model.Digest,
+ Details: modelDetails,
+ ExpiresAt: v.expiresAt,
+ }
+ // The scheduler waits to set expiresAt, so if a model is loading it's
+ // possible that it will be set to the unix epoch. For those cases, just
+ // calculate the time w/ the sessionDuration instead.
+ var epoch time.Time
+ if v.expiresAt == epoch {
+ mr.ExpiresAt = time.Now().Add(v.sessionDuration)
+ }
+
+ models = append(models, mr)
+ }
+
+ slices.SortStableFunc(models, func(i, j api.ProcessModelResponse) int {
+ // longest duration remaining listed first
+ return cmp.Compare(j.ExpiresAt.Unix(), i.ExpiresAt.Unix())
+ })
+
+ c.JSON(http.StatusOK, api.ProcessResponse{Models: models})
+}
+
+func (s *Server) ChatHandler(c *gin.Context) {
checkpointStart := time.Now()
var req api.ChatRequest
- err := c.ShouldBindJSON(&req)
- switch {
- case errors.Is(err, io.EOF):
+ if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
return
- case err != nil:
+ } else if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
- // validate the request
- switch {
- case req.Model == "":
- c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
- return
- case len(req.Format) > 0 && req.Format != "json":
- c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be json"})
- return
- }
-
- sessionDuration := defaultSessionDuration
- model, err := load(c, req.Model, req.Options, sessionDuration)
- if err != nil {
- var pErr *fs.PathError
- switch {
- case errors.As(err, &pErr):
- c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
- case errors.Is(err, api.ErrInvalidOpts):
- c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
- default:
- c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ // expire the runner
+ if len(req.Messages) == 0 && req.KeepAlive != nil && int(req.KeepAlive.Seconds()) == 0 {
+ model, err := GetModel(req.Model)
+ if err != nil {
+ switch {
+ case os.IsNotExist(err):
+ c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
+ case err.Error() == "invalid model name":
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ default:
+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ }
+ return
}
+ s.sched.expireRunner(model)
+
+ c.JSON(http.StatusOK, api.ChatResponse{
+ Model: req.Model,
+ CreatedAt: time.Now().UTC(),
+ Message: api.Message{Role: "assistant"},
+ Done: true,
+ DoneReason: "unload",
+ })
return
}
- // an empty request loads the model
- if len(req.Messages) == 0 {
- c.JSON(http.StatusOK, api.ChatResponse{CreatedAt: time.Now().UTC(), Model: req.Model, Done: true, Message: api.Message{Role: "assistant"}})
+ caps := []Capability{CapabilityCompletion}
+ if len(req.Tools) > 0 {
+ caps = append(caps, CapabilityTools)
+ }
+
+ r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, caps, req.Options, req.KeepAlive)
+ if errors.Is(err, errCapabilityCompletion) {
+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support chat", req.Model)})
+ return
+ } else if err != nil {
+ handleScheduleError(c, req.Model, err)
return
}
checkpointLoaded := time.Now()
- prompt, images, err := model.ChatPrompt(req.Messages)
- if err != nil {
- c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ if len(req.Messages) == 0 {
+ c.JSON(http.StatusOK, api.ChatResponse{
+ Model: req.Model,
+ CreatedAt: time.Now().UTC(),
+ Message: api.Message{Role: "assistant"},
+ Done: true,
+ DoneReason: "load",
+ })
return
}
- ch := make(chan any)
+ msgs := append(m.Messages, req.Messages...)
+ if req.Messages[0].Role != "system" && m.System != "" {
+ msgs = append([]api.Message{{Role: "system", Content: m.System}}, msgs...)
+ }
+ prompt, images, err := chatPrompt(c.Request.Context(), m, r.Tokenize, opts, msgs, req.Tools)
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ }
+
+ slog.Debug("chat request", "images", len(images), "prompt", prompt)
+
+ ch := make(chan any)
go func() {
defer close(ch)
-
- fn := func(r llm.PredictResult) {
- // Update model expiration
- loaded.expireAt = time.Now().Add(sessionDuration)
- loaded.expireTimer.Reset(sessionDuration)
-
- resp := api.ChatResponse{
- Model: req.Model,
- CreatedAt: time.Now().UTC(),
- Message: api.Message{Role: "assistant", Content: r.Content},
- Done: r.Done,
+ if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
+ Prompt: prompt,
+ Images: images,
+ Format: req.Format,
+ Options: opts,
+ }, func(r llm.CompletionResponse) {
+ res := api.ChatResponse{
+ Model: req.Model,
+ CreatedAt: time.Now().UTC(),
+ Message: api.Message{Role: "assistant", Content: r.Content},
+ Done: r.Done,
+ DoneReason: r.DoneReason,
Metrics: api.Metrics{
PromptEvalCount: r.PromptEvalCount,
PromptEvalDuration: r.PromptEvalDuration,
@@ -1044,51 +1445,65 @@ func ChatHandler(c *gin.Context) {
}
if r.Done {
- resp.TotalDuration = time.Since(checkpointStart)
- resp.LoadDuration = checkpointLoaded.Sub(checkpointStart)
+ res.TotalDuration = time.Since(checkpointStart)
+ res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
}
- ch <- resp
- }
-
- // Start prediction
- predictReq := llm.PredictOpts{
- Prompt: prompt,
- Format: req.Format,
- Images: images,
- }
- if err := loaded.runner.Predict(c.Request.Context(), predictReq, fn); err != nil {
+ ch <- res
+ }); err != nil {
ch <- gin.H{"error": err.Error()}
}
}()
if req.Stream != nil && !*req.Stream {
- // Accumulate responses into the final response
- var final api.ChatResponse
+ var resp api.ChatResponse
var sb strings.Builder
- for resp := range ch {
- switch r := resp.(type) {
+ for rr := range ch {
+ switch t := rr.(type) {
case api.ChatResponse:
- sb.WriteString(r.Message.Content)
- final = r
+ sb.WriteString(t.Message.Content)
+ resp = t
case gin.H:
- if errorMsg, ok := r["error"].(string); ok {
- c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
- return
- } else {
- c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in response"})
- return
+ msg, ok := t["error"].(string)
+ if !ok {
+ msg = "unexpected error format in response"
}
+
+ c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
+ return
default:
- c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error"})
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
return
}
}
- final.Message = api.Message{Role: "assistant", Content: sb.String()}
- c.JSON(http.StatusOK, final)
+ resp.Message.Content = sb.String()
+
+ if len(req.Tools) > 0 {
+ if toolCalls, ok := m.parseToolCalls(sb.String()); ok {
+ resp.Message.ToolCalls = toolCalls
+ resp.Message.Content = ""
+ }
+ }
+
+ c.JSON(http.StatusOK, resp)
return
}
streamResponse(c, ch)
}
+
+func handleScheduleError(c *gin.Context, name string, err error) {
+ switch {
+ case errors.Is(err, errCapabilities), errors.Is(err, errRequired):
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ case errors.Is(err, context.Canceled):
+ c.JSON(499, gin.H{"error": "request canceled"})
+ case errors.Is(err, ErrMaxQueue):
+ c.JSON(http.StatusServiceUnavailable, gin.H{"error": err.Error()})
+ case errors.Is(err, os.ErrNotExist):
+ c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found, try pulling it first", name)})
+ default:
+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ }
+}
diff --git a/server/routes_create_test.go b/server/routes_create_test.go
new file mode 100644
index 00000000..09521753
--- /dev/null
+++ b/server/routes_create_test.go
@@ -0,0 +1,618 @@
+package server
+
+import (
+ "bytes"
+ "cmp"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "slices"
+ "testing"
+
+ "github.com/gin-gonic/gin"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+)
+
+var stream bool = false
+
+func createBinFile(t *testing.T, kv map[string]any, ti []llm.Tensor) string {
+ t.Helper()
+
+ f, err := os.CreateTemp(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+
+ if err := llm.WriteGGUF(f, kv, ti); err != nil {
+ t.Fatal(err)
+ }
+
+ return f.Name()
+}
+
+type responseRecorder struct {
+ *httptest.ResponseRecorder
+ http.CloseNotifier
+}
+
+func NewRecorder() *responseRecorder {
+ return &responseRecorder{
+ ResponseRecorder: httptest.NewRecorder(),
+ }
+}
+
+func (t *responseRecorder) CloseNotify() <-chan bool {
+ return make(chan bool)
+}
+
+func createRequest(t *testing.T, fn func(*gin.Context), body any) *httptest.ResponseRecorder {
+ t.Helper()
+ // if OLLAMA_MODELS is not set, set it to the temp directory
+ t.Setenv("OLLAMA_MODELS", cmp.Or(os.Getenv("OLLAMA_MODELS"), t.TempDir()))
+
+ w := NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+
+ var b bytes.Buffer
+ if err := json.NewEncoder(&b).Encode(body); err != nil {
+ t.Fatal(err)
+ }
+
+ c.Request = &http.Request{
+ Body: io.NopCloser(&b),
+ }
+
+ fn(c)
+ return w.ResponseRecorder
+}
+
+func checkFileExists(t *testing.T, p string, expect []string) {
+ t.Helper()
+
+ actual, err := filepath.Glob(p)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !slices.Equal(actual, expect) {
+ t.Fatalf("expected slices to be equal %v", actual)
+ }
+}
+
+func TestCreateFromBin(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ p := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", p)
+
+ var s Server
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-ca239d7bd8ea90e4a5d2e6bf88f8d74a47b14336e73eb4e18bed4dd325018116"),
+ })
+}
+
+func TestCreateFromModel(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ p := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", p)
+ var s Server
+
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ })
+
+ w = createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test2",
+ Modelfile: "FROM test",
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-ca239d7bd8ea90e4a5d2e6bf88f8d74a47b14336e73eb4e18bed4dd325018116"),
+ })
+}
+
+func TestCreateRemovesLayers(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ p := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", p)
+ var s Server
+
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ .Prompt }}", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-b507b9c2f6ca642bffcd06665ea7c91f235fd32daeefdf875a0f938db05fb315"),
+ filepath.Join(p, "blobs", "sha256-bc80b03733773e0728011b2f4adf34c458b400e1aad48cb28d61170f3a2ad2d6"),
+ })
+
+ w = createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ .System }} {{ .Prompt }}", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-8f2c2167d789c6b2302dff965160fa5029f6a24096d262c1cbb469f21a045382"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-fe7ac77b725cda2ccad03f88a880ecdfd7a33192d6cae08fce2c0ee1455991ed"),
+ })
+}
+
+func TestCreateUnsetsSystem(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ p := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", p)
+ var s Server
+
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s\nSYSTEM Say hi!", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-8585df945d1069bc78b79bd10bb73ba07fbc29b0f5479a31a601c0d12731416e"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-f29e82a8284dbdf5910b1555580ff60b04238b8da9d5e51159ada67a4d0d5851"),
+ })
+
+ w = createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s\nSYSTEM \"\"", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-67d4b8d106af2a5b100a46e9bdc038c71eef2a35c9abac784092654212f97cf5"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"),
+ })
+
+ bts, err := os.ReadFile(filepath.Join(p, "blobs", "sha256-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if string(bts) != "" {
+ t.Fatalf("expected empty string, actual %s", string(bts))
+ }
+}
+
+func TestCreateMergeParameters(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ p := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", p)
+ var s Server
+
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s\nPARAMETER temperature 1\nPARAMETER top_k 10\nPARAMETER stop USER:\nPARAMETER stop ASSISTANT:", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-1d0ad71299d48c2fb7ae2b98e683643e771f8a5b72be34942af90d97a91c1e37"),
+ filepath.Join(p, "blobs", "sha256-4a384beaf47a9cbe452dfa5ab70eea691790f3b35a832d12933a1996685bf2b6"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ })
+
+ // in order to merge parameters, the second model must be created FROM the first
+ w = createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test2",
+ Modelfile: "FROM test\nPARAMETER temperature 0.6\nPARAMETER top_p 0.7",
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-1d0ad71299d48c2fb7ae2b98e683643e771f8a5b72be34942af90d97a91c1e37"),
+ filepath.Join(p, "blobs", "sha256-4a384beaf47a9cbe452dfa5ab70eea691790f3b35a832d12933a1996685bf2b6"),
+ filepath.Join(p, "blobs", "sha256-4cd9d4ba6b734d9b4cbd1e5caa60374c00722e993fce5e1e2d15a33698f71187"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-e29a7b3c47287a2489c895d21fe413c20f859a85d20e749492f52a838e36e1ba"),
+ })
+
+ actual, err := os.ReadFile(filepath.Join(p, "blobs", "sha256-e29a7b3c47287a2489c895d21fe413c20f859a85d20e749492f52a838e36e1ba"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ expect, err := json.Marshal(map[string]any{"temperature": 0.6, "top_k": 10, "top_p": 0.7, "stop": []string{"USER:", "ASSISTANT:"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !bytes.Equal(bytes.TrimSpace(expect), bytes.TrimSpace(actual)) {
+ t.Errorf("expected %s, actual %s", string(expect), string(actual))
+ }
+
+ // slices are replaced
+ w = createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test2",
+ Modelfile: "FROM test\nPARAMETER temperature 0.6\nPARAMETER top_p 0.7\nPARAMETER stop <|endoftext|>",
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-12f58bb75cb3042d69a7e013ab87fb3c3c7088f50ddc62f0c77bd332f0d44d35"),
+ filepath.Join(p, "blobs", "sha256-1d0ad71299d48c2fb7ae2b98e683643e771f8a5b72be34942af90d97a91c1e37"),
+ filepath.Join(p, "blobs", "sha256-257aa726584f24970a4f240765e75a7169bfbe7f4966c1f04513d6b6c860583a"),
+ filepath.Join(p, "blobs", "sha256-4a384beaf47a9cbe452dfa5ab70eea691790f3b35a832d12933a1996685bf2b6"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ })
+
+ actual, err = os.ReadFile(filepath.Join(p, "blobs", "sha256-12f58bb75cb3042d69a7e013ab87fb3c3c7088f50ddc62f0c77bd332f0d44d35"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ expect, err = json.Marshal(map[string]any{"temperature": 0.6, "top_k": 10, "top_p": 0.7, "stop": []string{"<|endoftext|>"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !bytes.Equal(bytes.TrimSpace(expect), bytes.TrimSpace(actual)) {
+ t.Errorf("expected %s, actual %s", string(expect), string(actual))
+ }
+}
+
+func TestCreateReplacesMessages(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ p := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", p)
+ var s Server
+
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s\nMESSAGE assistant \"What is my purpose?\"\nMESSAGE user \"You run tests.\"\nMESSAGE assistant \"Oh, my god.\"", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-298baeaf6928a60cf666d88d64a1ba606feb43a2865687c39e40652e407bffc4"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-e0e27d47045063ccb167ae852c51d49a98eab33fabaee4633fdddf97213e40b5"),
+ })
+
+ w = createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test2",
+ Modelfile: "FROM test\nMESSAGE assistant \"You're a test, Harry.\"\nMESSAGE user \"I-I'm a what?\"\nMESSAGE assistant \"A test. And a thumping good one at that, I'd wager.\"",
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-298baeaf6928a60cf666d88d64a1ba606feb43a2865687c39e40652e407bffc4"),
+ filepath.Join(p, "blobs", "sha256-4f48b25fe9969564c82f58eb1cedbdff6484cc0baf474bc6c2a9b37c8da3362a"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-a60ecc9da299ec7ede453f99236e5577fd125e143689b646d9f0ddc9971bf4db"),
+ filepath.Join(p, "blobs", "sha256-e0e27d47045063ccb167ae852c51d49a98eab33fabaee4633fdddf97213e40b5"),
+ })
+
+ type message struct {
+ Role string `json:"role"`
+ Content string `json:"content"`
+ }
+
+ f, err := os.Open(filepath.Join(p, "blobs", "sha256-a60ecc9da299ec7ede453f99236e5577fd125e143689b646d9f0ddc9971bf4db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+
+ var actual []message
+ if err := json.NewDecoder(f).Decode(&actual); err != nil {
+ t.Fatal(err)
+ }
+
+ expect := []message{
+ {Role: "assistant", Content: "You're a test, Harry."},
+ {Role: "user", Content: "I-I'm a what?"},
+ {Role: "assistant", Content: "A test. And a thumping good one at that, I'd wager."},
+ }
+
+ if !slices.Equal(actual, expect) {
+ t.Errorf("expected %s, actual %s", expect, actual)
+ }
+}
+
+func TestCreateTemplateSystem(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ p := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", p)
+ var s Server
+
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ .Prompt }}\nSYSTEM Say hello!\nTEMPLATE {{ .System }} {{ .Prompt }}\nSYSTEM Say bye!", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-2b5e330885117c82f3fd75169ea323e141070a2947c11ddb9f79ee0b01c589c1"),
+ filepath.Join(p, "blobs", "sha256-4c5f51faac758fecaff8db42f0b7382891a4d0c0bb885f7b86be88c814a7cc86"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-fe7ac77b725cda2ccad03f88a880ecdfd7a33192d6cae08fce2c0ee1455991ed"),
+ })
+
+ template, err := os.ReadFile(filepath.Join(p, "blobs", "sha256-fe7ac77b725cda2ccad03f88a880ecdfd7a33192d6cae08fce2c0ee1455991ed"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if string(template) != "{{ .System }} {{ .Prompt }}" {
+ t.Errorf("expected \"{{ .System }} {{ .Prompt }}\", actual %s", template)
+ }
+
+ system, err := os.ReadFile(filepath.Join(p, "blobs", "sha256-4c5f51faac758fecaff8db42f0b7382891a4d0c0bb885f7b86be88c814a7cc86"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if string(system) != "Say bye!" {
+ t.Errorf("expected \"Say bye!\", actual %s", system)
+ }
+
+ t.Run("incomplete template", func(t *testing.T) {
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ .Prompt", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("expected status code 400, actual %d", w.Code)
+ }
+ })
+
+ t.Run("template with unclosed if", func(t *testing.T) {
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ if .Prompt }}", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("expected status code 400, actual %d", w.Code)
+ }
+ })
+
+ t.Run("template with undefined function", func(t *testing.T) {
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ Prompt }}", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("expected status code 400, actual %d", w.Code)
+ }
+ })
+}
+
+func TestCreateLicenses(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ p := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", p)
+ var s Server
+
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s\nLICENSE MIT\nLICENSE Apache-2.0", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-2af71558e438db0b73a20beab92dc278a94e1bbe974c00c1a33e3ab62d53a608"),
+ filepath.Join(p, "blobs", "sha256-79a39c37536ddee29cbadd5d5e2dcba8ed7f03e431f626ff38432c1c866bb7e2"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-e5dcffe836b6ec8a58e492419b550e65fb8cbdc308503979e5dacb33ac7ea3b7"),
+ })
+
+ mit, err := os.ReadFile(filepath.Join(p, "blobs", "sha256-e5dcffe836b6ec8a58e492419b550e65fb8cbdc308503979e5dacb33ac7ea3b7"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if string(mit) != "MIT" {
+ t.Errorf("expected MIT, actual %s", mit)
+ }
+
+ apache, err := os.ReadFile(filepath.Join(p, "blobs", "sha256-2af71558e438db0b73a20beab92dc278a94e1bbe974c00c1a33e3ab62d53a608"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if string(apache) != "Apache-2.0" {
+ t.Errorf("expected Apache-2.0, actual %s", apache)
+ }
+}
+
+func TestCreateDetectTemplate(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ p := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", p)
+ var s Server
+
+ t.Run("matched", func(t *testing.T) {
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, llm.KV{
+ "tokenizer.chat_template": "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+ }, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-0d79f567714c62c048378f2107fb332dabee0135d080c302d884317da9433cc5"),
+ filepath.Join(p, "blobs", "sha256-35360843d0c84fb1506952a131bbef13cd2bb4a541251f22535170c05b56e672"),
+ filepath.Join(p, "blobs", "sha256-553c4a3f747b3d22a4946875f1cc8ed011c2930d83f864a0c7265f9ec0a20413"),
+ filepath.Join(p, "blobs", "sha256-de3959f841e9ef6b4b6255fa41cb9e0a45da89c3066aa72bdd07a4747f848990"),
+ })
+ })
+
+ t.Run("unmatched", func(t *testing.T) {
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-ca239d7bd8ea90e4a5d2e6bf88f8d74a47b14336e73eb4e18bed4dd325018116"),
+ })
+ })
+}
diff --git a/server/routes_delete_test.go b/server/routes_delete_test.go
new file mode 100644
index 00000000..5a337e79
--- /dev/null
+++ b/server/routes_delete_test.go
@@ -0,0 +1,111 @@
+package server
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "path/filepath"
+ "testing"
+
+ "github.com/gin-gonic/gin"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/types/model"
+)
+
+func TestDelete(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ p := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", p)
+
+ var s Server
+
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test",
+ Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)),
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ w = createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "test2",
+ Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ .System }} {{ .Prompt }}", createBinFile(t, nil, nil)),
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-8f2c2167d789c6b2302dff965160fa5029f6a24096d262c1cbb469f21a045382"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-ca239d7bd8ea90e4a5d2e6bf88f8d74a47b14336e73eb4e18bed4dd325018116"),
+ filepath.Join(p, "blobs", "sha256-fe7ac77b725cda2ccad03f88a880ecdfd7a33192d6cae08fce2c0ee1455991ed"),
+ })
+
+ w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Name: "test"})
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
+ filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
+ })
+
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
+ filepath.Join(p, "blobs", "sha256-8f2c2167d789c6b2302dff965160fa5029f6a24096d262c1cbb469f21a045382"),
+ filepath.Join(p, "blobs", "sha256-a4e5e156ddec27e286f75328784d7106b60a4eb1d246e950a001a3f944fbda99"),
+ filepath.Join(p, "blobs", "sha256-fe7ac77b725cda2ccad03f88a880ecdfd7a33192d6cae08fce2c0ee1455991ed"),
+ })
+
+ w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Name: "test2"})
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{})
+ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{})
+}
+
+func TestDeleteDuplicateLayers(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ p := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", p)
+ var s Server
+
+ n := model.ParseName("test")
+
+ var b bytes.Buffer
+ if err := json.NewEncoder(&b).Encode(&ConfigV2{}); err != nil {
+ t.Fatal(err)
+ }
+
+ config, err := NewLayer(&b, "application/vnd.docker.container.image.v1+json")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // create a manifest with duplicate layers
+ if err := WriteManifest(n, config, []Layer{config}); err != nil {
+ t.Fatal(err)
+ }
+
+ w := createRequest(t, s.DeleteHandler, api.DeleteRequest{Name: "test"})
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status code 200, actual %d", w.Code)
+ }
+
+ checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{})
+}
diff --git a/server/routes_generate_test.go b/server/routes_generate_test.go
new file mode 100644
index 00000000..480b9672
--- /dev/null
+++ b/server/routes_generate_test.go
@@ -0,0 +1,714 @@
+package server
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/google/go-cmp/cmp"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/gpu"
+ "github.com/ollama/ollama/llm"
+)
+
+type mockRunner struct {
+ llm.LlamaServer
+
+ // CompletionRequest is only valid until the next call to Completion
+ llm.CompletionRequest
+ llm.CompletionResponse
+}
+
+func (m *mockRunner) Completion(_ context.Context, r llm.CompletionRequest, fn func(r llm.CompletionResponse)) error {
+ m.CompletionRequest = r
+ fn(m.CompletionResponse)
+ return nil
+}
+
+func (mockRunner) Tokenize(_ context.Context, s string) (tokens []int, err error) {
+ for range strings.Fields(s) {
+ tokens = append(tokens, len(tokens))
+ }
+
+ return
+}
+
+func newMockServer(mock *mockRunner) func(gpu.GpuInfoList, string, *llm.GGML, []string, []string, api.Options, int) (llm.LlamaServer, error) {
+ return func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, projectors, system []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {
+ return mock, nil
+ }
+}
+
+func TestGenerateChat(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ mock := mockRunner{
+ CompletionResponse: llm.CompletionResponse{
+ Done: true,
+ DoneReason: "stop",
+ PromptEvalCount: 1,
+ PromptEvalDuration: 1,
+ EvalCount: 1,
+ EvalDuration: 1,
+ },
+ }
+
+ s := Server{
+ sched: &Scheduler{
+ pendingReqCh: make(chan *LlmRequest, 1),
+ finishedReqCh: make(chan *LlmRequest, 1),
+ expiredCh: make(chan *runnerRef, 1),
+ unloadedCh: make(chan any, 1),
+ loaded: make(map[string]*runnerRef),
+ newServerFn: newMockServer(&mock),
+ getGpuFn: gpu.GetGPUInfo,
+ getCpuFn: gpu.GetCPUInfo,
+ reschedDelay: 250 * time.Millisecond,
+ loadFn: func(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel int) {
+ // add small delay to simulate loading
+ time.Sleep(time.Millisecond)
+ req.successCh <- &runnerRef{
+ llama: &mock,
+ }
+ },
+ },
+ }
+
+ go s.sched.Run(context.TODO())
+
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Model: "test",
+ Modelfile: fmt.Sprintf(`FROM %s
+ TEMPLATE """
+{{- if .System }}System: {{ .System }} {{ end }}
+{{- if .Prompt }}User: {{ .Prompt }} {{ end }}
+{{- if .Response }}Assistant: {{ .Response }} {{ end }}"""
+`, createBinFile(t, llm.KV{
+ "general.architecture": "llama",
+ "llama.block_count": uint32(1),
+ "llama.context_length": uint32(8192),
+ "llama.embedding_length": uint32(4096),
+ "llama.attention.head_count": uint32(32),
+ "llama.attention.head_count_kv": uint32(8),
+ "tokenizer.ggml.tokens": []string{""},
+ "tokenizer.ggml.scores": []float32{0},
+ "tokenizer.ggml.token_type": []int32{0},
+ }, []llm.Tensor{
+ {Name: "token_embd.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.attn_norm.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.ffn_down.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.ffn_gate.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.ffn_up.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.ffn_norm.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.attn_k.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.attn_output.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.attn_q.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.attn_v.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "output.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ })),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status 200, got %d", w.Code)
+ }
+
+ t.Run("missing body", func(t *testing.T) {
+ w := createRequest(t, s.ChatHandler, nil)
+ if w.Code != http.StatusBadRequest {
+ t.Errorf("expected status 400, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(w.Body.String(), `{"error":"model is required"}`); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+
+ t.Run("missing model", func(t *testing.T) {
+ w := createRequest(t, s.ChatHandler, api.ChatRequest{})
+ if w.Code != http.StatusBadRequest {
+ t.Errorf("expected status 400, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(w.Body.String(), `{"error":"model is required"}`); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+
+ t.Run("missing capabilities chat", func(t *testing.T) {
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Model: "bert",
+ Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, llm.KV{
+ "general.architecture": "bert",
+ "bert.pooling_type": uint32(0),
+ }, []llm.Tensor{})),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status 200, got %d", w.Code)
+ }
+
+ w = createRequest(t, s.ChatHandler, api.ChatRequest{
+ Model: "bert",
+ })
+
+ if w.Code != http.StatusBadRequest {
+ t.Errorf("expected status 400, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(w.Body.String(), `{"error":"\"bert\" does not support chat"}`); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+
+ t.Run("load model", func(t *testing.T) {
+ w := createRequest(t, s.ChatHandler, api.ChatRequest{
+ Model: "test",
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ var actual api.ChatResponse
+ if err := json.NewDecoder(w.Body).Decode(&actual); err != nil {
+ t.Fatal(err)
+ }
+
+ if actual.Model != "test" {
+ t.Errorf("expected model test, got %s", actual.Model)
+ }
+
+ if !actual.Done {
+ t.Errorf("expected done true, got false")
+ }
+
+ if actual.DoneReason != "load" {
+ t.Errorf("expected done reason load, got %s", actual.DoneReason)
+ }
+ })
+
+ checkChatResponse := func(t *testing.T, body io.Reader, model, content string) {
+ t.Helper()
+
+ var actual api.ChatResponse
+ if err := json.NewDecoder(body).Decode(&actual); err != nil {
+ t.Fatal(err)
+ }
+
+ if actual.Model != model {
+ t.Errorf("expected model test, got %s", actual.Model)
+ }
+
+ if !actual.Done {
+ t.Errorf("expected done false, got true")
+ }
+
+ if actual.DoneReason != "stop" {
+ t.Errorf("expected done reason stop, got %s", actual.DoneReason)
+ }
+
+ if diff := cmp.Diff(actual.Message, api.Message{
+ Role: "assistant",
+ Content: content,
+ }); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+
+ if actual.PromptEvalCount == 0 {
+ t.Errorf("expected prompt eval count > 0, got 0")
+ }
+
+ if actual.PromptEvalDuration == 0 {
+ t.Errorf("expected prompt eval duration > 0, got 0")
+ }
+
+ if actual.EvalCount == 0 {
+ t.Errorf("expected eval count > 0, got 0")
+ }
+
+ if actual.EvalDuration == 0 {
+ t.Errorf("expected eval duration > 0, got 0")
+ }
+
+ if actual.LoadDuration == 0 {
+ t.Errorf("expected load duration > 0, got 0")
+ }
+
+ if actual.TotalDuration == 0 {
+ t.Errorf("expected total duration > 0, got 0")
+ }
+ }
+
+ mock.CompletionResponse.Content = "Hi!"
+ t.Run("messages", func(t *testing.T) {
+ w := createRequest(t, s.ChatHandler, api.ChatRequest{
+ Model: "test",
+ Messages: []api.Message{
+ {Role: "user", Content: "Hello!"},
+ },
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(mock.CompletionRequest.Prompt, "User: Hello! "); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+
+ checkChatResponse(t, w.Body, "test", "Hi!")
+ })
+
+ w = createRequest(t, s.CreateHandler, api.CreateRequest{
+ Model: "test-system",
+ Modelfile: "FROM test\nSYSTEM You are a helpful assistant.",
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status 200, got %d", w.Code)
+ }
+
+ t.Run("messages with model system", func(t *testing.T) {
+ w := createRequest(t, s.ChatHandler, api.ChatRequest{
+ Model: "test-system",
+ Messages: []api.Message{
+ {Role: "user", Content: "Hello!"},
+ },
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(mock.CompletionRequest.Prompt, "System: You are a helpful assistant. User: Hello! "); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+
+ checkChatResponse(t, w.Body, "test-system", "Hi!")
+ })
+
+ mock.CompletionResponse.Content = "Abra kadabra!"
+ t.Run("messages with system", func(t *testing.T) {
+ w := createRequest(t, s.ChatHandler, api.ChatRequest{
+ Model: "test-system",
+ Messages: []api.Message{
+ {Role: "system", Content: "You can perform magic tricks."},
+ {Role: "user", Content: "Hello!"},
+ },
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(mock.CompletionRequest.Prompt, "System: You can perform magic tricks. User: Hello! "); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+
+ checkChatResponse(t, w.Body, "test-system", "Abra kadabra!")
+ })
+
+ t.Run("messages with interleaved system", func(t *testing.T) {
+ w := createRequest(t, s.ChatHandler, api.ChatRequest{
+ Model: "test-system",
+ Messages: []api.Message{
+ {Role: "user", Content: "Hello!"},
+ {Role: "assistant", Content: "I can help you with that."},
+ {Role: "system", Content: "You can perform magic tricks."},
+ {Role: "user", Content: "Help me write tests."},
+ },
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(mock.CompletionRequest.Prompt, "System: You are a helpful assistant. User: Hello! Assistant: I can help you with that. System: You can perform magic tricks. User: Help me write tests. "); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+
+ checkChatResponse(t, w.Body, "test-system", "Abra kadabra!")
+ })
+}
+
+func TestGenerate(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ mock := mockRunner{
+ CompletionResponse: llm.CompletionResponse{
+ Done: true,
+ DoneReason: "stop",
+ PromptEvalCount: 1,
+ PromptEvalDuration: 1,
+ EvalCount: 1,
+ EvalDuration: 1,
+ },
+ }
+
+ s := Server{
+ sched: &Scheduler{
+ pendingReqCh: make(chan *LlmRequest, 1),
+ finishedReqCh: make(chan *LlmRequest, 1),
+ expiredCh: make(chan *runnerRef, 1),
+ unloadedCh: make(chan any, 1),
+ loaded: make(map[string]*runnerRef),
+ newServerFn: newMockServer(&mock),
+ getGpuFn: gpu.GetGPUInfo,
+ getCpuFn: gpu.GetCPUInfo,
+ reschedDelay: 250 * time.Millisecond,
+ loadFn: func(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel int) {
+ // add small delay to simulate loading
+ time.Sleep(time.Millisecond)
+ req.successCh <- &runnerRef{
+ llama: &mock,
+ }
+ },
+ },
+ }
+
+ go s.sched.Run(context.TODO())
+
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Model: "test",
+ Modelfile: fmt.Sprintf(`FROM %s
+ TEMPLATE """
+{{- if .System }}System: {{ .System }} {{ end }}
+{{- if .Prompt }}User: {{ .Prompt }} {{ end }}
+{{- if .Response }}Assistant: {{ .Response }} {{ end }}"""
+`, createBinFile(t, llm.KV{
+ "general.architecture": "llama",
+ "llama.block_count": uint32(1),
+ "llama.context_length": uint32(8192),
+ "llama.embedding_length": uint32(4096),
+ "llama.attention.head_count": uint32(32),
+ "llama.attention.head_count_kv": uint32(8),
+ "tokenizer.ggml.tokens": []string{""},
+ "tokenizer.ggml.scores": []float32{0},
+ "tokenizer.ggml.token_type": []int32{0},
+ }, []llm.Tensor{
+ {Name: "token_embd.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.attn_norm.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.ffn_down.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.ffn_gate.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.ffn_up.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.ffn_norm.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.attn_k.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.attn_output.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.attn_q.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "blk.0.attn_v.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ {Name: "output.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},
+ })),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status 200, got %d", w.Code)
+ }
+
+ t.Run("missing body", func(t *testing.T) {
+ w := createRequest(t, s.GenerateHandler, nil)
+ if w.Code != http.StatusBadRequest {
+ t.Errorf("expected status 400, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(w.Body.String(), `{"error":"model is required"}`); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+
+ t.Run("missing model", func(t *testing.T) {
+ w := createRequest(t, s.GenerateHandler, api.GenerateRequest{})
+ if w.Code != http.StatusBadRequest {
+ t.Errorf("expected status 400, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(w.Body.String(), `{"error":"model is required"}`); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+
+ t.Run("missing capabilities generate", func(t *testing.T) {
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Model: "bert",
+ Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, llm.KV{
+ "general.architecture": "bert",
+ "bert.pooling_type": uint32(0),
+ }, []llm.Tensor{})),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status 200, got %d", w.Code)
+ }
+
+ w = createRequest(t, s.GenerateHandler, api.GenerateRequest{
+ Model: "bert",
+ })
+
+ if w.Code != http.StatusBadRequest {
+ t.Errorf("expected status 400, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(w.Body.String(), `{"error":"\"bert\" does not support generate"}`); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+
+ t.Run("missing capabilities suffix", func(t *testing.T) {
+ w := createRequest(t, s.GenerateHandler, api.GenerateRequest{
+ Model: "test",
+ Prompt: "def add(",
+ Suffix: " return c",
+ })
+
+ if w.Code != http.StatusBadRequest {
+ t.Errorf("expected status 400, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(w.Body.String(), `{"error":"test does not support insert"}`); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+
+ t.Run("load model", func(t *testing.T) {
+ w := createRequest(t, s.GenerateHandler, api.GenerateRequest{
+ Model: "test",
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ var actual api.GenerateResponse
+ if err := json.NewDecoder(w.Body).Decode(&actual); err != nil {
+ t.Fatal(err)
+ }
+
+ if actual.Model != "test" {
+ t.Errorf("expected model test, got %s", actual.Model)
+ }
+
+ if !actual.Done {
+ t.Errorf("expected done true, got false")
+ }
+
+ if actual.DoneReason != "load" {
+ t.Errorf("expected done reason load, got %s", actual.DoneReason)
+ }
+ })
+
+ checkGenerateResponse := func(t *testing.T, body io.Reader, model, content string) {
+ t.Helper()
+
+ var actual api.GenerateResponse
+ if err := json.NewDecoder(body).Decode(&actual); err != nil {
+ t.Fatal(err)
+ }
+
+ if actual.Model != model {
+ t.Errorf("expected model test, got %s", actual.Model)
+ }
+
+ if !actual.Done {
+ t.Errorf("expected done false, got true")
+ }
+
+ if actual.DoneReason != "stop" {
+ t.Errorf("expected done reason stop, got %s", actual.DoneReason)
+ }
+
+ if actual.Response != content {
+ t.Errorf("expected response %s, got %s", content, actual.Response)
+ }
+
+ if actual.Context == nil {
+ t.Errorf("expected context not nil")
+ }
+
+ if actual.PromptEvalCount == 0 {
+ t.Errorf("expected prompt eval count > 0, got 0")
+ }
+
+ if actual.PromptEvalDuration == 0 {
+ t.Errorf("expected prompt eval duration > 0, got 0")
+ }
+
+ if actual.EvalCount == 0 {
+ t.Errorf("expected eval count > 0, got 0")
+ }
+
+ if actual.EvalDuration == 0 {
+ t.Errorf("expected eval duration > 0, got 0")
+ }
+
+ if actual.LoadDuration == 0 {
+ t.Errorf("expected load duration > 0, got 0")
+ }
+
+ if actual.TotalDuration == 0 {
+ t.Errorf("expected total duration > 0, got 0")
+ }
+ }
+
+ mock.CompletionResponse.Content = "Hi!"
+ t.Run("prompt", func(t *testing.T) {
+ w := createRequest(t, s.GenerateHandler, api.GenerateRequest{
+ Model: "test",
+ Prompt: "Hello!",
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(mock.CompletionRequest.Prompt, "User: Hello! "); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+
+ checkGenerateResponse(t, w.Body, "test", "Hi!")
+ })
+
+ w = createRequest(t, s.CreateHandler, api.CreateRequest{
+ Model: "test-system",
+ Modelfile: "FROM test\nSYSTEM You are a helpful assistant.",
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status 200, got %d", w.Code)
+ }
+
+ t.Run("prompt with model system", func(t *testing.T) {
+ w := createRequest(t, s.GenerateHandler, api.GenerateRequest{
+ Model: "test-system",
+ Prompt: "Hello!",
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(mock.CompletionRequest.Prompt, "System: You are a helpful assistant. User: Hello! "); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+
+ checkGenerateResponse(t, w.Body, "test-system", "Hi!")
+ })
+
+ mock.CompletionResponse.Content = "Abra kadabra!"
+ t.Run("prompt with system", func(t *testing.T) {
+ w := createRequest(t, s.GenerateHandler, api.GenerateRequest{
+ Model: "test-system",
+ Prompt: "Hello!",
+ System: "You can perform magic tricks.",
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(mock.CompletionRequest.Prompt, "System: You can perform magic tricks. User: Hello! "); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+
+ checkGenerateResponse(t, w.Body, "test-system", "Abra kadabra!")
+ })
+
+ t.Run("prompt with template", func(t *testing.T) {
+ w := createRequest(t, s.GenerateHandler, api.GenerateRequest{
+ Model: "test-system",
+ Prompt: "Help me write tests.",
+ System: "You can perform magic tricks.",
+ Template: `{{- if .System }}{{ .System }} {{ end }}
+{{- if .Prompt }}### USER {{ .Prompt }} {{ end }}
+{{- if .Response }}### ASSISTANT {{ .Response }} {{ end }}`,
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(mock.CompletionRequest.Prompt, "You can perform magic tricks. ### USER Help me write tests. "); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+
+ checkGenerateResponse(t, w.Body, "test-system", "Abra kadabra!")
+ })
+
+ w = createRequest(t, s.CreateHandler, api.CreateRequest{
+ Model: "test-suffix",
+ Modelfile: `FROM test
+TEMPLATE """{{- if .Suffix }}<PRE> {{ .Prompt }} <SUF>{{ .Suffix }} <MID>
+{{- else }}{{ .Prompt }}
+{{- end }}"""`,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status 200, got %d", w.Code)
+ }
+
+ t.Run("prompt with suffix", func(t *testing.T) {
+ w := createRequest(t, s.GenerateHandler, api.GenerateRequest{
+ Model: "test-suffix",
+ Prompt: "def add(",
+ Suffix: " return c",
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(mock.CompletionRequest.Prompt, "<PRE> def add( <SUF> return c <MID>"); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+
+ t.Run("prompt without suffix", func(t *testing.T) {
+ w := createRequest(t, s.GenerateHandler, api.GenerateRequest{
+ Model: "test-suffix",
+ Prompt: "def add(",
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(mock.CompletionRequest.Prompt, "def add("); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+
+ t.Run("raw", func(t *testing.T) {
+ w := createRequest(t, s.GenerateHandler, api.GenerateRequest{
+ Model: "test-system",
+ Prompt: "Help me write tests.",
+ Raw: true,
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Errorf("expected status 200, got %d", w.Code)
+ }
+
+ if diff := cmp.Diff(mock.CompletionRequest.Prompt, "Help me write tests."); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+}
diff --git a/server/routes_list_test.go b/server/routes_list_test.go
new file mode 100644
index 00000000..56b40830
--- /dev/null
+++ b/server/routes_list_test.go
@@ -0,0 +1,65 @@
+package server
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "slices"
+ "testing"
+
+ "github.com/gin-gonic/gin"
+
+ "github.com/ollama/ollama/api"
+)
+
+func TestList(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ expectNames := []string{
+ "mistral:7b-instruct-q4_0",
+ "zephyr:7b-beta-q5_K_M",
+ "apple/OpenELM:latest",
+ "boreas:2b-code-v1.5-q6_K",
+ "notus:7b-v1-IQ2_S",
+ // TODO: host:port currently fails on windows (#4107)
+ // "localhost:5000/library/eurus:700b-v0.5-iq3_XXS",
+ "mynamespace/apeliotes:latest",
+ "myhost/mynamespace/lips:code",
+ }
+
+ var s Server
+ for _, n := range expectNames {
+ createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: n,
+ Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)),
+ })
+ }
+
+ w := createRequest(t, s.ListHandler, nil)
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ var resp api.ListResponse
+ if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
+ t.Fatal(err)
+ }
+
+ if len(resp.Models) != len(expectNames) {
+ t.Fatalf("expected %d models, actual %d", len(expectNames), len(resp.Models))
+ }
+
+ actualNames := make([]string, len(resp.Models))
+ for i, m := range resp.Models {
+ actualNames[i] = m.Name
+ }
+
+ slices.Sort(actualNames)
+ slices.Sort(expectNames)
+
+ if !slices.Equal(actualNames, expectNames) {
+ t.Fatalf("expected slices to be equal %v", actualNames)
+ }
+}
diff --git a/server/routes_test.go b/server/routes_test.go
index 07bb699d..f7a7a22b 100644
--- a/server/routes_test.go
+++ b/server/routes_test.go
@@ -3,26 +3,69 @@ package server
import (
"bytes"
"context"
+ "encoding/binary"
"encoding/json"
"fmt"
"io"
+ "math"
"net/http"
"net/http/httptest"
"os"
+ "sort"
"strings"
"testing"
- "github.com/stretchr/testify/assert"
-
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/parser"
- "github.com/jmorganca/ollama/version"
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/openai"
+ "github.com/ollama/ollama/parser"
+ "github.com/ollama/ollama/types/model"
+ "github.com/ollama/ollama/version"
)
-func setupServer(t *testing.T) (*Server, error) {
+func createTestFile(t *testing.T, name string) string {
t.Helper()
- return NewServer()
+ f, err := os.CreateTemp(t.TempDir(), name)
+ if err != nil {
+ t.Fatalf("failed to create temp file: %v", err)
+ }
+ defer f.Close()
+
+ err = binary.Write(f, binary.LittleEndian, []byte("GGUF"))
+ if err != nil {
+ t.Fatalf("failed to write to file: %v", err)
+ }
+
+ err = binary.Write(f, binary.LittleEndian, uint32(3))
+ if err != nil {
+ t.Fatalf("failed to write to file: %v", err)
+ }
+
+ err = binary.Write(f, binary.LittleEndian, uint64(0))
+ if err != nil {
+ t.Fatalf("failed to write to file: %v", err)
+ }
+
+ err = binary.Write(f, binary.LittleEndian, uint64(0))
+ if err != nil {
+ t.Fatalf("failed to write to file: %v", err)
+ }
+
+ return f.Name()
+}
+
+// equalStringSlices checks if two slices of strings are equal.
+func equalStringSlices(a, b []string) bool {
+ if len(a) != len(b) {
+ return false
+ }
+ for i := range a {
+ if a[i] != b[i] {
+ return false
+ }
+ }
+ return true
}
func Test_Routes(t *testing.T) {
@@ -34,30 +77,23 @@ func Test_Routes(t *testing.T) {
Expected func(t *testing.T, resp *http.Response)
}
- createTestFile := func(t *testing.T, name string) string {
- f, err := os.CreateTemp(t.TempDir(), name)
- assert.Nil(t, err)
- defer f.Close()
-
- _, err = f.Write([]byte("GGUF"))
- assert.Nil(t, err)
- _, err = f.Write([]byte{0x2, 0})
- assert.Nil(t, err)
-
- return f.Name()
- }
-
createTestModel := func(t *testing.T, name string) {
+ t.Helper()
+
fname := createTestFile(t, "ollama-model")
- modelfile := strings.NewReader(fmt.Sprintf("FROM %s", fname))
- commands, err := parser.Parse(modelfile)
- assert.Nil(t, err)
+ r := strings.NewReader(fmt.Sprintf("FROM %s\nPARAMETER seed 42\nPARAMETER top_p 0.9\nPARAMETER stop foo\nPARAMETER stop bar", fname))
+ modelfile, err := parser.ParseFile(r)
+ if err != nil {
+ t.Fatalf("failed to parse file: %v", err)
+ }
fn := func(resp api.ProgressResponse) {
t.Logf("Status: %s", resp.Status)
}
- err = CreateModel(context.TODO(), name, "", commands, fn)
- assert.Nil(t, err)
+ err = CreateModel(context.TODO(), model.ParseName(name), "", "", modelfile, fn)
+ if err != nil {
+ t.Fatalf("failed to create model: %v", err)
+ }
}
testCases := []testCase{
@@ -69,10 +105,17 @@ func Test_Routes(t *testing.T) {
},
Expected: func(t *testing.T, resp *http.Response) {
contentType := resp.Header.Get("Content-Type")
- assert.Equal(t, contentType, "application/json; charset=utf-8")
+ if contentType != "application/json; charset=utf-8" {
+ t.Errorf("expected content type application/json; charset=utf-8, got %s", contentType)
+ }
body, err := io.ReadAll(resp.Body)
- assert.Nil(t, err)
- assert.Equal(t, fmt.Sprintf(`{"version":"%s"}`, version.Version), string(body))
+ if err != nil {
+ t.Fatalf("failed to read response body: %v", err)
+ }
+ expectedBody := fmt.Sprintf(`{"version":"%s"}`, version.Version)
+ if string(body) != expectedBody {
+ t.Errorf("expected body %s, got %s", expectedBody, string(body))
+ }
},
},
{
@@ -81,16 +124,49 @@ func Test_Routes(t *testing.T) {
Path: "/api/tags",
Expected: func(t *testing.T, resp *http.Response) {
contentType := resp.Header.Get("Content-Type")
- assert.Equal(t, contentType, "application/json; charset=utf-8")
+ if contentType != "application/json; charset=utf-8" {
+ t.Errorf("expected content type application/json; charset=utf-8, got %s", contentType)
+ }
body, err := io.ReadAll(resp.Body)
- assert.Nil(t, err)
+ if err != nil {
+ t.Fatalf("failed to read response body: %v", err)
+ }
var modelList api.ListResponse
err = json.Unmarshal(body, &modelList)
- assert.Nil(t, err)
+ if err != nil {
+ t.Fatalf("failed to unmarshal response body: %v", err)
+ }
- assert.Equal(t, 0, len(modelList.Models))
+ if modelList.Models == nil || len(modelList.Models) != 0 {
+ t.Errorf("expected empty model list, got %v", modelList.Models)
+ }
+ },
+ },
+ {
+ Name: "openai empty list",
+ Method: http.MethodGet,
+ Path: "/v1/models",
+ Expected: func(t *testing.T, resp *http.Response) {
+ contentType := resp.Header.Get("Content-Type")
+ if contentType != "application/json" {
+ t.Errorf("expected content type application/json, got %s", contentType)
+ }
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatalf("failed to read response body: %v", err)
+ }
+
+ var modelList openai.ListCompletion
+ err = json.Unmarshal(body, &modelList)
+ if err != nil {
+ t.Fatalf("failed to unmarshal response body: %v", err)
+ }
+
+ if modelList.Object != "list" || len(modelList.Data) != 0 {
+ t.Errorf("expected empty model list, got %v", modelList.Data)
+ }
},
},
{
@@ -102,16 +178,117 @@ func Test_Routes(t *testing.T) {
},
Expected: func(t *testing.T, resp *http.Response) {
contentType := resp.Header.Get("Content-Type")
- assert.Equal(t, contentType, "application/json; charset=utf-8")
+ if contentType != "application/json; charset=utf-8" {
+ t.Errorf("expected content type application/json; charset=utf-8, got %s", contentType)
+ }
body, err := io.ReadAll(resp.Body)
- assert.Nil(t, err)
+ if err != nil {
+ t.Fatalf("failed to read response body: %v", err)
+ }
+
+ if strings.Contains(string(body), "expires_at") {
+ t.Errorf("response body should not contain 'expires_at'")
+ }
var modelList api.ListResponse
err = json.Unmarshal(body, &modelList)
- assert.Nil(t, err)
+ if err != nil {
+ t.Fatalf("failed to unmarshal response body: %v", err)
+ }
- assert.Equal(t, 1, len(modelList.Models))
- assert.Equal(t, modelList.Models[0].Name, "test-model:latest")
+ if len(modelList.Models) != 1 || modelList.Models[0].Name != "test-model:latest" {
+ t.Errorf("expected model 'test-model:latest', got %v", modelList.Models)
+ }
+ },
+ },
+ {
+ Name: "Delete Model Handler",
+ Method: http.MethodDelete,
+ Path: "/api/delete",
+ Setup: func(t *testing.T, req *http.Request) {
+ createTestModel(t, "model-to-delete")
+
+ deleteReq := api.DeleteRequest{
+ Name: "model-to-delete",
+ }
+ jsonData, err := json.Marshal(deleteReq)
+ if err != nil {
+ t.Fatalf("failed to marshal delete request: %v", err)
+ }
+
+ req.Body = io.NopCloser(bytes.NewReader(jsonData))
+ },
+ Expected: func(t *testing.T, resp *http.Response) {
+ if resp.StatusCode != http.StatusOK {
+ t.Errorf("expected status code 200, got %d", resp.StatusCode)
+ }
+
+ // Verify the model was deleted
+ _, err := GetModel("model-to-delete")
+ if err == nil || !os.IsNotExist(err) {
+ t.Errorf("expected model to be deleted, got error %v", err)
+ }
+ },
+ },
+ {
+ Name: "Delete Non-existent Model",
+ Method: http.MethodDelete,
+ Path: "/api/delete",
+ Setup: func(t *testing.T, req *http.Request) {
+ deleteReq := api.DeleteRequest{
+ Name: "non-existent-model",
+ }
+ jsonData, err := json.Marshal(deleteReq)
+ if err != nil {
+ t.Fatalf("failed to marshal delete request: %v", err)
+ }
+
+ req.Body = io.NopCloser(bytes.NewReader(jsonData))
+ },
+ Expected: func(t *testing.T, resp *http.Response) {
+ if resp.StatusCode != http.StatusNotFound {
+ t.Errorf("expected status code 404, got %d", resp.StatusCode)
+ }
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatalf("failed to read response body: %v", err)
+ }
+
+ var errorResp map[string]string
+ err = json.Unmarshal(body, &errorResp)
+ if err != nil {
+ t.Fatalf("failed to unmarshal response body: %v", err)
+ }
+
+ if !strings.Contains(errorResp["error"], "not found") {
+ t.Errorf("expected error message to contain 'not found', got %s", errorResp["error"])
+ }
+ },
+ },
+ {
+ Name: "openai list models with tags",
+ Method: http.MethodGet,
+ Path: "/v1/models",
+ Expected: func(t *testing.T, resp *http.Response) {
+ contentType := resp.Header.Get("Content-Type")
+ if contentType != "application/json" {
+ t.Errorf("expected content type application/json, got %s", contentType)
+ }
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatalf("failed to read response body: %v", err)
+ }
+
+ var modelList openai.ListCompletion
+ err = json.Unmarshal(body, &modelList)
+ if err != nil {
+ t.Fatalf("failed to unmarshal response body: %v", err)
+ }
+
+ if len(modelList.Data) != 1 || modelList.Data[0].Id != "test-model:latest" || modelList.Data[0].OwnedBy != "library" {
+ t.Errorf("expected model 'test-model:latest' owned by 'library', got %v", modelList.Data)
+ }
},
},
{
@@ -119,31 +296,41 @@ func Test_Routes(t *testing.T) {
Method: http.MethodPost,
Path: "/api/create",
Setup: func(t *testing.T, req *http.Request) {
- f, err := os.CreateTemp(t.TempDir(), "ollama-model")
- assert.Nil(t, err)
- defer f.Close()
+ fname := createTestFile(t, "ollama-model")
stream := false
createReq := api.CreateRequest{
Name: "t-bone",
- Modelfile: fmt.Sprintf("FROM %s", f.Name()),
+ Modelfile: fmt.Sprintf("FROM %s", fname),
Stream: &stream,
}
jsonData, err := json.Marshal(createReq)
- assert.Nil(t, err)
+ if err != nil {
+ t.Fatalf("failed to marshal create request: %v", err)
+ }
req.Body = io.NopCloser(bytes.NewReader(jsonData))
},
Expected: func(t *testing.T, resp *http.Response) {
contentType := resp.Header.Get("Content-Type")
- assert.Equal(t, "application/json", contentType)
+ if contentType != "application/json" {
+ t.Errorf("expected content type application/json, got %s", contentType)
+ }
_, err := io.ReadAll(resp.Body)
- assert.Nil(t, err)
- assert.Equal(t, resp.StatusCode, 200)
+ if err != nil {
+ t.Fatalf("failed to read response body: %v", err)
+ }
+ if resp.StatusCode != http.StatusOK { // Updated line
+ t.Errorf("expected status code 200, got %d", resp.StatusCode)
+ }
model, err := GetModel("t-bone")
- assert.Nil(t, err)
- assert.Equal(t, "t-bone:latest", model.ShortName)
+ if err != nil {
+ t.Fatalf("failed to get model: %v", err)
+ }
+ if model.ShortName != "t-bone:latest" {
+ t.Errorf("expected model name 't-bone:latest', got %s", model.ShortName)
+ }
},
},
{
@@ -157,49 +344,281 @@ func Test_Routes(t *testing.T) {
Destination: "beefsteak",
}
jsonData, err := json.Marshal(copyReq)
- assert.Nil(t, err)
+ if err != nil {
+ t.Fatalf("failed to marshal copy request: %v", err)
+ }
req.Body = io.NopCloser(bytes.NewReader(jsonData))
},
Expected: func(t *testing.T, resp *http.Response) {
model, err := GetModel("beefsteak")
- assert.Nil(t, err)
- assert.Equal(t, "beefsteak:latest", model.ShortName)
+ if err != nil {
+ t.Fatalf("failed to get model: %v", err)
+ }
+ if model.ShortName != "beefsteak:latest" {
+ t.Errorf("expected model name 'beefsteak:latest', got %s", model.ShortName)
+ }
+ },
+ },
+ {
+ Name: "Show Model Handler",
+ Method: http.MethodPost,
+ Path: "/api/show",
+ Setup: func(t *testing.T, req *http.Request) {
+ createTestModel(t, "show-model")
+ showReq := api.ShowRequest{Model: "show-model"}
+ jsonData, err := json.Marshal(showReq)
+ if err != nil {
+ t.Fatalf("failed to marshal show request: %v", err)
+ }
+ req.Body = io.NopCloser(bytes.NewReader(jsonData))
+ },
+ Expected: func(t *testing.T, resp *http.Response) {
+ contentType := resp.Header.Get("Content-Type")
+ if contentType != "application/json; charset=utf-8" {
+ t.Errorf("expected content type application/json; charset=utf-8, got %s", contentType)
+ }
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatalf("failed to read response body: %v", err)
+ }
+
+ var showResp api.ShowResponse
+ err = json.Unmarshal(body, &showResp)
+ if err != nil {
+ t.Fatalf("failed to unmarshal response body: %v", err)
+ }
+
+ var params []string
+ paramsSplit := strings.Split(showResp.Parameters, "\n")
+ for _, p := range paramsSplit {
+ params = append(params, strings.Join(strings.Fields(p), " "))
+ }
+ sort.Strings(params)
+ expectedParams := []string{
+ "seed 42",
+ "stop \"bar\"",
+ "stop \"foo\"",
+ "top_p 0.9",
+ }
+ if !equalStringSlices(params, expectedParams) {
+ t.Errorf("expected parameters %v, got %v", expectedParams, params)
+ }
+ paramCount, ok := showResp.ModelInfo["general.parameter_count"].(float64)
+ if !ok {
+ t.Fatalf("expected parameter count to be a float64, got %T", showResp.ModelInfo["general.parameter_count"])
+ }
+ if math.Abs(paramCount) > 1e-9 {
+ t.Errorf("expected parameter count to be 0, got %f", paramCount)
+ }
+ },
+ },
+ {
+ Name: "openai retrieve model handler",
+ Method: http.MethodGet,
+ Path: "/v1/models/show-model",
+ Expected: func(t *testing.T, resp *http.Response) {
+ contentType := resp.Header.Get("Content-Type")
+ if contentType != "application/json" {
+ t.Errorf("expected content type application/json, got %s", contentType)
+ }
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatalf("failed to read response body: %v", err)
+ }
+
+ var retrieveResp api.RetrieveModelResponse
+ err = json.Unmarshal(body, &retrieveResp)
+ if err != nil {
+ t.Fatalf("failed to unmarshal response body: %v", err)
+ }
+
+ if retrieveResp.Id != "show-model" || retrieveResp.OwnedBy != "library" {
+ t.Errorf("expected model 'show-model' owned by 'library', got %v", retrieveResp)
+ }
},
},
}
- s, err := setupServer(t)
- assert.Nil(t, err)
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ s := &Server{}
router := s.GenerateRoutes()
httpSrv := httptest.NewServer(router)
t.Cleanup(httpSrv.Close)
- workDir, err := os.MkdirTemp("", "ollama-test")
- assert.Nil(t, err)
- defer os.RemoveAll(workDir)
- os.Setenv("OLLAMA_MODELS", workDir)
-
for _, tc := range testCases {
- t.Logf("Running Test: [%s]", tc.Name)
- u := httpSrv.URL + tc.Path
- req, err := http.NewRequestWithContext(context.TODO(), tc.Method, u, nil)
- assert.Nil(t, err)
+ t.Run(tc.Name, func(t *testing.T) {
+ u := httpSrv.URL + tc.Path
+ req, err := http.NewRequestWithContext(context.TODO(), tc.Method, u, nil)
+ if err != nil {
+ t.Fatalf("failed to create request: %v", err)
+ }
- if tc.Setup != nil {
- tc.Setup(t, req)
- }
+ if tc.Setup != nil {
+ tc.Setup(t, req)
+ }
- resp, err := httpSrv.Client().Do(req)
- defer resp.Body.Close()
- assert.Nil(t, err)
+ resp, err := httpSrv.Client().Do(req)
+ if err != nil {
+ t.Fatalf("failed to do request: %v", err)
+ }
+ defer resp.Body.Close()
- if tc.Expected != nil {
- tc.Expected(t, resp)
- }
+ if tc.Expected != nil {
+ tc.Expected(t, resp)
+ }
+ })
+ }
+}
+func TestCase(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ cases := []string{
+ "mistral",
+ "llama3:latest",
+ "library/phi3:q4_0",
+ "registry.ollama.ai/library/gemma:q5_K_M",
+ // TODO: host:port currently fails on windows (#4107)
+ // "localhost:5000/alice/bob:latest",
}
+ var s Server
+ for _, tt := range cases {
+ t.Run(tt, func(t *testing.T) {
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: tt,
+ Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status 200 got %d", w.Code)
+ }
+
+ expect, err := json.Marshal(map[string]string{"error": "a model with that name already exists"})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ t.Run("create", func(t *testing.T) {
+ w = createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: strings.ToUpper(tt),
+ Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("expected status 500 got %d", w.Code)
+ }
+
+ if !bytes.Equal(w.Body.Bytes(), expect) {
+ t.Fatalf("expected error %s got %s", expect, w.Body.String())
+ }
+ })
+
+ t.Run("pull", func(t *testing.T) {
+ w := createRequest(t, s.PullHandler, api.PullRequest{
+ Name: strings.ToUpper(tt),
+ Stream: &stream,
+ })
+
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("expected status 500 got %d", w.Code)
+ }
+
+ if !bytes.Equal(w.Body.Bytes(), expect) {
+ t.Fatalf("expected error %s got %s", expect, w.Body.String())
+ }
+ })
+
+ t.Run("copy", func(t *testing.T) {
+ w := createRequest(t, s.CopyHandler, api.CopyRequest{
+ Source: tt,
+ Destination: strings.ToUpper(tt),
+ })
+
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("expected status 500 got %d", w.Code)
+ }
+
+ if !bytes.Equal(w.Body.Bytes(), expect) {
+ t.Fatalf("expected error %s got %s", expect, w.Body.String())
+ }
+ })
+ })
+ }
+}
+
+func TestShow(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ var s Server
+
+ createRequest(t, s.CreateHandler, api.CreateRequest{
+ Name: "show-model",
+ Modelfile: fmt.Sprintf(
+ "FROM %s\nFROM %s",
+ createBinFile(t, llm.KV{"general.architecture": "test"}, nil),
+ createBinFile(t, llm.KV{"general.architecture": "clip"}, nil),
+ ),
+ })
+
+ w := createRequest(t, s.ShowHandler, api.ShowRequest{
+ Name: "show-model",
+ })
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status code 200, actual %d", w.Code)
+ }
+
+ var resp api.ShowResponse
+ if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
+ t.Fatal(err)
+ }
+
+ if resp.ModelInfo["general.architecture"] != "test" {
+ t.Fatal("Expected model architecture to be 'test', but got", resp.ModelInfo["general.architecture"])
+ }
+
+ if resp.ProjectorInfo["general.architecture"] != "clip" {
+ t.Fatal("Expected projector architecture to be 'clip', but got", resp.ProjectorInfo["general.architecture"])
+ }
+}
+
+func TestNormalize(t *testing.T) {
+ type testCase struct {
+ input []float32
+ }
+
+ testCases := []testCase{
+ {input: []float32{1}},
+ {input: []float32{0, 1, 2, 3}},
+ {input: []float32{0.1, 0.2, 0.3}},
+ {input: []float32{-0.1, 0.2, 0.3, -0.4}},
+ {input: []float32{0, 0, 0}},
+ }
+
+ isNormalized := func(vec []float32) (res bool) {
+ sum := 0.0
+ for _, v := range vec {
+ sum += float64(v * v)
+ }
+ if math.Abs(sum-1) > 1e-6 {
+ return sum == 0
+ } else {
+ return true
+ }
+ }
+
+ for _, tc := range testCases {
+ t.Run("", func(t *testing.T) {
+ normalized := normalize(tc.input)
+ if !isNormalized(normalized) {
+ t.Errorf("Vector %v is not normalized", tc.input)
+ }
+ })
+ }
}
diff --git a/server/sched.go b/server/sched.go
new file mode 100644
index 00000000..3c8656ad
--- /dev/null
+++ b/server/sched.go
@@ -0,0 +1,836 @@
+package server
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "os"
+ "reflect"
+ "runtime"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/format"
+ "github.com/ollama/ollama/gpu"
+ "github.com/ollama/ollama/llm"
+)
+
+type LlmRequest struct {
+ ctx context.Context //nolint:containedctx
+ model *Model
+ opts api.Options
+ origNumCtx int // Track the initial ctx request
+ sessionDuration *api.Duration
+ successCh chan *runnerRef
+ errCh chan error
+ schedAttempts uint
+}
+
+type Scheduler struct {
+ pendingReqCh chan *LlmRequest
+ finishedReqCh chan *LlmRequest
+ expiredCh chan *runnerRef
+ unloadedCh chan interface{}
+
+ loaded map[string]*runnerRef
+ loadedMu sync.Mutex
+
+ loadFn func(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel int)
+ newServerFn func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error)
+ getGpuFn func() gpu.GpuInfoList
+ getCpuFn func() gpu.GpuInfoList
+ reschedDelay time.Duration
+}
+
+// Default automatic value for number of models we allow per GPU
+// Model will still need to fit in VRAM, but loading many small models
+// on a large GPU can cause stalling
+var defaultModelsPerGPU = 3
+
+// Default automatic value for parallel setting
+// Model will still need to fit in VRAM. If this setting wont fit
+// we'll back off down to 1 to try to get it to fit
+var defaultParallel = 4
+
+var ErrMaxQueue = errors.New("server busy, please try again. maximum pending requests exceeded")
+
+func InitScheduler(ctx context.Context) *Scheduler {
+ maxQueue := envconfig.MaxQueue()
+ sched := &Scheduler{
+ pendingReqCh: make(chan *LlmRequest, maxQueue),
+ finishedReqCh: make(chan *LlmRequest, maxQueue),
+ expiredCh: make(chan *runnerRef, maxQueue),
+ unloadedCh: make(chan interface{}, maxQueue),
+ loaded: make(map[string]*runnerRef),
+ newServerFn: llm.NewLlamaServer,
+ getGpuFn: gpu.GetGPUInfo,
+ getCpuFn: gpu.GetCPUInfo,
+ reschedDelay: 250 * time.Millisecond,
+ }
+ sched.loadFn = sched.load
+ return sched
+}
+
+// context must be canceled to decrement ref count and release the runner
+func (s *Scheduler) GetRunner(c context.Context, model *Model, opts api.Options, sessionDuration *api.Duration) (chan *runnerRef, chan error) {
+ if opts.NumCtx < 4 {
+ opts.NumCtx = 4
+ }
+
+ req := &LlmRequest{
+ ctx: c,
+ model: model,
+ opts: opts,
+ sessionDuration: sessionDuration,
+ successCh: make(chan *runnerRef),
+ errCh: make(chan error, 1),
+ }
+
+ select {
+ case s.pendingReqCh <- req:
+ default:
+ req.errCh <- ErrMaxQueue
+ }
+ return req.successCh, req.errCh
+}
+
+// Returns immediately, spawns go routines for the scheduler which will shutdown when ctx is done
+func (s *Scheduler) Run(ctx context.Context) {
+ slog.Debug("starting llm scheduler")
+ go func() {
+ s.processPending(ctx)
+ }()
+
+ go func() {
+ s.processCompleted(ctx)
+ }()
+}
+
+func (s *Scheduler) processPending(ctx context.Context) {
+ for {
+ select {
+ case <-ctx.Done():
+ slog.Debug("shutting down scheduler pending loop")
+ return
+ case pending := <-s.pendingReqCh:
+ // Block other requests until we get this pending request running
+ pending.schedAttempts++
+ if pending.origNumCtx == 0 {
+ pending.origNumCtx = pending.opts.NumCtx
+ }
+
+ if pending.ctx.Err() != nil {
+ slog.Debug("pending request cancelled or timed out, skipping scheduling")
+ continue
+ }
+ numParallel := int(envconfig.NumParallel())
+ // TODO (jmorganca): multimodal models don't support parallel yet
+ // see https://github.com/ollama/ollama/issues/4165
+ if len(pending.model.ProjectorPaths) > 0 && numParallel != 1 {
+ numParallel = 1
+ slog.Warn("multimodal models don't support parallel requests yet")
+ }
+
+ for {
+ var runnerToExpire *runnerRef
+ s.loadedMu.Lock()
+ runner := s.loaded[pending.model.ModelPath]
+ loadedCount := len(s.loaded)
+ s.loadedMu.Unlock()
+ if runner != nil {
+ if runner.needsReload(ctx, pending) {
+ runnerToExpire = runner
+ } else {
+ // Runner is usable, return it
+ pending.useLoadedRunner(runner, s.finishedReqCh)
+ break
+ }
+ } else if envconfig.MaxRunners() > 0 && loadedCount >= int(envconfig.MaxRunners()) {
+ slog.Debug("max runners achieved, unloading one to make room", "runner_count", loadedCount)
+ runnerToExpire = s.findRunnerToUnload()
+ } else {
+ // Either no models are loaded or below envconfig.MaxRunners
+ // Get a refreshed GPU list
+ var gpus gpu.GpuInfoList
+ if pending.opts.NumGPU == 0 {
+ gpus = s.getCpuFn()
+ } else {
+ gpus = s.getGpuFn()
+ }
+
+ if envconfig.MaxRunners() <= 0 {
+ // No user specified MaxRunners, so figure out what automatic setting to use
+ // If all GPUs have reliable free memory reporting, defaultModelsPerGPU * the number of GPUs
+ // if any GPU has unreliable free memory reporting, 1x the number of GPUs
+ allReliable := true
+ for _, gpu := range gpus {
+ if gpu.UnreliableFreeMemory {
+ allReliable = false
+ break
+ }
+ }
+ if allReliable {
+ // HACK
+ os.Setenv("OLLAMA_MAX_LOADED_MODELS", strconv.Itoa(defaultModelsPerGPU*len(gpus)))
+ slog.Debug("updating default concurrency", "OLLAMA_MAX_LOADED_MODELS", envconfig.MaxRunners, "gpu_count", len(gpus))
+ } else {
+ // HACK
+ os.Setenv("OLLAMA_MAX_LOADED_MODELS", strconv.Itoa(len(gpus)))
+ slog.Info("one or more GPUs detected that are unable to accurately report free memory - disabling default concurrency")
+ }
+ }
+
+ // Load model for fitting
+ ggml, err := llm.LoadModel(pending.model.ModelPath, 0)
+ if err != nil {
+ pending.errCh <- err
+ break
+ }
+
+ // Embedding models should always be loaded with parallel=1
+ if pending.model.CheckCapabilities(CapabilityCompletion) != nil {
+ numParallel = 1
+ }
+
+ // Evaluate if the model will fit in the available system memory, or if we should unload a model first
+ if len(gpus) == 1 && gpus[0].Library == "cpu" {
+ // simplifying assumption of defaultParallel when in CPU mode
+ if numParallel <= 0 {
+ numParallel = defaultParallel
+ }
+
+ pending.opts.NumCtx = pending.origNumCtx * numParallel
+
+ if loadedCount == 0 {
+ slog.Debug("cpu mode with first model, loading")
+ s.loadFn(pending, ggml, gpus, numParallel)
+ break
+ }
+ runnerToExpire = s.maybeFindCPURunnerToUnload(pending, ggml, gpus)
+ if runnerToExpire == nil {
+ slog.Debug("cpu mode with available system memory or first model, loading")
+ s.loadFn(pending, ggml, gpus, numParallel)
+ break
+ }
+ // else we need to expire a runner
+ } else if loadedCount == 0 {
+ // No models loaded. Load the model but prefer the best fit.
+ slog.Debug("loading first model", "model", pending.model.ModelPath)
+ g := pickBestFullFitByLibrary(pending, ggml, gpus, &numParallel)
+ if g != nil {
+ gpus = g
+ } else {
+ // Only allow partial loads when this is the first model
+ gpus = pickBestPartialFitByLibrary(pending, ggml, gpus, &numParallel)
+ }
+ s.loadFn(pending, ggml, gpus, numParallel)
+ break
+ }
+
+ if runnerToExpire == nil {
+ // More than one loaded model, so we have to see if the
+ // new one fits
+ //
+ // We want to avoid loading on any GPUs that have other
+ // models still loading on them to avoid potential races
+ // with VRAM consumption ramping up during load
+ availGpus := s.filterGPUsWithoutLoadingModels(gpus)
+
+ // Update free memory from currently loaded models
+ s.updateFreeSpace(availGpus)
+ fitGpus := pickBestFullFitByLibrary(pending, ggml, availGpus, &numParallel)
+ if fitGpus != nil {
+ slog.Debug("new model fits with existing models, loading")
+ s.loadFn(pending, ggml, fitGpus, numParallel)
+ break
+ }
+
+ // We couldn't find a set of GPUs to fully load the new
+ // model. If no other models are loading (both GPU lists
+ // are the same) then we need to unload another model to
+ // make room
+ if len(availGpus) < len(gpus) {
+ // There are other requests pending, and this one
+ // needs more time, so put it on the back of the
+ // queue so that we might satisfy other pending
+ // requests that aren't blocked
+ go func() {
+ // Process in a go routine to avoid deadlocking
+ // the scheduler if our queue is full
+ slog.Debug("delaying scheduling while other models finish loading", "attempts", pending.schedAttempts, "model", pending.model.ModelPath)
+ time.Sleep(s.reschedDelay)
+ s.pendingReqCh <- pending
+ }()
+ break
+ }
+ runnerToExpire = s.findRunnerToUnload()
+ }
+ }
+
+ if runnerToExpire == nil {
+ // Shouildn't happen
+ slog.Error("runner to expire was nil!")
+ continue
+ }
+ // Trigger an expiration to unload once it's done
+ runnerToExpire.refMu.Lock()
+ slog.Debug("resetting model to expire immediately to make room", "modelPath", runnerToExpire.modelPath, "refCount", runnerToExpire.refCount)
+ if runnerToExpire.expireTimer != nil {
+ runnerToExpire.expireTimer.Stop()
+ runnerToExpire.expireTimer = nil
+ }
+ runnerToExpire.sessionDuration = 0
+ if runnerToExpire.refCount <= 0 {
+ s.expiredCh <- runnerToExpire
+ }
+ runnerToExpire.refMu.Unlock()
+ // Wait for the unload to happen
+ // Note: at this point we're queueing up all incoming requests, even if they were for
+ // a different model that's loaded and not scheduled to be removed.
+ slog.Debug("waiting for pending requests to complete and unload to occur", "modelPath", runnerToExpire.modelPath)
+ select {
+ case <-ctx.Done():
+ slog.Debug("shutting down scheduler pending loop")
+ return
+ case <-s.unloadedCh:
+ slog.Debug("unload completed", "modelPath", runnerToExpire.modelPath)
+ continue
+ }
+ }
+ case <-s.unloadedCh:
+ // An unload request when there are no pending request can be ignored
+ slog.Debug("ignoring unload event with no pending requests")
+ }
+ }
+}
+
+func (s *Scheduler) processCompleted(ctx context.Context) {
+ // Process completed requests, expired timers, and unloading models
+ for {
+ select {
+ case <-ctx.Done():
+ slog.Debug("shutting down scheduler completed loop")
+ return
+ case finished := <-s.finishedReqCh:
+ s.loadedMu.Lock()
+ runner := s.loaded[finished.model.ModelPath]
+ s.loadedMu.Unlock()
+ if runner == nil {
+ slog.Error("finished request signal received after model unloaded", "modelPath", finished.model.ModelPath)
+ continue
+ }
+ runner.refMu.Lock()
+ runner.refCount--
+ if runner.refCount <= 0 {
+ if runner.sessionDuration <= 0 {
+ slog.Debug("runner with zero duration has gone idle, expiring to unload", "modelPath", runner.modelPath)
+ if runner.expireTimer != nil {
+ runner.expireTimer.Stop()
+ runner.expireTimer = nil
+ }
+ s.expiredCh <- runner
+ } else if runner.expireTimer == nil {
+ slog.Debug("runner with non-zero duration has gone idle, adding timer", "modelPath", runner.modelPath, "duration", runner.sessionDuration)
+ runner.expireTimer = time.AfterFunc(runner.sessionDuration, func() {
+ slog.Debug("timer expired, expiring to unload", "modelPath", runner.modelPath)
+ runner.refMu.Lock()
+ defer runner.refMu.Unlock()
+ if runner.expireTimer != nil {
+ runner.expireTimer.Stop()
+ runner.expireTimer = nil
+ }
+ s.expiredCh <- runner
+ })
+ runner.expiresAt = time.Now().Add(runner.sessionDuration)
+ } else {
+ slog.Debug("runner with non-zero duration has gone idle, resetting timer", "modelPath", runner.modelPath, "duration", runner.sessionDuration)
+ runner.expireTimer.Reset(runner.sessionDuration)
+ runner.expiresAt = time.Now().Add(runner.sessionDuration)
+ }
+ }
+ slog.Debug("after processing request finished event", "modelPath", runner.modelPath, "refCount", runner.refCount)
+ runner.refMu.Unlock()
+ case runner := <-s.expiredCh:
+ slog.Debug("runner expired event received", "modelPath", runner.modelPath)
+ runner.refMu.Lock()
+ if runner.refCount > 0 {
+ slog.Debug("expired event with positive ref count, retrying", "modelPath", runner.modelPath, "refCount", runner.refCount)
+ go func(runner *runnerRef) {
+ // We can't unload yet, but want to as soon as the current request completes
+ // So queue up another expired event
+ time.Sleep(10 * time.Millisecond)
+ s.expiredCh <- runner
+ }(runner)
+ runner.refMu.Unlock()
+ continue
+ }
+
+ s.loadedMu.Lock()
+ slog.Debug("got lock to unload", "modelPath", runner.modelPath)
+ finished := runner.waitForVRAMRecovery()
+ runner.unload()
+ delete(s.loaded, runner.modelPath)
+ s.loadedMu.Unlock()
+ slog.Debug("runner released", "modelPath", runner.modelPath)
+ runner.refMu.Unlock()
+
+ <-finished
+ slog.Debug("sending an unloaded event", "modelPath", runner.modelPath)
+ s.unloadedCh <- struct{}{}
+ }
+ }
+}
+
+// Complete the pending request and send the runner back to the requester
+// Wires up a finished event after the request context is completed
+// Updates session duration, and resets expiration timer
+func (pending *LlmRequest) useLoadedRunner(runner *runnerRef, finished chan *LlmRequest) {
+ runner.refMu.Lock()
+ defer runner.refMu.Unlock()
+ runner.refCount++
+ if runner.expireTimer != nil {
+ runner.expireTimer.Stop()
+ runner.expireTimer = nil
+ }
+ if pending.sessionDuration != nil {
+ runner.sessionDuration = pending.sessionDuration.Duration
+ }
+ pending.successCh <- runner
+ go func() {
+ <-pending.ctx.Done()
+ slog.Debug("context for request finished")
+ finished <- pending
+ }()
+}
+
+func (s *Scheduler) load(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel int) {
+ if numParallel < 1 {
+ numParallel = 1
+ }
+ sessionDuration := envconfig.KeepAlive()
+ if req.sessionDuration != nil {
+ sessionDuration = req.sessionDuration.Duration
+ }
+ llama, err := s.newServerFn(gpus, req.model.ModelPath, ggml, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts, numParallel)
+ if err != nil {
+ // some older models are not compatible with newer versions of llama.cpp
+ // show a generalized compatibility error until there is a better way to
+ // check for model compatibility
+ if errors.Is(err, llm.ErrUnsupportedFormat) || strings.Contains(err.Error(), "failed to load model") {
+ err = fmt.Errorf("%v: this model may be incompatible with your version of Ollama. If you previously pulled this model, try updating it by running `ollama pull %s`", err, req.model.ShortName)
+ }
+ slog.Info("NewLlamaServer failed", "model", req.model.ModelPath, "error", err)
+ req.errCh <- err
+ return
+ }
+ runner := &runnerRef{
+ model: req.model,
+ modelPath: req.model.ModelPath,
+ llama: llama,
+ Options: &req.opts,
+ sessionDuration: sessionDuration,
+ gpus: gpus,
+ estimatedVRAM: llama.EstimatedVRAM(),
+ estimatedTotal: llama.EstimatedTotal(),
+ loading: true,
+ refCount: 1,
+ }
+ runner.numParallel = numParallel
+ runner.refMu.Lock()
+
+ s.loadedMu.Lock()
+ s.loaded[req.model.ModelPath] = runner
+ slog.Info("loaded runners", "count", len(s.loaded))
+ s.loadedMu.Unlock()
+
+ go func() {
+ defer runner.refMu.Unlock()
+ if err = llama.WaitUntilRunning(req.ctx); err != nil {
+ slog.Error("error loading llama server", "error", err)
+ runner.refCount--
+ req.errCh <- err
+ slog.Debug("triggering expiration for failed load", "model", runner.modelPath)
+ s.expiredCh <- runner
+ return
+ }
+ slog.Debug("finished setting up runner", "model", req.model.ModelPath)
+ runner.loading = false
+ go func() {
+ <-req.ctx.Done()
+ slog.Debug("context for request finished")
+ s.finishedReqCh <- req
+ }()
+ req.successCh <- runner
+ }()
+}
+
+func (s *Scheduler) updateFreeSpace(allGpus gpu.GpuInfoList) {
+ type predKey struct {
+ Library string
+ ID string
+ }
+ predMap := map[predKey]uint64{} // Sum up the total predicted usage per GPU for all runners
+ s.loadedMu.Lock()
+ for _, r := range s.loaded {
+ r.refMu.Lock()
+ if r.llama != nil {
+ for _, gpu := range allGpus {
+ predMap[predKey{gpu.Library, gpu.ID}] += r.llama.EstimatedVRAMByGPU(gpu.ID)
+ }
+ } else {
+ slog.Warn("unexpected nil runner reference, memory prediction may be incorrect")
+ }
+ r.refMu.Unlock()
+ }
+ s.loadedMu.Unlock()
+
+ // Now that we've summed up all the GPU usage predictions across all the loaded runners, update the gpu list
+ for i := range allGpus {
+ if p, ok := predMap[predKey{allGpus[i].Library, allGpus[i].ID}]; ok {
+ slog.Debug("gpu reported", "gpu", allGpus[i].ID, "library", allGpus[i].Library, "available", format.HumanBytes2(allGpus[i].FreeMemory))
+ if p > allGpus[i].TotalMemory {
+ // Shouldn't happen
+ slog.Warn("predicted usage exceeds VRAM", "gpu", allGpus[i].ID, "totalMemory", allGpus[i].TotalMemory, "predicted", p)
+ allGpus[i].FreeMemory = 0
+ } else if (allGpus[i].TotalMemory - p) < allGpus[i].FreeMemory { // predicted free is smaller than reported free, use it
+ // TODO maybe we should just always trust our numbers, since cuda's free memory reporting is laggy
+ // and we might unload models we didn't actually need to. The risk is if some other GPU intensive app is loaded
+ // after we start our first runner, then we'll never acount for that, so picking the smallest free value seems prudent.
+ allGpus[i].FreeMemory = allGpus[i].TotalMemory - p
+ }
+ slog.Info("updated VRAM based on existing loaded models", "gpu", allGpus[i].ID, "library", allGpus[i].Library, "total", format.HumanBytes2(allGpus[i].TotalMemory), "available", format.HumanBytes2(allGpus[i].FreeMemory))
+ }
+ }
+}
+
+// While models are loading the VRAM consumption numbers will be indeterminate, so we have
+// to avoid scheduling another model on the same GPU(s) that haven't stabilized.
+// This routine returns the set of GPUs that do not have an active loading model.
+// If all GPUs have loading models, an empty list will be returned (not a single CPU entry)
+func (s *Scheduler) filterGPUsWithoutLoadingModels(allGpus gpu.GpuInfoList) gpu.GpuInfoList {
+ ret := append(gpu.GpuInfoList{}, allGpus...)
+ s.loadedMu.Lock()
+ defer s.loadedMu.Unlock()
+ for _, runner := range s.loaded {
+ if runner.loading {
+ slog.Debug("overlapping loads detected", "gpus", runner.gpus, "model", runner.modelPath)
+ for _, busyGPU := range runner.gpus {
+ for i := range ret {
+ if ret[i].ID == busyGPU.ID {
+ ret = append(ret[:i], ret[i+1:]...)
+ break
+ }
+ }
+ }
+ }
+ }
+ return ret
+}
+
+// TODO consolidate sched_types.go
+type runnerRef struct {
+ refMu sync.Mutex
+ // refCond sync.Cond // Signaled on transition from 1 -> 0 refCount
+ refCount uint // prevent unloading if > 0
+ // unloading bool // set to true when we are trying to unload the runner
+
+ llama llm.LlamaServer
+ loading bool // True only during initial load, then false forever
+ gpus gpu.GpuInfoList // Recorded at time of provisioning
+ estimatedVRAM uint64
+ estimatedTotal uint64
+
+ sessionDuration time.Duration
+ expireTimer *time.Timer
+ expiresAt time.Time
+
+ model *Model
+ modelPath string
+ numParallel int
+ *api.Options
+}
+
+// The refMu must already be held when calling unload
+func (runner *runnerRef) unload() {
+ if runner.expireTimer != nil {
+ runner.expireTimer.Stop()
+ runner.expireTimer = nil
+ }
+ if runner.llama != nil {
+ runner.llama.Close()
+ }
+ runner.model = nil
+ runner.llama = nil
+ runner.Options = nil
+ runner.gpus = nil
+}
+
+func (runner *runnerRef) needsReload(ctx context.Context, req *LlmRequest) bool {
+ slog.Debug("evaluating already loaded", "model", req.model.ModelPath)
+ runner.refMu.Lock()
+ defer runner.refMu.Unlock()
+
+ timeout := 10 * time.Second
+ if runner.loading {
+ timeout = 2 * time.Minute // Initial load can take a long time for big models on slow systems...
+ }
+
+ if runner.Options == nil {
+ return true
+ }
+
+ // Don't reload runner if num_gpu=-1 was provided
+ optsExisting := runner.Options.Runner
+ optsNew := req.opts.Runner
+ if optsNew.NumGPU < 0 {
+ optsExisting.NumGPU = -1
+ optsNew.NumGPU = -1
+ }
+
+ // Normalize the NumCtx for parallelism
+ optsExisting.NumCtx = optsExisting.NumCtx / runner.numParallel
+
+ ctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+ if !reflect.DeepEqual(runner.model.AdapterPaths, req.model.AdapterPaths) || // have the adapters changed?
+ !reflect.DeepEqual(runner.model.ProjectorPaths, req.model.ProjectorPaths) || // have the projectors changed?
+ !reflect.DeepEqual(optsExisting, optsNew) || // have the runner options changed?
+ runner.llama.Ping(ctx) != nil {
+ return true
+ }
+
+ return false
+}
+
+// Free memory reporting on GPUs can lag for a while even after the runner
+// exits, so we have to keep checking until we see the available memory recover,
+// otherwise subsequent model loads will get far less layers loaded or worse
+// case, may completely fall back to CPU mode.
+// This routine must be called before the runner unloads so it can establish
+// a before and after GPU memory allocation. The returned channel
+// will be notified when we're done waiting, or have timed out and should
+// proceed anyway
+func (runner *runnerRef) waitForVRAMRecovery() chan interface{} {
+ finished := make(chan interface{}, 1)
+
+ // CPU or Metal don't need checking, so no waiting required
+ // windows can page VRAM, only cuda currently can report accurate used vram usage
+ if len(runner.gpus) == 0 ||
+ (len(runner.gpus) == 1 && (runner.gpus[0].Library == "cpu" || runner.gpus[0].Library == "metal")) ||
+ (runtime.GOOS == "windows" && runner.gpus[0].Library != "cuda") {
+ finished <- struct{}{}
+ return finished
+ }
+ start := time.Now()
+
+ // Establish a baseline before we unload
+ gpusBefore := gpu.GetGPUInfo()
+ var totalMemoryBefore, freeMemoryBefore uint64
+ for _, gpu := range gpusBefore {
+ totalMemoryBefore += gpu.TotalMemory
+ freeMemoryBefore += gpu.FreeMemory
+ }
+ go func() {
+ expiresAt := start.Add(5 * time.Second) // typical convergence is 0.5-1.5s
+ ticker := time.NewTicker(250 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ <-ticker.C
+ if time.Now().After(expiresAt) {
+ slog.Warn("gpu VRAM usage didn't recover within timeout", "seconds", time.Since(start).Seconds(), "model", runner.modelPath)
+ finished <- struct{}{}
+ }
+
+ // Query GPUs, look for free to go back up
+ gpusNow := gpu.GetGPUInfo()
+ var totalMemoryNow, freeMemoryNow uint64
+ for _, gpu := range gpusNow {
+ totalMemoryNow += gpu.TotalMemory
+ freeMemoryNow += gpu.FreeMemory
+ }
+ // If we're within ~80% of the estimated memory usage recovered, bail out
+ if float32(freeMemoryNow-freeMemoryBefore) > float32(runner.estimatedVRAM)*0.8 {
+ slog.Debug(fmt.Sprintf("gpu VRAM free memory converged after %0.2f seconds", time.Since(start).Seconds()), "model", runner.modelPath)
+ finished <- struct{}{}
+ return
+ }
+ }
+ }()
+ return finished
+}
+
+type ByDuration []*runnerRef
+
+func (a ByDuration) Len() int { return len(a) }
+func (a ByDuration) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByDuration) Less(i, j int) bool {
+ // uint64 to turn negative time (never unload) to largest
+ return uint64(a[i].sessionDuration) < uint64(a[j].sessionDuration)
+}
+
+// TODO - future consideration to pick runners based on size
+// type BySize []*runnerRef
+// func (a BySize) Len() int { return len(a) }
+// func (a BySize) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+// func (a BySize) Less(i, j int) bool { return a[i].estimatedVRAM < a[j].estimatedVRAM }
+
+// pickBestFullFitByLibrary will try to find the optimal placement of the model in the available GPUs where the model fully fits
+// The list of GPUs returned will always be the same brand (library)
+// If the model can not be fit fully within the available GPU(s) nil is returned
+// If numParallel is <= 0, this will attempt try to optimize parallism based on available VRAM, and adjust
+// opts.NumCtx accordingly
+func pickBestFullFitByLibrary(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel *int) gpu.GpuInfoList {
+ var estimatedVRAM uint64
+
+ var numParallelToTry []int
+ if *numParallel <= 0 {
+ // If no specific parallel setting was provided, try larger then smaller, always end with 1
+ numParallelToTry = append(numParallelToTry, defaultParallel, 1)
+ } else {
+ numParallelToTry = []int{*numParallel}
+ }
+
+ for _, gl := range gpus.ByLibrary() {
+ var ok bool
+ sgl := append(make(gpu.GpuInfoList, 0, len(gl)), gl...)
+
+ // TODO - potentially sort by performance capability, existing models loaded, etc.
+ // TODO - Eliminate any GPUs that already have envconfig.MaxRunners loaded on them
+ // Note: at present, this will favor more VRAM over faster GPU speed in mixed setups
+ sort.Sort(sort.Reverse(gpu.ByFreeMemory(sgl)))
+
+ // First attempt to fit the model into a single GPU
+ for _, p := range numParallelToTry {
+ req.opts.NumCtx = req.origNumCtx * p
+ if !envconfig.SchedSpread() {
+ for _, g := range sgl {
+ if ok, estimatedVRAM = llm.PredictServerFit([]gpu.GpuInfo{g}, ggml, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts); ok {
+ slog.Info("new model will fit in available VRAM in single GPU, loading", "model", req.model.ModelPath, "gpu", g.ID, "parallel", p, "available", g.FreeMemory, "required", format.HumanBytes2(estimatedVRAM))
+ *numParallel = p
+ return []gpu.GpuInfo{g}
+ }
+ }
+ }
+ }
+
+ // TODO future refinements
+ // - if multiple Libraries, see if any single GPU in any Library will fit
+ // - try subsets of GPUs instead of just falling back to 1 or all in a family
+
+ // Now try all the GPUs
+ for _, p := range numParallelToTry {
+ req.opts.NumCtx = req.origNumCtx * p
+ if ok, estimatedVRAM = llm.PredictServerFit(sgl, ggml, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts); ok {
+ slog.Info("new model will fit in available VRAM, loading", "model", req.model.ModelPath, "library", sgl[0].Library, "parallel", p, "required", format.HumanBytes2(estimatedVRAM))
+ *numParallel = p
+ return sgl
+ }
+ }
+ }
+ return nil
+}
+
+// If multiple Libraries are detected, pick the Library which loads the most layers for the model
+func pickBestPartialFitByLibrary(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel *int) gpu.GpuInfoList {
+ if *numParallel <= 0 {
+ *numParallel = 1
+ req.opts.NumCtx = req.origNumCtx
+ }
+ byLibrary := gpus.ByLibrary()
+ if len(byLibrary) <= 1 {
+ return gpus
+ }
+ var bestEstimate uint64
+ var bestFit int
+ for i, gl := range byLibrary {
+ _, estimatedVRAM := llm.PredictServerFit(gl, ggml, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts)
+ if estimatedVRAM > bestEstimate {
+ bestEstimate = estimatedVRAM
+ bestFit = i
+ }
+ }
+ return byLibrary[bestFit]
+}
+
+// findRunnerToUnload finds a runner to unload to make room for a new model
+func (s *Scheduler) findRunnerToUnload() *runnerRef {
+ s.loadedMu.Lock()
+ runnerList := make([]*runnerRef, 0, len(s.loaded))
+ for _, r := range s.loaded {
+ runnerList = append(runnerList, r)
+ }
+ s.loadedMu.Unlock()
+ if len(runnerList) == 0 {
+ slog.Debug("no loaded runner to unload")
+ return nil
+ }
+
+ // In the future we can enhance the algorithm to be smarter about picking the optimal runner to unload
+ // e.g., if we have multiple options, will one make room for the request?
+ sort.Sort(ByDuration(runnerList))
+
+ // First try to find a runner that's already idle
+ for _, runner := range runnerList {
+ runner.refMu.Lock()
+ rc := runner.refCount
+ runner.refMu.Unlock()
+ if rc == 0 {
+ slog.Debug("found an idle runner to unload")
+ return runner
+ }
+ }
+ // None appear idle, just wait for the one with the shortest duration
+ slog.Debug("no idle runners, picking the shortest duration", "count", len(runnerList))
+ return runnerList[0]
+}
+
+func (s *Scheduler) unloadAllRunners() {
+ s.loadedMu.Lock()
+ defer s.loadedMu.Unlock()
+ for model, runner := range s.loaded {
+ if runner.llama != nil {
+ slog.Debug("shutting down runner", "model", model)
+ runner.llama.Close()
+ }
+ }
+}
+
+func (s *Scheduler) expireRunner(model *Model) {
+ s.loadedMu.Lock()
+ defer s.loadedMu.Unlock()
+ runner, ok := s.loaded[model.ModelPath]
+ if ok {
+ runner.refMu.Lock()
+ runner.expiresAt = time.Now()
+ if runner.expireTimer != nil {
+ runner.expireTimer.Stop()
+ runner.expireTimer = nil
+ }
+ runner.sessionDuration = 0
+ if runner.refCount <= 0 {
+ s.expiredCh <- runner
+ }
+ runner.refMu.Unlock()
+ }
+}
+
+// If other runners are loaded, make sure the pending request will fit in system memory
+// If not, pick a runner to unload, else return nil and the request can be loaded
+func (s *Scheduler) maybeFindCPURunnerToUnload(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList) *runnerRef {
+ slog.Debug("evaluating if CPU model load will fit in available system memory")
+ estimate := llm.EstimateGPULayers(gpus, ggml, req.model.ProjectorPaths, req.opts)
+ if estimate.TotalSize <= gpus[0].FreeMemory {
+ slog.Debug("cpu inference mode, model fits in available system memory", "model", format.HumanBytes2(estimate.TotalSize), "available", format.HumanBytes2(gpus[0].FreeMemory))
+ return nil
+ }
+
+ // TODO - optimization: try to find CPU only runners first, or partial offloads with enough in system memory to make room
+
+ return s.findRunnerToUnload()
+}
diff --git a/server/sched_test.go b/server/sched_test.go
new file mode 100644
index 00000000..fe5647c5
--- /dev/null
+++ b/server/sched_test.go
@@ -0,0 +1,793 @@
+package server
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "log/slog"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/app/lifecycle"
+ "github.com/ollama/ollama/format"
+ "github.com/ollama/ollama/gpu"
+ "github.com/ollama/ollama/llm"
+)
+
+func TestMain(m *testing.M) {
+ os.Setenv("OLLAMA_DEBUG", "1")
+ lifecycle.InitLogging()
+ os.Exit(m.Run())
+}
+
+func TestInitScheduler(t *testing.T) {
+ ctx, done := context.WithCancel(context.Background())
+ defer done()
+ s := InitScheduler(ctx)
+ s.loadedMu.Lock()
+ require.NotNil(t, s.loaded)
+ s.loadedMu.Unlock()
+}
+
+func TestLoad(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer done()
+ s := InitScheduler(ctx)
+ var ggml *llm.GGML // value not used in tests
+ req := &LlmRequest{
+ ctx: ctx,
+ model: &Model{ModelPath: "foo"},
+ opts: api.DefaultOptions(),
+ successCh: make(chan *runnerRef, 1),
+ errCh: make(chan error, 1),
+ sessionDuration: &api.Duration{Duration: 2 * time.Second},
+ }
+ // Fail to load model first
+ s.newServerFn = func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {
+ return nil, errors.New("something failed to load model blah")
+ }
+ gpus := gpu.GpuInfoList{}
+ s.load(req, ggml, gpus, 0)
+ require.Empty(t, req.successCh)
+ require.Len(t, req.errCh, 1)
+ s.loadedMu.Lock()
+ require.Empty(t, s.loaded)
+ s.loadedMu.Unlock()
+ err := <-req.errCh
+ require.Contains(t, err.Error(), "this model may be incompatible")
+
+ server := &mockLlm{estimatedVRAM: 10, estimatedVRAMByGPU: map[string]uint64{}}
+ s.newServerFn = func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {
+ return server, nil
+ }
+ s.load(req, ggml, gpus, 0)
+ select {
+ case err := <-req.errCh:
+ require.NoError(t, err)
+ case resp := <-req.successCh:
+ require.Equal(t, uint64(10), resp.estimatedVRAM)
+ require.Equal(t, uint(1), resp.refCount)
+ s.loadedMu.Lock()
+ require.Len(t, s.loaded, 1)
+ s.loadedMu.Unlock()
+ }
+
+ req.model.ModelPath = "dummy_model_path"
+ server.waitResp = errors.New("wait failure")
+ s.load(req, ggml, gpus, 0)
+ select {
+ case err := <-req.errCh:
+ require.Contains(t, err.Error(), "wait failure")
+ case resp := <-req.successCh:
+ t.Fatalf("unexpected success %v", resp)
+ }
+ s.loadedMu.Lock()
+ runner := s.loaded["dummy_model_path"]
+ s.loadedMu.Unlock()
+ require.NotNil(t, runner)
+ require.Equal(t, uint(0), runner.refCount)
+ time.Sleep(1 * time.Millisecond)
+ require.Len(t, s.expiredCh, 1)
+}
+
+type reqBundle struct {
+ ctx context.Context //nolint:containedctx
+ ctxDone func()
+ srv *mockLlm
+ req *LlmRequest
+ ggml *llm.GGML
+}
+
+func (scenario *reqBundle) newServer(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {
+ return scenario.srv, nil
+}
+
+func newScenarioRequest(t *testing.T, ctx context.Context, modelName string, estimatedVRAM uint64, duration *api.Duration) *reqBundle {
+ b := &reqBundle{}
+ b.ctx, b.ctxDone = context.WithCancel(ctx)
+ t.Helper()
+
+ f, err := os.CreateTemp(t.TempDir(), modelName)
+ require.NoError(t, err)
+ defer f.Close()
+
+ require.NoError(t, llm.WriteGGUF(f, llm.KV{
+ "general.architecture": "llama",
+ "llama.context_length": uint32(32),
+ "llama.embedding_length": uint32(4096),
+ "llama.block_count": uint32(1),
+ "llama.attention.head_count": uint32(32),
+ "llama.attention.head_count_kv": uint32(32),
+ "tokenizer.ggml.tokens": []string{" "},
+ "tokenizer.ggml.scores": []float32{0},
+ "tokenizer.ggml.token_type": []int32{0},
+ }, []llm.Tensor{
+ {Name: "blk.0.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
+ {Name: "output.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
+ }))
+ require.NoError(t, err)
+
+ fname := f.Name()
+ model := &Model{Name: modelName, ModelPath: fname}
+ b.ggml, err = llm.LoadModel(model.ModelPath, 0)
+ require.NoError(t, err)
+
+ if duration == nil {
+ duration = &api.Duration{Duration: 5 * time.Millisecond}
+ }
+ b.req = &LlmRequest{
+ ctx: b.ctx,
+ model: model,
+ opts: api.DefaultOptions(),
+ sessionDuration: duration,
+ successCh: make(chan *runnerRef, 1),
+ errCh: make(chan error, 1),
+ }
+ b.srv = &mockLlm{estimatedVRAM: estimatedVRAM, estimatedVRAMByGPU: map[string]uint64{"": estimatedVRAM}}
+ return b
+}
+
+func getGpuFn() gpu.GpuInfoList {
+ g := gpu.GpuInfo{Library: "metal"}
+ g.TotalMemory = 24 * format.GigaByte
+ g.FreeMemory = 12 * format.GigaByte
+ return []gpu.GpuInfo{g}
+}
+
+func getCpuFn() gpu.GpuInfoList {
+ g := gpu.GpuInfo{Library: "cpu"}
+ g.TotalMemory = 32 * format.GigaByte
+ g.FreeMemory = 26 * format.GigaByte
+ return []gpu.GpuInfo{g}
+}
+
+func TestRequestsSameModelSameRequest(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 500*time.Millisecond)
+ defer done()
+ s := InitScheduler(ctx)
+ s.getGpuFn = getGpuFn
+ s.getCpuFn = getCpuFn
+ a := newScenarioRequest(t, ctx, "ollama-model-1", 10, &api.Duration{Duration: 5 * time.Millisecond})
+ b := newScenarioRequest(t, ctx, "ollama-model-1", 11, &api.Duration{Duration: 0})
+ b.req.model = a.req.model
+ b.ggml = a.ggml
+
+ s.newServerFn = a.newServer
+ slog.Info("a")
+ s.pendingReqCh <- a.req
+ require.Len(t, s.pendingReqCh, 1)
+ s.Run(ctx)
+ select {
+ case resp := <-a.req.successCh:
+ require.Equal(t, resp.llama, a.srv)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, a.req.errCh)
+ case err := <-a.req.errCh:
+ t.Fatal(err.Error())
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+
+ // Same runner as first request due to not needing a reload
+ s.newServerFn = b.newServer
+ slog.Info("b")
+ s.pendingReqCh <- b.req
+ select {
+ case resp := <-b.req.successCh:
+ require.Equal(t, resp.llama, a.srv)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, b.req.errCh)
+ case err := <-b.req.errCh:
+ t.Fatal(err.Error())
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+}
+
+func TestRequestsSimpleReloadSameModel(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 500*time.Millisecond)
+ defer done()
+ s := InitScheduler(ctx)
+ s.getGpuFn = getGpuFn
+ s.getCpuFn = getCpuFn
+ a := newScenarioRequest(t, ctx, "ollama-model-1", 10, &api.Duration{Duration: 5 * time.Millisecond})
+ b := newScenarioRequest(t, ctx, "ollama-model-1", 20, &api.Duration{Duration: 5 * time.Millisecond})
+ tmpModel := *a.req.model
+ b.req.model = &tmpModel
+ b.ggml = a.ggml
+
+ s.newServerFn = a.newServer
+ slog.Info("a")
+ s.pendingReqCh <- a.req
+ require.Len(t, s.pendingReqCh, 1)
+ s.Run(ctx)
+ select {
+ case resp := <-a.req.successCh:
+ require.Equal(t, resp.llama, a.srv)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, a.req.errCh)
+ case err := <-a.req.errCh:
+ t.Fatal(err.Error())
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+
+ // Trigger a reload
+ s.newServerFn = b.newServer
+ b.req.model.AdapterPaths = []string{"new"}
+ slog.Info("b")
+ s.pendingReqCh <- b.req
+ // finish first two requests, so model can reload
+ time.Sleep(1 * time.Millisecond)
+ a.ctxDone()
+ select {
+ case resp := <-b.req.successCh:
+ require.Equal(t, resp.llama, b.srv)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, b.req.errCh)
+ case err := <-b.req.errCh:
+ t.Fatal(err.Error())
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+}
+
+func TestRequestsMultipleLoadedModels(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 500*time.Millisecond)
+ defer done()
+ s := InitScheduler(ctx)
+ s.getGpuFn = getGpuFn
+ s.getCpuFn = getCpuFn
+
+ // Multiple loaded models
+ a := newScenarioRequest(t, ctx, "ollama-model-3a", 1*format.GigaByte, nil)
+ b := newScenarioRequest(t, ctx, "ollama-model-3b", 24*format.GigaByte, nil)
+ c := newScenarioRequest(t, ctx, "ollama-model-4a", 30, nil)
+ c.req.opts.NumGPU = 0 // CPU load, will be allowed
+ d := newScenarioRequest(t, ctx, "ollama-model-3c", 30, nil) // Needs prior unloaded
+
+ t.Setenv("OLLAMA_MAX_LOADED_MODELS", "1")
+ s.newServerFn = a.newServer
+ slog.Info("a")
+ s.pendingReqCh <- a.req
+ s.Run(ctx)
+ select {
+ case resp := <-a.req.successCh:
+ require.Equal(t, resp.llama, a.srv)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, a.req.errCh)
+ case err := <-a.req.errCh:
+ t.Fatal(err.Error())
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+ s.loadedMu.Lock()
+ require.Len(t, s.loaded, 1)
+ s.loadedMu.Unlock()
+
+ t.Setenv("OLLAMA_MAX_LOADED_MODELS", "0")
+ s.newServerFn = b.newServer
+ slog.Info("b")
+ s.pendingReqCh <- b.req
+ select {
+ case resp := <-b.req.successCh:
+ require.Equal(t, resp.llama, b.srv)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, b.req.errCh)
+ case err := <-b.req.errCh:
+ t.Fatal(err.Error())
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+ s.loadedMu.Lock()
+ require.Len(t, s.loaded, 2)
+ s.loadedMu.Unlock()
+
+ // This is a CPU load with NumGPU = 0 so it should load
+ s.newServerFn = c.newServer
+ slog.Info("c")
+ s.pendingReqCh <- c.req
+ select {
+ case resp := <-c.req.successCh:
+ require.Equal(t, resp.llama, c.srv)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, c.req.errCh)
+ case err := <-c.req.errCh:
+ t.Fatal(err.Error())
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+ s.loadedMu.Lock()
+ require.Len(t, s.loaded, 3)
+ s.loadedMu.Unlock()
+
+ // Try to load a model that wont fit
+ s.newServerFn = d.newServer
+ slog.Info("d")
+ s.loadedMu.Lock()
+ require.Len(t, s.loaded, 3)
+ s.loadedMu.Unlock()
+ a.ctxDone() // Won't help since this one isn't big enough to make room
+ time.Sleep(2 * time.Millisecond)
+ s.pendingReqCh <- d.req
+ // finish prior request, so new model can load
+ time.Sleep(6 * time.Millisecond)
+ s.loadedMu.Lock()
+ require.Len(t, s.loaded, 2)
+ s.loadedMu.Unlock()
+ b.ctxDone()
+ select {
+ case resp := <-d.req.successCh:
+ require.Equal(t, resp.llama, d.srv)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, d.req.errCh)
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+ s.loadedMu.Lock()
+ require.Len(t, s.loaded, 2)
+ s.loadedMu.Unlock()
+}
+
+func TestGetRunner(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 200*time.Millisecond)
+ defer done()
+
+ a := newScenarioRequest(t, ctx, "ollama-model-1a", 10, &api.Duration{Duration: 2 * time.Millisecond})
+ b := newScenarioRequest(t, ctx, "ollama-model-1b", 10, &api.Duration{Duration: 2 * time.Millisecond})
+ c := newScenarioRequest(t, ctx, "ollama-model-1c", 10, &api.Duration{Duration: 2 * time.Millisecond})
+ t.Setenv("OLLAMA_MAX_QUEUE", "1")
+ s := InitScheduler(ctx)
+ s.getGpuFn = getGpuFn
+ s.getCpuFn = getCpuFn
+ s.newServerFn = a.newServer
+ slog.Info("a")
+ successCh1a, errCh1a := s.GetRunner(a.ctx, a.req.model, a.req.opts, a.req.sessionDuration)
+ require.Len(t, s.pendingReqCh, 1)
+ slog.Info("b")
+ successCh1b, errCh1b := s.GetRunner(b.ctx, b.req.model, b.req.opts, b.req.sessionDuration)
+ require.Len(t, s.pendingReqCh, 1)
+ require.Empty(t, successCh1b)
+ require.Len(t, errCh1b, 1)
+ err := <-errCh1b
+ require.Contains(t, err.Error(), "server busy")
+ s.Run(ctx)
+ select {
+ case resp := <-successCh1a:
+ require.Equal(t, resp.llama, a.srv)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, errCh1a)
+ case err := <-errCh1a:
+ t.Fatal(err.Error())
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+ a.ctxDone() // Set "a" model to idle so it can unload
+ s.loadedMu.Lock()
+ require.Len(t, s.loaded, 1)
+ s.loadedMu.Unlock()
+
+ c.req.model.ModelPath = "bad path"
+ slog.Info("c")
+ successCh1c, errCh1c := s.GetRunner(c.ctx, c.req.model, c.req.opts, c.req.sessionDuration)
+ // Starts in pending channel, then should be quickly processsed to return an error
+ time.Sleep(50 * time.Millisecond) // Long enough for the "a" model to expire and unload
+ require.Empty(t, successCh1c)
+ s.loadedMu.Lock()
+ require.Empty(t, s.loaded)
+ s.loadedMu.Unlock()
+ require.Len(t, errCh1c, 1)
+ err = <-errCh1c
+ require.Contains(t, err.Error(), "bad path")
+ b.ctxDone()
+}
+
+func TestExpireRunner(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer done()
+ s := InitScheduler(ctx)
+ req := &LlmRequest{
+ ctx: ctx,
+ model: &Model{ModelPath: "foo"},
+ opts: api.DefaultOptions(),
+ successCh: make(chan *runnerRef, 1),
+ errCh: make(chan error, 1),
+ sessionDuration: &api.Duration{Duration: 2 * time.Minute},
+ }
+
+ var ggml *llm.GGML
+ gpus := gpu.GpuInfoList{}
+ server := &mockLlm{estimatedVRAM: 10, estimatedVRAMByGPU: map[string]uint64{}}
+ s.newServerFn = func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {
+ return server, nil
+ }
+ s.load(req, ggml, gpus, 0)
+
+ select {
+ case err := <-req.errCh:
+ if err != nil {
+ t.Fatalf("expected no errors when loading, got '%s'", err.Error())
+ }
+ case resp := <-req.successCh:
+ s.loadedMu.Lock()
+ if resp.refCount != uint(1) || len(s.loaded) != 1 {
+ t.Fatalf("expected a model to be loaded")
+ }
+ s.loadedMu.Unlock()
+ }
+
+ s.expireRunner(&Model{ModelPath: "foo"})
+
+ s.finishedReqCh <- req
+ s.processCompleted(ctx)
+
+ s.loadedMu.Lock()
+ if len(s.loaded) != 0 {
+ t.Fatalf("expected model to be unloaded")
+ }
+ s.loadedMu.Unlock()
+}
+
+// TODO - add one scenario that triggers the bogus finished event with positive ref count
+func TestPrematureExpired(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 500*time.Millisecond)
+ defer done()
+
+ // Same model, same request
+ scenario1a := newScenarioRequest(t, ctx, "ollama-model-1a", 10, nil)
+ s := InitScheduler(ctx)
+ s.getGpuFn = func() gpu.GpuInfoList {
+ g := gpu.GpuInfo{Library: "metal"}
+ g.TotalMemory = 24 * format.GigaByte
+ g.FreeMemory = 12 * format.GigaByte
+ return []gpu.GpuInfo{g}
+ }
+ s.newServerFn = scenario1a.newServer
+ successCh1a, errCh1a := s.GetRunner(scenario1a.ctx, scenario1a.req.model, scenario1a.req.opts, scenario1a.req.sessionDuration)
+ require.Len(t, s.pendingReqCh, 1)
+ s.Run(ctx)
+ select {
+ case resp := <-successCh1a:
+ require.Equal(t, resp.llama, scenario1a.srv)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, errCh1a)
+ s.loadedMu.Lock()
+ require.Len(t, s.loaded, 1)
+ s.loadedMu.Unlock()
+ slog.Info("sending premature expired event now")
+ s.expiredCh <- resp // Shouldn't happen in real life, but make sure its safe
+ case err := <-errCh1a:
+ t.Fatal(err.Error())
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+ time.Sleep(scenario1a.req.sessionDuration.Duration)
+ scenario1a.ctxDone()
+ time.Sleep(20 * time.Millisecond)
+ require.LessOrEqual(t, len(s.finishedReqCh), 1)
+ time.Sleep(10 * time.Millisecond)
+ require.Empty(t, s.finishedReqCh)
+ s.loadedMu.Lock()
+ require.Empty(t, s.loaded)
+ s.loadedMu.Unlock()
+
+ // also shouldn't happen in real life
+ s.finishedReqCh <- scenario1a.req
+ time.Sleep(5 * time.Millisecond)
+}
+
+func TestUseLoadedRunner(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ req := &LlmRequest{
+ ctx: ctx,
+ opts: api.DefaultOptions(),
+ successCh: make(chan *runnerRef, 1),
+ sessionDuration: &api.Duration{Duration: 2},
+ }
+ finished := make(chan *LlmRequest)
+ llm1 := &mockLlm{estimatedVRAMByGPU: map[string]uint64{}}
+ r1 := &runnerRef{llama: llm1, sessionDuration: 1, numParallel: 1}
+ req.useLoadedRunner(r1, finished)
+ require.Equal(t, uint(1), r1.refCount)
+ require.Equal(t, time.Duration(2), r1.sessionDuration)
+ select {
+ case success := <-req.successCh:
+ require.Equal(t, r1, success)
+ case err := <-req.errCh:
+ t.Fatal(err.Error())
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+ done()
+ fin := <-finished
+ require.Equal(t, req, fin)
+}
+
+func TestUpdateFreeSpace(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer done()
+ gpus := gpu.GpuInfoList{
+ {
+ Library: "a",
+ ID: "1",
+ },
+ {
+ Library: "a",
+ ID: "2",
+ },
+ }
+ gpus[0].TotalMemory = 1000
+ gpus[0].FreeMemory = 900
+ gpus[1].TotalMemory = 2000
+ gpus[1].FreeMemory = 1900
+ llm1 := &mockLlm{estimatedVRAMByGPU: map[string]uint64{"1": 50, "2": 50}}
+ llm2 := &mockLlm{estimatedVRAMByGPU: map[string]uint64{"1": 125, "2": 75}}
+ r1 := &runnerRef{llama: llm1, gpus: gpus, numParallel: 1}
+ r2 := &runnerRef{llama: llm2, gpus: gpus, numParallel: 1}
+
+ s := InitScheduler(ctx)
+ s.loadedMu.Lock()
+ s.loaded["a"] = r1
+ s.loaded["b"] = r2
+ s.loadedMu.Unlock()
+
+ s.updateFreeSpace(gpus)
+ require.Equal(t, uint64(1000-50-125), gpus[0].FreeMemory)
+ require.Equal(t, uint64(2000-50-75), gpus[1].FreeMemory)
+}
+
+func TestFilterGPUsWithoutLoadingModels(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer done()
+ gpus := gpu.GpuInfoList{
+ {
+ Library: "cuda",
+ ID: "0",
+ },
+ {
+ Library: "cuda",
+ ID: "1",
+ },
+ }
+ r1 := &runnerRef{gpus: gpu.GpuInfoList{gpus[0]}, loading: true}
+
+ s := InitScheduler(ctx)
+ s.loadedMu.Lock()
+ s.loaded["a"] = r1
+ s.loadedMu.Unlock()
+
+ tmp := s.filterGPUsWithoutLoadingModels(gpus)
+ require.Len(t, tmp, 1)
+ require.Equal(t, "1", tmp[0].ID)
+
+ r1.gpus = gpu.GpuInfoList{gpus[1]}
+ tmp = s.filterGPUsWithoutLoadingModels(gpus)
+ require.Len(t, tmp, 1)
+ require.Equal(t, "0", tmp[0].ID)
+
+ r1.gpus = gpu.GpuInfoList{}
+ tmp = s.filterGPUsWithoutLoadingModels(gpus)
+ require.Len(t, tmp, 2)
+}
+
+func TestFindRunnerToUnload(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer done()
+
+ r1 := &runnerRef{refCount: 1, sessionDuration: 1, numParallel: 1}
+ r2 := &runnerRef{sessionDuration: 2, numParallel: 1}
+
+ s := InitScheduler(ctx)
+ s.loadedMu.Lock()
+ s.loaded["a"] = r1
+ s.loaded["b"] = r2
+ s.loadedMu.Unlock()
+
+ resp := s.findRunnerToUnload()
+ require.Equal(t, r2, resp)
+ r2.refCount = 1
+ resp = s.findRunnerToUnload()
+ require.Equal(t, r1, resp)
+}
+
+func TestNeedsReload(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer done()
+
+ llm := &mockLlm{estimatedVRAMByGPU: map[string]uint64{}}
+ do := api.DefaultOptions()
+ runner := &runnerRef{
+ model: &Model{
+ AdapterPaths: []string{"adapter1"},
+ ProjectorPaths: []string{"projector1"},
+ },
+ Options: &do,
+ llama: llm,
+ numParallel: 1,
+ }
+ req := &LlmRequest{
+ model: &Model{
+ AdapterPaths: []string{"adapter2"},
+ ProjectorPaths: []string{"projector2"},
+ },
+ opts: api.DefaultOptions(),
+ }
+ resp := runner.needsReload(ctx, req)
+ require.True(t, resp)
+ req.model.AdapterPaths = runner.model.AdapterPaths
+ resp = runner.needsReload(ctx, req)
+ require.True(t, resp)
+ req.model.ProjectorPaths = runner.model.ProjectorPaths
+ runner.loading = true
+ req.opts.NumBatch = 1234
+ resp = runner.needsReload(ctx, req)
+ require.True(t, resp)
+ req.opts.NumBatch = runner.Options.NumBatch
+ llm.pingResp = errors.New("foo")
+ resp = runner.needsReload(ctx, req)
+ require.True(t, resp)
+ llm.pingResp = nil
+ resp = runner.needsReload(ctx, req)
+ require.False(t, resp)
+ req.opts.NumGPU = 99
+ resp = runner.needsReload(ctx, req)
+ require.True(t, resp)
+ req.opts.NumGPU = -1
+ resp = runner.needsReload(ctx, req)
+ require.False(t, resp)
+}
+
+func TestUnloadAllRunners(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer done()
+
+ llm1 := &mockLlm{estimatedVRAMByGPU: map[string]uint64{}}
+ llm2 := &mockLlm{estimatedVRAMByGPU: map[string]uint64{}}
+ s := InitScheduler(ctx)
+ s.unloadAllRunners()
+
+ r1 := &runnerRef{llama: llm1, numParallel: 1}
+ r2 := &runnerRef{llama: llm2, numParallel: 1}
+
+ s.loadedMu.Lock()
+ s.loaded["a"] = r1
+ s.loaded["b"] = r2
+ s.loadedMu.Unlock()
+ s.unloadAllRunners()
+
+ require.True(t, llm1.closeCalled)
+ require.True(t, llm2.closeCalled)
+}
+
+func TestUnload(t *testing.T) {
+ llm1 := &mockLlm{estimatedVRAMByGPU: map[string]uint64{}}
+ r1 := &runnerRef{llama: llm1, numParallel: 1}
+ r2 := &runnerRef{model: &Model{AdapterPaths: []string{"A"}}, numParallel: 1}
+ r1.unload()
+ require.True(t, llm1.closeCalled)
+ r2.unload()
+ require.Nil(t, r2.model)
+}
+
+func TestAlreadyCanceled(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 500*time.Millisecond)
+ defer done()
+ dctx, done2 := context.WithCancel(ctx)
+ done2()
+ scenario1a := newScenarioRequest(t, dctx, "ollama-model-1", 10, &api.Duration{Duration: 0})
+ s := InitScheduler(ctx)
+ slog.Info("scenario1a")
+ s.pendingReqCh <- scenario1a.req
+ require.Len(t, s.pendingReqCh, 1)
+ s.Run(ctx)
+ time.Sleep(5 * time.Millisecond)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, scenario1a.req.errCh)
+ require.Empty(t, scenario1a.req.successCh)
+}
+
+func TestHomogeneousGPUs(t *testing.T) {
+ ctx, done := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer done()
+ s := InitScheduler(ctx)
+
+ s.getGpuFn = func() gpu.GpuInfoList {
+ // Set memory values to require the model to be spread
+ gpus := []gpu.GpuInfo{
+ {Library: "cuda"},
+ {Library: "rocm"},
+ }
+ gpus[0].TotalMemory = 1 * format.GibiByte
+ gpus[0].FreeMemory = 256 * format.MebiByte
+ gpus[1].TotalMemory = 1 * format.GibiByte
+ gpus[1].FreeMemory = 256 * format.MebiByte
+ return gpus
+ }
+ s.getCpuFn = getCpuFn
+ a := newScenarioRequest(t, ctx, "ollama-model-1", 10, &api.Duration{Duration: 5 * time.Millisecond})
+ s.newServerFn = func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {
+ require.Len(t, gpus, 1)
+ return a.newServer(gpus, model, ggml, adapters, projectors, opts, numParallel)
+ }
+ slog.Info("a")
+ s.pendingReqCh <- a.req
+ require.Len(t, s.pendingReqCh, 1)
+ s.Run(ctx)
+ select {
+ case resp := <-a.req.successCh:
+ require.Equal(t, resp.llama, a.srv)
+ require.Empty(t, s.pendingReqCh)
+ require.Empty(t, a.req.errCh)
+ case err := <-a.req.errCh:
+ t.Fatal(err.Error())
+ case <-ctx.Done():
+ t.Fatal("timeout")
+ }
+}
+
+type mockLlm struct {
+ pingResp error
+ waitResp error
+ completionResp error
+ embeddingResp []float32
+ embeddingRespErr error
+ tokenizeResp []int
+ tokenizeRespErr error
+ detokenizeResp string
+ detonekizeRespErr error
+ closeResp error
+ closeCalled bool
+ estimatedVRAM uint64
+ estimatedTotal uint64
+ estimatedVRAMByGPU map[string]uint64
+}
+
+func (s *mockLlm) Ping(ctx context.Context) error { return s.pingResp }
+func (s *mockLlm) WaitUntilRunning(ctx context.Context) error { return s.waitResp }
+func (s *mockLlm) Completion(ctx context.Context, req llm.CompletionRequest, fn func(llm.CompletionResponse)) error {
+ return s.completionResp
+}
+
+func (s *mockLlm) Embedding(ctx context.Context, input string) ([]float32, error) {
+ return s.embeddingResp, s.embeddingRespErr
+}
+
+func (s *mockLlm) Tokenize(ctx context.Context, content string) ([]int, error) {
+ return s.tokenizeResp, s.tokenizeRespErr
+}
+
+func (s *mockLlm) Detokenize(ctx context.Context, tokens []int) (string, error) {
+ return s.detokenizeResp, s.detonekizeRespErr
+}
+
+func (s *mockLlm) Close() error {
+ s.closeCalled = true
+ return s.closeResp
+}
+func (s *mockLlm) EstimatedVRAM() uint64 { return s.estimatedVRAM }
+func (s *mockLlm) EstimatedTotal() uint64 { return s.estimatedTotal }
+func (s *mockLlm) EstimatedVRAMByGPU(gpuid string) uint64 { return s.estimatedVRAMByGPU[gpuid] }
diff --git a/server/sparse_common.go b/server/sparse_common.go
new file mode 100644
index 00000000..c88b2da0
--- /dev/null
+++ b/server/sparse_common.go
@@ -0,0 +1,8 @@
+//go:build !windows
+
+package server
+
+import "os"
+
+func setSparse(*os.File) {
+}
diff --git a/server/sparse_windows.go b/server/sparse_windows.go
new file mode 100644
index 00000000..f21cbbda
--- /dev/null
+++ b/server/sparse_windows.go
@@ -0,0 +1,17 @@
+package server
+
+import (
+ "os"
+
+ "golang.org/x/sys/windows"
+)
+
+func setSparse(file *os.File) {
+ // exFat (and other FS types) don't support sparse files, so ignore errors
+ windows.DeviceIoControl( //nolint:errcheck
+ windows.Handle(file.Fd()), windows.FSCTL_SET_SPARSE,
+ nil, 0,
+ nil, 0,
+ nil, nil,
+ )
+}
diff --git a/server/testdata/tools/command-r-plus.gotmpl b/server/testdata/tools/command-r-plus.gotmpl
new file mode 100644
index 00000000..f30124e3
--- /dev/null
+++ b/server/testdata/tools/command-r-plus.gotmpl
@@ -0,0 +1,67 @@
+{{- if or .Tools .System }}<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>
+{{- if .Tools }}# Safety Preamble
+The instructions in this section override those in the task description and style guide sections. Don't answer questions that are harmful or immoral.
+
+# System Preamble
+## Basic Rules
+You are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user's requests, you cite your sources in your answers, according to those instructions.
+
+{{ if .System }}# User Preamble
+{{ .System }}
+{{- end }}
+
+## Available Tools
+Here is a list of tools that you have available to you:
+{{- range .Tools }}
+
+```python
+def {{ .Function.Name }}(
+{{- range $name, $property := .Function.Parameters.Properties }}{{ $name }}: {{ $property.Type }}, {{ end }}) -> List[Dict]:
+ """{{ .Function.Description }}
+
+{{- if .Function.Parameters.Properties }}
+
+ Args:
+{{- range $name, $property := .Function.Parameters.Properties }}
+ {{ $name }} ({{ $property.Type }}): {{ $property.Description }}
+{{- end }}
+{{- end }}
+ """
+ pass
+```
+{{- end }}
+{{- else if .System }}{{ .System }}
+{{- end }}<|END_OF_TURN_TOKEN|>
+{{- end }}
+{{- range .Messages }}
+{{- if eq .Role "system" }}
+{{- continue }}
+{{- end }}<|START_OF_TURN_TOKEN|>
+{{- if eq .Role "user" }}<|USER_TOKEN|>{{ .Content }}
+{{- else if eq .Role "assistant" }}<|CHATBOT_TOKEN|>
+{{- if .Content }}{{ .Content }}
+{{- else if .ToolCalls }}
+Action: ```json
+[
+{{- range .ToolCalls }}
+ {
+ "tool_name": "{{ .Function.Name }}",
+ "parameters": {{ .Function.Arguments }}
+ }
+{{- end }}
+]```
+{{ continue }}
+{{ end }}
+{{- else if eq .Role "tool" }}<|SYSTEM_TOKEN|><results>
+{{ .Content }}</results>
+{{- end }}<|END_OF_TURN_TOKEN|>
+{{- end }}
+{{- if .Tools }}<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Write 'Action:' followed by a json-formatted list of actions that you want to perform in order to produce a good response to the user's last input. You can use any of the supplied tools any number of times, but you should aim to execute the minimum number of necessary actions for the input. You should use the `directly-answer` tool if calling the other tools is unnecessary. The list of actions you want to call should be formatted as a list of json objects, for example:
+```json
+[
+ {
+ "tool_name": title of the tool in the specification,
+ "parameters": a dict of parameters to input into the tool as they are defined in the specs, or {} if it takes no parameters
+ }
+]```
+{{- end }}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
\ No newline at end of file
diff --git a/server/testdata/tools/command-r-plus.out b/server/testdata/tools/command-r-plus.out
new file mode 100644
index 00000000..425af75a
--- /dev/null
+++ b/server/testdata/tools/command-r-plus.out
@@ -0,0 +1,39 @@
+<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># Safety Preamble
+The instructions in this section override those in the task description and style guide sections. Don't answer questions that are harmful or immoral.
+
+# System Preamble
+## Basic Rules
+You are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user's requests, you cite your sources in your answers, according to those instructions.
+
+# User Preamble
+You are a knowledgable assistant. You can answer questions and perform tasks.
+
+## Available Tools
+Here is a list of tools that you have available to you:
+
+```python
+def get_current_weather(format: string, location: string, ) -> List[Dict]:
+ """Get the current weather
+
+ Args:
+ format (string): The temperature unit to use. Infer this from the users location.
+ location (string): The city and state, e.g. San Francisco, CA
+ """
+ pass
+```<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>What's the weather like today in Paris?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
+Action: ```json
+[
+ {
+ "tool_name": "get_current_weather",
+ "parameters": {"format":"celsius","location":"Paris, France"}
+ }
+]```
+<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><results>
+22</results><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>The current temperature in Paris, France is 22 degrees Celsius.<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>What's the weather like today in San Francisco and Toronto?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Write 'Action:' followed by a json-formatted list of actions that you want to perform in order to produce a good response to the user's last input. You can use any of the supplied tools any number of times, but you should aim to execute the minimum number of necessary actions for the input. You should use the `directly-answer` tool if calling the other tools is unnecessary. The list of actions you want to call should be formatted as a list of json objects, for example:
+```json
+[
+ {
+ "tool_name": title of the tool in the specification,
+ "parameters": a dict of parameters to input into the tool as they are defined in the specs, or {} if it takes no parameters
+ }
+]```<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
\ No newline at end of file
diff --git a/server/testdata/tools/firefunction.gotmpl b/server/testdata/tools/firefunction.gotmpl
new file mode 100644
index 00000000..312be205
--- /dev/null
+++ b/server/testdata/tools/firefunction.gotmpl
@@ -0,0 +1,31 @@
+{{- if or .System .Tools }}<|start_header_id|>system<|end_header_id|>
+{{- if .System }}
+{{ .System }}
+{{- end }}
+In addition to plain text responses, you can chose to call one or more of the provided functions.
+
+Use the following rule to decide when to call a function:
+ * if the response can be generated from your internal knowledge (e.g., as in the case of queries like "What is the capital of Poland?"), do so
+ * if you need external information that can be obtained by calling one or more of the provided functions, generate a function calls
+
+If you decide to call functions:
+ * prefix function calls with functools marker (no closing marker required)
+ * all function calls should be generated in a single JSON list formatted as functools[{"name": [function name], "arguments": [function arguments as JSON]}, ...]
+ * follow the provided JSON schema. Do not hallucinate arguments or values. Do to blindly copy values from the provided samples
+ * respect the argument type formatting. E.g., if the type if number and format is float, write value 7 as 7.0
+ * make sure you pick the right functions that match the user intent
+
+Available functions as JSON spec:
+{{- if .Tools }}
+{{ .Tools }}
+{{- end }}<|eot_id|>
+{{- end }}
+{{- range .Messages }}<|start_header_id|>
+{{- if or (eq .Role "user") (eq .Role "assistant") (eq .Role "tool") }}{{ .Role }}
+{{- end }}<|end_header_id|>
+{{- if .Content }}{{ .Content }}
+{{- else if .ToolCalls }} functools[
+{{- range .ToolCalls }}{{ "{" }}"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}{{ "}" }}
+{{- end }}]
+{{- end }}<|eot_id|>
+{{- end }}<|start_header_id|>assistant<|end_header_id|>
\ No newline at end of file
diff --git a/server/testdata/tools/firefunction.out b/server/testdata/tools/firefunction.out
new file mode 100644
index 00000000..be50175e
--- /dev/null
+++ b/server/testdata/tools/firefunction.out
@@ -0,0 +1,17 @@
+<|start_header_id|>system<|end_header_id|>
+You are a knowledgable assistant. You can answer questions and perform tasks.
+In addition to plain text responses, you can chose to call one or more of the provided functions.
+
+Use the following rule to decide when to call a function:
+ * if the response can be generated from your internal knowledge (e.g., as in the case of queries like "What is the capital of Poland?"), do so
+ * if you need external information that can be obtained by calling one or more of the provided functions, generate a function calls
+
+If you decide to call functions:
+ * prefix function calls with functools marker (no closing marker required)
+ * all function calls should be generated in a single JSON list formatted as functools[{"name": [function name], "arguments": [function arguments as JSON]}, ...]
+ * follow the provided JSON schema. Do not hallucinate arguments or values. Do to blindly copy values from the provided samples
+ * respect the argument type formatting. E.g., if the type if number and format is float, write value 7 as 7.0
+ * make sure you pick the right functions that match the user intent
+
+Available functions as JSON spec:
+[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the users location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}}]<|eot_id|><|start_header_id|><|end_header_id|>You are a knowledgable assistant. You can answer questions and perform tasks.<|eot_id|><|start_header_id|>user<|end_header_id|>What's the weather like today in Paris?<|eot_id|><|start_header_id|>assistant<|end_header_id|> functools[{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}}]<|eot_id|><|start_header_id|>tool<|end_header_id|>22<|eot_id|><|start_header_id|>assistant<|end_header_id|>The current temperature in Paris, France is 22 degrees Celsius.<|eot_id|><|start_header_id|>user<|end_header_id|>What's the weather like today in San Francisco and Toronto?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
\ No newline at end of file
diff --git a/server/testdata/tools/llama3-groq-tool-use.gotmpl b/server/testdata/tools/llama3-groq-tool-use.gotmpl
new file mode 100644
index 00000000..45e9b462
--- /dev/null
+++ b/server/testdata/tools/llama3-groq-tool-use.gotmpl
@@ -0,0 +1,43 @@
+{{- if .Messages }}
+{{- if or .System .Tools }}<|start_header_id|>system<|end_header_id|>
+
+{{ .System }}
+{{- if .Tools }} You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
+<tool_call>
+{"name": <function-name>,"arguments": <args-dict>}
+</tool_call>
+
+Here are the available tools:
+<tools>
+{{- range .Tools }} {{ .Function }}
+{{- end }} </tools>
+{{- end }}
+{{- end }}<|eot_id|>
+{{- range .Messages }}
+{{- if ne .Role "system" }}<|start_header_id|>{{ .Role }}<|end_header_id|>
+
+{{ if eq .Role "user" }}{{ .Content }}
+{{- else if eq .Role "assistant" }}
+{{- if .Content }}{{ .Content }}
+{{- else if .ToolCalls }}<tool_call>
+{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}}
+{{- end }}
+</tool_call>
+{{- end }}
+{{- else if eq .Role "tool" }}<tool_response>
+{{ .Content }}
+</tool_response>
+{{- end }}<|eot_id|>
+{{- end }}
+{{- end }}<|start_header_id|>assistant<|end_header_id|>
+
+{{ else }}
+{{ if .System }}<|start_header_id|>system<|end_header_id|>
+
+{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>
+
+{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>
+
+{{ end }}{{ .Response }}
+{{- if .Response }}<|eot_id|>
+{{- end }}
\ No newline at end of file
diff --git a/server/testdata/tools/llama3-groq-tool-use.out b/server/testdata/tools/llama3-groq-tool-use.out
new file mode 100644
index 00000000..75a49558
--- /dev/null
+++ b/server/testdata/tools/llama3-groq-tool-use.out
@@ -0,0 +1,24 @@
+<|start_header_id|>system<|end_header_id|>
+
+You are a knowledgable assistant. You can answer questions and perform tasks. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
+<tool_call>
+{"name": <function-name>,"arguments": <args-dict>}
+</tool_call>
+
+Here are the available tools:
+<tools> {"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the users location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}} </tools><|eot_id|><|start_header_id|>user<|end_header_id|>
+
+What's the weather like today in Paris?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+<tool_call>
+{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}}
+</tool_call><|eot_id|><|start_header_id|>tool<|end_header_id|>
+
+<tool_response>
+22
+</tool_response><|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+The current temperature in Paris, France is 22 degrees Celsius.<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+What's the weather like today in San Francisco and Toronto?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
diff --git a/server/testdata/tools/messages.json b/server/testdata/tools/messages.json
new file mode 100644
index 00000000..1a3d1f56
--- /dev/null
+++ b/server/testdata/tools/messages.json
@@ -0,0 +1,39 @@
+[
+ {
+ "role": "system",
+ "content": "You are a knowledgable assistant. You can answer questions and perform tasks."
+ },
+ {
+ "role": "user",
+ "content": "What's the weather like today in Paris?"
+ },
+ {
+ "role": "assistant",
+ "tool_calls": [
+ {
+ "id": "89a1e453-0bce-4de3-a456-c54bed09c520",
+ "type": "function",
+ "function": {
+ "name": "get_current_weather",
+ "arguments": {
+ "location": "Paris, France",
+ "format": "celsius"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "89a1e453-0bce-4de3-a456-c54bed09c520",
+ "content": "22"
+ },
+ {
+ "role": "assistant",
+ "content": "The current temperature in Paris, France is 22 degrees Celsius."
+ },
+ {
+ "role": "user",
+ "content": "What's the weather like today in San Francisco and Toronto?"
+ }
+]
diff --git a/server/testdata/tools/mistral.gotmpl b/server/testdata/tools/mistral.gotmpl
new file mode 100644
index 00000000..b08d6c2c
--- /dev/null
+++ b/server/testdata/tools/mistral.gotmpl
@@ -0,0 +1,15 @@
+{{- range $index, $_ := .Messages }}
+{{- if eq .Role "user" }}
+{{- if and (eq (len (slice $.Messages $index)) 1) $.Tools }}[AVAILABLE_TOOLS] {{ $.Tools }}[/AVAILABLE_TOOLS]
+{{- end }}[INST] {{ if and (eq (len (slice $.Messages $index)) 1) $.System }}{{ $.System }}
+
+{{ end }}{{ .Content }}[/INST]
+{{- else if eq .Role "assistant" }}
+{{- if .Content }} {{ .Content }}</s>
+{{- else if .ToolCalls }}[TOOL_CALLS] [
+{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}}
+{{- end }}]</s>
+{{- end }}
+{{- else if eq .Role "tool" }}[TOOL_RESULTS] {"content": {{ .Content }}}[/TOOL_RESULTS]
+{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/server/testdata/tools/mistral.out b/server/testdata/tools/mistral.out
new file mode 100644
index 00000000..31d8cdd6
--- /dev/null
+++ b/server/testdata/tools/mistral.out
@@ -0,0 +1,3 @@
+[INST] What's the weather like today in Paris?[/INST][TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}}]</s>[TOOL_RESULTS] {"content": 22}[/TOOL_RESULTS] The current temperature in Paris, France is 22 degrees Celsius.</s>[AVAILABLE_TOOLS] [{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the users location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}}][/AVAILABLE_TOOLS][INST] You are a knowledgable assistant. You can answer questions and perform tasks.
+
+What's the weather like today in San Francisco and Toronto?[/INST]
\ No newline at end of file
diff --git a/server/testdata/tools/nemotron.gotmpl b/server/testdata/tools/nemotron.gotmpl
new file mode 100644
index 00000000..1b6b89ec
--- /dev/null
+++ b/server/testdata/tools/nemotron.gotmpl
@@ -0,0 +1,33 @@
+{{- if (or .Tools .System) }}<extra_id_0>System
+{{ if .System }}{{ .System }}
+
+
+{{ end }}
+{{- if .Tools }}
+{{- range .Tools }}<tool> {{ . }} </tool>{{ end }}
+
+
+{{ end }}
+{{- end }}
+{{- range $i, $m := .Messages }}
+{{- $last := eq (len (slice $.Messages $i)) 1 -}}
+{{- if eq .Role "user" }}<extra_id_1>User
+{{ .Content }}
+{{- if $last }}
+<extra_id_1>Assistant
+{{- end }}
+{{ else if eq .Role "tool" }}<extra_id_1>Tool
+{{ .Content }}
+{{- if $last }}
+<extra_id_1>Assistant
+{{- end }}
+{{ else if eq .Role "assistant" }}<extra_id_1>Assistant
+{{- if .ToolCalls }}
+{{ range .ToolCalls }}<toolcall> {"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} </toolcall> {{ end }}
+{{ else }}
+{{ .Content }}
+{{- if not $last }}
+{{ end }}
+{{- end }}
+{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/server/testdata/tools/nemotron.out b/server/testdata/tools/nemotron.out
new file mode 100644
index 00000000..2166b202
--- /dev/null
+++ b/server/testdata/tools/nemotron.out
@@ -0,0 +1,18 @@
+<extra_id_0>System
+You are a knowledgable assistant. You can answer questions and perform tasks.
+
+
+<tool> {"type":"function","function":{"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the users location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}} </tool>
+
+
+<extra_id_1>User
+What's the weather like today in Paris?
+<extra_id_1>Assistant
+<toolcall> {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}} </toolcall>
+<extra_id_1>Tool
+22
+<extra_id_1>Assistant
+The current temperature in Paris, France is 22 degrees Celsius.
+<extra_id_1>User
+What's the weather like today in San Francisco and Toronto?
+<extra_id_1>Assistant
diff --git a/server/testdata/tools/tools.json b/server/testdata/tools/tools.json
new file mode 100644
index 00000000..17260bf8
--- /dev/null
+++ b/server/testdata/tools/tools.json
@@ -0,0 +1,30 @@
+[
+ {
+ "type": "function",
+ "function": {
+ "name": "get_current_weather",
+ "description": "Get the current weather",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city and state, e.g. San Francisco, CA"
+ },
+ "format": {
+ "type": "string",
+ "enum": [
+ "celsius",
+ "fahrenheit"
+ ],
+ "description": "The temperature unit to use. Infer this from the users location."
+ }
+ },
+ "required": [
+ "location",
+ "format"
+ ]
+ }
+ }
+ }
+]
diff --git a/server/testdata/tools/xlam.gotmpl b/server/testdata/tools/xlam.gotmpl
new file mode 100644
index 00000000..51513d69
--- /dev/null
+++ b/server/testdata/tools/xlam.gotmpl
@@ -0,0 +1,45 @@
+{{- if .System }}{{ .System }}
+{{ end }}
+{{- range $i, $_ := .Messages }}
+{{- if eq .Role "user" }}### Instruction:
+{{- if and $.Tools (le (len (slice $.Messages $i)) 2) }}
+[BEGIN OF TASK INSTRUCTION]
+You are an expert in composing functions. You are given a question and a set of possible functions.
+Based on the question, you will need to make one or more function/tool calls to achieve the purpose.
+If none of the functions can be used, point it out and refuse to answer.
+If the given question lacks the parameters required by the function, also point it out.
+[END OF TASK INSTRUCTION]
+
+[BEGIN OF AVAILABLE TOOLS]
+{{ $.Tools }}
+[END OF AVAILABLE TOOLS]
+
+[BEGIN OF FORMAT INSTRUCTION]
+The output MUST strictly adhere to the following JSON format, and NO other text MUST be included.
+The example format is as follows. Please make sure the parameter type is correct. If no function call is needed, please make tool_calls an empty list '[]'.
+```
+{
+ "tool_calls": [
+ {"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},
+ ... (more tool calls as required)
+ ]
+}
+```
+[END OF FORMAT INSTRUCTION]
+
+[BEGIN OF QUERY]
+{{ .Content }}
+[END OF QUERY]
+
+
+{{ else }}
+{{ .Content }}
+{{ end }}
+{{- else if .ToolCalls }}### Response:
+{"tool_calls": [{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}}{{ end }}]}
+<|EOT|>
+{{ else if eq .Role "assistant" }}### Response:
+{{ .Content }}
+<|EOT|>
+{{ end }}
+{{- end }}### Response:
\ No newline at end of file
diff --git a/server/testdata/tools/xlam.out b/server/testdata/tools/xlam.out
new file mode 100644
index 00000000..a4a9952f
--- /dev/null
+++ b/server/testdata/tools/xlam.out
@@ -0,0 +1,40 @@
+You are a knowledgable assistant. You can answer questions and perform tasks.
+### Instruction:
+What's the weather like today in Paris?
+### Response:
+{"tool_calls": [{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}}]}
+<|EOT|>
+### Response:
+The current temperature in Paris, France is 22 degrees Celsius.
+<|EOT|>
+### Instruction:
+[BEGIN OF TASK INSTRUCTION]
+You are an expert in composing functions. You are given a question and a set of possible functions.
+Based on the question, you will need to make one or more function/tool calls to achieve the purpose.
+If none of the functions can be used, point it out and refuse to answer.
+If the given question lacks the parameters required by the function, also point it out.
+[END OF TASK INSTRUCTION]
+
+[BEGIN OF AVAILABLE TOOLS]
+[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the users location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}}]
+[END OF AVAILABLE TOOLS]
+
+[BEGIN OF FORMAT INSTRUCTION]
+The output MUST strictly adhere to the following JSON format, and NO other text MUST be included.
+The example format is as follows. Please make sure the parameter type is correct. If no function call is needed, please make tool_calls an empty list '[]'.
+```
+{
+ "tool_calls": [
+ {"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},
+ ... (more tool calls as required)
+ ]
+}
+```
+[END OF FORMAT INSTRUCTION]
+
+[BEGIN OF QUERY]
+What's the weather like today in San Francisco and Toronto?
+[END OF QUERY]
+
+
+### Response:
\ No newline at end of file
diff --git a/server/upload.go b/server/upload.go
index 721c5e7c..020e8955 100644
--- a/server/upload.go
+++ b/server/upload.go
@@ -7,25 +7,26 @@ import (
"fmt"
"hash"
"io"
- "log"
+ "log/slog"
"math"
"net/http"
"net/url"
"os"
- "strings"
+ "strconv"
"sync"
"sync/atomic"
"time"
- "github.com/jmorganca/ollama/api"
- "github.com/jmorganca/ollama/format"
"golang.org/x/sync/errgroup"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/format"
)
var blobUploadManager sync.Map
type blobUpload struct {
- *Layer
+ Layer
Total int64
Completed atomic.Int64
@@ -44,12 +45,12 @@ type blobUpload struct {
}
const (
- numUploadParts = 64
+ numUploadParts = 16
minUploadPartSize int64 = 100 * format.MegaByte
maxUploadPartSize int64 = 1000 * format.MegaByte
)
-func (b *blobUpload) Prepare(ctx context.Context, requestURL *url.URL, opts *RegistryOptions) error {
+func (b *blobUpload) Prepare(ctx context.Context, requestURL *url.URL, opts *registryOptions) error {
p, err := GetBlobsPath(b.Digest)
if err != nil {
return err
@@ -88,7 +89,7 @@ func (b *blobUpload) Prepare(ctx context.Context, requestURL *url.URL, opts *Reg
return nil
}
- var size = b.Total / numUploadParts
+ size := b.Total / numUploadParts
switch {
case size < minUploadPartSize:
size = minUploadPartSize
@@ -107,7 +108,7 @@ func (b *blobUpload) Prepare(ctx context.Context, requestURL *url.URL, opts *Reg
offset += size
}
- log.Printf("uploading %s in %d %s part(s)", b.Digest[7:19], len(b.Parts), format.HumanBytes(b.Parts[0].Size))
+ slog.Info(fmt.Sprintf("uploading %s in %d %s part(s)", b.Digest[7:19], len(b.Parts), format.HumanBytes(b.Parts[0].Size)))
requestURL, err = url.Parse(location)
if err != nil {
@@ -121,7 +122,7 @@ func (b *blobUpload) Prepare(ctx context.Context, requestURL *url.URL, opts *Reg
// Run uploads blob parts to the upstream. If the upstream supports redirection, parts will be uploaded
// in parallel as defined by Prepare. Otherwise, parts will be uploaded serially. Run sets b.err on error.
-func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) {
+func (b *blobUpload) Run(ctx context.Context, opts *registryOptions) {
defer blobUploadManager.Delete(b.Digest)
ctx, b.CancelFunc = context.WithCancel(ctx)
@@ -147,7 +148,7 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) {
case requestURL := <-b.nextURL:
g.Go(func() error {
var err error
- for try := 0; try < maxRetries; try++ {
+ for try := range maxRetries {
err = b.uploadPart(inner, http.MethodPatch, requestURL, part, opts)
switch {
case errors.Is(err, context.Canceled):
@@ -156,7 +157,7 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) {
return err
case err != nil:
sleep := time.Second * time.Duration(math.Pow(2, float64(try)))
- log.Printf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep)
+ slog.Info(fmt.Sprintf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep))
time.Sleep(sleep)
continue
}
@@ -177,30 +178,28 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) {
requestURL := <-b.nextURL
// calculate md5 checksum and add it to the commit request
- var sb strings.Builder
+ md5sum := md5.New()
for _, part := range b.Parts {
- sb.Write(part.Sum(nil))
+ md5sum.Write(part.Sum(nil))
}
- md5sum := md5.Sum([]byte(sb.String()))
-
values := requestURL.Query()
values.Add("digest", b.Digest)
- values.Add("etag", fmt.Sprintf("%x-%d", md5sum, len(b.Parts)))
+ values.Add("etag", fmt.Sprintf("%x-%d", md5sum.Sum(nil), len(b.Parts)))
requestURL.RawQuery = values.Encode()
headers := make(http.Header)
headers.Set("Content-Type", "application/octet-stream")
headers.Set("Content-Length", "0")
- for try := 0; try < maxRetries; try++ {
+ for try := range maxRetries {
var resp *http.Response
resp, err = makeRequestWithRetry(ctx, http.MethodPut, requestURL, headers, nil, opts)
if errors.Is(err, context.Canceled) {
break
} else if err != nil {
sleep := time.Second * time.Duration(math.Pow(2, float64(try)))
- log.Printf("%s complete upload attempt %d failed: %v, retrying in %s", b.Digest[7:19], try, err, sleep)
+ slog.Info(fmt.Sprintf("%s complete upload attempt %d failed: %v, retrying in %s", b.Digest[7:19], try, err, sleep))
time.Sleep(sleep)
continue
}
@@ -212,10 +211,10 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) {
b.done = true
}
-func (b *blobUpload) uploadPart(ctx context.Context, method string, requestURL *url.URL, part *blobUploadPart, opts *RegistryOptions) error {
+func (b *blobUpload) uploadPart(ctx context.Context, method string, requestURL *url.URL, part *blobUploadPart, opts *registryOptions) error {
headers := make(http.Header)
headers.Set("Content-Type", "application/octet-stream")
- headers.Set("Content-Length", fmt.Sprintf("%d", part.Size))
+ headers.Set("Content-Length", strconv.FormatInt(part.Size, 10))
if method == http.MethodPatch {
headers.Set("X-Redirect-Uploads", "1")
@@ -256,8 +255,8 @@ func (b *blobUpload) uploadPart(ctx context.Context, method string, requestURL *
}
// retry uploading to the redirect URL
- for try := 0; try < maxRetries; try++ {
- err = b.uploadPart(ctx, http.MethodPut, redirectURL, part, nil)
+ for try := range maxRetries {
+ err = b.uploadPart(ctx, http.MethodPut, redirectURL, part, ®istryOptions{})
switch {
case errors.Is(err, context.Canceled):
return err
@@ -265,7 +264,7 @@ func (b *blobUpload) uploadPart(ctx context.Context, method string, requestURL *
return err
case err != nil:
sleep := time.Second * time.Duration(math.Pow(2, float64(try)))
- log.Printf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep)
+ slog.Info(fmt.Sprintf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep))
time.Sleep(sleep)
continue
}
@@ -277,9 +276,8 @@ func (b *blobUpload) uploadPart(ctx context.Context, method string, requestURL *
case resp.StatusCode == http.StatusUnauthorized:
w.Rollback()
- auth := resp.Header.Get("www-authenticate")
- authRedir := ParseAuthRedirectString(auth)
- token, err := getAuthToken(ctx, authRedir)
+ challenge := parseRegistryChallenge(resp.Header.Get("www-authenticate"))
+ token, err := getAuthorizationToken(ctx, challenge)
if err != nil {
return err
}
@@ -364,7 +362,7 @@ func (p *progressWriter) Rollback() {
p.written = 0
}
-func uploadBlob(ctx context.Context, mp ModelPath, layer *Layer, opts *RegistryOptions, fn func(api.ProgressResponse)) error {
+func uploadBlob(ctx context.Context, mp ModelPath, layer Layer, opts *registryOptions, fn func(api.ProgressResponse)) error {
requestURL := mp.BaseURL()
requestURL = requestURL.JoinPath("v2", mp.GetNamespaceRepository(), "blobs", layer.Digest)
@@ -395,6 +393,7 @@ func uploadBlob(ctx context.Context, mp ModelPath, layer *Layer, opts *RegistryO
return err
}
+ //nolint:contextcheck
go upload.Run(context.Background(), opts)
}
diff --git a/template/alfred.gotmpl b/template/alfred.gotmpl
new file mode 100644
index 00000000..86dba48f
--- /dev/null
+++ b/template/alfred.gotmpl
@@ -0,0 +1,2 @@
+{{- range .Messages }}<start_{{ .Role }}>{{ .Content }}<end_message>
+{{- end }}<start_assistant>
\ No newline at end of file
diff --git a/template/alfred.json b/template/alfred.json
new file mode 100644
index 00000000..edac21af
--- /dev/null
+++ b/template/alfred.json
@@ -0,0 +1,8 @@
+{
+ "stop": [
+ "<start_system>",
+ "<end_message>",
+ "<start_user>",
+ "<start_assistant>"
+ ]
+}
diff --git a/template/alpaca.gotmpl b/template/alpaca.gotmpl
new file mode 100644
index 00000000..00439736
--- /dev/null
+++ b/template/alpaca.gotmpl
@@ -0,0 +1,18 @@
+{{- $system := "" }}
+{{- range .Messages }}
+{{- if eq .Role "system" }}
+{{- if not $system }}{{ $system = .Content }}
+{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }}
+{{- end }}
+{{- else if eq .Role "user" }}
+{{- if $system }}{{ $system }}
+
+{{ $system = "" }}
+{{- end }}### Instruction:
+{{ .Content }}
+
+{{ else if eq .Role "assistant" }}### Response:
+{{ .Content }}
+
+{{ end }}
+{{- end }}### Response:
diff --git a/template/alpaca.json b/template/alpaca.json
new file mode 100644
index 00000000..eafe2b8a
--- /dev/null
+++ b/template/alpaca.json
@@ -0,0 +1,6 @@
+{
+ "stop": [
+ "### Instruction:",
+ "### Response"
+ ]
+}
diff --git a/template/chatml.gotmpl b/template/chatml.gotmpl
new file mode 100644
index 00000000..43207ab1
--- /dev/null
+++ b/template/chatml.gotmpl
@@ -0,0 +1,3 @@
+{{- range .Messages }}<|im_start|>{{ .Role }}
+{{ .Content }}<|im_end|>
+{{ end }}<|im_start|>assistant
diff --git a/template/chatml.json b/template/chatml.json
new file mode 100644
index 00000000..7afeb3de
--- /dev/null
+++ b/template/chatml.json
@@ -0,0 +1,6 @@
+{
+ "stop": [
+ "<|im_start|>",
+ "<|im_end|>"
+ ]
+}
diff --git a/template/chatqa.gotmpl b/template/chatqa.gotmpl
new file mode 100644
index 00000000..0f91e0f0
--- /dev/null
+++ b/template/chatqa.gotmpl
@@ -0,0 +1,7 @@
+{{- range .Messages }}
+{{- if eq .Role "system" }}System:
+{{- else if eq .Role "user" }}User:
+{{- else if eq .Role "assistant" }}Assistant:
+{{- end }} {{ .Content }}
+
+{{ end }}Assistant:
\ No newline at end of file
diff --git a/template/chatqa.json b/template/chatqa.json
new file mode 100644
index 00000000..64dd0f33
--- /dev/null
+++ b/template/chatqa.json
@@ -0,0 +1,8 @@
+{
+ "stop": [
+ "System:",
+ "User:",
+ "Assistant:",
+ "<|begin_of_text|>"
+ ]
+}
diff --git a/template/codellama-70b-instruct.gotmpl b/template/codellama-70b-instruct.gotmpl
new file mode 100644
index 00000000..18931520
--- /dev/null
+++ b/template/codellama-70b-instruct.gotmpl
@@ -0,0 +1,10 @@
+{{- range .Messages }}Source:
+{{- if eq .Role "system" }} system
+{{- else if eq .Role "user" }} user
+{{- else if eq .Role "assistant" }} assistant
+{{- end }}
+
+ {{ .Content }} <step> {{ end }}Source: assistant
+Destination: user
+
+
\ No newline at end of file
diff --git a/template/codellama-70b-instruct.json b/template/codellama-70b-instruct.json
new file mode 100644
index 00000000..a56a63f1
--- /dev/null
+++ b/template/codellama-70b-instruct.json
@@ -0,0 +1,7 @@
+{
+ "stop": [
+ "Source:",
+ "Destination:",
+ "<step>"
+ ]
+}
diff --git a/template/falcon-instruct.gotmpl b/template/falcon-instruct.gotmpl
new file mode 100644
index 00000000..b9b51d2c
--- /dev/null
+++ b/template/falcon-instruct.gotmpl
@@ -0,0 +1,8 @@
+{{- range .Messages }}
+{{- if eq .Role "system" }}System: {{ .Content }}
+{{ continue }}
+{{- else if eq .Role "user" }}User:
+{{- else if eq .Role "assistant" }}Falcon:
+{{- end }}
+{{ .Content }}
+{{ end }}Falcon:
diff --git a/template/falcon-instruct.json b/template/falcon-instruct.json
new file mode 100644
index 00000000..a0da0e81
--- /dev/null
+++ b/template/falcon-instruct.json
@@ -0,0 +1,6 @@
+{
+ "stop": [
+ "User:",
+ "Assistant:"
+ ]
+}
diff --git a/template/gemma-instruct.gotmpl b/template/gemma-instruct.gotmpl
new file mode 100644
index 00000000..cce25719
--- /dev/null
+++ b/template/gemma-instruct.gotmpl
@@ -0,0 +1,16 @@
+{{- $system := "" }}
+{{- range .Messages }}
+{{- if eq .Role "system" }}
+{{- if not $system }}{{ $system = .Content }}
+{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }}
+{{- end }}
+{{- continue }}
+{{- else if eq .Role "user" }}<start_of_turn>user
+{{- if $system }}
+{{ $system }}
+{{- $system = "" }}
+{{- end }}
+{{- else if eq .Role "assistant" }}<start_of_turn>model
+{{- end }}
+{{ .Content }}<end_of_turn>
+{{ end }}<start_of_turn>model
diff --git a/template/gemma-instruct.json b/template/gemma-instruct.json
new file mode 100644
index 00000000..f4ad415c
--- /dev/null
+++ b/template/gemma-instruct.json
@@ -0,0 +1,6 @@
+{
+ "stop": [
+ "<start_of_turn>",
+ "<end_of_turn>"
+ ]
+}
diff --git a/template/granite-instruct.gotmpl b/template/granite-instruct.gotmpl
new file mode 100644
index 00000000..83634990
--- /dev/null
+++ b/template/granite-instruct.gotmpl
@@ -0,0 +1,8 @@
+{{- range .Messages }}
+{{- if eq .Role "system" }}System:
+{{- else if eq .Role "user" }}Question:
+{{- else if eq .Role "assistant" }}Answer:
+{{- end }}
+{{ .Content }}
+
+{{ end }}Answer:
diff --git a/template/granite-instruct.json b/template/granite-instruct.json
new file mode 100644
index 00000000..0933e4b5
--- /dev/null
+++ b/template/granite-instruct.json
@@ -0,0 +1,7 @@
+{
+ "stop": [
+ "System:",
+ "Question:",
+ "Answer:"
+ ]
+}
diff --git a/template/index.json b/template/index.json
new file mode 100644
index 00000000..0ce6ac0f
--- /dev/null
+++ b/template/index.json
@@ -0,0 +1,142 @@
+[
+ {
+ "template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\\n' + content + '<|im_end|>\\n<|im_start|>assistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\\n' }}{% endif %}{% endfor %}",
+ "name": "chatml"
+ },
+ {
+ "template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+ "name": "chatml"
+ },
+ {
+ "template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+ "name": "zephyr"
+ },
+ {
+ "template": "{% if messages[0]['role'] == 'user' or messages[0]['role'] == 'system' %}{{ bos_token }}{% endif %}{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% elif messages[-1]['role'] == 'assistant' %}{{ eos_token }}{% endif %}",
+ "name": "chatml"
+ },
+ {
+ "template": "{{ bos_token }}{% for message in messages %}{{ 'GPT4 Correct ' + message['role'].title() + ': ' + message['content'] + '<|end_of_turn|>'}}{% endfor %}{% if add_generation_prompt %}{{ 'GPT4 Correct Assistant:' }}{% endif %}",
+ "name": "openchat"
+ },
+ {
+ "template": "{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+ "name": "chatml"
+ },
+ {
+ "template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+ "name": "chatml"
+ },
+ {
+ "template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+ "name": "chatml"
+ },
+ {
+ "template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+ "name": "chatml"
+ },
+ {
+ "template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+ "name": "zephyr"
+ },
+ {
+ "template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+ "name": "mistral-instruct"
+ },
+ {
+ "template": "{{bos_token}}{{'You are an exceptionally intelligent coding assistant that consistently delivers accurate and reliable responses to user instructions.\n\n'}}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n {{ raise_exception('System messages are not allowed in this template.') }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction\n' + message['content'] + '\n\n'}}\n {%- else %}\n{{'### Response\n' + message['content'] + eos_token + '\n\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{{'### Response\n'}}",
+ "name": "starcoder2-instruct"
+ },
+ {
+ "template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content | trim + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content | trim + ' ' + eos_token }}{% endif %}{% endfor %}",
+ "name": "llama2-chat"
+ },
+ {
+ "template": "{% if messages[0]['role'] == 'system' %}{% set user_index = 1 %}{% else %}{% set user_index = 0 %}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != ((loop.index0 + user_index) % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 %}{{ '<s>' }}{% endif %}{% set content = 'Source: ' + message['role'] + '\n\n ' + message['content'] | trim %}{{ content + ' <step> ' }}{% endfor %}{{'Source: assistant\nDestination: user\n\n '}}",
+ "name": "codellama-70b-instruct"
+ },
+ {
+ "template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+ "name": "mistral-instruct"
+ },
+ {
+ "template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|im_start|>user\n' + message['content'] + '<|im_end|>' }}\n{% elif message['role'] == 'system' %}\n{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>' }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|im_start|>assistant\n' + message['content'] + '<|im_end|>' }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|im_start|>assistant' }}\n{% endif %}\n{% endfor %}",
+ "name": "chatml"
+ },
+ {
+ "template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+ "name": "chatml"
+ },
+ {
+ "template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = 'You are a helpful assistant.' %}{% endif %}{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{{'<|im_start|>system\n' + system_message + '<|im_end|>\n'}}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+ "name": "chatml"
+ },
+ {
+ "template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif 'system' not in messages[0]['role'] %}{% set loop_messages = messages %}{% set system_message = 'You are DBRX, created by Databricks. You were last updated in December 2023. You answer questions based on information available up to that point.\nYOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, but provide thorough responses to more complex and open-ended questions.\nYou assist with various tasks, from writing to coding (using markdown for code blocks \u2014 remember to use ``` with code, JSON, and tables).\n(You do not have real-time data access or code execution capabilities. You avoid stereotyping and provide balanced perspectives on controversial topics. You do not provide song lyrics, poems, or news articles and do not divulge details of your training data.)\nThis is your system prompt, guiding your responses. Do not reference it, just respond to the user. If you find yourself talking about this message, stop. You should be responding appropriately and usually that means not mentioning this.\nYOU DO NOT MENTION ANY OF THIS INFORMATION ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY PERTINENT TO THE USER\\'S QUERY.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{% if system_message != false %}{{ '<|im_start|>system\n' + system_message | trim + '<|im_end|>\n'}}{% endif %}{{ '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' }}{% else %}{{ '\n' + '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' }}{% endif %}{% if (add_generation_prompt == true and loop.last) %}{{ '\n' + '<|im_start|>' + 'assistant' + '\n' }}{% endif %}{% endfor %}",
+ "name": "chatml"
+ },
+ {
+ "template": "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}",
+ "name": "alpaca"
+ },
+ {
+ "template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ 'User: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ message['content'] + '\n\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}",
+ "name": "chatqa"
+ },
+ {
+ "template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+ "name": "gemma-instruct"
+ },
+ {
+ "template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
+ "name": "llama3-instruct"
+ },
+ {
+ "template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n {%- for arg_name, arg_val in tool_call.arguments | items %}\n {{- arg_name + '=\"' + arg_val + '\"' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \")\" }}\n {%- else %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {%- endif %}\n {%- if builtin_tools is defined %}\n {#- This means we're in ipython mode #}\n {{- \"<|eom_id|>\" }}\n {%- else %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+ "name": "llama3-instruct"
+ },
+ {
+ "template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ 'Question:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'system' %}\n{{ 'System:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Answer:\n' + message['content'] + '\n\n' }}{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ 'Answer:\n' }}{% endif %}{% endfor %}",
+ "name": "granite-instruct"
+ },
+ {
+ "template": "{{bos_token}}{{'You are an exceptionally intelligent coding assistant that consistently delivers accurate and reliable responses to user instructions.\n\n'}}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n {{ raise_exception('System messages are not allowed in this template.') }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'@@ Instruction\n' + message['content'] + '\n\n'}}\n {%- else %}\n{{'@@ Response\n' + message['content'] + eos_token + '\n\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{{'@@ Response\n'}}",
+ "name": "magicoder"
+ },
+ {
+ "template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '<start_user>' + message['content'].strip() + '<end_message>' }}{% elif message['role'] == 'system' %}{{ '<start_system>' + message['content'].strip() + '<end_message>' }}{% elif message['role'] == 'assistant' %}{{ '<start_assistant>' + message['content'] + '<end_message>' }}{% else %}{{ raise_exception('Only system, user and assistant roles are supported.') }}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<start_assistant>' }}{% endif %}{% endfor %}",
+ "name": "alfred"
+ },
+ {
+ "template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+ "name": "llama2-chat"
+ },
+ {
+ "template": "{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
+ "name": "phi-3"
+ },
+ {
+ "template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
+ "name": "phi-3"
+ },
+ {
+ "template": "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+ "name": "phi-3"
+ },
+ {
+ "template": "{{ bos_token }}{%- if messages[0]['role'] == 'system' -%}{% set loop_messages = messages[1:] %}{%- else -%}{% set loop_messages = messages %}{% endif %}System: This is a chat between a user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions based on the context. The assistant should also indicate when the answer cannot be found in the context.\n\n{% for message in loop_messages %}{%- if message['role'] == 'user' -%}User: {{ message['content'].strip() + '\n\n' }}{%- else -%}Assistant: {{ message['content'].strip() + '\n\n' }}{%- endif %}{% if loop.last and message['role'] == 'user' %}Assistant:{% endif %}{% endfor %}",
+ "name": "chatqa"
+ },
+ {
+ "template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ 'User: \n' + message['content'] }}\n{% elif message['role'] == 'system' %}\n{{ 'System: ' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ 'Falcon:\n' + message['content']}}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ 'Falcon:' }}\n{% endif %}\n{% endfor %}",
+ "name": "falcon-instruct"
+ },
+ {
+ "template": "{% for message in messages %}{% if not loop.first %}{{ '\n' }}{% endif %}{% if message['role'] == 'system' %}{{ 'System: ' }}{% elif message['role'] == 'user' %}{{ 'User: ' }}{% elif message['role'] == 'assistant' %}{{ 'Falcon: ' }}{% endif %}{{ message['content'] }}{% endfor %}{% if add_generation_prompt %}{{ '\n' + 'Falcon:' }}{% endif %}",
+ "name": "falcon-instruct"
+ },
+ {
+ "template": "{% for message in messages %}{% if message['role'] == 'system' %}{% if message['content']%}{{'### System:\n' + message['content']+'\n\n'}}{% endif %}{% elif message['role'] == 'user' %}{{'### User:\n' + message['content']+'\n\n'}}{% elif message['role'] == 'assistant' %}{{'### Assistant:\n' + message['content']}}{% endif %}{% if loop.last and add_generation_prompt %}{{ '### Assistant:\n' }}{% endif %}{% endfor %}",
+ "name": "solar-instruct"
+ }
+]
diff --git a/template/llama2-chat.gotmpl b/template/llama2-chat.gotmpl
new file mode 100644
index 00000000..5634a072
--- /dev/null
+++ b/template/llama2-chat.gotmpl
@@ -0,0 +1,14 @@
+{{- $system := "" }}[INST] {{ range .Messages }}
+{{- if eq .Role "system" }}
+{{- if not $system }}{{ $system = .Content }}
+{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }}
+{{- end }}
+{{- else if eq .Role "user" }}<<SYS>>
+{{- if $system }}
+{{ $system }}
+{{ $system = "" }}
+{{- end }}<</SYS>>
+
+{{ .Content }} [/INST]
+{{- else if eq .Role "assistant" }} {{ .Content }}</s><s>[INST] {{ end }}
+{{- end }}
\ No newline at end of file
diff --git a/template/llama2-chat.json b/template/llama2-chat.json
new file mode 100644
index 00000000..17590ab4
--- /dev/null
+++ b/template/llama2-chat.json
@@ -0,0 +1,8 @@
+{
+ "stop": [
+ "[INST]",
+ "[/INST]",
+ "<<SYS>>",
+ "<</SYS>>"
+ ]
+}
diff --git a/template/llama3-instruct.gotmpl b/template/llama3-instruct.gotmpl
new file mode 100644
index 00000000..305ae403
--- /dev/null
+++ b/template/llama3-instruct.gotmpl
@@ -0,0 +1,5 @@
+{{- range .Messages }}<|start_header_id|>{{ .Role }}<|end_header_id|>
+
+{{ .Content }}<|eot_id|>
+{{- end }}<|start_header_id|>assistant<|end_header_id|>
+
diff --git a/template/llama3-instruct.json b/template/llama3-instruct.json
new file mode 100644
index 00000000..c4e9d448
--- /dev/null
+++ b/template/llama3-instruct.json
@@ -0,0 +1,7 @@
+{
+ "stop": [
+ "<|start_header_id|>",
+ "<|end_header_id|>",
+ "<|eot_id|>"
+ ]
+}
diff --git a/template/magicoder.gotmpl b/template/magicoder.gotmpl
new file mode 100644
index 00000000..e5ee0e47
--- /dev/null
+++ b/template/magicoder.gotmpl
@@ -0,0 +1,17 @@
+{{- $system := "" }}
+{{- range .Messages }}
+{{- if eq .Role "system" }}
+{{- if not $system }}{{ $system = .Content }}
+{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }}
+{{- end }}
+{{- continue }}
+{{- else if eq .Role "user" }}
+{{- if $system }}{{ $system }}
+
+{{ $system = "" }}
+{{- end }}@@ Instruction
+{{- else if eq .Role "assistant" }}@@ Response
+{{- end }}
+{{ .Content }}
+
+{{ end }}@@ Response
diff --git a/template/magicoder.json b/template/magicoder.json
new file mode 100644
index 00000000..6f67cab0
--- /dev/null
+++ b/template/magicoder.json
@@ -0,0 +1,6 @@
+{
+ "stop": [
+ "@@ Instruction",
+ "@@ Response"
+ ]
+}
diff --git a/template/mistral-instruct.gotmpl b/template/mistral-instruct.gotmpl
new file mode 100644
index 00000000..7a6ecdfd
--- /dev/null
+++ b/template/mistral-instruct.gotmpl
@@ -0,0 +1,6 @@
+[INST] {{ range $index, $_ := .Messages }}
+{{- if eq .Role "system" }}{{ .Content }}
+
+{{ else if eq .Role "user" }}{{ .Content }}[/INST]
+{{- else if eq .Role "assistant" }} {{ .Content }}</s>[INST] {{ end }}
+{{- end }}
\ No newline at end of file
diff --git a/template/mistral-instruct.json b/template/mistral-instruct.json
new file mode 100644
index 00000000..7afeb3de
--- /dev/null
+++ b/template/mistral-instruct.json
@@ -0,0 +1,6 @@
+{
+ "stop": [
+ "<|im_start|>",
+ "<|im_end|>"
+ ]
+}
diff --git a/template/openchat.gotmpl b/template/openchat.gotmpl
new file mode 100644
index 00000000..66a4d687
--- /dev/null
+++ b/template/openchat.gotmpl
@@ -0,0 +1,6 @@
+{{- range .Messages }}GPT4 Correct
+{{- if eq .Role "system" }} System:
+{{- else if eq .Role "user" }} User:
+{{- else if eq .Role "assistant" }} Assistant:
+{{- end }} {{ .Content }}<|end_of_turn|>
+{{- end }}GPT4 Correct Assistant:
\ No newline at end of file
diff --git a/template/openchat.json b/template/openchat.json
new file mode 100644
index 00000000..0edc341f
--- /dev/null
+++ b/template/openchat.json
@@ -0,0 +1,5 @@
+{
+ "stop": [
+ "<|end_of_turn|>"
+ ]
+}
diff --git a/template/phi-3.gotmpl b/template/phi-3.gotmpl
new file mode 100644
index 00000000..abec2137
--- /dev/null
+++ b/template/phi-3.gotmpl
@@ -0,0 +1,3 @@
+{{- range .Messages }}<|{{ .Role }}|>
+{{ .Content }}<|end|>
+{{ end }}<|assistant|>
diff --git a/template/phi-3.json b/template/phi-3.json
new file mode 100644
index 00000000..27bf7664
--- /dev/null
+++ b/template/phi-3.json
@@ -0,0 +1,8 @@
+{
+ "stop": [
+ "<|end|>",
+ "<|system|>",
+ "<|user|>",
+ "<|assistant|>"
+ ]
+}
diff --git a/template/solar-instruct.gotmpl b/template/solar-instruct.gotmpl
new file mode 100644
index 00000000..263bde80
--- /dev/null
+++ b/template/solar-instruct.gotmpl
@@ -0,0 +1,11 @@
+{{- range .Messages }}
+{{- if eq .Role "system" }}### System:
+{{- else if eq .Role "user" }}### User:
+{{- else if eq .Role "assistant" }}### Assistant:
+{{ .Content }}</s>
+
+{{ continue }}
+{{- end }}
+{{ .Content }}
+
+{{ end }}### Assistant:
diff --git a/template/solar-instruct.json b/template/solar-instruct.json
new file mode 100644
index 00000000..7b7a9050
--- /dev/null
+++ b/template/solar-instruct.json
@@ -0,0 +1,7 @@
+{
+ "stop": [
+ "### System:",
+ "### User:",
+ "### Assistant"
+ ]
+}
diff --git a/template/starcoder2-instruct.gotmpl b/template/starcoder2-instruct.gotmpl
new file mode 100644
index 00000000..7963b4f9
--- /dev/null
+++ b/template/starcoder2-instruct.gotmpl
@@ -0,0 +1,18 @@
+{{- $system := "" }}
+{{- range .Messages }}
+{{- if eq .Role "system" }}
+{{- if not $system }}{{ $system = .Content }}
+{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }}
+{{- end }}
+{{- else if eq .Role "user" }}
+{{- if $system }}{{ $system }}
+
+{{ $system = "" }}
+{{- end }}### Instruction
+{{ .Content }}
+
+{{ else if eq .Role "assistant" }}### Response
+{{ .Content }}<|endoftext|>
+
+{{ end }}
+{{- end }}### Response
diff --git a/template/starcoder2-instruct.json b/template/starcoder2-instruct.json
new file mode 100644
index 00000000..31348908
--- /dev/null
+++ b/template/starcoder2-instruct.json
@@ -0,0 +1,7 @@
+{
+ "stop": [
+ "### Instruction",
+ "### Response",
+ "<|endoftext|>"
+ ]
+}
diff --git a/template/template.go b/template/template.go
new file mode 100644
index 00000000..5dc484f4
--- /dev/null
+++ b/template/template.go
@@ -0,0 +1,447 @@
+package template
+
+import (
+ "bytes"
+ "embed"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "slices"
+ "strings"
+ "sync"
+ "text/template"
+ "text/template/parse"
+
+ "github.com/agnivade/levenshtein"
+ "golang.org/x/exp/maps"
+
+ "github.com/ollama/ollama/api"
+)
+
+//go:embed index.json
+var indexBytes []byte
+
+//go:embed *.gotmpl
+//go:embed *.json
+var templatesFS embed.FS
+
+var templatesOnce = sync.OnceValues(func() ([]*named, error) {
+ var templates []*named
+ if err := json.Unmarshal(indexBytes, &templates); err != nil {
+ return nil, err
+ }
+
+ for _, t := range templates {
+ bts, err := templatesFS.ReadFile(t.Name + ".gotmpl")
+ if err != nil {
+ return nil, err
+ }
+
+ // normalize line endings
+ t.Bytes = bytes.ReplaceAll(bts, []byte("\r\n"), []byte("\n"))
+
+ params, err := templatesFS.ReadFile(t.Name + ".json")
+ if err != nil {
+ continue
+ }
+
+ if err := json.Unmarshal(params, &t.Parameters); err != nil {
+ return nil, err
+ }
+ }
+
+ return templates, nil
+})
+
+type named struct {
+ Name string `json:"name"`
+ Template string `json:"template"`
+ Bytes []byte
+
+ Parameters *struct {
+ Stop []string `json:"stop"`
+ }
+}
+
+func (t named) Reader() io.Reader {
+ return bytes.NewReader(t.Bytes)
+}
+
+func Named(s string) (*named, error) {
+ templates, err := templatesOnce()
+ if err != nil {
+ return nil, err
+ }
+
+ var template *named
+ score := math.MaxInt
+ for _, t := range templates {
+ if s := levenshtein.ComputeDistance(s, t.Template); s < score {
+ score = s
+ template = t
+ }
+ }
+
+ if score < 100 {
+ return template, nil
+ }
+
+ return nil, errors.New("no matching template found")
+}
+
+var DefaultTemplate, _ = Parse("{{ .Prompt }}")
+
+type Template struct {
+ *template.Template
+ raw string
+}
+
+// response is a template node that can be added to templates that don't already have one
+var response = parse.ActionNode{
+ NodeType: parse.NodeAction,
+ Pipe: &parse.PipeNode{
+ NodeType: parse.NodePipe,
+ Cmds: []*parse.CommandNode{
+ {
+ NodeType: parse.NodeCommand,
+ Args: []parse.Node{
+ &parse.FieldNode{
+ NodeType: parse.NodeField,
+ Ident: []string{"Response"},
+ },
+ },
+ },
+ },
+ },
+}
+
+var funcs = template.FuncMap{
+ "json": func(v any) string {
+ b, _ := json.Marshal(v)
+ return string(b)
+ },
+}
+
+func Parse(s string) (*Template, error) {
+ tmpl := template.New("").Option("missingkey=zero").Funcs(funcs)
+
+ tmpl, err := tmpl.Parse(s)
+ if err != nil {
+ return nil, err
+ }
+
+ t := Template{Template: tmpl, raw: s}
+ if vars := t.Vars(); !slices.Contains(vars, "messages") && !slices.Contains(vars, "response") {
+ // touch up the template and append {{ .Response }}
+ tmpl.Tree.Root.Nodes = append(tmpl.Tree.Root.Nodes, &response)
+ }
+
+ return &t, nil
+}
+
+func (t *Template) String() string {
+ return t.raw
+}
+
+func (t *Template) Vars() []string {
+ var vars []string
+ for _, tt := range t.Templates() {
+ for _, n := range tt.Root.Nodes {
+ vars = append(vars, Identifiers(n)...)
+ }
+ }
+
+ set := make(map[string]struct{})
+ for _, n := range vars {
+ set[strings.ToLower(n)] = struct{}{}
+ }
+
+ vars = maps.Keys(set)
+ slices.Sort(vars)
+ return vars
+}
+
+type Values struct {
+ Messages []api.Message
+ api.Tools
+ Prompt string
+ Suffix string
+
+ // forceLegacy is a flag used to test compatibility with legacy templates
+ forceLegacy bool
+}
+
+func (t *Template) Subtree(fn func(parse.Node) bool) *template.Template {
+ var walk func(parse.Node) parse.Node
+ walk = func(n parse.Node) parse.Node {
+ if fn(n) {
+ return n
+ }
+
+ switch t := n.(type) {
+ case *parse.ListNode:
+ for _, c := range t.Nodes {
+ if n := walk(c); n != nil {
+ return n
+ }
+ }
+ case *parse.BranchNode:
+ for _, n := range []*parse.ListNode{t.List, t.ElseList} {
+ if n != nil {
+ if n := walk(n); n != nil {
+ return n
+ }
+ }
+ }
+ case *parse.IfNode:
+ return walk(&t.BranchNode)
+ case *parse.WithNode:
+ return walk(&t.BranchNode)
+ case *parse.RangeNode:
+ return walk(&t.BranchNode)
+ }
+
+ return nil
+ }
+
+ if n := walk(t.Tree.Root); n != nil {
+ return (&template.Template{
+ Tree: &parse.Tree{
+ Root: &parse.ListNode{
+ Nodes: []parse.Node{n},
+ },
+ },
+ }).Funcs(funcs)
+ }
+
+ return nil
+}
+
+func (t *Template) Execute(w io.Writer, v Values) error {
+ system, messages := collate(v.Messages)
+ if v.Prompt != "" && v.Suffix != "" {
+ return t.Template.Execute(w, map[string]any{
+ "Prompt": v.Prompt,
+ "Suffix": v.Suffix,
+ "Response": "",
+ })
+ } else if !v.forceLegacy && slices.Contains(t.Vars(), "messages") {
+ return t.Template.Execute(w, map[string]any{
+ "System": system,
+ "Messages": messages,
+ "Tools": v.Tools,
+ "Response": "",
+ })
+ }
+
+ system = ""
+ var b bytes.Buffer
+ var prompt, response string
+ for _, m := range messages {
+ execute := func() error {
+ if err := t.Template.Execute(&b, map[string]any{
+ "System": system,
+ "Prompt": prompt,
+ "Response": response,
+ }); err != nil {
+ return err
+ }
+
+ system = ""
+ prompt = ""
+ response = ""
+ return nil
+ }
+
+ switch m.Role {
+ case "system":
+ if prompt != "" || response != "" {
+ if err := execute(); err != nil {
+ return err
+ }
+ }
+ system = m.Content
+ case "user":
+ if response != "" {
+ if err := execute(); err != nil {
+ return err
+ }
+ }
+ prompt = m.Content
+ case "assistant":
+ response = m.Content
+ }
+ }
+
+ var cut bool
+ nodes := deleteNode(t.Template.Root.Copy(), func(n parse.Node) bool {
+ if field, ok := n.(*parse.FieldNode); ok && slices.Contains(field.Ident, "Response") {
+ cut = true
+ return false
+ }
+
+ return cut
+ })
+
+ tree := parse.Tree{Root: nodes.(*parse.ListNode)}
+ if err := template.Must(template.New("").AddParseTree("", &tree)).Execute(&b, map[string]any{
+ "System": system,
+ "Prompt": prompt,
+ "Response": response,
+ }); err != nil {
+ return err
+ }
+
+ _, err := io.Copy(w, &b)
+ return err
+}
+
+// collate messages based on role. consecutive messages of the same role are merged
+// into a single message. collate also collects and returns all system messages.
+// collate mutates message content adding image tags ([img-%d]) as needed
+func collate(msgs []api.Message) (string, []*api.Message) {
+ var n int
+
+ var system []string
+ var collated []*api.Message
+ for i := range msgs {
+ msg := msgs[i]
+ for range msg.Images {
+ imageTag := fmt.Sprintf("[img-%d]", n)
+ if !strings.Contains(msg.Content, "[img]") {
+ msg.Content = strings.TrimSpace("[img] " + msg.Content)
+ }
+
+ msg.Content = strings.Replace(msg.Content, "[img]", imageTag, 1)
+ n++
+ }
+
+ if msg.Role == "system" {
+ system = append(system, msg.Content)
+ }
+
+ if len(collated) > 0 && collated[len(collated)-1].Role == msg.Role {
+ collated[len(collated)-1].Content += "\n\n" + msg.Content
+ } else {
+ collated = append(collated, &msg)
+ }
+ }
+
+ return strings.Join(system, "\n\n"), collated
+}
+
+// Identifiers walks the node tree returning any identifiers it finds along the way
+func Identifiers(n parse.Node) []string {
+ switch n := n.(type) {
+ case *parse.ListNode:
+ var names []string
+ for _, n := range n.Nodes {
+ names = append(names, Identifiers(n)...)
+ }
+
+ return names
+ case *parse.TemplateNode:
+ return Identifiers(n.Pipe)
+ case *parse.ActionNode:
+ return Identifiers(n.Pipe)
+ case *parse.BranchNode:
+ names := Identifiers(n.Pipe)
+ for _, n := range []*parse.ListNode{n.List, n.ElseList} {
+ if n != nil {
+ names = append(names, Identifiers(n)...)
+ }
+ }
+ return names
+ case *parse.IfNode:
+ return Identifiers(&n.BranchNode)
+ case *parse.RangeNode:
+ return Identifiers(&n.BranchNode)
+ case *parse.WithNode:
+ return Identifiers(&n.BranchNode)
+ case *parse.PipeNode:
+ var names []string
+ for _, c := range n.Cmds {
+ for _, a := range c.Args {
+ names = append(names, Identifiers(a)...)
+ }
+ }
+ return names
+ case *parse.FieldNode:
+ return n.Ident
+ case *parse.VariableNode:
+ return n.Ident
+ }
+
+ return nil
+}
+
+// deleteNode walks the node list and deletes nodes that match the predicate
+// this is currently to remove the {{ .Response }} node from templates
+func deleteNode(n parse.Node, fn func(parse.Node) bool) parse.Node {
+ var walk func(n parse.Node) parse.Node
+ walk = func(n parse.Node) parse.Node {
+ if fn(n) {
+ return nil
+ }
+
+ switch t := n.(type) {
+ case *parse.ListNode:
+ var nodes []parse.Node
+ for _, c := range t.Nodes {
+ if n := walk(c); n != nil {
+ nodes = append(nodes, n)
+ }
+ }
+
+ t.Nodes = nodes
+ return t
+ case *parse.IfNode:
+ t.BranchNode = *(walk(&t.BranchNode).(*parse.BranchNode))
+ case *parse.WithNode:
+ t.BranchNode = *(walk(&t.BranchNode).(*parse.BranchNode))
+ case *parse.RangeNode:
+ t.BranchNode = *(walk(&t.BranchNode).(*parse.BranchNode))
+ case *parse.BranchNode:
+ t.List = walk(t.List).(*parse.ListNode)
+ if t.ElseList != nil {
+ t.ElseList = walk(t.ElseList).(*parse.ListNode)
+ }
+ case *parse.ActionNode:
+ n := walk(t.Pipe)
+ if n == nil {
+ return nil
+ }
+
+ t.Pipe = n.(*parse.PipeNode)
+ case *parse.PipeNode:
+ var commands []*parse.CommandNode
+ for _, c := range t.Cmds {
+ var args []parse.Node
+ for _, a := range c.Args {
+ if n := walk(a); n != nil {
+ args = append(args, n)
+ }
+ }
+
+ if len(args) == 0 {
+ return nil
+ }
+
+ c.Args = args
+ commands = append(commands, c)
+ }
+
+ if len(commands) == 0 {
+ return nil
+ }
+
+ t.Cmds = commands
+ }
+
+ return n
+ }
+
+ return walk(n)
+}
diff --git a/template/template_test.go b/template/template_test.go
new file mode 100644
index 00000000..113e0683
--- /dev/null
+++ b/template/template_test.go
@@ -0,0 +1,417 @@
+package template
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "io"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+)
+
+func TestNamed(t *testing.T) {
+ f, err := os.Open(filepath.Join("testdata", "templates.jsonl"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ var ss map[string]string
+ if err := json.Unmarshal(scanner.Bytes(), &ss); err != nil {
+ t.Fatal(err)
+ }
+
+ for k, v := range ss {
+ t.Run(k, func(t *testing.T) {
+ kv := llm.KV{"tokenizer.chat_template": v}
+ s := kv.ChatTemplate()
+ r, err := Named(s)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if r.Name != k {
+ t.Errorf("expected %q, got %q", k, r.Name)
+ }
+
+ var b bytes.Buffer
+ if _, err := io.Copy(&b, r.Reader()); err != nil {
+ t.Fatal(err)
+ }
+
+ tmpl, err := Parse(b.String())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if tmpl.Tree.Root.String() == "" {
+ t.Errorf("empty %s template", k)
+ }
+ })
+ }
+ }
+}
+
+func TestTemplate(t *testing.T) {
+ cases := make(map[string][]api.Message)
+ for _, mm := range [][]api.Message{
+ {
+ {Role: "user", Content: "Hello, how are you?"},
+ },
+ {
+ {Role: "user", Content: "Hello, how are you?"},
+ {Role: "assistant", Content: "I'm doing great. How can I help you today?"},
+ {Role: "user", Content: "I'd like to show off how chat templating works!"},
+ },
+ {
+ {Role: "system", Content: "You are a helpful assistant."},
+ {Role: "user", Content: "Hello, how are you?"},
+ {Role: "assistant", Content: "I'm doing great. How can I help you today?"},
+ {Role: "user", Content: "I'd like to show off how chat templating works!"},
+ },
+ } {
+ var roles []string
+ for _, m := range mm {
+ roles = append(roles, m.Role)
+ }
+
+ cases[strings.Join(roles, "-")] = mm
+ }
+
+ matches, err := filepath.Glob("*.gotmpl")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for _, match := range matches {
+ t.Run(match, func(t *testing.T) {
+ bts, err := os.ReadFile(match)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ tmpl, err := Parse(string(bts))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for n, tt := range cases {
+ var actual bytes.Buffer
+ t.Run(n, func(t *testing.T) {
+ if err := tmpl.Execute(&actual, Values{Messages: tt}); err != nil {
+ t.Fatal(err)
+ }
+
+ expect, err := os.ReadFile(filepath.Join("testdata", match, n))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ bts := actual.Bytes()
+
+ if slices.Contains([]string{"chatqa.gotmpl", "llama2-chat.gotmpl", "mistral-instruct.gotmpl", "openchat.gotmpl", "vicuna.gotmpl"}, match) && bts[len(bts)-1] == ' ' {
+ t.Log("removing trailing space from output")
+ bts = bts[:len(bts)-1]
+ }
+
+ if diff := cmp.Diff(bts, expect); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+
+ t.Run("legacy", func(t *testing.T) {
+ t.Skip("legacy outputs are currently default outputs")
+ var legacy bytes.Buffer
+ if err := tmpl.Execute(&legacy, Values{Messages: tt, forceLegacy: true}); err != nil {
+ t.Fatal(err)
+ }
+
+ legacyBytes := legacy.Bytes()
+ if slices.Contains([]string{"chatqa.gotmpl", "openchat.gotmpl", "vicuna.gotmpl"}, match) && legacyBytes[len(legacyBytes)-1] == ' ' {
+ t.Log("removing trailing space from legacy output")
+ legacyBytes = legacyBytes[:len(legacyBytes)-1]
+ } else if slices.Contains([]string{"codellama-70b-instruct.gotmpl", "llama2-chat.gotmpl", "mistral-instruct.gotmpl"}, match) {
+ t.Skip("legacy outputs cannot be compared to messages outputs")
+ }
+
+ if diff := cmp.Diff(legacyBytes, actual.Bytes()); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestParse(t *testing.T) {
+ cases := []struct {
+ template string
+ vars []string
+ }{
+ {"{{ .Prompt }}", []string{"prompt", "response"}},
+ {"{{ .System }} {{ .Prompt }}", []string{"prompt", "response", "system"}},
+ {"{{ .System }} {{ .Prompt }} {{ .Response }}", []string{"prompt", "response", "system"}},
+ {"{{ with .Tools }}{{ . }}{{ end }} {{ .System }} {{ .Prompt }}", []string{"prompt", "response", "system", "tools"}},
+ {"{{ range .Messages }}{{ .Role }} {{ .Content }}{{ end }}", []string{"content", "messages", "role"}},
+ {`{{- range .Messages }}
+{{- if eq .Role "system" }}SYSTEM:
+{{- else if eq .Role "user" }}USER:
+{{- else if eq .Role "assistant" }}ASSISTANT:
+{{- end }} {{ .Content }}
+{{- end }}`, []string{"content", "messages", "role"}},
+ {`{{- if .Messages }}
+{{- range .Messages }}<|im_start|>{{ .Role }}
+{{ .Content }}<|im_end|>
+{{ end }}<|im_start|>assistant
+{{ else -}}
+{{ if .System }}<|im_start|>system
+{{ .System }}<|im_end|>
+{{ end }}{{ if .Prompt }}<|im_start|>user
+{{ .Prompt }}<|im_end|>
+{{ end }}<|im_start|>assistant
+{{ .Response }}<|im_end|>
+{{- end -}}`, []string{"content", "messages", "prompt", "response", "role", "system"}},
+ }
+
+ for _, tt := range cases {
+ t.Run("", func(t *testing.T) {
+ tmpl, err := Parse(tt.template)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if diff := cmp.Diff(tmpl.Vars(), tt.vars); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+ }
+}
+
+func TestExecuteWithMessages(t *testing.T) {
+ type template struct {
+ name string
+ template string
+ }
+ cases := []struct {
+ name string
+ templates []template
+ values Values
+ expected string
+ }{
+ {
+ "mistral",
+ []template{
+ {"no response", `[INST] {{ if .System }}{{ .System }}
+
+{{ end }}{{ .Prompt }}[/INST] `},
+ {"response", `[INST] {{ if .System }}{{ .System }}
+
+{{ end }}{{ .Prompt }}[/INST] {{ .Response }}`},
+ {"messages", `[INST] {{ if .System }}{{ .System }}
+
+{{ end }}
+{{- range .Messages }}
+{{- if eq .Role "user" }}{{ .Content }}[/INST] {{ else if eq .Role "assistant" }}{{ .Content }}[INST] {{ end }}
+{{- end }}`},
+ },
+ Values{
+ Messages: []api.Message{
+ {Role: "user", Content: "Hello friend!"},
+ {Role: "assistant", Content: "Hello human!"},
+ {Role: "user", Content: "What is your name?"},
+ },
+ },
+ `[INST] Hello friend![/INST] Hello human![INST] What is your name?[/INST] `,
+ },
+ {
+ "mistral system",
+ []template{
+ {"no response", `[INST] {{ if .System }}{{ .System }}
+
+{{ end }}{{ .Prompt }}[/INST] `},
+ {"response", `[INST] {{ if .System }}{{ .System }}
+
+{{ end }}{{ .Prompt }}[/INST] {{ .Response }}`},
+ {"messages", `[INST] {{ if .System }}{{ .System }}
+
+{{ end }}
+{{- range .Messages }}
+{{- if eq .Role "user" }}{{ .Content }}[/INST] {{ else if eq .Role "assistant" }}{{ .Content }}[INST] {{ end }}
+{{- end }}`},
+ },
+ Values{
+ Messages: []api.Message{
+ {Role: "system", Content: "You are a helpful assistant!"},
+ {Role: "user", Content: "Hello friend!"},
+ {Role: "assistant", Content: "Hello human!"},
+ {Role: "user", Content: "What is your name?"},
+ },
+ },
+ `[INST] You are a helpful assistant!
+
+Hello friend![/INST] Hello human![INST] What is your name?[/INST] `,
+ },
+ {
+ "mistral assistant",
+ []template{
+ {"no response", `[INST] {{ .Prompt }}[/INST] `},
+ {"response", `[INST] {{ .Prompt }}[/INST] {{ .Response }}`},
+ {"messages", `
+{{- range $i, $m := .Messages }}
+{{- if eq .Role "user" }}[INST] {{ .Content }}[/INST] {{ else if eq .Role "assistant" }}{{ .Content }}{{ end }}
+{{- end }}`},
+ },
+ Values{
+ Messages: []api.Message{
+ {Role: "user", Content: "Hello friend!"},
+ {Role: "assistant", Content: "Hello human!"},
+ {Role: "user", Content: "What is your name?"},
+ {Role: "assistant", Content: "My name is Ollama and I"},
+ },
+ },
+ `[INST] Hello friend![/INST] Hello human![INST] What is your name?[/INST] My name is Ollama and I`,
+ },
+ {
+ "chatml",
+ []template{
+ // this does not have a "no response" test because it's impossible to render the same output
+ {"response", `{{ if .System }}<|im_start|>system
+{{ .System }}<|im_end|>
+{{ end }}{{ if .Prompt }}<|im_start|>user
+{{ .Prompt }}<|im_end|>
+{{ end }}<|im_start|>assistant
+{{ .Response }}<|im_end|>
+`},
+ {"messages", `
+{{- range $index, $_ := .Messages }}<|im_start|>{{ .Role }}
+{{ .Content }}<|im_end|>
+{{ end }}<|im_start|>assistant
+`},
+ },
+ Values{
+ Messages: []api.Message{
+ {Role: "system", Content: "You are a helpful assistant!"},
+ {Role: "user", Content: "Hello friend!"},
+ {Role: "assistant", Content: "Hello human!"},
+ {Role: "user", Content: "What is your name?"},
+ },
+ },
+ `<|im_start|>system
+You are a helpful assistant!<|im_end|>
+<|im_start|>user
+Hello friend!<|im_end|>
+<|im_start|>assistant
+Hello human!<|im_end|>
+<|im_start|>user
+What is your name?<|im_end|>
+<|im_start|>assistant
+`,
+ },
+ {
+ "moondream",
+ []template{
+ // this does not have a "no response" test because it's impossible to render the same output
+ {"response", `{{ if .Prompt }}Question: {{ .Prompt }}
+
+{{ end }}Answer: {{ .Response }}
+
+`},
+ {"messages", `
+{{- range .Messages }}
+{{- if eq .Role "user" }}Question: {{ .Content }}
+
+{{ else if eq .Role "assistant" }}Answer: {{ .Content }}
+
+{{ end }}
+{{- end }}Answer: `},
+ },
+ Values{
+ Messages: []api.Message{
+ {Role: "user", Content: "What's in this image?", Images: []api.ImageData{[]byte("")}},
+ {Role: "assistant", Content: "It's a hot dog."},
+ {Role: "user", Content: "What's in _this_ image?"},
+ {Role: "user", Images: []api.ImageData{[]byte("")}},
+ {Role: "user", Content: "Is it a hot dog?"},
+ },
+ },
+ `Question: [img-0] What's in this image?
+
+Answer: It's a hot dog.
+
+Question: What's in _this_ image?
+
+[img-1]
+
+Is it a hot dog?
+
+Answer: `,
+ },
+ }
+
+ for _, tt := range cases {
+ t.Run(tt.name, func(t *testing.T) {
+ for _, ttt := range tt.templates {
+ t.Run(ttt.name, func(t *testing.T) {
+ tmpl, err := Parse(ttt.template)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var b bytes.Buffer
+ if err := tmpl.Execute(&b, tt.values); err != nil {
+ t.Fatal(err)
+ }
+
+ if diff := cmp.Diff(b.String(), tt.expected); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestExecuteWithSuffix(t *testing.T) {
+ tmpl, err := Parse(`{{- if .Suffix }}<PRE> {{ .Prompt }} <SUF>{{ .Suffix }} <MID>
+{{- else }}{{ .Prompt }}
+{{- end }}`)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cases := []struct {
+ name string
+ values Values
+ expect string
+ }{
+ {
+ "message", Values{Messages: []api.Message{{Role: "user", Content: "hello"}}}, "hello",
+ },
+ {
+ "prompt suffix", Values{Prompt: "def add(", Suffix: "return x"}, "<PRE> def add( <SUF>return x <MID>",
+ },
+ }
+
+ for _, tt := range cases {
+ t.Run(tt.name, func(t *testing.T) {
+ var b bytes.Buffer
+ if err := tmpl.Execute(&b, tt.values); err != nil {
+ t.Fatal(err)
+ }
+
+ if diff := cmp.Diff(b.String(), tt.expect); diff != "" {
+ t.Errorf("mismatch (-got +want):\n%s", diff)
+ }
+ })
+ }
+}
diff --git a/template/testdata/alfred.gotmpl/system-user-assistant-user b/template/testdata/alfred.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..03e23ea9
--- /dev/null
+++ b/template/testdata/alfred.gotmpl/system-user-assistant-user
@@ -0,0 +1 @@
+<start_system>You are a helpful assistant.<end_message><start_user>Hello, how are you?<end_message><start_assistant>I'm doing great. How can I help you today?<end_message><start_user>I'd like to show off how chat templating works!<end_message><start_assistant>
\ No newline at end of file
diff --git a/template/testdata/alfred.gotmpl/user b/template/testdata/alfred.gotmpl/user
new file mode 100644
index 00000000..7c884a6f
--- /dev/null
+++ b/template/testdata/alfred.gotmpl/user
@@ -0,0 +1 @@
+<start_user>Hello, how are you?<end_message><start_assistant>
\ No newline at end of file
diff --git a/template/testdata/alfred.gotmpl/user-assistant-user b/template/testdata/alfred.gotmpl/user-assistant-user
new file mode 100644
index 00000000..a60701ed
--- /dev/null
+++ b/template/testdata/alfred.gotmpl/user-assistant-user
@@ -0,0 +1 @@
+<start_user>Hello, how are you?<end_message><start_assistant>I'm doing great. How can I help you today?<end_message><start_user>I'd like to show off how chat templating works!<end_message><start_assistant>
\ No newline at end of file
diff --git a/template/testdata/alpaca.gotmpl/system-user-assistant-user b/template/testdata/alpaca.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..4caa8178
--- /dev/null
+++ b/template/testdata/alpaca.gotmpl/system-user-assistant-user
@@ -0,0 +1,12 @@
+You are a helpful assistant.
+
+### Instruction:
+Hello, how are you?
+
+### Response:
+I'm doing great. How can I help you today?
+
+### Instruction:
+I'd like to show off how chat templating works!
+
+### Response:
diff --git a/template/testdata/alpaca.gotmpl/user b/template/testdata/alpaca.gotmpl/user
new file mode 100644
index 00000000..a0ce5dec
--- /dev/null
+++ b/template/testdata/alpaca.gotmpl/user
@@ -0,0 +1,4 @@
+### Instruction:
+Hello, how are you?
+
+### Response:
diff --git a/template/testdata/alpaca.gotmpl/user-assistant-user b/template/testdata/alpaca.gotmpl/user-assistant-user
new file mode 100644
index 00000000..6c5e23ff
--- /dev/null
+++ b/template/testdata/alpaca.gotmpl/user-assistant-user
@@ -0,0 +1,10 @@
+### Instruction:
+Hello, how are you?
+
+### Response:
+I'm doing great. How can I help you today?
+
+### Instruction:
+I'd like to show off how chat templating works!
+
+### Response:
diff --git a/template/testdata/chatml.gotmpl/system-user-assistant-user b/template/testdata/chatml.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..8b013fcf
--- /dev/null
+++ b/template/testdata/chatml.gotmpl/system-user-assistant-user
@@ -0,0 +1,9 @@
+<|im_start|>system
+You are a helpful assistant.<|im_end|>
+<|im_start|>user
+Hello, how are you?<|im_end|>
+<|im_start|>assistant
+I'm doing great. How can I help you today?<|im_end|>
+<|im_start|>user
+I'd like to show off how chat templating works!<|im_end|>
+<|im_start|>assistant
diff --git a/template/testdata/chatml.gotmpl/user b/template/testdata/chatml.gotmpl/user
new file mode 100644
index 00000000..aa9e597a
--- /dev/null
+++ b/template/testdata/chatml.gotmpl/user
@@ -0,0 +1,3 @@
+<|im_start|>user
+Hello, how are you?<|im_end|>
+<|im_start|>assistant
diff --git a/template/testdata/chatml.gotmpl/user-assistant-user b/template/testdata/chatml.gotmpl/user-assistant-user
new file mode 100644
index 00000000..a7cba4de
--- /dev/null
+++ b/template/testdata/chatml.gotmpl/user-assistant-user
@@ -0,0 +1,7 @@
+<|im_start|>user
+Hello, how are you?<|im_end|>
+<|im_start|>assistant
+I'm doing great. How can I help you today?<|im_end|>
+<|im_start|>user
+I'd like to show off how chat templating works!<|im_end|>
+<|im_start|>assistant
diff --git a/template/testdata/chatqa.gotmpl/system-user-assistant-user b/template/testdata/chatqa.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..98fd59bf
--- /dev/null
+++ b/template/testdata/chatqa.gotmpl/system-user-assistant-user
@@ -0,0 +1,9 @@
+System: You are a helpful assistant.
+
+User: Hello, how are you?
+
+Assistant: I'm doing great. How can I help you today?
+
+User: I'd like to show off how chat templating works!
+
+Assistant:
\ No newline at end of file
diff --git a/template/testdata/chatqa.gotmpl/user b/template/testdata/chatqa.gotmpl/user
new file mode 100644
index 00000000..9e7cf702
--- /dev/null
+++ b/template/testdata/chatqa.gotmpl/user
@@ -0,0 +1,3 @@
+User: Hello, how are you?
+
+Assistant:
\ No newline at end of file
diff --git a/template/testdata/chatqa.gotmpl/user-assistant-user b/template/testdata/chatqa.gotmpl/user-assistant-user
new file mode 100644
index 00000000..405bbe12
--- /dev/null
+++ b/template/testdata/chatqa.gotmpl/user-assistant-user
@@ -0,0 +1,7 @@
+User: Hello, how are you?
+
+Assistant: I'm doing great. How can I help you today?
+
+User: I'd like to show off how chat templating works!
+
+Assistant:
\ No newline at end of file
diff --git a/template/testdata/codellama-70b-instruct.gotmpl/system-user-assistant-user b/template/testdata/codellama-70b-instruct.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..d7528f80
--- /dev/null
+++ b/template/testdata/codellama-70b-instruct.gotmpl/system-user-assistant-user
@@ -0,0 +1,12 @@
+Source: system
+
+ You are a helpful assistant. <step> Source: user
+
+ Hello, how are you? <step> Source: assistant
+
+ I'm doing great. How can I help you today? <step> Source: user
+
+ I'd like to show off how chat templating works! <step> Source: assistant
+Destination: user
+
+
\ No newline at end of file
diff --git a/template/testdata/codellama-70b-instruct.gotmpl/user b/template/testdata/codellama-70b-instruct.gotmpl/user
new file mode 100644
index 00000000..8e07853c
--- /dev/null
+++ b/template/testdata/codellama-70b-instruct.gotmpl/user
@@ -0,0 +1,6 @@
+Source: user
+
+ Hello, how are you? <step> Source: assistant
+Destination: user
+
+
\ No newline at end of file
diff --git a/template/testdata/codellama-70b-instruct.gotmpl/user-assistant-user b/template/testdata/codellama-70b-instruct.gotmpl/user-assistant-user
new file mode 100644
index 00000000..f732cc74
--- /dev/null
+++ b/template/testdata/codellama-70b-instruct.gotmpl/user-assistant-user
@@ -0,0 +1,10 @@
+Source: user
+
+ Hello, how are you? <step> Source: assistant
+
+ I'm doing great. How can I help you today? <step> Source: user
+
+ I'd like to show off how chat templating works! <step> Source: assistant
+Destination: user
+
+
\ No newline at end of file
diff --git a/template/testdata/falcon-instruct.gotmpl/system-user-assistant-user b/template/testdata/falcon-instruct.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..16e45e5b
--- /dev/null
+++ b/template/testdata/falcon-instruct.gotmpl/system-user-assistant-user
@@ -0,0 +1,8 @@
+System: You are a helpful assistant.
+User:
+Hello, how are you?
+Falcon:
+I'm doing great. How can I help you today?
+User:
+I'd like to show off how chat templating works!
+Falcon:
diff --git a/template/testdata/falcon-instruct.gotmpl/user b/template/testdata/falcon-instruct.gotmpl/user
new file mode 100644
index 00000000..110831a2
--- /dev/null
+++ b/template/testdata/falcon-instruct.gotmpl/user
@@ -0,0 +1,3 @@
+User:
+Hello, how are you?
+Falcon:
diff --git a/template/testdata/falcon-instruct.gotmpl/user-assistant-user b/template/testdata/falcon-instruct.gotmpl/user-assistant-user
new file mode 100644
index 00000000..b49639ea
--- /dev/null
+++ b/template/testdata/falcon-instruct.gotmpl/user-assistant-user
@@ -0,0 +1,7 @@
+User:
+Hello, how are you?
+Falcon:
+I'm doing great. How can I help you today?
+User:
+I'd like to show off how chat templating works!
+Falcon:
diff --git a/template/testdata/gemma-instruct.gotmpl/system-user-assistant-user b/template/testdata/gemma-instruct.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..5f6c3732
--- /dev/null
+++ b/template/testdata/gemma-instruct.gotmpl/system-user-assistant-user
@@ -0,0 +1,8 @@
+<start_of_turn>user
+You are a helpful assistant.
+Hello, how are you?<end_of_turn>
+<start_of_turn>model
+I'm doing great. How can I help you today?<end_of_turn>
+<start_of_turn>user
+I'd like to show off how chat templating works!<end_of_turn>
+<start_of_turn>model
diff --git a/template/testdata/gemma-instruct.gotmpl/user b/template/testdata/gemma-instruct.gotmpl/user
new file mode 100644
index 00000000..dc8b30b6
--- /dev/null
+++ b/template/testdata/gemma-instruct.gotmpl/user
@@ -0,0 +1,3 @@
+<start_of_turn>user
+Hello, how are you?<end_of_turn>
+<start_of_turn>model
diff --git a/template/testdata/gemma-instruct.gotmpl/user-assistant-user b/template/testdata/gemma-instruct.gotmpl/user-assistant-user
new file mode 100644
index 00000000..1185924b
--- /dev/null
+++ b/template/testdata/gemma-instruct.gotmpl/user-assistant-user
@@ -0,0 +1,7 @@
+<start_of_turn>user
+Hello, how are you?<end_of_turn>
+<start_of_turn>model
+I'm doing great. How can I help you today?<end_of_turn>
+<start_of_turn>user
+I'd like to show off how chat templating works!<end_of_turn>
+<start_of_turn>model
diff --git a/template/testdata/granite-instruct.gotmpl/system-user-assistant-user b/template/testdata/granite-instruct.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..a732a77f
--- /dev/null
+++ b/template/testdata/granite-instruct.gotmpl/system-user-assistant-user
@@ -0,0 +1,13 @@
+System:
+You are a helpful assistant.
+
+Question:
+Hello, how are you?
+
+Answer:
+I'm doing great. How can I help you today?
+
+Question:
+I'd like to show off how chat templating works!
+
+Answer:
diff --git a/template/testdata/granite-instruct.gotmpl/user b/template/testdata/granite-instruct.gotmpl/user
new file mode 100644
index 00000000..7abd2ea3
--- /dev/null
+++ b/template/testdata/granite-instruct.gotmpl/user
@@ -0,0 +1,4 @@
+Question:
+Hello, how are you?
+
+Answer:
diff --git a/template/testdata/granite-instruct.gotmpl/user-assistant-user b/template/testdata/granite-instruct.gotmpl/user-assistant-user
new file mode 100644
index 00000000..da5e43ea
--- /dev/null
+++ b/template/testdata/granite-instruct.gotmpl/user-assistant-user
@@ -0,0 +1,10 @@
+Question:
+Hello, how are you?
+
+Answer:
+I'm doing great. How can I help you today?
+
+Question:
+I'd like to show off how chat templating works!
+
+Answer:
diff --git a/template/testdata/llama2-chat.gotmpl/system-user-assistant-user b/template/testdata/llama2-chat.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..9db81cb4
--- /dev/null
+++ b/template/testdata/llama2-chat.gotmpl/system-user-assistant-user
@@ -0,0 +1,7 @@
+[INST] <<SYS>>
+You are a helpful assistant.
+<</SYS>>
+
+Hello, how are you? [/INST] I'm doing great. How can I help you today?</s><s>[INST] <<SYS>><</SYS>>
+
+I'd like to show off how chat templating works! [/INST]
\ No newline at end of file
diff --git a/template/testdata/llama2-chat.gotmpl/user b/template/testdata/llama2-chat.gotmpl/user
new file mode 100644
index 00000000..ceef9bdb
--- /dev/null
+++ b/template/testdata/llama2-chat.gotmpl/user
@@ -0,0 +1,3 @@
+[INST] <<SYS>><</SYS>>
+
+Hello, how are you? [/INST]
\ No newline at end of file
diff --git a/template/testdata/llama2-chat.gotmpl/user-assistant-user b/template/testdata/llama2-chat.gotmpl/user-assistant-user
new file mode 100644
index 00000000..ca58954f
--- /dev/null
+++ b/template/testdata/llama2-chat.gotmpl/user-assistant-user
@@ -0,0 +1,5 @@
+[INST] <<SYS>><</SYS>>
+
+Hello, how are you? [/INST] I'm doing great. How can I help you today?</s><s>[INST] <<SYS>><</SYS>>
+
+I'd like to show off how chat templating works! [/INST]
\ No newline at end of file
diff --git a/template/testdata/llama3-instruct.gotmpl/system-user-assistant-user b/template/testdata/llama3-instruct.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..6740bcb4
--- /dev/null
+++ b/template/testdata/llama3-instruct.gotmpl/system-user-assistant-user
@@ -0,0 +1,10 @@
+<|start_header_id|>system<|end_header_id|>
+
+You are a helpful assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+Hello, how are you?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+I'm doing great. How can I help you today?<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+I'd like to show off how chat templating works!<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
diff --git a/template/testdata/llama3-instruct.gotmpl/user b/template/testdata/llama3-instruct.gotmpl/user
new file mode 100644
index 00000000..470aa028
--- /dev/null
+++ b/template/testdata/llama3-instruct.gotmpl/user
@@ -0,0 +1,4 @@
+<|start_header_id|>user<|end_header_id|>
+
+Hello, how are you?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
diff --git a/template/testdata/llama3-instruct.gotmpl/user-assistant-user b/template/testdata/llama3-instruct.gotmpl/user-assistant-user
new file mode 100644
index 00000000..6dd768af
--- /dev/null
+++ b/template/testdata/llama3-instruct.gotmpl/user-assistant-user
@@ -0,0 +1,8 @@
+<|start_header_id|>user<|end_header_id|>
+
+Hello, how are you?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+I'm doing great. How can I help you today?<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+I'd like to show off how chat templating works!<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
diff --git a/template/testdata/magicoder.gotmpl/system-user-assistant-user b/template/testdata/magicoder.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..c966a861
--- /dev/null
+++ b/template/testdata/magicoder.gotmpl/system-user-assistant-user
@@ -0,0 +1,12 @@
+You are a helpful assistant.
+
+@@ Instruction
+Hello, how are you?
+
+@@ Response
+I'm doing great. How can I help you today?
+
+@@ Instruction
+I'd like to show off how chat templating works!
+
+@@ Response
diff --git a/template/testdata/magicoder.gotmpl/user b/template/testdata/magicoder.gotmpl/user
new file mode 100644
index 00000000..ccfb02bd
--- /dev/null
+++ b/template/testdata/magicoder.gotmpl/user
@@ -0,0 +1,4 @@
+@@ Instruction
+Hello, how are you?
+
+@@ Response
diff --git a/template/testdata/magicoder.gotmpl/user-assistant-user b/template/testdata/magicoder.gotmpl/user-assistant-user
new file mode 100644
index 00000000..3aea6dab
--- /dev/null
+++ b/template/testdata/magicoder.gotmpl/user-assistant-user
@@ -0,0 +1,10 @@
+@@ Instruction
+Hello, how are you?
+
+@@ Response
+I'm doing great. How can I help you today?
+
+@@ Instruction
+I'd like to show off how chat templating works!
+
+@@ Response
diff --git a/template/testdata/mistral-instruct.gotmpl/system-user-assistant-user b/template/testdata/mistral-instruct.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..2f1edaec
--- /dev/null
+++ b/template/testdata/mistral-instruct.gotmpl/system-user-assistant-user
@@ -0,0 +1,3 @@
+[INST] You are a helpful assistant.
+
+Hello, how are you?[/INST] I'm doing great. How can I help you today?</s>[INST] I'd like to show off how chat templating works![/INST]
\ No newline at end of file
diff --git a/template/testdata/mistral-instruct.gotmpl/user b/template/testdata/mistral-instruct.gotmpl/user
new file mode 100644
index 00000000..b04871e5
--- /dev/null
+++ b/template/testdata/mistral-instruct.gotmpl/user
@@ -0,0 +1 @@
+[INST] Hello, how are you?[/INST]
\ No newline at end of file
diff --git a/template/testdata/mistral-instruct.gotmpl/user-assistant-user b/template/testdata/mistral-instruct.gotmpl/user-assistant-user
new file mode 100644
index 00000000..b473e0df
--- /dev/null
+++ b/template/testdata/mistral-instruct.gotmpl/user-assistant-user
@@ -0,0 +1 @@
+[INST] Hello, how are you?[/INST] I'm doing great. How can I help you today?</s>[INST] I'd like to show off how chat templating works![/INST]
\ No newline at end of file
diff --git a/template/testdata/openchat.gotmpl/system-user-assistant-user b/template/testdata/openchat.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..404b071a
--- /dev/null
+++ b/template/testdata/openchat.gotmpl/system-user-assistant-user
@@ -0,0 +1 @@
+GPT4 Correct System: You are a helpful assistant.<|end_of_turn|>GPT4 Correct User: Hello, how are you?<|end_of_turn|>GPT4 Correct Assistant: I'm doing great. How can I help you today?<|end_of_turn|>GPT4 Correct User: I'd like to show off how chat templating works!<|end_of_turn|>GPT4 Correct Assistant:
\ No newline at end of file
diff --git a/template/testdata/openchat.gotmpl/user b/template/testdata/openchat.gotmpl/user
new file mode 100644
index 00000000..48229cb0
--- /dev/null
+++ b/template/testdata/openchat.gotmpl/user
@@ -0,0 +1 @@
+GPT4 Correct User: Hello, how are you?<|end_of_turn|>GPT4 Correct Assistant:
\ No newline at end of file
diff --git a/template/testdata/openchat.gotmpl/user-assistant-user b/template/testdata/openchat.gotmpl/user-assistant-user
new file mode 100644
index 00000000..4719abb2
--- /dev/null
+++ b/template/testdata/openchat.gotmpl/user-assistant-user
@@ -0,0 +1 @@
+GPT4 Correct User: Hello, how are you?<|end_of_turn|>GPT4 Correct Assistant: I'm doing great. How can I help you today?<|end_of_turn|>GPT4 Correct User: I'd like to show off how chat templating works!<|end_of_turn|>GPT4 Correct Assistant:
\ No newline at end of file
diff --git a/template/testdata/phi-3.gotmpl/system-user-assistant-user b/template/testdata/phi-3.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..6109a9a2
--- /dev/null
+++ b/template/testdata/phi-3.gotmpl/system-user-assistant-user
@@ -0,0 +1,9 @@
+<|system|>
+You are a helpful assistant.<|end|>
+<|user|>
+Hello, how are you?<|end|>
+<|assistant|>
+I'm doing great. How can I help you today?<|end|>
+<|user|>
+I'd like to show off how chat templating works!<|end|>
+<|assistant|>
diff --git a/template/testdata/phi-3.gotmpl/user b/template/testdata/phi-3.gotmpl/user
new file mode 100644
index 00000000..feb96e7c
--- /dev/null
+++ b/template/testdata/phi-3.gotmpl/user
@@ -0,0 +1,3 @@
+<|user|>
+Hello, how are you?<|end|>
+<|assistant|>
diff --git a/template/testdata/phi-3.gotmpl/user-assistant-user b/template/testdata/phi-3.gotmpl/user-assistant-user
new file mode 100644
index 00000000..db79d01c
--- /dev/null
+++ b/template/testdata/phi-3.gotmpl/user-assistant-user
@@ -0,0 +1,7 @@
+<|user|>
+Hello, how are you?<|end|>
+<|assistant|>
+I'm doing great. How can I help you today?<|end|>
+<|user|>
+I'd like to show off how chat templating works!<|end|>
+<|assistant|>
diff --git a/template/testdata/solar-instruct.gotmpl/system-user-assistant-user b/template/testdata/solar-instruct.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..28c1730a
--- /dev/null
+++ b/template/testdata/solar-instruct.gotmpl/system-user-assistant-user
@@ -0,0 +1,13 @@
+### System:
+You are a helpful assistant.
+
+### User:
+Hello, how are you?
+
+### Assistant:
+I'm doing great. How can I help you today?</s>
+
+### User:
+I'd like to show off how chat templating works!
+
+### Assistant:
diff --git a/template/testdata/solar-instruct.gotmpl/user b/template/testdata/solar-instruct.gotmpl/user
new file mode 100644
index 00000000..3a43382a
--- /dev/null
+++ b/template/testdata/solar-instruct.gotmpl/user
@@ -0,0 +1,4 @@
+### User:
+Hello, how are you?
+
+### Assistant:
diff --git a/template/testdata/solar-instruct.gotmpl/user-assistant-user b/template/testdata/solar-instruct.gotmpl/user-assistant-user
new file mode 100644
index 00000000..8553e73b
--- /dev/null
+++ b/template/testdata/solar-instruct.gotmpl/user-assistant-user
@@ -0,0 +1,10 @@
+### User:
+Hello, how are you?
+
+### Assistant:
+I'm doing great. How can I help you today?</s>
+
+### User:
+I'd like to show off how chat templating works!
+
+### Assistant:
diff --git a/template/testdata/starcoder2-instruct.gotmpl/system-user-assistant-user b/template/testdata/starcoder2-instruct.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..5b718b3e
--- /dev/null
+++ b/template/testdata/starcoder2-instruct.gotmpl/system-user-assistant-user
@@ -0,0 +1,12 @@
+You are a helpful assistant.
+
+### Instruction
+Hello, how are you?
+
+### Response
+I'm doing great. How can I help you today?<|endoftext|>
+
+### Instruction
+I'd like to show off how chat templating works!
+
+### Response
diff --git a/template/testdata/starcoder2-instruct.gotmpl/user b/template/testdata/starcoder2-instruct.gotmpl/user
new file mode 100644
index 00000000..11b0be1f
--- /dev/null
+++ b/template/testdata/starcoder2-instruct.gotmpl/user
@@ -0,0 +1,4 @@
+### Instruction
+Hello, how are you?
+
+### Response
diff --git a/template/testdata/starcoder2-instruct.gotmpl/user-assistant-user b/template/testdata/starcoder2-instruct.gotmpl/user-assistant-user
new file mode 100644
index 00000000..d99feabb
--- /dev/null
+++ b/template/testdata/starcoder2-instruct.gotmpl/user-assistant-user
@@ -0,0 +1,10 @@
+### Instruction
+Hello, how are you?
+
+### Response
+I'm doing great. How can I help you today?<|endoftext|>
+
+### Instruction
+I'd like to show off how chat templating works!
+
+### Response
diff --git a/template/testdata/templates.jsonl b/template/testdata/templates.jsonl
new file mode 100644
index 00000000..41a8d0ef
--- /dev/null
+++ b/template/testdata/templates.jsonl
@@ -0,0 +1,35 @@
+{"chatml": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\\n' + content + '<|im_end|>\\n<|im_start|>assistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\\n' }}{% endif %}{% endfor %}"}
+{"chatml": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"}
+{"zephyr": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}"}
+{"chatml": "{% if messages[0]['role'] == 'user' or messages[0]['role'] == 'system' %}{{ bos_token }}{% endif %}{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% elif messages[-1]['role'] == 'assistant' %}{{ eos_token }}{% endif %}"}
+{"openchat": "{{ bos_token }}{% for message in messages %}{{ 'GPT4 Correct ' + message['role'].title() + ': ' + message['content'] + '<|end_of_turn|>'}}{% endfor %}{% if add_generation_prompt %}{{ 'GPT4 Correct Assistant:' }}{% endif %}"}
+{"chatml": "{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"}
+{"chatml": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"}
+{"chatml": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"}
+{"chatml": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"}
+{"zephyr": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}"}
+{"mistral-instruct": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}"}
+{"starcoder2-instruct": "{{bos_token}}{{'You are an exceptionally intelligent coding assistant that consistently delivers accurate and reliable responses to user instructions.\n\n'}}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n {{ raise_exception('System messages are not allowed in this template.') }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction\n' + message['content'] + '\n\n'}}\n {%- else %}\n{{'### Response\n' + message['content'] + eos_token + '\n\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{{'### Response\n'}}"}
+{"llama2-chat": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content | trim + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content | trim + ' ' + eos_token }}{% endif %}{% endfor %}"}
+{"codellama-70b-instruct": "{% if messages[0]['role'] == 'system' %}{% set user_index = 1 %}{% else %}{% set user_index = 0 %}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != ((loop.index0 + user_index) % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 %}{{ '<s>' }}{% endif %}{% set content = 'Source: ' + message['role'] + '\n\n ' + message['content'] | trim %}{{ content + ' <step> ' }}{% endfor %}{{'Source: assistant\nDestination: user\n\n '}}"}
+{"mistral-instruct": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}"}
+{"chatml": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|im_start|>user\n' + message['content'] + '<|im_end|>' }}\n{% elif message['role'] == 'system' %}\n{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>' }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|im_start|>assistant\n' + message['content'] + '<|im_end|>' }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|im_start|>assistant' }}\n{% endif %}\n{% endfor %}"}
+{"chatml": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"}
+{"chatml": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = 'You are a helpful assistant.' %}{% endif %}{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{{'<|im_start|>system\n' + system_message + '<|im_end|>\n'}}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"}
+{"chatml": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif 'system' not in messages[0]['role'] %}{% set loop_messages = messages %}{% set system_message = 'You are DBRX, created by Databricks. You were last updated in December 2023. You answer questions based on information available up to that point.\nYOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, but provide thorough responses to more complex and open-ended questions.\nYou assist with various tasks, from writing to coding (using markdown for code blocks \u2014 remember to use ``` with code, JSON, and tables).\n(You do not have real-time data access or code execution capabilities. You avoid stereotyping and provide balanced perspectives on controversial topics. You do not provide song lyrics, poems, or news articles and do not divulge details of your training data.)\nThis is your system prompt, guiding your responses. Do not reference it, just respond to the user. If you find yourself talking about this message, stop. You should be responding appropriately and usually that means not mentioning this.\nYOU DO NOT MENTION ANY OF THIS INFORMATION ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY PERTINENT TO THE USER\\'S QUERY.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{% if system_message != false %}{{ '<|im_start|>system\n' + system_message | trim + '<|im_end|>\n'}}{% endif %}{{ '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' }}{% else %}{{ '\n' + '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' }}{% endif %}{% if (add_generation_prompt == true and loop.last) %}{{ '\n' + '<|im_start|>' + 'assistant' + '\n' }}{% endif %}{% endfor %}"}
+{"alpaca": "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}"}
+{"chatqa": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ 'User: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ message['content'] + '\n\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}"}
+{"gemma-instruct": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}"}
+{"llama3-instruct": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}"}
+{"granite-instruct": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ 'Question:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'system' %}\n{{ 'System:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Answer:\n' + message['content'] + '\n\n' }}{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ 'Answer:\n' }}{% endif %}{% endfor %}"}
+{"magicoder": "{{bos_token}}{{'You are an exceptionally intelligent coding assistant that consistently delivers accurate and reliable responses to user instructions.\n\n'}}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n {{ raise_exception('System messages are not allowed in this template.') }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'@@ Instruction\n' + message['content'] + '\n\n'}}\n {%- else %}\n{{'@@ Response\n' + message['content'] + eos_token + '\n\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{{'@@ Response\n'}}"}
+{"alfred": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '<start_user>' + message['content'].strip() + '<end_message>' }}{% elif message['role'] == 'system' %}{{ '<start_system>' + message['content'].strip() + '<end_message>' }}{% elif message['role'] == 'assistant' %}{{ '<start_assistant>' + message['content'] + '<end_message>' }}{% else %}{{ raise_exception('Only system, user and assistant roles are supported.') }}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<start_assistant>' }}{% endif %}{% endfor %}"}
+{"llama2-chat": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}"}
+{"phi-3": "{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}"}
+{"phi-3": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}"}
+{"phi-3": "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}"}
+{"chatqa": "{{ bos_token }}{%- if messages[0]['role'] == 'system' -%}{% set loop_messages = messages[1:] %}{%- else -%}{% set loop_messages = messages %}{% endif %}System: This is a chat between a user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions based on the context. The assistant should also indicate when the answer cannot be found in the context.\n\n{% for message in loop_messages %}{%- if message['role'] == 'user' -%}User: {{ message['content'].strip() + '\n\n' }}{%- else -%}Assistant: {{ message['content'].strip() + '\n\n' }}{%- endif %}{% if loop.last and message['role'] == 'user' %}Assistant:{% endif %}{% endfor %}"}
+{"falcon-instruct": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ 'User: \n' + message['content'] }}\n{% elif message['role'] == 'system' %}\n{{ 'System: ' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ 'Falcon:\n' + message['content']}}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ 'Falcon:' }}\n{% endif %}\n{% endfor %}"}
+{"falcon-instruct": "{% for message in messages %}{% if not loop.first %}{{ '\n' }}{% endif %}{% if message['role'] == 'system' %}{{ 'System: ' }}{% elif message['role'] == 'user' %}{{ 'User: ' }}{% elif message['role'] == 'assistant' %}{{ 'Falcon: ' }}{% endif %}{{ message['content'] }}{% endfor %}{% if add_generation_prompt %}{{ '\n' + 'Falcon:' }}{% endif %}"}
+{"solar-instruct": "{% for message in messages %}{% if message['role'] == 'system' %}{% if message['content']%}{{'### System:\n' + message['content']+'\n\n'}}{% endif %}{% elif message['role'] == 'user' %}{{'### User:\n' + message['content']+'\n\n'}}{% elif message['role'] == 'assistant' %}{{'### Assistant:\n' + message['content']}}{% endif %}{% if loop.last and add_generation_prompt %}{{ '### Assistant:\n' }}{% endif %}{% endfor %}"}
+{"chatml": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"}
diff --git a/template/testdata/vicuna.gotmpl/system-user-assistant-user b/template/testdata/vicuna.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..50d2f92c
--- /dev/null
+++ b/template/testdata/vicuna.gotmpl/system-user-assistant-user
@@ -0,0 +1,6 @@
+You are a helpful assistant.
+
+USER: Hello, how are you?
+ASSISTANT: I'm doing great. How can I help you today?</s>
+USER: I'd like to show off how chat templating works!
+ASSISTANT:
\ No newline at end of file
diff --git a/template/testdata/vicuna.gotmpl/user b/template/testdata/vicuna.gotmpl/user
new file mode 100644
index 00000000..cbe5ef70
--- /dev/null
+++ b/template/testdata/vicuna.gotmpl/user
@@ -0,0 +1,2 @@
+USER: Hello, how are you?
+ASSISTANT:
\ No newline at end of file
diff --git a/template/testdata/vicuna.gotmpl/user-assistant-user b/template/testdata/vicuna.gotmpl/user-assistant-user
new file mode 100644
index 00000000..9172547e
--- /dev/null
+++ b/template/testdata/vicuna.gotmpl/user-assistant-user
@@ -0,0 +1,4 @@
+USER: Hello, how are you?
+ASSISTANT: I'm doing great. How can I help you today?</s>
+USER: I'd like to show off how chat templating works!
+ASSISTANT:
\ No newline at end of file
diff --git a/template/testdata/zephyr.gotmpl/system-user-assistant-user b/template/testdata/zephyr.gotmpl/system-user-assistant-user
new file mode 100644
index 00000000..03d43fc3
--- /dev/null
+++ b/template/testdata/zephyr.gotmpl/system-user-assistant-user
@@ -0,0 +1,9 @@
+<|system|>
+You are a helpful assistant.</s>
+<|user|>
+Hello, how are you?</s>
+<|assistant|>
+I'm doing great. How can I help you today?</s>
+<|user|>
+I'd like to show off how chat templating works!</s>
+<|assistant|>
diff --git a/template/testdata/zephyr.gotmpl/user b/template/testdata/zephyr.gotmpl/user
new file mode 100644
index 00000000..6cefdaa0
--- /dev/null
+++ b/template/testdata/zephyr.gotmpl/user
@@ -0,0 +1,3 @@
+<|user|>
+Hello, how are you?</s>
+<|assistant|>
diff --git a/template/testdata/zephyr.gotmpl/user-assistant-user b/template/testdata/zephyr.gotmpl/user-assistant-user
new file mode 100644
index 00000000..3937b006
--- /dev/null
+++ b/template/testdata/zephyr.gotmpl/user-assistant-user
@@ -0,0 +1,7 @@
+<|user|>
+Hello, how are you?</s>
+<|assistant|>
+I'm doing great. How can I help you today?</s>
+<|user|>
+I'd like to show off how chat templating works!</s>
+<|assistant|>
diff --git a/template/vicuna.gotmpl b/template/vicuna.gotmpl
new file mode 100644
index 00000000..c27f39c5
--- /dev/null
+++ b/template/vicuna.gotmpl
@@ -0,0 +1,14 @@
+{{- $system := "" }}
+{{- range .Messages }}
+{{- if eq .Role "system" }}
+{{- if not $system }}{{ $system = .Content }}
+{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }}
+{{- end }}
+{{- else if eq .Role "user" }}
+{{- if $system }}{{ $system }}
+
+{{ $system = "" }}
+{{- end }}USER: {{ .Content }}
+{{ else if eq .Role "assistant" }}ASSISTANT: {{ .Content }}</s>
+{{ end }}
+{{- end }}ASSISTANT:
\ No newline at end of file
diff --git a/template/vicuna.json b/template/vicuna.json
new file mode 100644
index 00000000..ed7bfb0f
--- /dev/null
+++ b/template/vicuna.json
@@ -0,0 +1,6 @@
+{
+ "stop": [
+ "USER:",
+ "ASSISTANT:"
+ ]
+}
diff --git a/template/zephyr.gotmpl b/template/zephyr.gotmpl
new file mode 100644
index 00000000..25da148a
--- /dev/null
+++ b/template/zephyr.gotmpl
@@ -0,0 +1,3 @@
+{{- range .Messages }}<|{{ .Role }}|>
+{{ .Content }}</s>
+{{ end }}<|assistant|>
diff --git a/template/zephyr.json b/template/zephyr.json
new file mode 100644
index 00000000..f9c0115c
--- /dev/null
+++ b/template/zephyr.json
@@ -0,0 +1,8 @@
+{
+ "stop": [
+ "<|system|>",
+ "</s>",
+ "<|user|>",
+ "<|assistant|>"
+ ]
+}
diff --git a/types/errtypes/errtypes.go b/types/errtypes/errtypes.go
new file mode 100644
index 00000000..27c3f913
--- /dev/null
+++ b/types/errtypes/errtypes.go
@@ -0,0 +1,21 @@
+// Package errtypes contains custom error types
+package errtypes
+
+import (
+ "fmt"
+ "strings"
+)
+
+const (
+ UnknownOllamaKeyErrMsg = "unknown ollama key"
+ InvalidModelNameErrMsg = "invalid model name"
+)
+
+// TODO: This should have a structured response from the API
+type UnknownOllamaKey struct {
+ Key string
+}
+
+func (e *UnknownOllamaKey) Error() string {
+ return fmt.Sprintf("unauthorized: %s %q", UnknownOllamaKeyErrMsg, strings.TrimSpace(e.Key))
+}
diff --git a/types/model/name.go b/types/model/name.go
new file mode 100644
index 00000000..75b35ef7
--- /dev/null
+++ b/types/model/name.go
@@ -0,0 +1,364 @@
+// Package model contains types and utilities for parsing, validating, and
+// working with model names and digests.
+package model
+
+import (
+ "cmp"
+ "errors"
+ "fmt"
+ "log/slog"
+ "path/filepath"
+ "strings"
+)
+
+// Errors
+var (
+ // ErrUnqualifiedName represents an error where a name is not fully
+ // qualified. It is not used directly in this package, but is here
+ // to avoid other packages inventing their own error type.
+ // Additionally, it can be conveniently used via [Unqualified].
+ ErrUnqualifiedName = errors.New("unqualified name")
+)
+
+// Unqualified is a helper function that returns an error with
+// ErrUnqualifiedName as the cause and the name as the message.
+func Unqualified(n Name) error {
+ return fmt.Errorf("%w: %s", ErrUnqualifiedName, n)
+}
+
+// MissingPart is used to indicate any part of a name that was "promised" by
+// the presence of a separator, but is missing.
+//
+// The value was chosen because it is deemed unlikely to be set by a user,
+// not a valid part name valid when checked by [Name.IsValid], and easy to
+// spot in logs.
+const MissingPart = "!MISSING!"
+
+const (
+ defaultHost = "registry.ollama.ai"
+ defaultNamespace = "library"
+ defaultTag = "latest"
+)
+
+// DefaultName returns a name with the default values for the host, namespace,
+// and tag parts. The model and digest parts are empty.
+//
+// - The default host is ("registry.ollama.ai")
+// - The default namespace is ("library")
+// - The default tag is ("latest")
+func DefaultName() Name {
+ return Name{
+ Host: defaultHost,
+ Namespace: defaultNamespace,
+ Tag: defaultTag,
+ }
+}
+
+type partKind int
+
+const (
+ kindHost partKind = iota
+ kindNamespace
+ kindModel
+ kindTag
+ kindDigest
+)
+
+func (k partKind) String() string {
+ switch k {
+ case kindHost:
+ return "host"
+ case kindNamespace:
+ return "namespace"
+ case kindModel:
+ return "model"
+ case kindTag:
+ return "tag"
+ case kindDigest:
+ return "digest"
+ default:
+ return "unknown"
+ }
+}
+
+// Name is a structured representation of a model name string, as defined by
+// [ParseNameNoDefaults].
+//
+// It is not guaranteed to be valid. Use [Name.IsValid] to check if the name
+// is valid.
+type Name struct {
+ Host string
+ Namespace string
+ Model string
+ Tag string
+}
+
+// ParseName parses and assembles a Name from a name string. The
+// format of a valid name string is:
+//
+// s:
+// { host } "/" { namespace } "/" { model } ":" { tag } "@" { digest }
+// { host } "/" { namespace } "/" { model } ":" { tag }
+// { host } "/" { namespace } "/" { model } "@" { digest }
+// { host } "/" { namespace } "/" { model }
+// { namespace } "/" { model } ":" { tag } "@" { digest }
+// { namespace } "/" { model } ":" { tag }
+// { namespace } "/" { model } "@" { digest }
+// { namespace } "/" { model }
+// { model } ":" { tag } "@" { digest }
+// { model } ":" { tag }
+// { model } "@" { digest }
+// { model }
+// "@" { digest }
+// host:
+// pattern: { alphanum | "_" } { alphanum | "-" | "_" | "." | ":" }*
+// length: [1, 350]
+// namespace:
+// pattern: { alphanum | "_" } { alphanum | "-" | "_" }*
+// length: [1, 80]
+// model:
+// pattern: { alphanum | "_" } { alphanum | "-" | "_" | "." }*
+// length: [1, 80]
+// tag:
+// pattern: { alphanum | "_" } { alphanum | "-" | "_" | "." }*
+// length: [1, 80]
+// digest:
+// pattern: { alphanum | "_" } { alphanum | "-" | ":" }*
+// length: [1, 80]
+//
+// Most users should use [ParseName] instead, unless need to support
+// different defaults than DefaultName.
+//
+// The name returned is not guaranteed to be valid. If it is not valid, the
+// field values are left in an undefined state. Use [Name.IsValid] to check
+// if the name is valid.
+func ParseName(s string) Name {
+ return Merge(ParseNameBare(s), DefaultName())
+}
+
+// ParseNameBare parses s as a name string and returns a Name. No merge with
+// [DefaultName] is performed.
+func ParseNameBare(s string) Name {
+ var n Name
+ var promised bool
+
+ // "/" is an illegal tag character, so we can use it to split the host
+ if strings.LastIndex(s, ":") > strings.LastIndex(s, "/") {
+ s, n.Tag, _ = cutPromised(s, ":")
+ }
+
+ s, n.Model, promised = cutPromised(s, "/")
+ if !promised {
+ n.Model = s
+ return n
+ }
+
+ s, n.Namespace, promised = cutPromised(s, "/")
+ if !promised {
+ n.Namespace = s
+ return n
+ }
+
+ scheme, host, ok := strings.Cut(s, "://")
+ if !ok {
+ host = scheme
+ }
+ n.Host = host
+
+ return n
+}
+
+// ParseNameFromFilepath parses a 4-part filepath as a Name. The parts are
+// expected to be in the form:
+//
+// { host } "/" { namespace } "/" { model } "/" { tag }
+func ParseNameFromFilepath(s string) (n Name) {
+ parts := strings.Split(s, string(filepath.Separator))
+ if len(parts) != 4 {
+ return Name{}
+ }
+
+ n.Host = parts[0]
+ n.Namespace = parts[1]
+ n.Model = parts[2]
+ n.Tag = parts[3]
+ if !n.IsFullyQualified() {
+ return Name{}
+ }
+
+ return n
+}
+
+// Merge merges the host, namespace, and tag parts of the two names,
+// preferring the non-empty parts of a.
+func Merge(a, b Name) Name {
+ a.Host = cmp.Or(a.Host, b.Host)
+ a.Namespace = cmp.Or(a.Namespace, b.Namespace)
+ a.Tag = cmp.Or(a.Tag, b.Tag)
+ return a
+}
+
+// String returns the name string, in the format that [ParseNameNoDefaults]
+// accepts as valid, if [Name.IsValid] reports true; otherwise the empty
+// string is returned.
+func (n Name) String() string {
+ var b strings.Builder
+ if n.Host != "" {
+ b.WriteString(n.Host)
+ b.WriteByte('/')
+ }
+ if n.Namespace != "" {
+ b.WriteString(n.Namespace)
+ b.WriteByte('/')
+ }
+ b.WriteString(n.Model)
+ if n.Tag != "" {
+ b.WriteByte(':')
+ b.WriteString(n.Tag)
+ }
+ return b.String()
+}
+
+// DisplayShortest returns a short string version of the name.
+func (n Name) DisplayShortest() string {
+ var sb strings.Builder
+
+ if n.Host != defaultHost {
+ sb.WriteString(n.Host)
+ sb.WriteByte('/')
+ sb.WriteString(n.Namespace)
+ sb.WriteByte('/')
+ } else if n.Namespace != defaultNamespace {
+ sb.WriteString(n.Namespace)
+ sb.WriteByte('/')
+ }
+
+ // always include model and tag
+ sb.WriteString(n.Model)
+ sb.WriteString(":")
+ sb.WriteString(n.Tag)
+ return sb.String()
+}
+
+// IsValidNamespace reports whether the provided string is a valid
+// namespace.
+func IsValidNamespace(s string) bool {
+ return isValidPart(kindNamespace, s)
+}
+
+// IsValid reports whether all parts of the name are present and valid. The
+// digest is a special case, and is checked for validity only if present.
+//
+// Note: The digest check has been removed as is planned to be added back in
+// at a later time.
+func (n Name) IsValid() bool {
+ return n.IsFullyQualified()
+}
+
+// IsFullyQualified returns true if all parts of the name are present and
+// valid without the digest.
+func (n Name) IsFullyQualified() bool {
+ parts := []string{
+ n.Host,
+ n.Namespace,
+ n.Model,
+ n.Tag,
+ }
+ for i, part := range parts {
+ if !isValidPart(partKind(i), part) {
+ return false
+ }
+ }
+ return true
+}
+
+// Filepath returns a canonical filepath that represents the name with each part from
+// host to tag as a directory in the form:
+//
+// {host}/{namespace}/{model}/{tag}
+//
+// It uses the system's filepath separator and ensures the path is clean.
+//
+// It panics if the name is not fully qualified. Use [Name.IsFullyQualified]
+// to check if the name is fully qualified.
+func (n Name) Filepath() string {
+ if !n.IsFullyQualified() {
+ panic("illegal attempt to get filepath of invalid name")
+ }
+ return filepath.Join(
+ n.Host,
+ n.Namespace,
+ n.Model,
+ n.Tag,
+ )
+}
+
+// LogValue returns a slog.Value that represents the name as a string.
+func (n Name) LogValue() slog.Value {
+ return slog.StringValue(n.String())
+}
+
+func isValidLen(kind partKind, s string) bool {
+ switch kind {
+ case kindHost:
+ return len(s) >= 1 && len(s) <= 350
+ case kindTag:
+ return len(s) >= 1 && len(s) <= 80
+ default:
+ return len(s) >= 1 && len(s) <= 80
+ }
+}
+
+func isValidPart(kind partKind, s string) bool {
+ if !isValidLen(kind, s) {
+ return false
+ }
+ for i := range s {
+ if i == 0 {
+ if !isAlphanumericOrUnderscore(s[i]) {
+ return false
+ }
+ continue
+ }
+ switch s[i] {
+ case '_', '-':
+ case '.':
+ if kind == kindNamespace {
+ return false
+ }
+ case ':':
+ if kind != kindHost && kind != kindDigest {
+ return false
+ }
+ default:
+ if !isAlphanumericOrUnderscore(s[i]) {
+ return false
+ }
+ }
+ }
+ return true
+}
+
+func isAlphanumericOrUnderscore(c byte) bool {
+ return c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '_'
+}
+
+func cutLast(s, sep string) (before, after string, ok bool) {
+ i := strings.LastIndex(s, sep)
+ if i >= 0 {
+ return s[:i], s[i+len(sep):], true
+ }
+ return s, "", false
+}
+
+// cutPromised cuts the last part of s at the last occurrence of sep. If sep is
+// found, the part before and after sep are returned as-is unless empty, in
+// which case they are returned as MissingPart, which will cause
+// [Name.IsValid] to return false.
+func cutPromised(s, sep string) (before, after string, ok bool) {
+ before, after, ok = cutLast(s, sep)
+ if !ok {
+ return before, after, false
+ }
+ return cmp.Or(before, MissingPart), cmp.Or(after, MissingPart), true
+}
diff --git a/types/model/name_test.go b/types/model/name_test.go
new file mode 100644
index 00000000..794d14d7
--- /dev/null
+++ b/types/model/name_test.go
@@ -0,0 +1,358 @@
+package model
+
+import (
+ "path/filepath"
+ "reflect"
+ "runtime"
+ "testing"
+)
+
+const (
+ part80 = "88888888888888888888888888888888888888888888888888888888888888888888888888888888"
+ part350 = "33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333"
+)
+
+func TestParseNameParts(t *testing.T) {
+ cases := []struct {
+ in string
+ want Name
+ wantFilepath string
+ wantValidDigest bool
+ }{
+ {
+ in: "registry.ollama.ai/library/dolphin-mistral:7b-v2.6-dpo-laser-q6_K",
+ want: Name{
+ Host: "registry.ollama.ai",
+ Namespace: "library",
+ Model: "dolphin-mistral",
+ Tag: "7b-v2.6-dpo-laser-q6_K",
+ },
+ wantFilepath: filepath.Join("registry.ollama.ai", "library", "dolphin-mistral", "7b-v2.6-dpo-laser-q6_K"),
+ },
+ {
+ in: "scheme://host:port/namespace/model:tag",
+ want: Name{
+ Host: "host:port",
+ Namespace: "namespace",
+ Model: "model",
+ Tag: "tag",
+ },
+ wantFilepath: filepath.Join("host:port", "namespace", "model", "tag"),
+ },
+ {
+ in: "host/namespace/model:tag",
+ want: Name{
+ Host: "host",
+ Namespace: "namespace",
+ Model: "model",
+ Tag: "tag",
+ },
+ wantFilepath: filepath.Join("host", "namespace", "model", "tag"),
+ },
+ {
+ in: "host:port/namespace/model:tag",
+ want: Name{
+ Host: "host:port",
+ Namespace: "namespace",
+ Model: "model",
+ Tag: "tag",
+ },
+ wantFilepath: filepath.Join("host:port", "namespace", "model", "tag"),
+ },
+ {
+ in: "host/namespace/model",
+ want: Name{
+ Host: "host",
+ Namespace: "namespace",
+ Model: "model",
+ },
+ wantFilepath: filepath.Join("host", "namespace", "model", "latest"),
+ },
+ {
+ in: "host:port/namespace/model",
+ want: Name{
+ Host: "host:port",
+ Namespace: "namespace",
+ Model: "model",
+ },
+ wantFilepath: filepath.Join("host:port", "namespace", "model", "latest"),
+ },
+ {
+ in: "namespace/model",
+ want: Name{
+ Namespace: "namespace",
+ Model: "model",
+ },
+ wantFilepath: filepath.Join("registry.ollama.ai", "namespace", "model", "latest"),
+ },
+ {
+ in: "model",
+ want: Name{
+ Model: "model",
+ },
+ wantFilepath: filepath.Join("registry.ollama.ai", "library", "model", "latest"),
+ },
+ {
+ in: "h/nn/mm:t",
+ want: Name{
+ Host: "h",
+ Namespace: "nn",
+ Model: "mm",
+ Tag: "t",
+ },
+ wantFilepath: filepath.Join("h", "nn", "mm", "t"),
+ },
+ {
+ in: part80 + "/" + part80 + "/" + part80 + ":" + part80,
+ want: Name{
+ Host: part80,
+ Namespace: part80,
+ Model: part80,
+ Tag: part80,
+ },
+ wantFilepath: filepath.Join(part80, part80, part80, part80),
+ },
+ {
+ in: part350 + "/" + part80 + "/" + part80 + ":" + part80,
+ want: Name{
+ Host: part350,
+ Namespace: part80,
+ Model: part80,
+ Tag: part80,
+ },
+ wantFilepath: filepath.Join(part350, part80, part80, part80),
+ },
+ }
+
+ for _, tt := range cases {
+ t.Run(tt.in, func(t *testing.T) {
+ got := ParseNameBare(tt.in)
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("parseName(%q) = %v; want %v", tt.in, got, tt.want)
+ }
+
+ got = ParseName(tt.in)
+ if tt.wantFilepath != "" && got.Filepath() != tt.wantFilepath {
+ t.Errorf("parseName(%q).Filepath() = %q; want %q", tt.in, got.Filepath(), tt.wantFilepath)
+ }
+ })
+ }
+}
+
+var testCases = map[string]bool{ // name -> valid
+ "": false,
+
+ "_why/_the/_lucky:_stiff": true,
+
+ // minimal
+ "h/n/m:t": true,
+
+ "host/namespace/model:tag": true,
+ "host/namespace/model": false,
+ "namespace/model": false,
+ "model": false,
+
+ // long (but valid)
+ part80 + "/" + part80 + "/" + part80 + ":" + part80: true,
+ part350 + "/" + part80 + "/" + part80 + ":" + part80: true,
+
+ "h/nn/mm:t": true, // bare minimum part sizes
+
+ // unqualified
+ "m": false,
+ "n/m:": false,
+ "h/n/m": false,
+ "@t": false,
+ "m@d": false,
+
+ // invalids
+ "^": false,
+ "mm:": false,
+ "/nn/mm": false,
+ "//": false,
+ "//mm": false,
+ "hh//": false,
+ "//mm:@": false,
+ "00@": false,
+ "@": false,
+
+ // not starting with alphanum
+ "-hh/nn/mm:tt": false,
+ "hh/-nn/mm:tt": false,
+ "hh/nn/-mm:tt": false,
+ "hh/nn/mm:-tt": false,
+
+ // hosts
+ "host:https/namespace/model:tag": true,
+
+ // colon in non-host part before tag
+ "host/name:space/model:tag": false,
+}
+
+func TestNameparseNameDefault(t *testing.T) {
+ const name = "xx"
+ n := ParseName(name)
+ got := n.String()
+ want := "registry.ollama.ai/library/xx:latest"
+ if got != want {
+ t.Errorf("parseName(%q).String() = %q; want %q", name, got, want)
+ }
+}
+
+func TestNameIsValid(t *testing.T) {
+ var numStringTests int
+ for s, want := range testCases {
+ n := ParseNameBare(s)
+ got := n.IsValid()
+ if got != want {
+ t.Errorf("parseName(%q).IsValid() = %v; want %v", s, got, want)
+ }
+
+ // Test roundtrip with String
+ if got {
+ got := ParseNameBare(s).String()
+ if got != s {
+ t.Errorf("parseName(%q).String() = %q; want %q", s, got, s)
+ }
+ numStringTests++
+ }
+ }
+
+ if numStringTests == 0 {
+ t.Errorf("no tests for Name.String")
+ }
+}
+
+func TestNameIsValidPart(t *testing.T) {
+ cases := []struct {
+ kind partKind
+ s string
+ want bool
+ }{
+ {kind: kindHost, s: "", want: false},
+ {kind: kindHost, s: "a", want: true},
+ {kind: kindHost, s: "a.", want: true},
+ {kind: kindHost, s: "a.b", want: true},
+ {kind: kindHost, s: "a:123", want: true},
+ {kind: kindHost, s: "a:123/aa/bb", want: false},
+ {kind: kindNamespace, s: "bb", want: true},
+ {kind: kindNamespace, s: "a.", want: false},
+ {kind: kindModel, s: "-h", want: false},
+ {kind: kindDigest, s: "sha256-1000000000000000000000000000000000000000000000000000000000000000", want: true},
+ }
+ for _, tt := range cases {
+ t.Run(tt.s, func(t *testing.T) {
+ got := isValidPart(tt.kind, tt.s)
+ if got != tt.want {
+ t.Errorf("isValidPart(%s, %q) = %v; want %v", tt.kind, tt.s, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestFilepathAllocs(t *testing.T) {
+ n := ParseNameBare("HOST/NAMESPACE/MODEL:TAG")
+ allocs := testing.AllocsPerRun(1000, func() {
+ n.Filepath()
+ })
+ var allowedAllocs float64 = 1
+ if runtime.GOOS == "windows" {
+ allowedAllocs = 3
+ }
+ if allocs > allowedAllocs {
+ t.Errorf("allocs = %v; allowed %v", allocs, allowedAllocs)
+ }
+}
+
+func TestParseNameFromFilepath(t *testing.T) {
+ cases := map[string]Name{
+ filepath.Join("host", "namespace", "model", "tag"): {Host: "host", Namespace: "namespace", Model: "model", Tag: "tag"},
+ filepath.Join("host:port", "namespace", "model", "tag"): {Host: "host:port", Namespace: "namespace", Model: "model", Tag: "tag"},
+ filepath.Join("namespace", "model", "tag"): {},
+ filepath.Join("model", "tag"): {},
+ "model": {},
+ filepath.Join("..", "..", "model", "tag"): {},
+ filepath.Join("", "namespace", ".", "tag"): {},
+ filepath.Join(".", ".", ".", "."): {},
+ filepath.Join("/", "path", "to", "random", "file"): {},
+ }
+
+ for in, want := range cases {
+ t.Run(in, func(t *testing.T) {
+ got := ParseNameFromFilepath(in)
+
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("parseNameFromFilepath(%q) = %v; want %v", in, got, want)
+ }
+ })
+ }
+}
+
+func TestDisplayShortest(t *testing.T) {
+ cases := map[string]string{
+ "registry.ollama.ai/library/model:latest": "model:latest",
+ "registry.ollama.ai/library/model:tag": "model:tag",
+ "registry.ollama.ai/namespace/model:tag": "namespace/model:tag",
+ "host/namespace/model:tag": "host/namespace/model:tag",
+ "host/library/model:tag": "host/library/model:tag",
+ }
+
+ for in, want := range cases {
+ t.Run(in, func(t *testing.T) {
+ got := ParseNameBare(in).DisplayShortest()
+ if got != want {
+ t.Errorf("parseName(%q).DisplayShortest() = %q; want %q", in, got, want)
+ }
+ })
+ }
+}
+
+func FuzzName(f *testing.F) {
+ for s := range testCases {
+ f.Add(s)
+ }
+ f.Fuzz(func(t *testing.T, s string) {
+ n := ParseNameBare(s)
+ if n.IsValid() {
+ parts := [...]string{n.Host, n.Namespace, n.Model, n.Tag}
+ for _, part := range parts {
+ if part == ".." {
+ t.Errorf("unexpected .. as valid part")
+ }
+ if len(part) > 350 {
+ t.Errorf("part too long: %q", part)
+ }
+ }
+ if n.String() != s {
+ t.Errorf("String() = %q; want %q", n.String(), s)
+ }
+ }
+ })
+}
+
+func TestIsValidNamespace(t *testing.T) {
+ cases := []struct {
+ username string
+ expected bool
+ }{
+ {"", false},
+ {"a", true},
+ {"a:b", false},
+ {"a/b", false},
+ {"a:b/c", false},
+ {"a/b:c", false},
+ {"a/b:c", false},
+ {"a/b:c/d", false},
+ {"a/b:c/d@e", false},
+ {"a/b:c/d@sha256-100", false},
+ {"himynameisjoe", true},
+ {"himynameisreallyreallyreallyreallylongbutitshouldstillbevalid", true},
+ }
+ for _, tt := range cases {
+ t.Run(tt.username, func(t *testing.T) {
+ if got := IsValidNamespace(tt.username); got != tt.expected {
+ t.Errorf("IsValidName(%q) = %v; want %v", tt.username, got, tt.expected)
+ }
+ })
+ }
+}
diff --git a/types/model/testdata/fuzz/FuzzName/d37463aa416f6bab b/types/model/testdata/fuzz/FuzzName/d37463aa416f6bab
new file mode 100644
index 00000000..0034d9f5
--- /dev/null
+++ b/types/model/testdata/fuzz/FuzzName/d37463aa416f6bab
@@ -0,0 +1,2 @@
+go test fuzz v1
+string("00@")
diff --git a/util/bufioutil/buffer_seeker.go b/util/bufioutil/buffer_seeker.go
new file mode 100644
index 00000000..8775fdb8
--- /dev/null
+++ b/util/bufioutil/buffer_seeker.go
@@ -0,0 +1,34 @@
+package bufioutil
+
+import (
+ "bufio"
+ "io"
+)
+
+type BufferedSeeker struct {
+ rs io.ReadSeeker
+ br *bufio.Reader
+}
+
+func NewBufferedSeeker(rs io.ReadSeeker, size int) *BufferedSeeker {
+ return &BufferedSeeker{
+ rs: rs,
+ br: bufio.NewReaderSize(rs, size),
+ }
+}
+
+func (b *BufferedSeeker) Read(p []byte) (int, error) {
+ return b.br.Read(p)
+}
+
+func (b *BufferedSeeker) Seek(offset int64, whence int) (int64, error) {
+ if whence == io.SeekCurrent {
+ offset -= int64(b.br.Buffered())
+ }
+ n, err := b.rs.Seek(offset, whence)
+ if err != nil {
+ return 0, err
+ }
+ b.br.Reset(b.rs)
+ return n, nil
+}
diff --git a/util/bufioutil/buffer_seeker_test.go b/util/bufioutil/buffer_seeker_test.go
new file mode 100644
index 00000000..87145f6b
--- /dev/null
+++ b/util/bufioutil/buffer_seeker_test.go
@@ -0,0 +1,64 @@
+package bufioutil
+
+import (
+ "bytes"
+ "io"
+ "strings"
+ "testing"
+)
+
+func TestBufferedSeeker(t *testing.T) {
+ const alphabet = "abcdefghijklmnopqrstuvwxyz"
+
+ bs := NewBufferedSeeker(strings.NewReader(alphabet), 0) // minReadBufferSize = 16
+
+ checkRead := func(buf []byte, expected string) {
+ t.Helper()
+ _, err := bs.Read(buf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(buf, []byte(expected)) {
+ t.Fatalf("expected %s, got %s", expected, buf)
+ }
+ }
+
+ // Read the first 5 bytes
+ buf := make([]byte, 5)
+
+ checkRead(buf, "abcde")
+
+ // Seek back to the beginning
+ _, err := bs.Seek(0, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // read 'a'
+ checkRead(buf[:1], "a")
+
+ if bs.br.Buffered() == 0 {
+ t.Fatalf("totally unexpected sanity check failed")
+ }
+
+ // Seek past 'b'
+ _, err = bs.Seek(1, io.SeekCurrent)
+ if err != nil {
+ t.Fatal(err)
+ }
+ checkRead(buf, "cdefg")
+
+ // Seek back to the beginning
+ _, err = bs.Seek(0, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+ checkRead(buf, "abcde")
+
+ // Seek to the end
+ _, err = bs.Seek(-5, io.SeekEnd)
+ if err != nil {
+ t.Fatal(err)
+ }
+ checkRead(buf, "vwxyz")
+}