third-party-mirrors/ollama
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x/mode: Path -> Name

Browse Source
This commit is contained in:
Blake Mizerany 2024-04-04 14:40:01 -07:00
parent 42cda9dd46
commit 7cd939690a
6 changed files with 93 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
x/build/build.go
View File

@ -52,7 +52,7 @@ func Open(dir string) (*Server, error) {
}
func (s *Server) Build(ref string, f model.File) error {
mp := model.ParsePath(ref)
mp := model.ParseName(ref)
if !mp.CompleteWithoutBuild() {
return fmt.Errorf("%w: %q", ErrIncompleteRef, ref)
}
@ -112,13 +112,13 @@ func (s *Server) LayerFile(digest string) (string, error) {
}
func (s *Server) ManifestData(ref string) ([]byte, error) {
data, _, err := s.resolve(model.ParsePath(ref))
data, _, err := s.resolve(model.ParseName(ref))
return data, err
}
// WeightFile returns the absolute path to the weights file for the given model ref.
func (s *Server) WeightsFile(ref string) (string, error) {
m, err := s.getManifest(model.ParsePath(ref))
m, err := s.getManifest(model.ParseName(ref))
if err != nil {
return "", err
}
@ -139,7 +139,7 @@ func (s *Server) WeightsFile(ref string) (string, error) {
// blob, and then have the ref point to that blob. This would simplify the
// code, allow us to have integrity checks on the manifest, and clean up
// this interface.
func (s *Server) resolve(ref model.Path) (data []byte, fileName string, err error) {
func (s *Server) resolve(ref model.Name) (data []byte, fileName string, err error) {
fileName, err = s.refFileName(ref)
if err != nil {
return nil, "", err
@ -158,11 +158,11 @@ func (s *Server) resolve(ref model.Path) (data []byte, fileName string, err erro
}
func (s *Server) SetManifestData(ref string, data []byte) error {
return s.setManifestData(model.ParsePath(ref), data)
return s.setManifestData(model.ParseName(ref), data)
}
// Set sets the data for the given ref.
func (s *Server) setManifestData(mp model.Path, data []byte) error {
func (s *Server) setManifestData(mp model.Name, data []byte) error {
path, err := s.refFileName(mp)
if err != nil {
return err
@ -176,7 +176,7 @@ func (s *Server) setManifestData(mp model.Path, data []byte) error {
return nil
}
func (s *Server) refFileName(mp model.Path) (string, error) {
func (s *Server) refFileName(mp model.Name) (string, error) {
if !mp.Complete() {
return "", fmt.Errorf("ref not fully qualified: %q", mp)
}
@ -196,7 +196,7 @@ type layerJSON struct {
Size int64 `json:"size"`
}
func (s *Server) getManifest(ref model.Path) (manifestJSON, error) {
func (s *Server) getManifest(ref model.Name) (manifestJSON, error) {
data, path, err := s.resolve(ref)
if err != nil {
return manifestJSON{}, err

2
x/build/internal/blobstore/store_test.go
View File

@ -68,7 +68,7 @@ func TestStoreBasicBlob(t *testing.T) {
}
// Check tags
name := model.ParsePath("registry.ollama.ai/library/test:latest+KQED")
name := model.ParseName("registry.ollama.ai/library/test:latest+KQED")
t.Logf("RESOLVING: %q", name.Parts())

2
x/model/file.go
View File

@ -20,7 +20,7 @@ type MessagePragma struct {
type File struct {
// From is a required pragma that specifies the source of the model,
// either on disk, or by reference (see blob.ParseRef).
// either on disk, or by reference (see model.ParseName).
From string
// Optional

90
x/model/path.go → x/model/name.go
View File

@ -7,35 +7,35 @@ import (
"strings"
)
const MaxPathLength = 255
const MaxNameLength = 255
type PathPart int
type NamePart int
// Levels of concreteness
const (
Invalid PathPart = iota
Domain
Invalid NamePart = iota
Registry
Namespace
Name
Short
Tag
Build
)
var kindNames = map[PathPart]string{
var kindNames = map[NamePart]string{
Invalid: "Invalid",
Domain: "Domain",
Registry: "Domain",
Namespace: "Namespace",
Name: "Name",
Short: "Name",
Tag: "Tag",
Build: "Build",
}
// Path is an opaque reference to a model.
// Name is an opaque reference to a model.
//
// It is comparable and can be used as a map key.
//
// Users or Path must check Valid before using it.
type Path struct {
// Users or Name must check Valid before using it.
type Name struct {
domain string
namespace string
name string
@ -46,7 +46,7 @@ type Path struct {
// Format returns a string representation of the ref with the given
// concreteness. If a part is missing, it is replaced with a loud
// placeholder.
func (r Path) Full() string {
func (r Name) Full() string {
r.domain = cmp.Or(r.domain, "!(MISSING DOMAIN)")
r.namespace = cmp.Or(r.namespace, "!(MISSING NAMESPACE)")
r.name = cmp.Or(r.name, "!(MISSING NAME)")
@ -55,21 +55,21 @@ func (r Path) Full() string {
return r.String()
}
func (r Path) NameAndTag() string {
func (r Name) ShortAndTag() string {
r.domain = ""
r.namespace = ""
r.build = ""
return r.String()
}
func (r Path) NameTagAndBuild() string {
func (r Name) ShortTagAndBuild() string {
r.domain = ""
r.namespace = ""
return r.String()
}
// String returns the fully qualified ref string.
func (r Path) String() string {
func (r Name) String() string {
var b strings.Builder
if r.domain != "" {
b.WriteString(r.domain)
@ -93,19 +93,19 @@ func (r Path) String() string {
// Complete reports whether the ref is fully qualified. That is it has a
// domain, namespace, name, tag, and build.
func (r Path) Complete() bool {
func (r Name) Complete() bool {
return r.Valid() && !slices.Contains(r.Parts(), "")
}
// CompleteWithoutBuild reports whether the ref would be complete if it had a
// valid build.
func (r Path) CompleteWithoutBuild() bool {
func (r Name) CompleteWithoutBuild() bool {
r.build = "x"
return r.Valid() && r.Complete()
}
// Less returns true if r is less concrete than o; false otherwise.
func (r Path) Less(o Path) bool {
func (r Name) Less(o Name) bool {
rp := r.Parts()
op := o.Parts()
for i := range rp {
@ -119,7 +119,7 @@ func (r Path) Less(o Path) bool {
// Parts returns the parts of the ref in order of concreteness.
//
// The length of the returned slice is always 5.
func (r Path) Parts() []string {
func (r Name) Parts() []string {
return []string{
r.domain,
r.namespace,
@ -129,13 +129,13 @@ func (r Path) Parts() []string {
}
}
func (r Path) Domain() string { return r.namespace }
func (r Path) Namespace() string { return r.namespace }
func (r Path) Name() string { return r.name }
func (r Path) Tag() string { return r.tag }
func (r Path) Build() string { return r.build }
func (r Name) Domain() string { return r.namespace }
func (r Name) Namespace() string { return r.namespace }
func (r Name) Name() string { return r.name }
func (r Name) Tag() string { return r.tag }
func (r Name) Build() string { return r.build }
// ParsePath parses a model path string into a Path.
// ParseName parses a model path string into a Name.
//
// Examples of valid paths:
//
@ -151,26 +151,26 @@ func (r Path) Build() string { return r.build }
// "example.com/mistral:7b+Q4_0+"
// "x/y/z/z:8n+I"
// ""
func ParsePath(s string) Path {
var r Path
for kind, part := range PathParts(s) {
func ParseName(s string) Name {
var r Name
for kind, part := range NameParts(s) {
switch kind {
case Domain:
case Registry:
r.domain = part
case Namespace:
r.namespace = part
case Name:
case Short:
r.name = part
case Tag:
r.tag = part
case Build:
r.build = strings.ToUpper(part)
case Invalid:
return Path{}
return Name{}
}
}
if !r.Valid() {
return Path{}
return Name{}
}
return r
}
@ -179,8 +179,8 @@ func ParsePath(s string) Path {
// The name is left untouched.
//
// Use this for merging a ref with a default ref.
func Merge(a, b Path) Path {
return Path{
func Merge(a, b Name) Name {
return Name{
// name is left untouched
name: a.name,
@ -192,7 +192,7 @@ func Merge(a, b Path) Path {
}
// WithBuild returns a copy of r with the build set to the given string.
func (r Path) WithBuild(build string) Path {
func (r Name) WithBuild(build string) Name {
r.build = build
return r
}
@ -202,8 +202,8 @@ func (r Path) WithBuild(build string) Path {
//
// It normalizes the input string by removing "http://" and "https://" only.
// No other normalization is done.
func PathParts(s string) iter.Seq2[PathPart, string] {
return func(yield func(PathPart, string) bool) {
func NameParts(s string) iter.Seq2[NamePart, string] {
return func(yield func(NamePart, string) bool) {
if strings.HasPrefix(s, "http://") {
s = s[len("http://"):]
}
@ -211,11 +211,11 @@ func PathParts(s string) iter.Seq2[PathPart, string] {
s = s[len("https://"):]
}
if len(s) > MaxPathLength || len(s) == 0 {
if len(s) > MaxNameLength || len(s) == 0 {
return
}
yieldValid := func(kind PathPart, part string) bool {
yieldValid := func(kind NamePart, part string) bool {
if !isValidPart(part) {
yield(Invalid, "")
return false
@ -243,15 +243,15 @@ func PathParts(s string) iter.Seq2[PathPart, string] {
if !yieldValid(Tag, s[i+1:j]) {
return
}
state, j = Name, i
state, j = Short, i
default:
yield(Invalid, "")
return
}
case '/':
switch state {
case Name, Tag, Build:
if !yieldValid(Name, s[i+1:j]) {
case Short, Tag, Build:
if !yieldValid(Short, s[i+1:j]) {
return
}
state, j = Namespace, i
@ -259,7 +259,7 @@ func PathParts(s string) iter.Seq2[PathPart, string] {
if !yieldValid(Namespace, s[i+1:j]) {
return
}
state, j = Domain, i
state, j = Registry, i
default:
yield(Invalid, "")
return
@ -275,14 +275,14 @@ func PathParts(s string) iter.Seq2[PathPart, string] {
if state <= Namespace {
yieldValid(state, s[:j])
} else {
yieldValid(Name, s[:j])
yieldValid(Short, s[:j])
}
}
}
// Valid returns true if the ref has a valid name. To know if a ref is
// "complete", use Complete.
func (r Path) Valid() bool {
func (r Name) Valid() bool {
// Parts ensures we only have valid parts, so no need to validate
// them here, only check if we have a name or not.
return r.name != ""

74
x/model/path_test.go → x/model/name_test.go
View File

@ -6,7 +6,7 @@ import (
"testing"
)
var testPaths = map[string]Path{
var testNames = map[string]Name{
"mistral:latest": {name: "mistral", tag: "latest"},
"mistral": {name: "mistral"},
"mistral:30B": {name: "mistral", tag: "30B"},
@ -36,33 +36,33 @@ var testPaths = map[string]Path{
"file:///etc/passwd:latest": {},
"file:///etc/passwd:latest+u": {},
strings.Repeat("a", MaxPathLength): {name: strings.Repeat("a", MaxPathLength)},
strings.Repeat("a", MaxPathLength+1): {},
strings.Repeat("a", MaxNameLength): {name: strings.Repeat("a", MaxNameLength)},
strings.Repeat("a", MaxNameLength+1): {},
}
func TestPathParts(t *testing.T) {
func TestNameParts(t *testing.T) {
const wantNumParts = 5
var p Path
var p Name
if len(p.Parts()) != wantNumParts {
t.Errorf("Parts() = %d; want %d", len(p.Parts()), wantNumParts)
}
}
func TestParsePath(t *testing.T) {
for s, want := range testPaths {
func TestParseName(t *testing.T) {
for s, want := range testNames {
for _, prefix := range []string{"", "https://", "http://"} {
// We should get the same results with or without the
// http(s) prefixes
s := prefix + s
t.Run(s, func(t *testing.T) {
got := ParsePath(s)
got := ParseName(s)
if got != want {
t.Errorf("ParsePath(%q) = %q; want %q", s, got, want)
t.Errorf("ParseName(%q) = %q; want %q", s, got, want)
}
// test round-trip
if ParsePath(got.String()) != got {
if ParseName(got.String()) != got {
t.Errorf("String() = %s; want %s", got.String(), s)
}
@ -76,7 +76,7 @@ func TestParsePath(t *testing.T) {
}
}
func TestPathComplete(t *testing.T) {
func TestName(t *testing.T) {
cases := []struct {
in string
complete bool
@ -92,8 +92,8 @@ func TestPathComplete(t *testing.T) {
for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) {
p := ParsePath(tt.in)
t.Logf("ParsePath(%q) = %#v", tt.in, p)
p := ParseName(tt.in)
t.Logf("ParseName(%q) = %#v", tt.in, p)
if g := p.Complete(); g != tt.complete {
t.Errorf("Complete(%q) = %v; want %v", tt.in, g, tt.complete)
}
@ -104,7 +104,7 @@ func TestPathComplete(t *testing.T) {
}
}
func TestPathStringVariants(t *testing.T) {
func TestNameStringVariants(t *testing.T) {
cases := []struct {
in string
nameAndTag string
@ -116,19 +116,19 @@ func TestPathStringVariants(t *testing.T) {
for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) {
p := ParsePath(tt.in)
t.Logf("ParsePath(%q) = %#v", tt.in, p)
if g := p.NameAndTag(); g != tt.nameAndTag {
t.Errorf("NameAndTag(%q) = %q; want %q", tt.in, g, tt.nameAndTag)
p := ParseName(tt.in)
t.Logf("ParseName(%q) = %#v", tt.in, p)
if g := p.ShortAndTag(); g != tt.nameAndTag {
t.Errorf("ShortAndTag(%q) = %q; want %q", tt.in, g, tt.nameAndTag)
}
if g := p.NameTagAndBuild(); g != tt.nameTagAndBuild {
t.Errorf("NameTagAndBuild(%q) = %q; want %q", tt.in, g, tt.nameTagAndBuild)
if g := p.ShortTagAndBuild(); g != tt.nameTagAndBuild {
t.Errorf("ShortTagAndBuild(%q) = %q; want %q", tt.in, g, tt.nameTagAndBuild)
}
})
}
}
func TestPathFull(t *testing.T) {
func TestNameFull(t *testing.T) {
const empty = "!(MISSING DOMAIN)/!(MISSING NAMESPACE)/!(MISSING NAME):!(MISSING TAG)+!(MISSING BUILD)"
cases := []struct {
@ -151,8 +151,8 @@ func TestPathFull(t *testing.T) {
for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) {
p := ParsePath(tt.in)
t.Logf("ParsePath(%q) = %#v", tt.in, p)
p := ParseName(tt.in)
t.Logf("ParseName(%q) = %#v", tt.in, p)
if g := p.Full(); g != tt.wantFull {
t.Errorf("Full(%q) = %q; want %q", tt.in, g, tt.wantFull)
}
@ -160,44 +160,44 @@ func TestPathFull(t *testing.T) {
}
}
func TestParsePathAllocs(t *testing.T) {
func TestParseNameAllocs(t *testing.T) {
// test allocations
var r Path
var r Name
allocs := testing.AllocsPerRun(1000, func() {
r = ParsePath("example.com/mistral:7b+Q4_0")
r = ParseName("example.com/mistral:7b+Q4_0")
})
_ = r
if allocs > 0 {
t.Errorf("ParsePath allocs = %v; want 0", allocs)
t.Errorf("ParseName allocs = %v; want 0", allocs)
}
}
func BenchmarkParsePath(b *testing.B) {
func BenchmarkParseName(b *testing.B) {
b.ReportAllocs()
var r Path
var r Name
for i := 0; i < b.N; i++ {
r = ParsePath("example.com/mistral:7b+Q4_0")
r = ParseName("example.com/mistral:7b+Q4_0")
}
_ = r
}
func FuzzParsePath(f *testing.F) {
func FuzzParseName(f *testing.F) {
f.Add("example.com/mistral:7b+Q4_0")
f.Add("example.com/mistral:7b+q4_0")
f.Add("example.com/mistral:7b+x")
f.Add("x/y/z:8n+I")
f.Fuzz(func(t *testing.T, s string) {
r0 := ParsePath(s)
r0 := ParseName(s)
if !r0.Valid() {
if r0 != (Path{}) {
if r0 != (Name{}) {
t.Errorf("expected invalid path to be zero value; got %#v", r0)
}
t.Skipf("invalid path: %q", s)
}
for _, p := range r0.Parts() {
if len(p) > MaxPathLength {
if len(p) > MaxNameLength {
t.Errorf("part too long: %q", p)
}
}
@ -206,7 +206,7 @@ func FuzzParsePath(f *testing.F) {
t.Errorf("String() did not round-trip with case insensitivity: %q\ngot = %q\nwant = %q", s, r0.String(), s)
}
r1 := ParsePath(r0.String())
r1 := ParseName(r0.String())
if r0 != r1 {
t.Errorf("round-trip mismatch: %+v != %+v", r0, r1)
}
@ -216,8 +216,8 @@ func FuzzParsePath(f *testing.F) {
func ExampleMerge() {
r := Merge(
ParsePath("mistral"),
ParsePath("registry.ollama.com/XXXXX:latest+Q4_0"),
ParseName("mistral"),
ParseName("registry.ollama.com/XXXXX:latest+Q4_0"),
)
fmt.Println(r)

2
x/registry/server.go
View File

@ -82,7 +82,7 @@ func (s *Server) handlePush(w http.ResponseWriter, r *http.Request) error {
return err
}
mp := model.ParsePath(pr.Name)
mp := model.ParseName(pr.Name)
if !mp.Complete() {
return oweb.Invalid("name", pr.Name, "must be complete")
}