x/model: move x/build.Ref -> x/model.Path

Also, update names and comments accordingly.
This commit is contained in:
Blake Mizerany 2024-04-04 14:02:31 -07:00
parent de72688b35
commit 58de2b8d4a
11 changed files with 88 additions and 94 deletions

View File

@ -1,4 +1,4 @@
package blob
package model
import (
"cmp"
@ -7,7 +7,7 @@ import (
"strings"
)
const MaxRefLength = 255
const MaxPathLength = 255
type PartKind int
@ -30,12 +30,12 @@ var kindNames = map[PartKind]string{
Build: "Build",
}
// Ref is an opaque reference to a blob.
// Path is an opaque reference to a model.
//
// It is comparable and can be used as a map key.
//
// Users or Ref must check Valid before using it.
type Ref struct {
// Users or Path must check Valid before using it.
type Path struct {
domain string
namespace string
name string
@ -46,7 +46,7 @@ type Ref struct {
// Format returns a string representation of the ref with the given
// concreteness. If a part is missing, it is replaced with a loud
// placeholder.
func (r Ref) Full() string {
func (r Path) Full() string {
r.domain = cmp.Or(r.domain, "!(MISSING DOMAIN)")
r.namespace = cmp.Or(r.namespace, "!(MISSING NAMESPACE)")
r.name = cmp.Or(r.name, "!(MISSING NAME)")
@ -55,21 +55,21 @@ func (r Ref) Full() string {
return r.String()
}
func (r Ref) NameAndTag() string {
func (r Path) NameAndTag() string {
r.domain = ""
r.namespace = ""
r.build = ""
return r.String()
}
func (r Ref) NameTagAndBuild() string {
func (r Path) NameTagAndBuild() string {
r.domain = ""
r.namespace = ""
return r.String()
}
// String returns the fully qualified ref string.
func (r Ref) String() string {
func (r Path) String() string {
var b strings.Builder
if r.domain != "" {
b.WriteString(r.domain)
@ -93,19 +93,19 @@ func (r Ref) String() string {
// Complete reports whether the ref is fully qualified. That is it has a
// domain, namespace, name, tag, and build.
func (r Ref) Complete() bool {
func (r Path) Complete() bool {
return r.Valid() && !slices.Contains(r.Parts(), "")
}
// CompleteWithoutBuild reports whether the ref would be complete if it had a
// valid build.
func (r Ref) CompleteWithoutBuild() bool {
func (r Path) CompleteWithoutBuild() bool {
r.build = "x"
return r.Valid() && r.Complete()
}
// Less returns true if r is less concrete than o; false otherwise.
func (r Ref) Less(o Ref) bool {
func (r Path) Less(o Path) bool {
rp := r.Parts()
op := o.Parts()
for i := range rp {
@ -119,7 +119,7 @@ func (r Ref) Less(o Ref) bool {
// Parts returns the parts of the ref in order of concreteness.
//
// The length of the returned slice is always 5.
func (r Ref) Parts() []string {
func (r Path) Parts() []string {
return []string{
r.domain,
r.namespace,
@ -129,36 +129,30 @@ func (r Ref) Parts() []string {
}
}
func (r Ref) Domain() string { return r.namespace }
func (r Ref) Namespace() string { return r.namespace }
func (r Ref) Name() string { return r.name }
func (r Ref) Tag() string { return r.tag }
func (r Ref) Build() string { return r.build }
func (r Path) Domain() string { return r.namespace }
func (r Path) Namespace() string { return r.namespace }
func (r Path) Name() string { return r.name }
func (r Path) Tag() string { return r.tag }
func (r Path) Build() string { return r.build }
// ParseRef parses a ref string into a Ref. A ref string is a name, an
// optional tag, and an optional build, separated by colons and pluses.
// ParsePath parses a model path string into a Path.
//
// The name must be valid ascii [a-zA-Z0-9_].
// The tag must be valid ascii [a-zA-Z0-9_].
// The build must be valid ascii [a-zA-Z0-9_].
// Examples of valid paths:
//
// It returns then zero value if the ref is invalid.
// "example.com/mistral:7b+x"
// "example.com/mistral:7b+Q4_0"
// "mistral:7b+x"
// "example.com/x/mistral:latest+Q4_0"
// "example.com/x/mistral:latest"
//
// // Valid Examples:
// ParseRef("mistral:latest") returns ("mistral", "latest", "")
// ParseRef("mistral") returns ("mistral", "", "")
// ParseRef("mistral:30B") returns ("mistral", "30B", "")
// ParseRef("mistral:7b") returns ("mistral", "7b", "")
// ParseRef("mistral:7b+Q4_0") returns ("mistral", "7b", "Q4_0")
// ParseRef("mistral+KQED") returns ("mistral", "latest", "KQED")
// ParseRef(".x.:7b+Q4_0:latest") returns (".x.", "7b", "Q4_0")
// ParseRef("-grok-f.oo:7b+Q4_0") returns ("-grok-f.oo", "7b", "Q4_0")
// Examples of invalid paths:
//
// // Invalid Examples:
// ParseRef("m stral") returns ("", "", "") // zero
// ParseRef("... 129 chars ...") returns ("", "", "") // zero
func ParseRef(s string) Ref {
var r Ref
// "example.com/mistral:7b+"
// "example.com/mistral:7b+Q4_0+"
// "x/y/z/z:8n+I"
// ""
func ParsePath(s string) Path {
var r Path
for kind, part := range Parts(s) {
switch kind {
case Domain:
@ -172,11 +166,11 @@ func ParseRef(s string) Ref {
case Build:
r.build = strings.ToUpper(part)
case Invalid:
return Ref{}
return Path{}
}
}
if !r.Valid() {
return Ref{}
return Path{}
}
return r
}
@ -185,8 +179,8 @@ func ParseRef(s string) Ref {
// The name is left untouched.
//
// Use this for merging a ref with a default ref.
func Merge(a, b Ref) Ref {
return Ref{
func Merge(a, b Path) Path {
return Path{
// name is left untouched
name: a.name,
@ -211,7 +205,7 @@ func Parts(s string) iter.Seq2[PartKind, string] {
s = s[len("https://"):]
}
if len(s) > MaxRefLength || len(s) == 0 {
if len(s) > MaxPathLength || len(s) == 0 {
return
}
@ -282,7 +276,7 @@ func Parts(s string) iter.Seq2[PartKind, string] {
// Valid returns true if the ref has a valid name. To know if a ref is
// "complete", use Complete.
func (r Ref) Valid() bool {
func (r Path) Valid() bool {
// Parts ensures we only have valid parts, so no need to validate
// them here, only check if we have a name or not.
return r.name != ""

View File

@ -1,4 +1,4 @@
package blob
package model
import (
"fmt"
@ -6,7 +6,7 @@ import (
"testing"
)
var testRefs = map[string]Ref{
var testPaths = map[string]Path{
"mistral:latest": {name: "mistral", tag: "latest"},
"mistral": {name: "mistral"},
"mistral:30B": {name: "mistral", tag: "30B"},
@ -36,33 +36,33 @@ var testRefs = map[string]Ref{
"file:///etc/passwd:latest": {},
"file:///etc/passwd:latest+u": {},
strings.Repeat("a", MaxRefLength): {name: strings.Repeat("a", MaxRefLength)},
strings.Repeat("a", MaxRefLength+1): {},
strings.Repeat("a", MaxPathLength): {name: strings.Repeat("a", MaxPathLength)},
strings.Repeat("a", MaxPathLength+1): {},
}
func TestRefParts(t *testing.T) {
func TestPathParts(t *testing.T) {
const wantNumParts = 5
var ref Ref
if len(ref.Parts()) != wantNumParts {
t.Errorf("Parts() = %d; want %d", len(ref.Parts()), wantNumParts)
var p Path
if len(p.Parts()) != wantNumParts {
t.Errorf("Parts() = %d; want %d", len(p.Parts()), wantNumParts)
}
}
func TestParseRef(t *testing.T) {
for s, want := range testRefs {
func TestParsePath(t *testing.T) {
for s, want := range testPaths {
for _, prefix := range []string{"", "https://", "http://"} {
// We should get the same results with or without the
// http(s) prefixes
s := prefix + s
t.Run(s, func(t *testing.T) {
got := ParseRef(s)
got := ParsePath(s)
if got != want {
t.Errorf("ParseRef(%q) = %q; want %q", s, got, want)
t.Errorf("ParsePath(%q) = %q; want %q", s, got, want)
}
// test round-trip
if ParseRef(got.String()) != got {
if ParsePath(got.String()) != got {
t.Errorf("String() = %s; want %s", got.String(), s)
}
@ -76,7 +76,7 @@ func TestParseRef(t *testing.T) {
}
}
func TestRefComplete(t *testing.T) {
func TestPathComplete(t *testing.T) {
cases := []struct {
in string
complete bool
@ -92,19 +92,19 @@ func TestRefComplete(t *testing.T) {
for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) {
ref := ParseRef(tt.in)
t.Logf("ParseRef(%q) = %#v", tt.in, ref)
if g := ref.Complete(); g != tt.complete {
p := ParsePath(tt.in)
t.Logf("ParsePath(%q) = %#v", tt.in, p)
if g := p.Complete(); g != tt.complete {
t.Errorf("Complete(%q) = %v; want %v", tt.in, g, tt.complete)
}
if g := ref.CompleteWithoutBuild(); g != tt.completeWithoutBuild {
if g := p.CompleteWithoutBuild(); g != tt.completeWithoutBuild {
t.Errorf("CompleteWithoutBuild(%q) = %v; want %v", tt.in, g, tt.completeWithoutBuild)
}
})
}
}
func TestRefStringVariants(t *testing.T) {
func TestPathStringVariants(t *testing.T) {
cases := []struct {
in string
nameAndTag string
@ -116,19 +116,19 @@ func TestRefStringVariants(t *testing.T) {
for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) {
ref := ParseRef(tt.in)
t.Logf("ParseRef(%q) = %#v", tt.in, ref)
if g := ref.NameAndTag(); g != tt.nameAndTag {
p := ParsePath(tt.in)
t.Logf("ParsePath(%q) = %#v", tt.in, p)
if g := p.NameAndTag(); g != tt.nameAndTag {
t.Errorf("NameAndTag(%q) = %q; want %q", tt.in, g, tt.nameAndTag)
}
if g := ref.NameTagAndBuild(); g != tt.nameTagAndBuild {
if g := p.NameTagAndBuild(); g != tt.nameTagAndBuild {
t.Errorf("NameTagAndBuild(%q) = %q; want %q", tt.in, g, tt.nameTagAndBuild)
}
})
}
}
func TestRefFull(t *testing.T) {
func TestPathFull(t *testing.T) {
const empty = "!(MISSING DOMAIN)/!(MISSING NAMESPACE)/!(MISSING NAME):!(MISSING TAG)+!(MISSING BUILD)"
cases := []struct {
@ -151,53 +151,53 @@ func TestRefFull(t *testing.T) {
for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) {
ref := ParseRef(tt.in)
t.Logf("ParseRef(%q) = %#v", tt.in, ref)
if g := ref.Full(); g != tt.wantFull {
p := ParsePath(tt.in)
t.Logf("ParsePath(%q) = %#v", tt.in, p)
if g := p.Full(); g != tt.wantFull {
t.Errorf("Full(%q) = %q; want %q", tt.in, g, tt.wantFull)
}
})
}
}
func TestParseRefAllocs(t *testing.T) {
func TestParsePathAllocs(t *testing.T) {
// test allocations
var r Ref
var r Path
allocs := testing.AllocsPerRun(1000, func() {
r = ParseRef("example.com/mistral:7b+Q4_0")
r = ParsePath("example.com/mistral:7b+Q4_0")
})
_ = r
if allocs > 0 {
t.Errorf("ParseRef allocs = %v; want 0", allocs)
t.Errorf("ParsePath allocs = %v; want 0", allocs)
}
}
func BenchmarkParseRef(b *testing.B) {
func BenchmarkParsePath(b *testing.B) {
b.ReportAllocs()
var r Ref
var r Path
for i := 0; i < b.N; i++ {
r = ParseRef("example.com/mistral:7b+Q4_0")
r = ParsePath("example.com/mistral:7b+Q4_0")
}
_ = r
}
func FuzzParseRef(f *testing.F) {
func FuzzParsePath(f *testing.F) {
f.Add("example.com/mistral:7b+Q4_0")
f.Add("example.com/mistral:7b+q4_0")
f.Add("example.com/mistral:7b+x")
f.Add("x/y/z:8n+I")
f.Fuzz(func(t *testing.T, s string) {
r0 := ParseRef(s)
r0 := ParsePath(s)
if !r0.Valid() {
if r0 != (Ref{}) {
t.Errorf("expected invalid ref to be zero value; got %#v", r0)
if r0 != (Path{}) {
t.Errorf("expected invalid path to be zero value; got %#v", r0)
}
t.Skipf("invalid ref: %q", s)
t.Skipf("invalid path: %q", s)
}
for _, p := range r0.Parts() {
if len(p) > MaxRefLength {
if len(p) > MaxPathLength {
t.Errorf("part too long: %q", p)
}
}
@ -206,7 +206,7 @@ func FuzzParseRef(f *testing.F) {
t.Errorf("String() did not round-trip with case insensitivity: %q\ngot = %q\nwant = %q", s, r0.String(), s)
}
r1 := ParseRef(r0.String())
r1 := ParsePath(r0.String())
if r0 != r1 {
t.Errorf("round-trip mismatch: %+v != %+v", r0, r1)
}
@ -216,8 +216,8 @@ func FuzzParseRef(f *testing.F) {
func ExampleMerge() {
r := Merge(
ParseRef("mistral"),
ParseRef("registry.ollama.com/XXXXX:latest+Q4_0"),
ParsePath("mistral"),
ParsePath("registry.ollama.com/XXXXX:latest+Q4_0"),
)
fmt.Println(r)

View File

@ -18,7 +18,7 @@ type Layer struct {
}
type PushRequest struct {
Ref string `json:"ref"`
Name string `json:"ref"`
Manifest json.RawMessage `json:"manifest"`
// Parts is a list of upload parts that the client upload in the previous

View File

@ -32,7 +32,7 @@ func (c *Client) Push(ctx context.Context, ref string, manifest []byte, p *PushP
p = cmp.Or(p, &PushParams{})
// TODO(bmizerany): backoff
v, err := ollama.Do[apitype.PushResponse](ctx, c.oclient(), "POST", "/v1/push", &apitype.PushRequest{
Ref: ref,
Name: ref,
Manifest: manifest,
CompleteParts: p.CompleteParts,
})

View File

@ -14,8 +14,8 @@ import (
"strconv"
"time"
"bllamo.com/build/blob"
"bllamo.com/client/ollama"
"bllamo.com/model"
"bllamo.com/oweb"
"bllamo.com/registry/apitype"
"bllamo.com/utils/upload"
@ -82,9 +82,9 @@ func (s *Server) handlePush(w http.ResponseWriter, r *http.Request) error {
return err
}
ref := blob.ParseRef(pr.Ref)
if !ref.Complete() {
return oweb.Invalid("name", pr.Ref, "must be complete")
mp := model.ParsePath(pr.Name)
if !mp.Complete() {
return oweb.Invalid("name", pr.Name, "must be complete")
}
m, err := oweb.DecodeUserJSON[apitype.Manifest]("manifest", bytes.NewReader(pr.Manifest))
@ -205,7 +205,7 @@ func (s *Server) handlePush(w http.ResponseWriter, r *http.Request) error {
if len(requirements) == 0 {
// Commit the manifest
body := bytes.NewReader(pr.Manifest)
path := path.Join("manifests", path.Join(ref.Parts()...))
path := path.Join("manifests", path.Join(mp.Parts()...))
_, err := s.mc().PutObject(r.Context(), bucketTODO, path, body, int64(len(pr.Manifest)), minio.PutObjectOptions{})
if err != nil {
return err