diff --git a/convert/convert_test.go b/convert/convert_test.go index 56b34f22..7dd489e2 100644 --- a/convert/convert_test.go +++ b/convert/convert_test.go @@ -89,7 +89,7 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestConvertFull(t *testing.T) { +func TestConvertModel(t *testing.T) { cases := []string{ "Meta-Llama-3-8B-Instruct", "Meta-Llama-3.1-8B-Instruct", diff --git a/convert/tokenizer.go b/convert/tokenizer.go index 429d36e7..14d6ba66 100644 --- a/convert/tokenizer.go +++ b/convert/tokenizer.go @@ -154,7 +154,6 @@ func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error) } type tokenizer struct { - Version string `json:"version"` AddedTokens []token `json:"added_tokens"` Model struct { Type string `json:"type"` @@ -252,7 +251,7 @@ func parseVocabulary(fsys fs.FS) (*Vocabulary, error) { return pattern.Func(fsys) } - return nil, errors.New("unknown tensor format") + return nil, errors.New("unknown tokenizer format") } type SpecialVocabulary struct {