From 6bc5c137581cdd825627e7ec3da308843c94e162 Mon Sep 17 00:00:00 2001 From: Vyacheslav Moskalev Date: Thu, 1 Aug 2024 15:45:41 +0700 Subject: [PATCH 1/5] Fix extra context concatenation in generate handler (#5980). --- server/routes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/routes.go b/server/routes.go index fb9b3091..b449136e 100644 --- a/server/routes.go +++ b/server/routes.go @@ -247,7 +247,7 @@ func (s *Server) GenerateHandler(c *gin.Context) { ch <- gin.H{"error": err.Error()} return } - res.Context = append(req.Context, tokens...) + res.Context = tokens } } From 49a54831397d1723af4fbcd4f0c5c68dadbc54d5 Mon Sep 17 00:00:00 2001 From: Vyacheslav Moskalev Date: Thu, 1 Aug 2024 19:25:56 +0700 Subject: [PATCH 2/5] Change the order of context and prompt. --- server/routes.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/server/routes.go b/server/routes.go index b449136e..65ba22ea 100644 --- a/server/routes.go +++ b/server/routes.go @@ -188,21 +188,22 @@ func (s *Server) GenerateHandler(c *gin.Context) { } var b bytes.Buffer - if err := tmpl.Execute(&b, values); err != nil { + var t bytes.Buffer + if err := tmpl.Execute(&t, values); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } if req.Context != nil { - s, err := r.Detokenize(c.Request.Context(), req.Context) + prev, err := r.Detokenize(c.Request.Context(), req.Context) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - - b.WriteString(s) + b.WriteString(prev) } + b.WriteString(t.String()); prompt = b.String() } @@ -242,12 +243,12 @@ func (s *Server) GenerateHandler(c *gin.Context) { res.LoadDuration = checkpointLoaded.Sub(checkpointStart) if !req.Raw { - tokens, err := r.Tokenize(c.Request.Context(), prompt+sb.String()) + tokens, err := r.Tokenize(c.Request.Context(), prompt + sb.String()) if err != nil { ch <- gin.H{"error": err.Error()} return } - res.Context = tokens + res.Context = tokens[:] } } From b0c216584c82b47fa91323468a2c58e79f96f0bb Mon Sep 17 00:00:00 2001 From: Vyacheslav Moskalev Date: Thu, 1 Aug 2024 19:43:44 +0700 Subject: [PATCH 3/5] Better types and naming closer to style. --- server/routes.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/routes.go b/server/routes.go index 65ba22ea..0d397aaa 100644 --- a/server/routes.go +++ b/server/routes.go @@ -187,9 +187,9 @@ func (s *Server) GenerateHandler(c *gin.Context) { values.Messages = append(msgs, api.Message{Role: "user", Content: req.Prompt}) } + var s string var b bytes.Buffer - var t bytes.Buffer - if err := tmpl.Execute(&t, values); err != nil { + if err := tmpl.Execute(&b, values); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } @@ -200,11 +200,11 @@ func (s *Server) GenerateHandler(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - b.WriteString(prev) + s += prev } - b.WriteString(t.String()); - prompt = b.String() + s += b.String(); + prompt = s } slog.Debug("generate request", "prompt", prompt, "images", images) From 3b5210548e957c5011233ae0e114131413362188 Mon Sep 17 00:00:00 2001 From: Vyacheslav Moskalev Date: Thu, 1 Aug 2024 19:56:15 +0700 Subject: [PATCH 4/5] Refactor code. Remove extra variable. --- server/routes.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/server/routes.go b/server/routes.go index 0d397aaa..8184db75 100644 --- a/server/routes.go +++ b/server/routes.go @@ -187,7 +187,6 @@ func (s *Server) GenerateHandler(c *gin.Context) { values.Messages = append(msgs, api.Message{Role: "user", Content: req.Prompt}) } - var s string var b bytes.Buffer if err := tmpl.Execute(&b, values); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) @@ -195,16 +194,15 @@ func (s *Server) GenerateHandler(c *gin.Context) { } if req.Context != nil { - prev, err := r.Detokenize(c.Request.Context(), req.Context) + s, err := r.Detokenize(c.Request.Context(), req.Context) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - s += prev + prompt = s + b.String() + } else { + prompt = b.String(); } - - s += b.String(); - prompt = s } slog.Debug("generate request", "prompt", prompt, "images", images) From 8a9f946ca76d04cfc964a3ffe5b919c0fb51915b Mon Sep 17 00:00:00 2001 From: Vyacheslav Moskalev Date: Fri, 2 Aug 2024 03:50:05 +0700 Subject: [PATCH 5/5] Refactor and format code. --- server/routes.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/server/routes.go b/server/routes.go index 8184db75..a745fb20 100644 --- a/server/routes.go +++ b/server/routes.go @@ -188,21 +188,21 @@ func (s *Server) GenerateHandler(c *gin.Context) { } var b bytes.Buffer - if err := tmpl.Execute(&b, values); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - if req.Context != nil { s, err := r.Detokenize(c.Request.Context(), req.Context) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - prompt = s + b.String() - } else { - prompt = b.String(); + b.WriteString(s) } + + if err := tmpl.Execute(&b, values); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + prompt = b.String() } slog.Debug("generate request", "prompt", prompt, "images", images) @@ -241,12 +241,12 @@ func (s *Server) GenerateHandler(c *gin.Context) { res.LoadDuration = checkpointLoaded.Sub(checkpointStart) if !req.Raw { - tokens, err := r.Tokenize(c.Request.Context(), prompt + sb.String()) + tokens, err := r.Tokenize(c.Request.Context(), prompt+sb.String()) if err != nil { ch <- gin.H{"error": err.Error()} return } - res.Context = tokens[:] + res.Context = tokens } }