diff --git a/server/download.go b/server/download.go
index bb791130..80914ae3 100644
--- a/server/download.go
+++ b/server/download.go
@@ -314,6 +314,7 @@ func downloadBlob(ctx context.Context, opts downloadOpts) error {
 		requestURL := opts.mp.BaseURL()
 		requestURL = requestURL.JoinPath("v2", opts.mp.GetNamespaceRepository(), "blobs", opts.digest)
 		if err := download.Prepare(ctx, requestURL, opts.regOpts); err != nil {
+			blobDownloadManager.Delete(opts.digest)
 			return err
 		}