third-party-mirrors/Discord-History-Tracker
1
0
Fork 0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2025-04-15 08:00:33 +03:00
Code Issues Projects Releases Wiki Activity

Stream downloaded files from database directly into HTTP server responses

Browse Source
This commit is contained in:
chylex 2024-04-17 11:51:37 +02:00
parent 70c04fc986
commit d79e6f53b4
No known key found for this signature in database
GPG Key ID: 4DE42C8F19A80548
11 changed files with 89 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/Server/Database/Repositories/IDownloadRepository.cs
View File

@ -25,7 +25,7 @@ public interface IDownloadRepository {
Task<DownloadWithData> HydrateWithData(Data.Download download);
Task<DownloadWithData?> GetSuccessfulDownloadWithData(string normalizedUrl);
Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor);
IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, CancellationToken cancellationToken = default);
@ -56,8 +56,8 @@ public interface IDownloadRepository {
return Task.FromResult(new DownloadWithData(download, Data: null));
}
public Task<DownloadWithData?> GetSuccessfulDownloadWithData(string normalizedUrl) {
return Task.FromResult<DownloadWithData?>(null);
public Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor) {
return Task.FromResult(false);
}
public IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, CancellationToken cancellationToken) {

30
app/Server/Database/Sqlite/Repositories/SqliteDownloadRepository.cs
View File

@ -205,12 +205,12 @@ sealed class SqliteDownloadRepository : BaseSqliteRepository, IDownloadRepositor
return new DownloadWithData(download, data);
}
public async Task<DownloadWithData?> GetSuccessfulDownloadWithData(string normalizedUrl) {
public async Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor) {
await using var conn = await pool.Take();
await using var cmd = conn.Command(
"""
SELECT dm.download_url, dm.type, db.blob FROM download_metadata dm
SELECT dm.download_url, dm.type, db.rowid FROM download_metadata dm
JOIN download_blobs db ON dm.normalized_url = db.normalized_url
WHERE dm.normalized_url = :normalized_url AND dm.status = :success IS NOT NULL
"""
@ -219,19 +219,25 @@ sealed class SqliteDownloadRepository : BaseSqliteRepository, IDownloadRepositor
cmd.AddAndSet(":normalized_url", SqliteType.Text, normalizedUrl);
cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
await using var reader = await cmd.ExecuteReaderAsync();
string downloadUrl;
string? type;
long rowid;
await using (var reader = await cmd.ExecuteReaderAsync()) {
if (!await reader.ReadAsync()) {
return false;
}
if (!await reader.ReadAsync()) {
return null;
downloadUrl = reader.GetString(0);
type = reader.IsDBNull(1) ? null : reader.GetString(1);
rowid = reader.GetInt64(2);
}
await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) {
await dataProcessor(new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, (ulong) blob.Length), blob);
}
var downloadUrl = reader.GetString(0);
var type = reader.IsDBNull(1) ? null : reader.GetString(1);
var data = (byte[]) reader[2];
var size = (ulong) data.LongLength;
var download = new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, size);
return new DownloadWithData(download, data);
return true;
}
public async IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, [EnumeratorCancellation] CancellationToken cancellationToken) {

24
app/Server/Endpoints/BaseEndpoint.cs
View File

@ -9,37 +9,37 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints;
abstract class BaseEndpoint {
abstract class BaseEndpoint(IDatabaseFile db) {
private static readonly Log Log = Log.ForType<BaseEndpoint>();
protected IDatabaseFile Db { get; }
protected BaseEndpoint(IDatabaseFile db) {
this.Db = db;
}
protected IDatabaseFile Db { get; } = db;
public async Task Handle(HttpContext ctx) {
var response = ctx.Response;
try {
response.StatusCode = (int) HttpStatusCode.OK;
var output = await Respond(ctx);
await output.WriteTo(response);
await Respond(ctx.Request, response);
} catch (HttpException e) {
Log.Error(e);
response.StatusCode = (int) e.StatusCode;
await response.WriteAsync(e.Message);
if (response.HasStarted) {
Log.Warn("Response has already started, cannot write status message: " + e.Message);
}
else {
await response.WriteAsync(e.Message);
}
} catch (Exception e) {
Log.Error(e);
response.StatusCode = (int) HttpStatusCode.InternalServerError;
}
}
protected abstract Task<IHttpOutput> Respond(HttpContext ctx);
protected abstract Task Respond(HttpRequest request, HttpResponse response);
protected static async Task<JsonElement> ReadJson(HttpContext ctx) {
protected static async Task<JsonElement> ReadJson(HttpRequest request) {
try {
return await ctx.Request.ReadFromJsonAsync(JsonElementContext.Default.JsonElement);
return await request.ReadFromJsonAsync(JsonElementContext.Default.JsonElement);
} catch (JsonException) {
throw new HttpException(HttpStatusCode.UnsupportedMediaType, "This endpoint only accepts JSON.");
}

15
app/Server/Endpoints/GetDownloadedFileEndpoint.cs
View File

@ -7,18 +7,13 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints;
sealed class GetDownloadedFileEndpoint : BaseEndpoint {
public GetDownloadedFileEndpoint(IDatabaseFile db) : base(db) {}
protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
string url = WebUtility.UrlDecode((string) ctx.Request.RouteValues["url"]!);
sealed class GetDownloadedFileEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
protected override async Task Respond(HttpRequest request, HttpResponse response) {
string url = WebUtility.UrlDecode((string) request.RouteValues["url"]!);
string normalizedUrl = DiscordCdn.NormalizeUrl(url);
if (await Db.Downloads.GetSuccessfulDownloadWithData(normalizedUrl) is { Download: {} download, Data: {} data }) {
return new HttpOutput.File(download.Type, data);
}
else {
return new HttpOutput.Redirect(url, permanent: false);
if (!await Db.Downloads.GetSuccessfulDownloadWithData(normalizedUrl, (download, stream) => response.WriteStreamAsync(download.Type, download.Size, stream))) {
response.Redirect(url, permanent: false);
}
}
}

22
app/Server/Endpoints/GetTrackingScriptEndpoint.cs
View File

@ -1,5 +1,5 @@
using System.Net.Mime;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using DHT.Server.Database;
@ -10,25 +10,19 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints;
sealed class GetTrackingScriptEndpoint : BaseEndpoint {
sealed class GetTrackingScriptEndpoint(IDatabaseFile db, ServerParameters parameters) : BaseEndpoint(db) {
private static ResourceLoader Resources { get; } = new (Assembly.GetExecutingAssembly());
private readonly ServerParameters serverParameters;
public GetTrackingScriptEndpoint(IDatabaseFile db, ServerParameters parameters) : base(db) {
serverParameters = parameters;
}
protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
protected override async Task Respond(HttpRequest request, HttpResponse response) {
string bootstrap = await Resources.ReadTextAsync("Tracker/bootstrap.js");
string script = bootstrap.Replace("= 0; /*[PORT]*/", "= " + serverParameters.Port + ";")
.Replace("/*[TOKEN]*/", HttpUtility.JavaScriptStringEncode(serverParameters.Token))
string script = bootstrap.Replace("= 0; /*[PORT]*/", "= " + parameters.Port + ";")
.Replace("/*[TOKEN]*/", HttpUtility.JavaScriptStringEncode(parameters.Token))
.Replace("/*[IMPORTS]*/", await Resources.ReadJoinedAsync("Tracker/scripts/", '\n'))
.Replace("/*[CSS-CONTROLLER]*/", await Resources.ReadTextAsync("Tracker/styles/controller.css"))
.Replace("/*[CSS-SETTINGS]*/", await Resources.ReadTextAsync("Tracker/styles/settings.css"))
.Replace("/*[DEBUGGER]*/", ctx.Request.Query.ContainsKey("debug") ? "debugger;" : "");
.Replace("/*[DEBUGGER]*/", request.Query.ContainsKey("debug") ? "debugger;" : "");
ctx.Response.Headers.Append("X-DHT", "1");
return new HttpOutput.File("text/javascript", Encoding.UTF8.GetBytes(script));
response.Headers.Append("X-DHT", "1");
await response.WriteTextAsync(MediaTypeNames.Text.JavaScript, script);
}
}

10
app/Server/Endpoints/TrackChannelEndpoint.cs
View File

@ -8,18 +8,14 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints;
sealed class TrackChannelEndpoint : BaseEndpoint {
public TrackChannelEndpoint(IDatabaseFile db) : base(db) {}
protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
var root = await ReadJson(ctx);
sealed class TrackChannelEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
protected override async Task Respond(HttpRequest request, HttpResponse response) {
var root = await ReadJson(request);
var server = ReadServer(root.RequireObject("server"), "server");
var channel = ReadChannel(root.RequireObject("channel"), "channel", server.Id);
await Db.Servers.Add([server]);
await Db.Channels.Add([channel]);
return HttpOutput.None;
}
private static Data.Server ReadServer(JsonElement json, string path) => new () {

10
app/Server/Endpoints/TrackMessagesEndpoint.cs
View File

@ -15,14 +15,12 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints;
sealed class TrackMessagesEndpoint : BaseEndpoint {
sealed class TrackMessagesEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
private const string HasNewMessages = "1";
private const string NoNewMessages = "0";
public TrackMessagesEndpoint(IDatabaseFile db) : base(db) {}
protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
var root = await ReadJson(ctx);
protected override async Task Respond(HttpRequest request, HttpResponse response) {
var root = await ReadJson(request);
if (root.ValueKind != JsonValueKind.Array) {
throw new HttpException(HttpStatusCode.BadRequest, "Expected root element to be an array.");
@ -43,7 +41,7 @@ sealed class TrackMessagesEndpoint : BaseEndpoint {
await Db.Messages.Add(messages);
return new HttpOutput.Text(anyNewMessages ? HasNewMessages : NoNewMessages);
await response.WriteTextAsync(anyNewMessages ? HasNewMessages : NoNewMessages);
}
private static Message ReadMessage(JsonElement json, string path) => new () {

10
app/Server/Endpoints/TrackUsersEndpoint.cs
View File

@ -8,11 +8,9 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints;
sealed class TrackUsersEndpoint : BaseEndpoint {
public TrackUsersEndpoint(IDatabaseFile db) : base(db) {}
protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
var root = await ReadJson(ctx);
sealed class TrackUsersEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
protected override async Task Respond(HttpRequest request, HttpResponse response) {
var root = await ReadJson(request);
if (root.ValueKind != JsonValueKind.Array) {
throw new HttpException(HttpStatusCode.BadRequest, "Expected root element to be an array.");
@ -26,8 +24,6 @@ sealed class TrackUsersEndpoint : BaseEndpoint {
}
await Db.Users.Add(users);
return HttpOutput.None;
}
private static User ReadUser(JsonElement json, string path) => new () {

33
app/Utils/Http/HttpExtensions.cs Normal file
View File

@ -0,0 +1,33 @@
using System.IO;
using System.Net.Mime;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
namespace DHT.Utils.Http;
public static class HttpExtensions {
public static Task WriteTextAsync(this HttpResponse response, string text) {
return WriteTextAsync(response, MediaTypeNames.Text.Plain, text);
}
public static async Task WriteTextAsync(this HttpResponse response, string contentType, string text) {
response.ContentType = contentType;
await response.StartAsync();
await response.WriteAsync(text, Encoding.UTF8);
}
public static async Task WriteFileAsync(this HttpResponse response, string? contentType, byte[] bytes) {
response.ContentType = contentType ?? string.Empty;
response.ContentLength = bytes.Length;
await response.StartAsync();
await response.Body.WriteAsync(bytes);
}
public static async Task WriteStreamAsync(this HttpResponse response, string? contentType, ulong? contentLength, Stream source) {
response.ContentType = contentType ?? string.Empty;
response.ContentLength = (long?) contentLength;
await response.StartAsync();
await source.CopyToAsync(response.Body);
}
}

35
app/Utils/Http/HttpOutput.cs
View File

@ -1,35 +0,0 @@
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
namespace DHT.Utils.Http;
public static class HttpOutput {
public static IHttpOutput None { get; } = new NoneImpl();
private sealed class NoneImpl : IHttpOutput {
public Task WriteTo(HttpResponse response) {
return Task.CompletedTask;
}
}
public sealed class Text(string text) : IHttpOutput {
public Task WriteTo(HttpResponse response) {
return response.WriteAsync(text, Encoding.UTF8);
}
}
public sealed class File(string? contentType, byte[] bytes) : IHttpOutput {
public async Task WriteTo(HttpResponse response) {
response.ContentType = contentType ?? string.Empty;
await response.Body.WriteAsync(bytes);
}
}
public sealed class Redirect(string url, bool permanent) : IHttpOutput {
public Task WriteTo(HttpResponse response) {
response.Redirect(url, permanent);
return Task.CompletedTask;
}
}
}

8
app/Utils/Http/IHttpOutput.cs
View File

@ -1,8 +0,0 @@
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
namespace DHT.Utils.Http;
public interface IHttpOutput {
Task WriteTo(HttpResponse response);
}