diff --git a/app/Resources/Viewer/scripts/bootstrap.mjs b/app/Resources/Viewer/scripts/bootstrap.mjs index 9a93020..d0c58ed 100644 --- a/app/Resources/Viewer/scripts/bootstrap.mjs +++ b/app/Resources/Viewer/scripts/bootstrap.mjs @@ -36,18 +36,69 @@ document.addEventListener("DOMContentLoaded", () => { gui.scrollMessagesToTop(); }); + async function fetchUrl(path, contentType) { + const response = await fetch("/" + path + "?token=" + encodeURIComponent(window.DHT_SERVER_TOKEN) + "&session=" + encodeURIComponent(window.DHT_SERVER_SESSION), { + method: "GET", + headers: { + "Content-Type": contentType, + }, + credentials: "omit", + redirect: "error", + }); + + if (!response.ok) { + throw "Unexpected response status: " + response.statusText; + } + + return response; + } + + async function processLines(response, callback) { + let body = ""; + + for await (const chunk of response.body.pipeThrough(new TextDecoderStream("utf-8"))) { + body += chunk; + + let startIndex = 0; + + while (true) { + const endIndex = body.indexOf("\n", startIndex); + if (endIndex === -1) { + break; + } + + callback(body.substring(startIndex, endIndex)); + startIndex = endIndex + 1; + } + + body = body.substring(startIndex); + } + + if (body !== "") { + callback(body); + } + } + async function loadData() { try { - const response = await fetch("/get-viewer-data?token=" + encodeURIComponent(window.DHT_SERVER_TOKEN) + "&session=" + encodeURIComponent(window.DHT_SERVER_SESSION), { - method: "GET", - headers: { - "Content-Type": "application/json", - }, - credentials: "omit", - redirect: "error", + const metadataResponse = await fetchUrl("get-viewer-metadata", "application/json"); + const metadataJson = await metadataResponse.json(); + + const messagesResponse = await fetchUrl("get-viewer-messages", "application/x-ndjson"); + const messages = {}; + + await processLines(messagesResponse, line => { + const message = JSON.parse(line); + const channel = message.c; + + const channelMessages = messages[channel] || (messages[channel] = {}); + channelMessages[message.id] = message; + + delete message.id; + delete message.c; }); - - state.uploadFile(await response.json()); + + state.uploadFile(metadataJson, messages); } catch (e) { console.error(e); alert("Could not load data, see console for details."); diff --git a/app/Resources/Viewer/scripts/processor.mjs b/app/Resources/Viewer/scripts/processor.mjs index 5a662db..0ba6ff6 100644 --- a/app/Resources/Viewer/scripts/processor.mjs +++ b/app/Resources/Viewer/scripts/processor.mjs @@ -5,7 +5,7 @@ import discord from "./discord.mjs"; // ------------------------ const filter = { - byUser: ((userindex) => message => message.u === userindex), + byUser: ((user) => message => message.u === user), byTime: ((timeStart, timeEnd) => message => message.t >= timeStart && message.t <= timeEnd), byContents: ((substr) => message => ("m" in message ? message.m : "").indexOf(substr) !== -1), byRegex: ((regex) => message => regex.test("m" in message ? message.m : "")), diff --git a/app/Resources/Viewer/scripts/state.mjs b/app/Resources/Viewer/scripts/state.mjs index a915f3b..476bbd6 100644 --- a/app/Resources/Viewer/scripts/state.mjs +++ b/app/Resources/Viewer/scripts/state.mjs @@ -6,8 +6,7 @@ export default (function() { /** * @type {{}} * @property {{}} users - * @property {String[]} userindex - * @property {{}[]} servers + * @property {{}} servers * @property {{}} channels */ let loadedFileMeta; @@ -20,20 +19,16 @@ export default (function() { let currentPage; let messagesPerPage; - const getUser = function(index) { - return loadedFileMeta.users[loadedFileMeta.userindex[index]] || { "name": "<unknown>" }; - }; - - const getUserId = function(index) { - return loadedFileMeta.userindex[index]; + const getUser = function(id) { + return loadedFileMeta.users[id] || { "name": "<unknown>" }; }; const getUserList = function() { return loadedFileMeta ? loadedFileMeta.users : []; }; - const getServer = function(index) { - return loadedFileMeta.servers[index] || { "name": "<unknown>", "type": "unknown" }; + const getServer = function(id) { + return loadedFileMeta.servers[id] || { "name": "<unknown>", "type": "unknown" }; }; const generateChannelHierarchy = function() { @@ -207,7 +202,7 @@ export default (function() { */ const message = messages[key]; const user = getUser(message.u); - const avatar = user.avatar ? { id: getUserId(message.u), path: user.avatar } : null; + const avatar = user.avatar ? { id: message.u, path: user.avatar } : null; const obj = { user, @@ -235,7 +230,7 @@ export default (function() { if ("r" in message) { const replyMessage = getMessageById(message.r); const replyUser = replyMessage ? getUser(replyMessage.u) : null; - const replyAvatar = replyUser && replyUser.avatar ? { id: getUserId(replyMessage.u), path: replyUser.avatar } : null; + const replyAvatar = replyUser && replyUser.avatar ? { id: replyMessage.u, path: replyUser.avatar } : null; obj["reply"] = replyMessage ? { "id": message.r, @@ -293,20 +288,17 @@ export default (function() { eventOnUsersRefreshed = callback; }, - /** - * @param {{ meta, data }} file - */ - uploadFile(file) { + uploadFile(meta, data) { if (loadedFileMeta != null) { throw "A file is already loaded!"; } - if (!file || typeof file.meta !== "object" || typeof file.data !== "object") { + if (typeof meta !== "object" || typeof data !== "object") { throw "Invalid file format!"; } - loadedFileMeta = file.meta; - loadedFileData = file.data; + loadedFileMeta = meta; + loadedFileData = data; loadedMessages = null; selectedChannel = null; @@ -419,7 +411,7 @@ export default (function() { setActiveFilter(filter) { switch (filter ? filter.type : "") { case "user": - filterFunction = processor.FILTER.byUser(loadedFileMeta.userindex.indexOf(filter.value)); + filterFunction = processor.FILTER.byUser(filter.value); break; case "contents": diff --git a/app/Server/Database/Export/Snowflake.cs b/app/Server/Database/Export/Snowflake.cs index 91146ed..5f28677 100644 --- a/app/Server/Database/Export/Snowflake.cs +++ b/app/Server/Database/Export/Snowflake.cs @@ -1,3 +1,5 @@ namespace DHT.Server.Database.Export; -readonly record struct Snowflake(ulong Id); +readonly record struct Snowflake(ulong Id) { + public static implicit operator Snowflake(ulong id) => new (id); +} diff --git a/app/Server/Database/Export/ViewerJson.cs b/app/Server/Database/Export/ViewerJson.cs index faf6ef8..24670df 100644 --- a/app/Server/Database/Export/ViewerJson.cs +++ b/app/Server/Database/Export/ViewerJson.cs @@ -3,14 +3,10 @@ using System.Text.Json.Serialization; namespace DHT.Server.Database.Export; -sealed class ViewerJson { - public required JsonMeta Meta { get; init; } - public required Dictionary> Data { get; init; } - +static class ViewerJson { public sealed class JsonMeta { public required Dictionary Users { get; init; } - public required List Userindex { get; init; } - public required List Servers { get; init; } + public required Dictionary Servers { get; init; } public required Dictionary Channels { get; init; } } @@ -30,7 +26,7 @@ sealed class ViewerJson { } public sealed class JsonChannel { - public required int Server { get; init; } + public required Snowflake Server { get; init; } public required string Name { get; init; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] @@ -47,7 +43,9 @@ sealed class ViewerJson { } public sealed class JsonMessage { - public required int U { get; init; } + public required Snowflake Id { get; init; } + public required Snowflake C { get; init; } + public required Snowflake U { get; init; } public required long T { get; init; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/app/Server/Database/Export/ViewerJsonExport.cs b/app/Server/Database/Export/ViewerJsonExport.cs index de912be..2faab63 100644 --- a/app/Server/Database/Export/ViewerJsonExport.cs +++ b/app/Server/Database/Export/ViewerJsonExport.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -14,106 +15,90 @@ namespace DHT.Server.Database.Export; static class ViewerJsonExport { private static readonly Log Log = Log.ForType(typeof(ViewerJsonExport)); - public static async Task Generate(Stream stream, IDatabaseFile db, MessageFilter? filter = null, CancellationToken cancellationToken = default) { + public static async Task GetMetadata(Stream stream, IDatabaseFile db, MessageFilter? filter = null, CancellationToken cancellationToken = default) { var perf = Log.Start(); - var includedUserIds = new HashSet(); - var includedChannelIds = new HashSet(); + var includedChannels = new List(); var includedServerIds = new HashSet(); - var includedMessages = await db.Messages.Get(filter, cancellationToken).ToListAsync(cancellationToken); - var includedChannels = new List(); - - foreach (var message in includedMessages) { - includedUserIds.Add(message.Sender); - includedChannelIds.Add(message.Channel); - } + var channelIdFilter = filter?.ChannelIds; await foreach (var channel in db.Channels.Get(cancellationToken)) { - if (includedChannelIds.Contains(channel.Id)) { + if (channelIdFilter == null || channelIdFilter.Contains(channel.Id)) { includedChannels.Add(channel); includedServerIds.Add(channel.Server); } } - var (users, userIndex, userIndices) = await GenerateUserList(db, includedUserIds, cancellationToken); - var (servers, serverIndices) = await GenerateServerList(db, includedServerIds, cancellationToken); - var channels = GenerateChannelList(includedChannels, serverIndices); + var users = await GenerateUserList(db, cancellationToken); + var servers = await GenerateServerList(db, includedServerIds, cancellationToken); + var channels = GenerateChannelList(includedChannels); + + var meta = new ViewerJson.JsonMeta { + Users = users, + Servers = servers, + Channels = channels + }; perf.Step("Collect database data"); - var value = new ViewerJson { - Meta = new ViewerJson.JsonMeta { - Users = users, - Userindex = userIndex, - Servers = servers, - Channels = channels - }, - Data = GenerateMessageList(includedMessages, userIndices) - }; - - perf.Step("Generate value object"); - - await JsonSerializer.SerializeAsync(stream, value, ViewerJsonContext.Default.ViewerJson, cancellationToken); + await JsonSerializer.SerializeAsync(stream, meta, ViewerJsonMetadataContext.Default.JsonMeta, cancellationToken); perf.Step("Serialize to JSON"); perf.End(); } - private static async Task<(Dictionary Users, List UserIndex, Dictionary UserIndices)> GenerateUserList(IDatabaseFile db, HashSet userIds, CancellationToken cancellationToken) { + public static async Task GetMessages(Stream stream, IDatabaseFile db, MessageFilter? filter = null, CancellationToken cancellationToken = default) { + var perf = Log.Start(); + + ReadOnlyMemory newLine = "\n"u8.ToArray(); + + await foreach(var message in GenerateMessageList(db, filter, cancellationToken)) { + await JsonSerializer.SerializeAsync(stream, message, ViewerJsonMessageContext.Default.JsonMessage, cancellationToken); + await stream.WriteAsync(newLine, cancellationToken); + } + + perf.Step("Generate and serialize messages to JSON"); + perf.End(); + } + + private static async Task> GenerateUserList(IDatabaseFile db, CancellationToken cancellationToken) { var users = new Dictionary(); - var userIndex = new List(); - var userIndices = new Dictionary(); await foreach (var user in db.Users.Get(cancellationToken)) { - var id = user.Id; - if (!userIds.Contains(id)) { - continue; - } - - var idSnowflake = new Snowflake(id); - userIndices[id] = users.Count; - userIndex.Add(idSnowflake); - - users[idSnowflake] = new ViewerJson.JsonUser { + users[user.Id] = new ViewerJson.JsonUser { Name = user.Name, Avatar = user.AvatarUrl, Tag = user.Discriminator }; } - return (users, userIndex, userIndices); + return users; } - private static async Task<(List Servers, Dictionary ServerIndices)> GenerateServerList(IDatabaseFile db, HashSet serverIds, CancellationToken cancellationToken) { - var servers = new List(); - var serverIndices = new Dictionary(); + private static async Task> GenerateServerList(IDatabaseFile db, HashSet serverIds, CancellationToken cancellationToken) { + var servers = new Dictionary(); await foreach (var server in db.Servers.Get(cancellationToken)) { - var id = server.Id; - if (!serverIds.Contains(id)) { + if (!serverIds.Contains(server.Id)) { continue; } - serverIndices[id] = servers.Count; - - servers.Add(new ViewerJson.JsonServer { + servers[server.Id] = new ViewerJson.JsonServer { Name = server.Name, Type = ServerTypes.ToJsonViewerString(server.Type) - }); + }; } - return (servers, serverIndices); + return servers; } - private static Dictionary GenerateChannelList(List includedChannels, Dictionary serverIndices) { + private static Dictionary GenerateChannelList(List includedChannels) { var channels = new Dictionary(); foreach (var channel in includedChannels) { - var channelIdSnowflake = new Snowflake(channel.Id); - - channels[channelIdSnowflake] = new ViewerJson.JsonChannel { - Server = serverIndices[channel.Server], + channels[channel.Id] = new ViewerJson.JsonChannel { + Server = channel.Server, Name = channel.Name, Parent = channel.ParentId?.ToString(), Position = channel.Position, @@ -125,51 +110,40 @@ static class ViewerJsonExport { return channels; } - private static Dictionary> GenerateMessageList(List includedMessages, Dictionary userIndices) { - var data = new Dictionary>(); + private static async IAsyncEnumerable GenerateMessageList(IDatabaseFile db, MessageFilter? filter, [EnumeratorCancellation] CancellationToken cancellationToken) { + await foreach (var message in db.Messages.Get(filter, cancellationToken)) { + yield return new ViewerJson.JsonMessage { + Id = message.Id, + C = message.Channel, + U = message.Sender, + T = message.Timestamp, + M = string.IsNullOrEmpty(message.Text) ? null : message.Text, + Te = message.EditTimestamp, + R = message.RepliedToId?.ToString(), - foreach (var grouping in includedMessages.GroupBy(static message => message.Channel)) { - var channelIdSnowflake = new Snowflake(grouping.Key); - var channelData = new Dictionary(); + A = message.Attachments.IsEmpty ? null : message.Attachments.Select(static attachment => { + var a = new ViewerJson.JsonMessageAttachment { + Url = attachment.DownloadUrl, + Name = Uri.TryCreate(attachment.NormalizedUrl, UriKind.Absolute, out var uri) ? Path.GetFileName(uri.LocalPath) : attachment.NormalizedUrl + }; - foreach (var message in grouping) { - var messageIdSnowflake = new Snowflake(message.Id); - - channelData[messageIdSnowflake] = new ViewerJson.JsonMessage { - U = userIndices[message.Sender], - T = message.Timestamp, - M = string.IsNullOrEmpty(message.Text) ? null : message.Text, - Te = message.EditTimestamp, - R = message.RepliedToId?.ToString(), - - A = message.Attachments.IsEmpty ? null : message.Attachments.Select(static attachment => { - var a = new ViewerJson.JsonMessageAttachment { - Url = attachment.DownloadUrl, - Name = Uri.TryCreate(attachment.NormalizedUrl, UriKind.Absolute, out var uri) ? Path.GetFileName(uri.LocalPath) : attachment.NormalizedUrl - }; + if (attachment is { Width: not null, Height: not null }) { + a.Width = attachment.Width; + a.Height = attachment.Height; + } - if (attachment is { Width: not null, Height: not null }) { - a.Width = attachment.Width; - a.Height = attachment.Height; - } + return a; + }).ToArray(), - return a; - }).ToArray(), - - E = message.Embeds.IsEmpty ? null : message.Embeds.Select(static embed => embed.Json).ToArray(), - - Re = message.Reactions.IsEmpty ? null : message.Reactions.Select(static reaction => new ViewerJson.JsonMessageReaction { - Id = reaction.EmojiId?.ToString(), - N = reaction.EmojiName, - A = reaction.EmojiFlags.HasFlag(EmojiFlags.Animated), - C = reaction.Count - }).ToArray() - }; - } + E = message.Embeds.IsEmpty ? null : message.Embeds.Select(static embed => embed.Json).ToArray(), - data[channelIdSnowflake] = channelData; + Re = message.Reactions.IsEmpty ? null : message.Reactions.Select(static reaction => new ViewerJson.JsonMessageReaction { + Id = reaction.EmojiId?.ToString(), + N = reaction.EmojiName, + A = reaction.EmojiFlags.HasFlag(EmojiFlags.Animated), + C = reaction.Count + }).ToArray() + }; } - - return data; } } diff --git a/app/Server/Database/Export/ViewerJsonMessageContext.cs b/app/Server/Database/Export/ViewerJsonMessageContext.cs new file mode 100644 index 0000000..df3ba68 --- /dev/null +++ b/app/Server/Database/Export/ViewerJsonMessageContext.cs @@ -0,0 +1,11 @@ +using System.Text.Json.Serialization; + +namespace DHT.Server.Database.Export; + +[JsonSourceGenerationOptions( + Converters = [typeof(SnowflakeJsonSerializer)], + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + GenerationMode = JsonSourceGenerationMode.Default +)] +[JsonSerializable(typeof(ViewerJson.JsonMessage))] +sealed partial class ViewerJsonMessageContext : JsonSerializerContext; diff --git a/app/Server/Database/Export/ViewerJsonContext.cs b/app/Server/Database/Export/ViewerJsonMetadataContext.cs similarity index 69% rename from app/Server/Database/Export/ViewerJsonContext.cs rename to app/Server/Database/Export/ViewerJsonMetadataContext.cs index de4dec2..18c3f6d 100644 --- a/app/Server/Database/Export/ViewerJsonContext.cs +++ b/app/Server/Database/Export/ViewerJsonMetadataContext.cs @@ -7,5 +7,5 @@ namespace DHT.Server.Database.Export; PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, GenerationMode = JsonSourceGenerationMode.Default )] -[JsonSerializable(typeof(ViewerJson))] -sealed partial class ViewerJsonContext : JsonSerializerContext; +[JsonSerializable(typeof(ViewerJson.JsonMeta))] +sealed partial class ViewerJsonMetadataContext : JsonSerializerContext; diff --git a/app/Server/Endpoints/BaseEndpoint.cs b/app/Server/Endpoints/BaseEndpoint.cs index cbea1e0..c6ef216 100644 --- a/app/Server/Endpoints/BaseEndpoint.cs +++ b/app/Server/Endpoints/BaseEndpoint.cs @@ -47,4 +47,13 @@ abstract class BaseEndpoint(IDatabaseFile db) { throw new HttpException(HttpStatusCode.UnsupportedMediaType, "This endpoint only accepts JSON."); } } + + protected static Guid GetSessionId(HttpRequest request) { + if (request.Query.TryGetValue("session", out var sessionIdValue) && sessionIdValue.Count == 1 && Guid.TryParse(sessionIdValue[0], out Guid sessionId)) { + return sessionId; + } + else { + throw new HttpException(HttpStatusCode.BadRequest, "Invalid session ID."); + } + } } diff --git a/app/Server/Endpoints/GetViewerDataEndpoint.cs b/app/Server/Endpoints/GetViewerDataEndpoint.cs deleted file mode 100644 index b3c8b66..0000000 --- a/app/Server/Endpoints/GetViewerDataEndpoint.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System; -using System.Net; -using System.Net.Mime; -using System.Threading; -using System.Threading.Tasks; -using DHT.Server.Database; -using DHT.Server.Database.Export; -using DHT.Server.Service.Viewer; -using DHT.Utils.Http; -using Microsoft.AspNetCore.Http; - -namespace DHT.Server.Endpoints; - -sealed class GetViewerDataEndpoint(IDatabaseFile db, ViewerSessions viewerSessions) : BaseEndpoint(db) { - protected override Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) { - if (!request.Query.TryGetValue("session", out var sessionIdValue) || sessionIdValue.Count != 1 || !Guid.TryParse(sessionIdValue[0], out Guid sessionId)) { - throw new HttpException(HttpStatusCode.BadRequest, "Invalid session ID."); - } - - response.ContentType = MediaTypeNames.Application.Json; - - var session = viewerSessions.Get(sessionId); - return ViewerJsonExport.Generate(response.Body, Db, session.MessageFilter, cancellationToken); - } -} diff --git a/app/Server/Endpoints/GetViewerMessagesEndpoint.cs b/app/Server/Endpoints/GetViewerMessagesEndpoint.cs new file mode 100644 index 0000000..3da01b3 --- /dev/null +++ b/app/Server/Endpoints/GetViewerMessagesEndpoint.cs @@ -0,0 +1,18 @@ +using System.Threading; +using System.Threading.Tasks; +using DHT.Server.Database; +using DHT.Server.Database.Export; +using DHT.Server.Service.Viewer; +using Microsoft.AspNetCore.Http; + +namespace DHT.Server.Endpoints; + +sealed class GetViewerMessagesEndpoint(IDatabaseFile db, ViewerSessions viewerSessions) : BaseEndpoint(db) { + protected override Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) { + var sessionId = GetSessionId(request); + var session = viewerSessions.Get(sessionId); + + response.ContentType = "application/x-ndjson"; + return ViewerJsonExport.GetMessages(response.Body, Db, session.MessageFilter, cancellationToken); + } +} diff --git a/app/Server/Endpoints/GetViewerMetadataEndpoint.cs b/app/Server/Endpoints/GetViewerMetadataEndpoint.cs new file mode 100644 index 0000000..3fee364 --- /dev/null +++ b/app/Server/Endpoints/GetViewerMetadataEndpoint.cs @@ -0,0 +1,19 @@ +using System.Net.Mime; +using System.Threading; +using System.Threading.Tasks; +using DHT.Server.Database; +using DHT.Server.Database.Export; +using DHT.Server.Service.Viewer; +using Microsoft.AspNetCore.Http; + +namespace DHT.Server.Endpoints; + +sealed class GetViewerMetadataEndpoint(IDatabaseFile db, ViewerSessions viewerSessions) : BaseEndpoint(db) { + protected override Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) { + var sessionId = GetSessionId(request); + var session = viewerSessions.Get(sessionId); + + response.ContentType = MediaTypeNames.Application.Json; + return ViewerJsonExport.GetMetadata(response.Body, Db, session.MessageFilter, cancellationToken); + } +} diff --git a/app/Server/Service/ServerStartup.cs b/app/Server/Service/ServerStartup.cs index b559141..a4275cb 100644 --- a/app/Server/Service/ServerStartup.cs +++ b/app/Server/Service/ServerStartup.cs @@ -51,7 +51,8 @@ sealed class Startup { app.UseRouting(); app.UseEndpoints(endpoints => { endpoints.MapGet("/get-tracking-script", new GetTrackingScriptEndpoint(db, parameters, resources).Handle); - endpoints.MapGet("/get-viewer-data", new GetViewerDataEndpoint(db, viewerSessions).Handle); + endpoints.MapGet("/get-viewer-metadata", new GetViewerMetadataEndpoint(db, viewerSessions).Handle); + endpoints.MapGet("/get-viewer-messages", new GetViewerMessagesEndpoint(db, viewerSessions).Handle); endpoints.MapGet("/get-downloaded-file/{url}", new GetDownloadedFileEndpoint(db).Handle); endpoints.MapPost("/track-channel", new TrackChannelEndpoint(db).Handle); endpoints.MapPost("/track-users", new TrackUsersEndpoint(db).Handle);