third-party-mirrors/Discord-History-Tracker
1
0
Fork 0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2025-04-12 06:50:01 +03:00
Code Issues Projects Releases Wiki Activity

Optimize viewer message export

Browse Source
This commit is contained in:
chylex 2025-03-18 16:06:51 +01:00
parent 38f79dee7d
commit 780d5ae421
No known key found for this signature in database
3 changed files with 77 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
app/Server/Database/Export/SnowflakeJsonSerializer.cs
View File

@ -1,16 +1,20 @@
using System;
using System.Buffers.Text;
using System.Diagnostics;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace DHT.Server.Database.Export;
sealed class SnowflakeJsonSerializer : JsonConverter<Snowflake> {
private const int MaxUlongStringLength = 20;
public override Snowflake Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
return new Snowflake(ulong.Parse(reader.GetString()!));
}
public override void Write(Utf8JsonWriter writer, Snowflake value, JsonSerializerOptions options) {
writer.WriteStringValue(value.Id.ToString());
writer.WriteStringValue(Format(value, stackalloc byte[MaxUlongStringLength]));
}
public override Snowflake ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
@ -18,6 +22,14 @@ sealed class SnowflakeJsonSerializer : JsonConverter<Snowflake> {
}
public override void WriteAsPropertyName(Utf8JsonWriter writer, Snowflake value, JsonSerializerOptions options) {
writer.WritePropertyName(value.Id.ToString());
writer.WritePropertyName(Format(value, stackalloc byte[MaxUlongStringLength]));
}
private static ReadOnlySpan<byte> Format(Snowflake value, Span<byte> destination) {
if (!Utf8Formatter.TryFormat(value.Id, destination, out int bytesWritten)) {
Debug.Fail("Failed to format Snowflake value.");
}
return destination[..bytesWritten];
}
}

6
app/Server/Database/Export/ViewerJson.cs
View File

@ -30,7 +30,7 @@ static class ViewerJson {
public required string Name { get; init; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Parent { get; init; }
public Snowflake? Parent { get; init; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? Position { get; init; }
@ -55,7 +55,7 @@ static class ViewerJson {
public long? Te { get; init; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? R { get; init; }
public Snowflake? R { get; init; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public JsonMessageAttachment[]? A { get; init; }
@ -80,7 +80,7 @@ static class ViewerJson {
public sealed class JsonMessageReaction {
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Id { get; init; }
public Snowflake? Id { get; init; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? N { get; init; }

101
app/Server/Database/Export/ViewerJsonExport.cs
View File

@ -2,13 +2,15 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text.Json;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Filters;
using DHT.Utils.Logging;
using Channel = System.Threading.Channels.Channel;
using DiscordChannel = DHT.Server.Data.Channel;
namespace DHT.Server.Database.Export;
@ -18,12 +20,12 @@ static class ViewerJsonExport {
public static async Task GetMetadata(Stream stream, IDatabaseFile db, MessageFilter? filter = null, CancellationToken cancellationToken = default) {
Perf perf = Log.Start();
var includedChannels = new List<Channel>();
var includedChannels = new List<DiscordChannel>();
var includedServerIds = new HashSet<ulong>();
HashSet<ulong>? channelIdFilter = filter?.ChannelIds;
await foreach (Channel channel in db.Channels.Get(cancellationToken)) {
await foreach (DiscordChannel channel in db.Channels.Get(cancellationToken)) {
if (channelIdFilter == null || channelIdFilter.Contains(channel.Id)) {
includedChannels.Add(channel);
includedServerIds.Add(channel.Server);
@ -53,11 +55,30 @@ static class ViewerJsonExport {
ReadOnlyMemory<byte> newLine = "\n"u8.ToArray();
await foreach (ViewerJson.JsonMessage message in GenerateMessageList(db, filter, cancellationToken)) {
await JsonSerializer.SerializeAsync(stream, message, ViewerJsonMessageContext.Default.JsonMessage, cancellationToken);
Channel<Message> channel = Channel.CreateBounded<Message>(new BoundedChannelOptions(32) {
SingleWriter = true,
SingleReader = true,
AllowSynchronousContinuations = true,
FullMode = BoundedChannelFullMode.Wait,
});
Task writerTask = Task.Run(async () => {
try {
await foreach (Message message in db.Messages.Get(filter, cancellationToken)) {
await channel.Writer.WriteAsync(message, cancellationToken);
}
} finally {
channel.Writer.Complete();
}
}, cancellationToken);
await foreach (Message message in channel.Reader.ReadAllAsync(cancellationToken)) {
await JsonSerializer.SerializeAsync(stream, ToJsonMessage(message), ViewerJsonMessageContext.Default.JsonMessage, cancellationToken);
await stream.WriteAsync(newLine, cancellationToken);
}
await writerTask;
perf.Step("Generate and serialize messages to JSON");
perf.End();
}
@ -93,14 +114,14 @@ static class ViewerJsonExport {
return servers;
}
private static Dictionary<Snowflake, ViewerJson.JsonChannel> GenerateChannelList(List<Channel> includedChannels) {
private static Dictionary<Snowflake, ViewerJson.JsonChannel> GenerateChannelList(List<DiscordChannel> includedChannels) {
var channels = new Dictionary<Snowflake, ViewerJson.JsonChannel>();
foreach (Channel channel in includedChannels) {
foreach (DiscordChannel channel in includedChannels) {
channels[channel.Id] = new ViewerJson.JsonChannel {
Server = channel.Server,
Name = channel.Name,
Parent = channel.ParentId?.ToString(),
Parent = channel.ParentId,
Position = channel.Position,
Topic = channel.Topic,
Nsfw = channel.Nsfw,
@ -110,40 +131,38 @@ static class ViewerJsonExport {
return channels;
}
private static async IAsyncEnumerable<ViewerJson.JsonMessage> GenerateMessageList(IDatabaseFile db, MessageFilter? filter, [EnumeratorCancellation] CancellationToken cancellationToken) {
await foreach (Message message in db.Messages.Get(filter, cancellationToken)) {
yield return new ViewerJson.JsonMessage {
Id = message.Id,
C = message.Channel,
U = message.Sender,
T = message.Timestamp,
M = string.IsNullOrEmpty(message.Text) ? null : message.Text,
Te = message.EditTimestamp,
R = message.RepliedToId?.ToString(),
private static ViewerJson.JsonMessage ToJsonMessage(Message message) {
return new ViewerJson.JsonMessage {
Id = message.Id,
C = message.Channel,
U = message.Sender,
T = message.Timestamp,
M = string.IsNullOrEmpty(message.Text) ? null : message.Text,
Te = message.EditTimestamp,
R = message.RepliedToId,
A = message.Attachments.IsEmpty ? null : message.Attachments.Select(static attachment => {
var a = new ViewerJson.JsonMessageAttachment {
Url = attachment.DownloadUrl,
Name = Uri.TryCreate(attachment.NormalizedUrl, UriKind.Absolute, out Uri? uri) ? Path.GetFileName(uri.LocalPath) : attachment.NormalizedUrl,
};
A = message.Attachments.IsEmpty ? null : message.Attachments.Select(static attachment => {
var a = new ViewerJson.JsonMessageAttachment {
Url = attachment.DownloadUrl,
Name = Uri.TryCreate(attachment.NormalizedUrl, UriKind.Absolute, out Uri? uri) ? Path.GetFileName(uri.LocalPath) : attachment.NormalizedUrl,
};
if (attachment is { Width: not null, Height: not null }) {
a.Width = attachment.Width;
a.Height = attachment.Height;
}
return a;
}).ToArray(),
if (attachment is { Width: not null, Height: not null }) {
a.Width = attachment.Width;
a.Height = attachment.Height;
}
E = message.Embeds.IsEmpty ? null : message.Embeds.Select(static embed => embed.Json).ToArray(),
Re = message.Reactions.IsEmpty ? null : message.Reactions.Select(static reaction => new ViewerJson.JsonMessageReaction {
Id = reaction.EmojiId?.ToString(),
N = reaction.EmojiName,
A = reaction.EmojiFlags.HasFlag(EmojiFlags.Animated),
C = reaction.Count,
}).ToArray(),
};
}
return a;
}).ToArray(),
E = message.Embeds.IsEmpty ? null : message.Embeds.Select(static embed => embed.Json).ToArray(),
Re = message.Reactions.IsEmpty ? null : message.Reactions.Select(static reaction => new ViewerJson.JsonMessageReaction {
Id = reaction.EmojiId,
N = reaction.EmojiName,
A = reaction.EmojiFlags.HasFlag(EmojiFlags.Animated),
C = reaction.Count,
}).ToArray(),
};
}
}