third-party-mirrors/Discord-History-Tracker
1
0
Fork 0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2025-04-14 15:47:14 +03:00
Code Issues Projects Releases Wiki Activity

Split edit timestamps and message reply ids to separate tables to reduce database size

Browse Source
This commit is contained in:
chylex 2022-03-05 20:34:36 +01:00
parent 6f1149ad5e
commit e420add646
No known key found for this signature in database
GPG Key ID: 4DE42C8F19A80548
4 changed files with 101 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
app/Server/Database/Sqlite/Schema.cs
View File

@ -5,7 +5,7 @@ using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite {
sealed class Schema {
internal const int Version = 2;
internal const int Version = 3;
private readonly SqliteConnection conn;
@ -97,6 +97,9 @@ namespace DHT.Server.Database.Sqlite {
emoji_flags INTEGER NOT NULL,
count INTEGER NOT NULL)");
CreateMessageEditTimestampTable();
CreateMessageRepliedToTable();
Execute("CREATE INDEX attachments_message_ix ON attachments(message_id)");
Execute("CREATE INDEX embeds_message_ix ON embeds(message_id)");
Execute("CREATE INDEX reactions_message_ix ON reactions(message_id)");
@ -104,12 +107,41 @@ namespace DHT.Server.Database.Sqlite {
Execute("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
}
private void CreateMessageEditTimestampTable() {
Execute(@"CREATE TABLE edit_timestamps (
message_id INTEGER PRIMARY KEY NOT NULL,
edit_timestamp INTEGER NOT NULL)");
}
private void CreateMessageRepliedToTable() {
Execute(@"CREATE TABLE replied_to (
message_id INTEGER PRIMARY KEY NOT NULL,
replied_to_id INTEGER NOT NULL)");
}
private void UpgradeSchemas(int dbVersion) {
Execute("UPDATE metadata SET value = " + Version + " WHERE key = 'version'");
if (dbVersion <= 1) {
Execute("ALTER TABLE channels ADD parent_id INTEGER");
}
if (dbVersion <= 2) {
CreateMessageEditTimestampTable();
CreateMessageRepliedToTable();
Execute(@"INSERT INTO edit_timestamps (message_id, edit_timestamp)
SELECT message_id, edit_timestamp FROM messages
WHERE edit_timestamp IS NOT NULL");
Execute(@"INSERT INTO replied_to (message_id, replied_to_id)
SELECT message_id, replied_to_id FROM messages
WHERE replied_to_id IS NOT NULL");
Execute("ALTER TABLE messages DROP COLUMN replied_to_id");
Execute("ALTER TABLE messages DROP COLUMN edit_timestamp");
Execute("VACUUM");
}
}
}
}

75
app/Server/Database/Sqlite/SqliteDatabaseFile.cs
View File

@ -6,6 +6,7 @@ using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Filters;
using DHT.Utils.Collections;
using DHT.Utils.Logging;
using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite {
@ -25,9 +26,11 @@ namespace DHT.Server.Database.Sqlite {
public string Path { get; }
public DatabaseStatistics Statistics { get; }
private readonly Log log;
private readonly SqliteConnection conn;
private SqliteDatabaseFile(string path, SqliteConnection conn) {
this.log = Log.ForType(typeof(SqliteDatabaseFile), System.IO.Path.GetFileName(path));
this.conn = conn;
this.Path = path;
this.Statistics = new DatabaseStatistics();
@ -56,6 +59,7 @@ namespace DHT.Server.Database.Sqlite {
}
public List<Data.Server> GetAllServers() {
var perf = log.Start();
var list = new List<Data.Server>();
using var cmd = conn.Command("SELECT id, name, type FROM servers");
@ -69,6 +73,7 @@ namespace DHT.Server.Database.Sqlite {
});
}
perf.End();
return list;
}
@ -137,6 +142,7 @@ namespace DHT.Server.Database.Sqlite {
}
public List<User> GetAllUsers() {
var perf = log.Start();
var list = new List<User>();
using var cmd = conn.Command("SELECT id, name, avatar_url, discriminator FROM users");
@ -151,10 +157,20 @@ namespace DHT.Server.Database.Sqlite {
});
}
perf.End();
return list;
}
public void AddMessages(Message[] messages) {
static SqliteCommand DeleteByMessageId(SqliteConnection conn, string tableName) {
return conn.Delete(tableName, ("message_id", SqliteType.Integer));
}
static void ExecuteDeleteByMessageId(SqliteCommand cmd, object id) {
cmd.Set(":message_id", id);
cmd.ExecuteNonQuery();
}
using var tx = conn.BeginTransaction();
using var messageCmd = conn.Upsert("messages", new[] {
@ -162,14 +178,25 @@ namespace DHT.Server.Database.Sqlite {
("sender_id", SqliteType.Integer),
("channel_id", SqliteType.Integer),
("text", SqliteType.Text),
("timestamp", SqliteType.Integer),
("edit_timestamp", SqliteType.Integer),
("replied_to_id", SqliteType.Integer)
("timestamp", SqliteType.Integer)
});
using var deleteAttachmentsCmd = conn.Delete("attachments", ("message_id", SqliteType.Integer));
using var deleteEmbedsCmd = conn.Delete("embeds", ("message_id", SqliteType.Integer));
using var deleteReactionsCmd = conn.Delete("reactions", ("message_id", SqliteType.Integer));
using var deleteEditTimestampCmd = DeleteByMessageId(conn, "edit_timestamps");
using var deleteRepliedToCmd = DeleteByMessageId(conn, "replied_to");
using var deleteAttachmentsCmd = DeleteByMessageId(conn, "attachments");
using var deleteEmbedsCmd = DeleteByMessageId(conn, "embeds");
using var deleteReactionsCmd = DeleteByMessageId(conn, "reactions");
using var editTimestampCmd = conn.Insert("edit_timestamps", new [] {
("message_id", SqliteType.Integer),
("edit_timestamp", SqliteType.Integer)
});
using var repliedToCmd = conn.Insert("replied_to", new [] {
("message_id", SqliteType.Integer),
("replied_to_id", SqliteType.Integer)
});
using var attachmentCmd = conn.Insert("attachments", new[] {
("message_id", SqliteType.Integer),
@ -201,18 +228,26 @@ namespace DHT.Server.Database.Sqlite {
messageCmd.Set(":channel_id", message.Channel);
messageCmd.Set(":text", message.Text);
messageCmd.Set(":timestamp", message.Timestamp);
messageCmd.Set(":edit_timestamp", message.EditTimestamp);
messageCmd.Set(":replied_to_id", message.RepliedToId);
messageCmd.ExecuteNonQuery();
deleteAttachmentsCmd.Set(":message_id", messageId);
deleteAttachmentsCmd.ExecuteNonQuery();
ExecuteDeleteByMessageId(deleteEditTimestampCmd, messageId);
ExecuteDeleteByMessageId(deleteRepliedToCmd, messageId);
deleteEmbedsCmd.Set(":message_id", messageId);
deleteEmbedsCmd.ExecuteNonQuery();
ExecuteDeleteByMessageId(deleteAttachmentsCmd, messageId);
ExecuteDeleteByMessageId(deleteEmbedsCmd, messageId);
ExecuteDeleteByMessageId(deleteReactionsCmd, messageId);
deleteReactionsCmd.Set(":message_id", messageId);
deleteReactionsCmd.ExecuteNonQuery();
if (message.EditTimestamp is {} timestamp) {
editTimestampCmd.Set(":message_id", messageId);
editTimestampCmd.Set(":edit_timestamp", timestamp);
editTimestampCmd.ExecuteNonQuery();
}
if (message.RepliedToId is {} repliedToId) {
repliedToCmd.Set(":message_id", messageId);
repliedToCmd.Set(":replied_to_id", repliedToId);
repliedToCmd.ExecuteNonQuery();
}
if (!message.Attachments.IsEmpty) {
foreach (var attachment in message.Attachments) {
@ -258,13 +293,18 @@ namespace DHT.Server.Database.Sqlite {
}
public List<Message> GetMessages(MessageFilter? filter = null) {
var perf = log.Start();
var list = new List<Message>();
var attachments = GetAllAttachments();
var embeds = GetAllEmbeds();
var reactions = GetAllReactions();
var list = new List<Message>();
using var cmd = conn.Command("SELECT message_id, sender_id, channel_id, text, timestamp, edit_timestamp, replied_to_id FROM messages" + filter.GenerateWhereClause());
using var cmd = conn.Command(@"
SELECT m.message_id, m.sender_id, m.channel_id, m.text, m.timestamp, et.edit_timestamp, rt.replied_to_id
FROM messages m
LEFT JOIN edit_timestamps et ON m.message_id = et.message_id
LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereClause("m"));
using var reader = cmd.ExecuteReader();
while (reader.Read()) {
@ -284,6 +324,7 @@ namespace DHT.Server.Database.Sqlite {
});
}
perf.End();
return list;
}

16
app/Server/Database/Sqlite/SqliteMessageFilter.cs
View File

@ -4,31 +4,35 @@ using DHT.Server.Data.Filters;
namespace DHT.Server.Database.Sqlite {
static class SqliteMessageFilter {
public static string GenerateWhereClause(this MessageFilter? filter, bool invert = false) {
public static string GenerateWhereClause(this MessageFilter? filter, string? tableAlias = null, bool invert = false) {
if (filter == null) {
return "";
}
if (tableAlias != null) {
tableAlias += ".";
}
List<string> conditions = new();
if (filter.StartDate != null) {
conditions.Add("timestamp >= " + new DateTimeOffset(filter.StartDate.Value).ToUnixTimeMilliseconds());
conditions.Add(tableAlias + "timestamp >= " + new DateTimeOffset(filter.StartDate.Value).ToUnixTimeMilliseconds());
}
if (filter.EndDate != null) {
conditions.Add("timestamp <= " + new DateTimeOffset(filter.EndDate.Value).ToUnixTimeMilliseconds());
conditions.Add(tableAlias + "timestamp <= " + new DateTimeOffset(filter.EndDate.Value).ToUnixTimeMilliseconds());
}
if (filter.ChannelIds != null) {
conditions.Add("channel_id IN (" + string.Join(",", filter.ChannelIds) + ")");
conditions.Add(tableAlias + "channel_id IN (" + string.Join(",", filter.ChannelIds) + ")");
}
if (filter.UserIds != null) {
conditions.Add("sender_id IN (" + string.Join(",", filter.UserIds) + ")");
conditions.Add(tableAlias + "sender_id IN (" + string.Join(",", filter.UserIds) + ")");
}
if (filter.MessageIds != null) {
conditions.Add("message_id IN (" + string.Join(",", filter.MessageIds) + ")");
conditions.Add(tableAlias + "message_id IN (" + string.Join(",", filter.MessageIds) + ")");
}
if (conditions.Count == 0) {

BIN
app/empty.dht
View File

Binary file not shown.