third-party-mirrors/Discord-History-Tracker
1
0
Fork 0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2025-04-12 06:50:01 +03:00
Code Issues Projects Releases Wiki Activity

Remove orphaned servers, channels, users, and attachments after removing messages

Browse Source
This commit is contained in:
chylex 2025-02-02 11:44:46 +01:00
parent 2ecb658e42
commit 261a583b50
No known key found for this signature in database
10 changed files with 106 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
app/Desktop/Main/Pages/ViewerPageModel.cs
View File

@ -12,10 +12,13 @@ using DHT.Desktop.Server;
using DHT.Server;
using DHT.Server.Data.Filters;
using DHT.Server.Service.Viewer;
using DHT.Utils.Logging;
namespace DHT.Desktop.Main.Pages;
sealed partial class ViewerPageModel : ObservableObject, IDisposable {
private static readonly Log Log = Log.ForType<ViewerPageModel>();
public bool DatabaseToolFilterModeKeep { get; set; } = true;
public bool DatabaseToolFilterModeRemove { get; set; } = false;
@ -74,6 +77,21 @@ sealed partial class ViewerPageModel : ObservableObject, IDisposable {
}
private async Task ApplyFilterToDatabase(MessageFilter filter, FilterRemovalMode removalMode) {
await ProgressDialog.ShowIndeterminate(window, "Apply Filters", "Removing messages...", _ => state.Db.Messages.Remove(filter, removalMode));
await ProgressDialog.Show(window, "Apply Filters", async (_, callback) => {
await callback.UpdateIndeterminate("Removing messages...");
Log.Info("Removed messages: " + await state.Db.Messages.Remove(filter, removalMode));
await callback.UpdateIndeterminate("Cleaning up attachments...");
Log.Info("Removed orphaned attachments: " + await state.Db.Messages.RemoveUnreachableAttachments());
await callback.UpdateIndeterminate("Cleaning up users...");
Log.Info("Removed orphaned users: " + await state.Db.Users.RemoveUnreachable());
await callback.UpdateIndeterminate("Cleaning up channels...");
Log.Info("Removed orphaned channels: " + await state.Db.Channels.RemoveUnreachable());
await callback.UpdateIndeterminate("Cleaning up servers...");
Log.Info("Removed orphaned servers: " + await state.Db.Servers.RemoveUnreachable());
});
}
}

6
app/Server/Database/Repositories/IChannelRepository.cs
View File

@ -17,6 +17,8 @@ public interface IChannelRepository {
IAsyncEnumerable<Channel> Get(CancellationToken cancellationToken = default);
Task<int> RemoveUnreachable();
internal sealed class Dummy : IChannelRepository {
public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@ -31,5 +33,9 @@ public interface IChannelRepository {
public IAsyncEnumerable<Channel> Get(CancellationToken cancellationToken) {
return AsyncEnumerable.Empty<Channel>();
}
public Task<int> RemoveUnreachable() {
return Task.FromResult(0);
}
}
}

12
app/Server/Database/Repositories/IMessageRepository.cs
View File

@ -20,7 +20,9 @@ public interface IMessageRepository {
IAsyncEnumerable<ulong> GetIds(MessageFilter? filter = null);
Task Remove(MessageFilter filter, FilterRemovalMode mode);
Task<int> Remove(MessageFilter filter, FilterRemovalMode mode);
Task<int> RemoveUnreachableAttachments();
internal sealed class Dummy : IMessageRepository {
public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@ -41,8 +43,12 @@ public interface IMessageRepository {
return AsyncEnumerable.Empty<ulong>();
}
public Task Remove(MessageFilter filter, FilterRemovalMode mode) {
return Task.CompletedTask;
public Task<int> Remove(MessageFilter filter, FilterRemovalMode mode) {
return Task.FromResult(0);
}
public Task<int> RemoveUnreachableAttachments() {
return Task.FromResult(0);
}
}
}

6
app/Server/Database/Repositories/IServerRepository.cs
View File

@ -16,6 +16,8 @@ public interface IServerRepository {
IAsyncEnumerable<Data.Server> Get(CancellationToken cancellationToken = default);
Task<int> RemoveUnreachable();
internal sealed class Dummy : IServerRepository {
public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@ -30,5 +32,9 @@ public interface IServerRepository {
public IAsyncEnumerable<Data.Server> Get(CancellationToken cancellationToken) {
return AsyncEnumerable.Empty<Data.Server>();
}
public Task<int> RemoveUnreachable() {
return Task.FromResult(0);
}
}
}

6
app/Server/Database/Repositories/IUserRepository.cs
View File

@ -17,6 +17,8 @@ public interface IUserRepository {
IAsyncEnumerable<User> Get(CancellationToken cancellationToken = default);
Task<int> RemoveUnreachable();
internal sealed class Dummy : IUserRepository {
public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@ -31,5 +33,9 @@ public interface IUserRepository {
public IAsyncEnumerable<User> Get(CancellationToken cancellationToken) {
return AsyncEnumerable.Empty<User>();
}
public Task<int> RemoveUnreachable() {
return Task.FromResult(0);
}
}
}

10
app/Server/Database/Sqlite/Repositories/SqliteChannelRepository.cs
View File

@ -73,4 +73,14 @@ sealed class SqliteChannelRepository : BaseSqliteRepository, IChannelRepository
};
}
}
public async Task<int> RemoveUnreachable() {
int removed;
await using (var conn = await pool.Take()) {
removed = await conn.ExecuteAsync("DELETE FROM channels WHERE id NOT IN (SELECT DISTINCT channel_id FROM messages)");
}
UpdateTotalCount();
return removed;
}
}

23
app/Server/Database/Sqlite/Repositories/SqliteMessageRepository.cs
View File

@ -303,17 +303,24 @@ sealed class SqliteMessageRepository(SqliteConnectionPool pool, SqliteDownloadRe
}
}
public async Task Remove(MessageFilter filter, FilterRemovalMode mode) {
public async Task<int> Remove(MessageFilter filter, FilterRemovalMode mode) {
int removed;
await using (var conn = await pool.Take()) {
await conn.ExecuteAsync(
$"""
-- noinspection SqlWithoutWhere
DELETE FROM messages
{filter.GenerateConditions(invert: mode == FilterRemovalMode.KeepMatching).BuildWhereClause()}
"""
);
removed = await conn.ExecuteAsync(
$"""
-- noinspection SqlWithoutWhere
DELETE FROM messages
{filter.GenerateConditions(invert: mode == FilterRemovalMode.KeepMatching).BuildWhereClause()}
"""
);
}
UpdateTotalCount();
return removed;
}
public async Task<int> RemoveUnreachableAttachments() {
await using var conn = await pool.Take();
return await conn.ExecuteAsync("DELETE FROM attachments WHERE attachment_id NOT IN (SELECT DISTINCT attachment_id FROM message_attachments)");
}
}

10
app/Server/Database/Sqlite/Repositories/SqliteServerRepository.cs
View File

@ -61,4 +61,14 @@ sealed class SqliteServerRepository : BaseSqliteRepository, IServerRepository {
};
}
}
public async Task<int> RemoveUnreachable() {
int removed;
await using (var conn = await pool.Take()) {
removed = await conn.ExecuteAsync("DELETE FROM servers WHERE id NOT IN (SELECT DISTINCT server FROM channels)");
}
UpdateTotalCount();
return removed;
}
}

10
app/Server/Database/Sqlite/Repositories/SqliteUserRepository.cs
View File

@ -77,4 +77,14 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
};
}
}
public async Task<int> RemoveUnreachable() {
int removed;
await using (var conn = await pool.Take()) {
removed = await conn.ExecuteAsync("DELETE FROM users WHERE id NOT IN (SELECT DISTINCT sender_id FROM messages)");
}
UpdateTotalCount();
return removed;
}
}

20
app/Server/Database/Sqlite/Schema/SqliteSchemaUpgradeTo10.cs
View File

@ -1,11 +1,14 @@
using System.Threading.Tasks;
using DHT.Server.Database.Sqlite.Utils;
using DHT.Utils.Logging;
namespace DHT.Server.Database.Sqlite.Schema;
sealed class SqliteSchemaUpgradeTo10 : ISchemaUpgrade {
private static readonly Log Log = Log.ForType<SqliteSchemaUpgradeTo10>();
async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
await reporter.MainWork("Migrating message embeds...", finishedItems: 0, totalItems: 5);
await reporter.MainWork("Migrating message embeds...", finishedItems: 0, totalItems: 6);
await conn.ExecuteAsync("""
CREATE TABLE message_embeds_new (
message_id INTEGER NOT NULL,
@ -15,7 +18,7 @@ sealed class SqliteSchemaUpgradeTo10 : ISchemaUpgrade {
""");
await conn.ExecuteAsync("INSERT INTO message_embeds_new (message_id, json) SELECT message_id, json FROM message_embeds WHERE message_id IN (SELECT DISTINCT message_id FROM messages)");
await reporter.MainWork("Migrating message reactions...", finishedItems: 1, totalItems: 5);
await reporter.MainWork("Migrating message reactions...", finishedItems: 1, totalItems: 6);
await conn.ExecuteAsync("""
CREATE TABLE message_reactions_new (
message_id INTEGER NOT NULL,
@ -28,7 +31,7 @@ sealed class SqliteSchemaUpgradeTo10 : ISchemaUpgrade {
""");
await conn.ExecuteAsync("INSERT INTO message_reactions_new (message_id, emoji_id, emoji_name, emoji_flags, count) SELECT message_id, emoji_id, emoji_name, emoji_flags, count FROM message_reactions WHERE message_id IN (SELECT DISTINCT message_id FROM messages)");
await reporter.MainWork("Migrating message edit timestamps...", finishedItems: 2, totalItems: 5);
await reporter.MainWork("Migrating message edit timestamps...", finishedItems: 2, totalItems: 6);
await conn.ExecuteAsync("""
CREATE TABLE message_edit_timestamps_new (
message_id INTEGER PRIMARY KEY NOT NULL,
@ -38,7 +41,7 @@ sealed class SqliteSchemaUpgradeTo10 : ISchemaUpgrade {
""");
await conn.ExecuteAsync("INSERT INTO message_edit_timestamps_new (message_id, edit_timestamp) SELECT message_id, edit_timestamp FROM message_edit_timestamps WHERE message_id IN (SELECT DISTINCT message_id FROM messages)");
await reporter.MainWork("Migrating message replies...", finishedItems: 3, totalItems: 5);
await reporter.MainWork("Migrating message replies...", finishedItems: 3, totalItems: 6);
await conn.ExecuteAsync("""
CREATE TABLE message_replied_to_new (
message_id INTEGER PRIMARY KEY NOT NULL,
@ -48,7 +51,7 @@ sealed class SqliteSchemaUpgradeTo10 : ISchemaUpgrade {
""");
await conn.ExecuteAsync("INSERT INTO message_replied_to_new (message_id, replied_to_id) SELECT message_id, replied_to_id FROM message_replied_to WHERE message_id IN (SELECT DISTINCT message_id FROM messages)");
await reporter.MainWork("Applying schema changes...", finishedItems: 4, totalItems: 5);
await reporter.MainWork("Applying schema changes...", finishedItems: 4, totalItems: 6);
await conn.ExecuteAsync("DROP TABLE message_embeds");
await conn.ExecuteAsync("ALTER TABLE message_embeds_new RENAME TO message_embeds");
@ -63,5 +66,12 @@ sealed class SqliteSchemaUpgradeTo10 : ISchemaUpgrade {
await conn.ExecuteAsync("DROP TABLE message_replied_to");
await conn.ExecuteAsync("ALTER TABLE message_replied_to_new RENAME TO message_replied_to");
await reporter.MainWork("Removing orphaned objects...", finishedItems: 5, totalItems: 6);
Log.Info("Removed orphaned attachments: " + await conn.ExecuteAsync("DELETE FROM attachments WHERE attachment_id NOT IN (SELECT DISTINCT attachment_id FROM message_attachments)"));
Log.Info("Removed orphaned users: " + await conn.ExecuteAsync("DELETE FROM users WHERE id NOT IN (SELECT DISTINCT sender_id FROM messages)"));
Log.Info("Removed orphaned channels: " + await conn.ExecuteAsync("DELETE FROM channels WHERE id NOT IN (SELECT DISTINCT channel_id FROM messages)"));
Log.Info("Removed orphaned servers: " + await conn.ExecuteAsync("DELETE FROM servers WHERE id NOT IN (SELECT DISTINCT server FROM channels)"));
}
}