Add option to export downloaded files

Closes #277

app/Desktop/Dialogs/File/FileDialogs.cs

@@ -8,6 +8,10 @@ using Avalonia.Platform.Storage;
 namespace DHT.Desktop.Dialogs.File;
 
 static class FileDialogs {
+	public static async Task<string[]> OpenFolders(this IStorageProvider storageProvider, FolderPickerOpenOptions options) {
+		return (await storageProvider.OpenFolderPickerAsync(options)).ToLocalPaths();
+	}
+	
 	public static async Task<string[]> OpenFiles(this IStorageProvider storageProvider, FilePickerOpenOptions options) {
 		return (await storageProvider.OpenFilePickerAsync(options)).ToLocalPaths();
 	}
@@ -26,11 +30,11 @@ static class FileDialogs {
 		return suggestedDirectory == null ? Task.FromResult<IStorageFolder?>(null) : window.StorageProvider.TryGetFolderFromPathAsync(suggestedDirectory);
 	}
 	
-	private static string ToLocalPath(this IStorageFile file) {
-		return file.TryGetLocalPath() ?? throw new NotSupportedException("Local filesystem is not supported.");
+	private static string ToLocalPath(this IStorageItem itme) {
+		return itme.TryGetLocalPath() ?? throw new NotSupportedException("Local filesystem is not supported.");
 	}
 	
-	private static string[] ToLocalPaths(this IReadOnlyList<IStorageFile> files) {
-		return files.Select(ToLocalPath).ToArray();
+	private static string[] ToLocalPaths(this IReadOnlyList<IStorageItem> items) {
+		return items.Select(ToLocalPath).ToArray();
 	}
 }

app/Desktop/Dialogs/Progress/IProgressCallback.cs

@@ -3,7 +3,7 @@ using System.Threading.Tasks;
 namespace DHT.Desktop.Dialogs.Progress;
 
 interface IProgressCallback {
-	Task Update(string message, int finishedItems, int totalItems);
+	Task Update(string message, long finishedItems, long totalItems);
 	Task UpdateIndeterminate(string message);
 	Task Hide();
 }

app/Desktop/Dialogs/Progress/ProgressDialogModel.cs

@@ -38,7 +38,7 @@ sealed class ProgressDialogModel {
 			this.item = item;
 		}
 		
-		public async Task Update(string message, int finishedItems, int totalItems) {
+		public async Task Update(string message, long finishedItems, long totalItems) {
 			await Dispatcher.UIThread.InvokeAsync(() => {
 				item.Message = message;
 				item.Items = totalItems == 0 ? string.Empty : finishedItems.Format() + " / " + totalItems.Format();

app/Desktop/Dialogs/Progress/ProgressItem.cs

@@ -23,7 +23,7 @@ sealed partial class ProgressItem : ObservableObject {
 	private string items = "";
 	
 	[ObservableProperty]
-	private int progress = 0;
+	private long progress = 0L;
 	
 	[ObservableProperty]
 	private bool isIndeterminate;

app/Desktop/Main/Pages/DebugPageModel.cs

@@ -51,7 +51,7 @@ sealed class DebugPageModel {
 	
 	private async Task GenerateRandomData(int channelCount, int userCount, int messageCount, IProgressCallback callback) {
 		int batchCount = (messageCount + BatchSize - 1) / BatchSize;
-		await callback.Update("Adding messages in batches of " + BatchSize, finishedItems: 0, batchCount);
+		await callback.Update("Adding messages in batches of " + BatchSize, finishedItems: 0, totalItems: batchCount);
 		
 		var rand = new Random();
 		var server = new DHT.Server.Data.Server {

app/Desktop/Main/Pages/DownloadsPage.axaml

@@ -33,8 +33,9 @@
     <StackPanel Orientation="Vertical">
         <WrapPanel Orientation="Horizontal">
             <Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" />
-            <Button Command="{Binding OnClickRetryFailedDownloads}" IsEnabled="{Binding IsRetryFailedOnDownloadsButtonEnabled}">Retry Failed Downloads</Button>
-            <Button Command="{Binding OnClickDeleteOrphanedDownloads}">Delete Orphaned Downloads</Button>
+            <Button Command="{Binding OnClickRetryFailed}" IsEnabled="{Binding IsRetryFailedOnDownloadsButtonEnabled}">Retry Failed</Button>
+            <Button Command="{Binding OnClickDeleteOrphaned}">Delete Orphaned</Button>
+            <Button Command="{Binding OnClickExportAll}" IsEnabled="{Binding HasSuccessfulDownloads}">Export All</Button>
         </WrapPanel>
         <StackPanel Orientation="Vertical" Spacing="20" Margin="0 10 0 0">
             <controls:DownloadItemFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !$parent[UserControl].((pages:DownloadsPageModel)DataContext).IsDownloading}" />

app/Desktop/Main/Pages/DownloadsPageModel.cs

@@ -4,9 +4,11 @@ using System.Collections.ObjectModel;
 using System.Reactive.Linq;
 using System.Threading.Tasks;
 using Avalonia.Controls;
+using Avalonia.Platform.Storage;
 using Avalonia.ReactiveUI;
 using CommunityToolkit.Mvvm.ComponentModel;
 using DHT.Desktop.Common;
+using DHT.Desktop.Dialogs.File;
 using DHT.Desktop.Dialogs.Message;
 using DHT.Desktop.Dialogs.Progress;
 using DHT.Desktop.Main.Controls;
@@ -33,6 +35,9 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 	[NotifyPropertyChangedFor(nameof(IsRetryFailedOnDownloadsButtonEnabled))]
 	private bool isRetryingFailedDownloads = false;
 	
+	[ObservableProperty(Setter = Access.Private)]
+	private bool hasSuccessfulDownloads;
+	
 	[ObservableProperty(Setter = Access.Private)]
 	[NotifyPropertyChangedFor(nameof(IsRetryFailedOnDownloadsButtonEnabled))]
 	private bool hasFailedDownloads;
@@ -148,7 +153,7 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 		RecomputeDownloadStatistics();
 	}
 	
-	public async Task OnClickRetryFailedDownloads() {
+	public async Task OnClickRetryFailed() {
 		IsRetryingFailedDownloads = true;
 		
 		try {
@@ -165,7 +170,7 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 		downloadStatisticsTask.Post(cancellationToken => state.Db.Downloads.GetStatistics(currentDownloadFilter ?? new DownloadItemFilter(), cancellationToken));
 	}
 	
-	public async Task OnClickDeleteOrphanedDownloads() {
+	public async Task OnClickDeleteOrphaned() {
 		const string Title = "Delete Orphaned Downloads";
 		
 		try {
@@ -212,6 +217,49 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 		}
 	}
 	
+	public async Task OnClickExportAll() {
+		const string Title = "Export Downloaded Files";
+		
+		string[] folders = await window.StorageProvider.OpenFolders(new FolderPickerOpenOptions {
+			Title = Title,
+			AllowMultiple = false,
+		});
+		
+		if (folders.Length != 1) {
+			return;
+		}
+		
+		string folderPath = folders[0];
+		
+		DownloadExporter exporter = new DownloadExporter(state.Db, folderPath);
+		DownloadExporter.Result result;
+		try {
+			result = await ProgressDialog.Show(window, Title, async (_, callback) => {
+				await callback.UpdateIndeterminate("Exporting downloaded files...");
+				return await exporter.Export(new ExportProgressReporter(callback));
+			});
+		} catch (Exception e) {
+			Log.Error(e);
+			await Dialog.ShowOk(window, Title, "Could not export downloaded files: " + e.Message);
+			return;
+		}
+		
+		string messageStart = "Exported " + result.SuccessfulCount.Pluralize("file");
+		
+		if (result.FailedCount > 0L) {
+			await Dialog.ShowOk(window, Title, messageStart + " (" + result.FailedCount.Format() + " failed).");
+		}
+		else {
+			await Dialog.ShowOk(window, Title, messageStart + ".");
+		}
+	}
+	
+	private sealed class ExportProgressReporter(IProgressCallback callback) : DownloadExporter.IProgressReporter {
+		public Task ReportProgress(long processedCount, long totalCount) {
+			return callback.Update("Exporting downloaded files...", processedCount, totalCount);
+		}
+	}
+	
 	private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
 		statisticsPending.Items = statusStatistics.PendingCount;
 		statisticsPending.Size = statusStatistics.PendingTotalSize;
@@ -229,6 +277,7 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 		statisticsSkipped.Size = statusStatistics.SkippedTotalSize;
 		statisticsSkipped.HasFilesWithUnknownSize = statusStatistics.SkippedWithUnknownSizeCount > 0;
 		
+		HasSuccessfulDownloads = statusStatistics.SuccessfulCount > 0;
 		HasFailedDownloads = statusStatistics.FailedCount > 0;
 	}
 	

app/Server/Database/Repositories/IDownloadRepository.cs

@@ -20,7 +20,7 @@ public interface IDownloadRepository {
 	
 	Task<DownloadStatusStatistics> GetStatistics(DownloadItemFilter nonSkippedFilter, CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<Data.Download> Get();
+	IAsyncEnumerable<Data.Download> Get(DownloadItemFilter? filter = null);
 	
 	Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor);
 	
@@ -51,7 +51,7 @@ public interface IDownloadRepository {
 			return Task.FromResult(new DownloadStatusStatistics());
 		}
 		
-		public IAsyncEnumerable<Data.Download> Get() {
+		public IAsyncEnumerable<Data.Download> Get(DownloadItemFilter? filter) {
 			return AsyncEnumerable.Empty<Data.Download>();
 		}
 		

app/Server/Database/Sqlite/Repositories/SqliteDownloadRepository.cs

@@ -201,10 +201,10 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 		};
 	}
 	
-	public async IAsyncEnumerable<Data.Download> Get() {
+	public async IAsyncEnumerable<Data.Download> Get(DownloadItemFilter? filter) {
 		await using var conn = await pool.Take();
 		
-		await using var cmd = conn.Command("SELECT normalized_url, download_url, status, type, size FROM download_metadata");
+		await using var cmd = conn.Command("SELECT normalized_url, download_url, status, type, size FROM download_metadata" + filter.GenerateConditions().BuildWhereClause());
 		await using var reader = await cmd.ExecuteReaderAsync();
 		
 		while (await reader.ReadAsync()) {

app/Server/Download/DownloadExporter.cs

@@ -0,0 +1,152 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Channels;
+using System.Threading.Tasks;
+using DHT.Server.Data;
+using DHT.Server.Data.Filters;
+using DHT.Server.Database;
+using DHT.Utils.Logging;
+using DHT.Utils.Tasks;
+using Channel = System.Threading.Channels.Channel;
+
+namespace DHT.Server.Download;
+
+public sealed partial class DownloadExporter(IDatabaseFile db, string folderPath) {
+	private static readonly Log Log = Log.ForType<DownloadExporter>();
+	
+	private const int Concurrency = 3;
+	
+	private static Channel<Data.Download> CreateExportChannel() {
+		return Channel.CreateBounded<Data.Download>(new BoundedChannelOptions(Concurrency * 4) {
+			SingleWriter = true,
+			SingleReader = false,
+			AllowSynchronousContinuations = true,
+			FullMode = BoundedChannelFullMode.Wait,
+		});
+	}
+	
+	public interface IProgressReporter {
+		Task ReportProgress(long processedCount, long totalCount);
+	}
+	
+	public readonly record struct Result(long SuccessfulCount, long FailedCount) {
+		internal static Result Combine(Result left, Result right) {
+			return new Result(left.SuccessfulCount + right.SuccessfulCount, left.FailedCount + right.FailedCount);
+		}
+	}
+	
+	public async Task<Result> Export(IProgressReporter reporter) {
+		DownloadItemFilter filter = new DownloadItemFilter {
+			IncludeStatuses = [DownloadStatus.Success]
+		};
+		
+		long totalCount = await db.Downloads.Count(filter);
+		
+		Channel<Data.Download> channel = CreateExportChannel();
+		ExportRunner exportRunner = new ExportRunner(db, folderPath, channel.Reader, reporter, totalCount);
+		
+		using CancellableTask progressTask = CancellableTask.Run(exportRunner.RunReportTask);
+		
+		List<Task<Result>> readerTasks = [];
+		for (int reader = 0; reader < Concurrency; reader++) {
+			readerTasks.Add(Task.Run(exportRunner.RunExportTask, CancellationToken.None));
+		}
+		
+		await foreach (Data.Download download in db.Downloads.Get(filter).WithCancellation(CancellationToken.None)) {
+			await channel.Writer.WriteAsync(download, CancellationToken.None);
+		}
+		
+		channel.Writer.Complete();
+		
+		Result result = (await Task.WhenAll(readerTasks)).Aggregate(Result.Combine);
+		
+		progressTask.Cancel();
+		await progressTask.Task;
+		
+		return result;
+	}
+	
+	private sealed partial class ExportRunner(IDatabaseFile db, string folderPath, ChannelReader<Data.Download> reader, IProgressReporter reporter, long totalCount) {
+		private long processedCount;
+		
+		public async Task RunReportTask(CancellationToken cancellationToken) {
+			try {
+				while (true) {
+					await reporter.ReportProgress(processedCount, totalCount);
+					await Task.Delay(TimeSpan.FromMilliseconds(25), cancellationToken);
+				}
+			} catch (OperationCanceledException) {
+				await reporter.ReportProgress(processedCount, totalCount);
+			}
+		}
+		
+		public async Task<Result> RunExportTask() {
+			long successfulCount = 0L;
+			long failedCount = 0L;
+			
+			await foreach (Data.Download download in reader.ReadAllAsync()) {
+				bool success;
+				try {
+					success = await db.Downloads.GetDownloadData(download.NormalizedUrl, stream => CopyToFile(download.NormalizedUrl, stream));
+				} catch (FileAlreadyExistsException) {
+					success = false;
+				} catch (Exception e) {
+					Log.Error(e);
+					success = false;
+				}
+				
+				if (success) {
+					++successfulCount;
+				}
+				else {
+					++failedCount;
+				}
+				
+				Interlocked.Increment(ref processedCount);
+			}
+			
+			return new Result(successfulCount, failedCount);
+		}
+		
+		private async Task CopyToFile(string normalizedUrl, Stream blobStream) {
+			string fileName = UrlToFileName(normalizedUrl);
+			string filePath = Path.Combine(folderPath, fileName);
+			
+			if (File.Exists(filePath)) {
+				Log.Error("Skipping existing file: " + fileName);
+				throw FileAlreadyExistsException.Instance;
+			}
+			
+			await using var fileStream = new FileStream(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.Read);
+			await blobStream.CopyToAsync(fileStream);
+		}
+		
+		[GeneratedRegex("[^a-zA-Z0-9_.-]")]
+		private static partial Regex DisallowedFileNameCharactersRegex();
+		
+		private static string UrlToFileName(string url) {
+			static string UriToFileName(Uri uri) {
+				string fileName = uri.AbsolutePath.TrimStart('/');
+				
+				if (uri.Query.Length > 0) {
+					int periodIndex = fileName.LastIndexOf('.');
+					return fileName.Insert(periodIndex == -1 ? fileName.Length : periodIndex, uri.Query.TrimEnd('&'));
+				}
+				else {
+					return fileName;
+				}
+			}
+			
+			string fileName = Uri.TryCreate(url, UriKind.Absolute, out var uri) ? UriToFileName(uri) : url;
+			return DisallowedFileNameCharactersRegex().Replace(fileName, "_");
+		}
+	}
+	
+	private sealed class FileAlreadyExistsException : Exception {
+		public static FileAlreadyExistsException Instance { get; } = new ();
+	}
+}

app/Utils/Tasks/CancellableTask.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace DHT.Utils.Tasks;
+
+public sealed class CancellableTask : IDisposable {
+	public static CancellableTask Run(Func<CancellationToken, Task> action) {
+		return new CancellableTask(action);
+	}
+	
+	public Task Task { get; }
+	
+	private readonly CancellationTokenSource cancellationTokenSource = new ();
+	
+	private CancellableTask(Func<CancellationToken, Task> action) {
+		CancellationToken cancellationToken = cancellationTokenSource.Token;
+		Task = Task.Run(() => action(cancellationToken));
+	}
+	
+	public void Cancel() {
+		cancellationTokenSource.Cancel();
+	}
+	
+	public void Dispose() {
+		cancellationTokenSource.Dispose();
+	}
+}