diff --git a/doc/MasterData.Docs/articles/developer-guide/export-formats.md b/doc/MasterData.Docs/articles/developer-guide/export-formats.md index 098712433..bb16fd6b1 100644 --- a/doc/MasterData.Docs/articles/developer-guide/export-formats.md +++ b/doc/MasterData.Docs/articles/developer-guide/export-formats.md @@ -46,18 +46,16 @@ public sealed class JsonExportFormat : IExportFormat } writer.WriteEndObject(); processed++; + context.Progress.Report(processed); if (processed % 1000 == 0) { await writer.FlushAsync(cancellationToken); - context.Progress.Report(new ExportProgress( - processed, context.TotalRecords, $"Exported {processed} records.")); } } writer.WriteEndArray(); await writer.FlushAsync(cancellationToken); - context.Progress.Report(new ExportProgress( - processed, context.TotalRecords, $"Exported {processed} records.")); + context.Progress.Complete(processed); } } ``` @@ -87,7 +85,7 @@ Add `WithExportFormat` to the application's existing registration chain. The for | `Rows` | Asynchronous sequence of repository rows. | | `UserValues` | Values supplied with the export request. | | `TotalRecords` | Expected count, which can be null. | -| `Progress` | Report `ExportProgress` with processed count, total and message. | +| `Progress` | Call `Report(processed)` for each row and `Complete(processed)` at the end. The reporter sends updates every 50 records and sends the final count if it has not already been reported, creating the progress object and message only when sending an update. | Enumerate rows once, honor cancellation and write incrementally for large exports. Keep the supplied stream open for the pipeline. The pipeline handles repository paging, naming, temporary-file cleanup and saving to `IFileStorage`; formats should not duplicate those steps. diff --git a/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs b/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs index be8a22bcb..92b892431 100644 --- a/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs +++ b/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs @@ -31,7 +31,7 @@ internal sealed class ExportJobHandler( IStringLocalizer localizer, IMasterDataUser masterDataUser) : BackgroundJobHandler { - private const int RecordsPerPage = 100_000; + private const int RecordsPerPage = 10_000; public override async Task ExecuteAsync( ExportRequest request, @@ -43,17 +43,39 @@ internal sealed class ExportJobHandler( var formElement = request.FormElement; var format = formats.GetRequired(request.FormatId); var columns = GetColumns(request.FormElement, request.ExportAllFields); - var source = await CreateSourceAsync(formElement, request, cancellationToken); + List> firstPage; + long totalRecords; + EntityParameters? parameters = null; + if (request.Rows is not null) + { + firstPage = request.Rows; + totalRecords = request.Rows.Count; + } + else + { + parameters = new EntityParameters + { + Filters = new Dictionary(request.Filters), + RecordsPerPage = RecordsPerPage, + OrderBy = OrderByData.FromString(request.OrderBy), + CurrentPage = 1 + }; + var result = await entityRepository.GetDictionaryListResultAsync(formElement, parameters); + cancellationToken.ThrowIfCancellationRequested(); + firstPage = result.Data; + totalRecords = result.TotalOfRecords; + } var context = new ExportContext { FormElement = formElement, Columns = columns, - Rows = GetRowsAsync(formElement, source, cancellationToken), + Rows = GetRowsAsync(formElement, firstPage, totalRecords, parameters, cancellationToken), UserValues = new Dictionary(request.UserValues), - TotalRecords = source.Total, - Progress = new Progress(current => progress.Report( - new BackgroundJobProgress(current.Percentage, current.Message, current))) + TotalRecords = totalRecords, + Progress = new ExportProgressReporter(totalRecords, + new Progress(current => progress.Report( + new BackgroundJobProgress(current.Percentage, current.Message, current)))) }; progress.Report(new BackgroundJobProgress(0, localizer["Retrieving records..."])); @@ -99,60 +121,39 @@ private List GetColumns(FormElement formElement, bool exportAl .ToList(); } - private async Task CreateSourceAsync( - FormElement formElement, - ExportRequest request, - CancellationToken cancellationToken) - { - if (request.Rows is not null) - return new ExportSource(request.Rows, request.Rows.Count, null); - - var parameters = new EntityParameters - { - Filters = new Dictionary(request.Filters), - RecordsPerPage = RecordsPerPage, - OrderBy = OrderByData.FromString(request.OrderBy), - CurrentPage = 1 - }; - var firstPage = await entityRepository.GetDictionaryListResultAsync(formElement, parameters); - cancellationToken.ThrowIfCancellationRequested(); - return new ExportSource( - firstPage.Data, - firstPage.TotalOfRecords, - parameters); - } - private async IAsyncEnumerable> GetRowsAsync( FormElement formElement, - ExportSource source, + List> firstPage, + long totalOfRecords, + EntityParameters? parameters, [EnumeratorCancellation] CancellationToken cancellationToken) { - var totalPages = source.Parameters is null ? 1 : - (int)Math.Ceiling((source.Total ?? 0) / (double)RecordsPerPage); + var totalPages = parameters is null ? 1 : + (int)Math.Ceiling(totalOfRecords / (double)RecordsPerPage); for (var page = 1; page <= Math.Max(1, totalPages); page++) { List> rows; if (page == 1) - rows = source.FirstPage; + rows = firstPage; else { var pageParameters = new EntityParameters { - Filters = source.Parameters!.Filters, - RecordsPerPage = source.Parameters.RecordsPerPage, - OrderBy = source.Parameters.OrderBy, + Filters = parameters!.Filters, + RecordsPerPage = parameters.RecordsPerPage, + OrderBy = parameters.OrderBy, CurrentPage = page }; - var result = await entityRepository.GetDictionaryListResultAsync(formElement, pageParameters); + var result = await entityRepository.GetDictionaryListResultAsync( + formElement, pageParameters, recoverTotalOfRecords: false); rows = result.Data; } foreach (var sourceRow in rows) { cancellationToken.ThrowIfCancellationRequested(); - var row = new Dictionary(sourceRow, StringComparer.OrdinalIgnoreCase); - yield return row; + yield return sourceRow; } } } @@ -170,14 +171,4 @@ private string GetFileName(FormElement formElement, string extension) name = HttpUtility.UrlEncode(name, Encoding.UTF8); return $"{name}_{DateTime.UtcNow:yyyyMMdd_HHmmss}.{extension.TrimStart('.').ToLowerInvariant()}"; } - - private sealed class ExportSource( - List> firstPage, - long? total, - EntityParameters? parameters) - { - public List> FirstPage { get; init; } = firstPage; - public long? Total { get; init; } = total; - public EntityParameters? Parameters { get; init; } = parameters; - } } diff --git a/src/MasterData.Core/DataManager/Exportation/ExportContext.cs b/src/MasterData.Core/DataManager/Exportation/ExportContext.cs index dc0d0bed9..3fba4f736 100644 --- a/src/MasterData.Core/DataManager/Exportation/ExportContext.cs +++ b/src/MasterData.Core/DataManager/Exportation/ExportContext.cs @@ -1,4 +1,3 @@ -using System; using System.Collections.Generic; using JJMasterData.Core.DataDictionary.Models; @@ -11,5 +10,5 @@ public sealed class ExportContext public required IAsyncEnumerable> Rows { get; init; } public required Dictionary UserValues { get; init; } public required long? TotalRecords { get; init; } - public required IProgress Progress { get; init; } + public required ExportProgressReporter Progress { get; init; } } diff --git a/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs b/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs new file mode 100644 index 000000000..76aac4ad6 --- /dev/null +++ b/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs @@ -0,0 +1,25 @@ +using System; + +namespace JJMasterData.Core.DataManager.Exportation; + +public sealed class ExportProgressReporter(long? totalRecords, IProgress progress) +{ + private const long ReportInterval = 50; + private long _lastReported; + + public void Report(long processed) => Report(processed, completed: false); + + public void Complete(long processed) => Report(processed, completed: true); + + private void Report(long processed, bool completed) + { + if (!completed && processed - _lastReported < ReportInterval) + return; + + if (processed == _lastReported) + return; + + _lastReported = processed; + progress.Report(new ExportProgress(processed, totalRecords, $"Exporting {processed:N0} records...")); + } +} diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs index f3ab43f98..9a083cfc2 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs @@ -1,16 +1,13 @@ using System.IO; -using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -internal sealed class CsvExportFormat(FieldFormattingService fieldFormattingService) - : DelimitedTextExportFormat(fieldFormattingService) +internal sealed class CsvExportFormat : DelimitedTextExportFormat { public override string Id => "csv"; public override string DisplayName => "CSV"; public override string FileExtension => "csv"; - public override string ContentType => "text/csv"; - + protected override string GetDelimiter(CsvExportOptions options) => options.Delimiter switch { CsvExportDelimiter.Semicolon => ";", diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs index 36294131f..ee9f7218d 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs @@ -1,24 +1,22 @@ +using System.Globalization; using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; using CsvHelper; using CsvHelper.Configuration; -using JJMasterData.Core.DataDictionary.Models; using JJMasterData.Core.DataManager.Exportation.Abstractions; -using JJMasterData.Core.DataManager.Models; using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -internal abstract class DelimitedTextExportFormat(FieldFormattingService fieldFormattingService) : IExportFormat where TOptions : ExportFormatOptions, new() +internal abstract class DelimitedTextExportFormat : IExportFormat where TOptions : ExportFormatOptions, new() { protected abstract string GetDelimiter(TOptions options); public abstract string Id { get; } public abstract string DisplayName { get; } public abstract string FileExtension { get; } - public abstract string ContentType { get; } public async Task WriteAsync( ExportContext context, @@ -28,7 +26,7 @@ public async Task WriteAsync( { await using var textWriter = new StreamWriter(output, new UTF8Encoding(true), leaveOpen: true); await using var csv = new CsvWriter(textWriter, - new CsvConfiguration(System.Globalization.CultureInfo.CurrentCulture) + new CsvConfiguration(CultureInfo.CurrentCulture) { Delimiter = GetDelimiter(options), HasHeaderRecord = false @@ -44,19 +42,18 @@ public async Task WriteAsync( long processed = 0; await foreach (var row in context.Rows.WithCancellation(cancellationToken)) { - var formState = new FormStateData(row, context.UserValues, PageState.List); foreach (var field in context.Columns) { - var value = await fieldFormattingService.FormatGridValueAsync( - new FormElementFieldSelector(context.FormElement, field.Name), formState); + row.TryGetValue(field.Name, out var rawValue); + var value = FieldFormattingService.FormatValue(field, rawValue); csv.WriteField(value); } await csv.NextRecordAsync(); processed++; - context.Progress.Report(new ExportProgress( - processed, context.TotalRecords, $"Exporting {processed:N0} records...")); + context.Progress.Report(processed); } + context.Progress.Complete(processed); await textWriter.FlushAsync(cancellationToken); } -} \ No newline at end of file +} diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs index 87dd45b66..6f81503f1 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs @@ -3,14 +3,12 @@ using System.Threading; using System.Threading.Tasks; using System.Web; -using JJMasterData.Core.DataDictionary.Models; using JJMasterData.Core.DataManager.Exportation.Abstractions; -using JJMasterData.Core.DataManager.Models; using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -public sealed class ExcelXlsExportFormat(FieldFormattingService fieldFormattingService) : IExportFormat +public sealed class ExcelXlsExportFormat : IExportFormat { public string Id => "excel"; public string DisplayName => "Excel (.xls)"; @@ -33,17 +31,17 @@ public async Task WriteAsync(ExportContext context, ExcelXlsExportOptions option await foreach (var row in context.Rows.WithCancellation(cancellationToken)) { await writer.WriteAsync(""); - var formState = new FormStateData(row, context.UserValues, PageState.List); foreach (var column in context.Columns) { - var value = await fieldFormattingService.FormatGridValueAsync( - new FormElementFieldSelector(context.FormElement, column.Name), formState); + row.TryGetValue(column.Name, out var rawValue); + var value = FieldFormattingService.FormatValue(column, rawValue); await writer.WriteAsync($"{HttpUtility.HtmlEncode(value)}"); } await writer.WriteAsync(""); processed++; - context.Progress.Report(new ExportProgress(processed, context.TotalRecords, $"Exporting {processed:N0} records...")); + context.Progress.Report(processed); } + context.Progress.Complete(processed); await writer.WriteAsync(""); await writer.FlushAsync(cancellationToken); } diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs index c006d9215..9fa1818e9 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using JJMasterData.Core.DataManager.Exportation; using MiniExcelLibs; namespace JJMasterData.Core.DataManager.Exportation.Formats; @@ -55,15 +54,13 @@ public override async Task ReadAsync(CancellationToken token = default) if (!await _enumerator.MoveNextAsync()) { _current = null; + context.Progress.Complete(_processed); return false; } _current = _enumerator.Current; _processed++; - context.Progress.Report(new ExportProgress( - _processed, - context.TotalRecords, - $"Exporting {_processed:N0} records...")); + context.Progress.Report(_processed); return true; } @@ -79,8 +76,8 @@ public override async ValueTask DisposeAsync() _disposed = true; _current = null; + if (_enumerator is not null) await _enumerator.DisposeAsync(); - GC.SuppressFinalize(this); } } diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs index 39d22296d..c41c23426 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs @@ -1,7 +1,6 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using JJMasterData.Core.DataManager.Exportation; using JJMasterData.Core.DataManager.Exportation.Abstractions; using MiniExcelLibs; using MiniExcelLibs.OpenXml; diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs index 18d046c11..cd961fad7 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs @@ -1,16 +1,13 @@ using System.IO; -using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -internal sealed class TextExportFormat(FieldFormattingService fieldFormattingService) - : DelimitedTextExportFormat(fieldFormattingService) +internal sealed class TextExportFormat : DelimitedTextExportFormat { public override string Id => "txt"; public override string DisplayName => "Text"; public override string FileExtension => "txt"; - public override string ContentType => "text/plain"; - + protected override string GetDelimiter(TextExportOptions options) => options.Delimiter switch { TextExportDelimiter.Tab => "\t", diff --git a/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs b/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs index 9933803e4..68f6088dd 100644 --- a/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs +++ b/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs @@ -39,7 +39,7 @@ public async ValueTask FormatGridValueAsync( break; case FormComponent.Number: case FormComponent.Slider: - stringValue = GetNumericValueAsString(field, value,$"N{field.NumberOfDecimalPlaces}"); + stringValue = GetNumericValueAsString(field, value, $"N{field.NumberOfDecimalPlaces}"); break; case FormComponent.Currency: stringValue = GetNumericValueAsString(field, value,$"C{field.NumberOfDecimalPlaces}"); @@ -63,7 +63,7 @@ public async ValueTask FormatGridValueAsync( if (field.EncodeHtml) stringValue = HttpUtility.HtmlEncode(stringValue); - + return stringValue.Trim(); } @@ -166,4 +166,4 @@ private static string GetNumericValueAsString(FormElementField field, object? va return stringValue ?? string.Empty; } -} \ No newline at end of file +} diff --git a/src/Plugins/MasterData.Pdf/PdfExportFormat.cs b/src/Plugins/MasterData.Pdf/PdfExportFormat.cs index 4b0b9bd1a..8255b03ce 100644 --- a/src/Plugins/MasterData.Pdf/PdfExportFormat.cs +++ b/src/Plugins/MasterData.Pdf/PdfExportFormat.cs @@ -62,11 +62,11 @@ public async Task WriteAsync( table.AddCell(Style(new Cell().Add(new Paragraph(value)), options, false, striped)); } processed++; - context.Progress.Report(new ExportProgress(processed, context.TotalRecords, - $"Exporting {processed:N0} records...")); + context.Progress.Report(processed); table.Flush(); } table.Complete(); + context.Progress.Complete(processed); } private static Cell Style(Cell cell, PdfExportOptions options, bool header, bool striped) diff --git a/test/MasterData.Core.Test/DataManager/Exports/ExcelXlsxExportFormatTests.cs b/test/MasterData.Core.Test/DataManager/Exports/ExcelXlsxExportFormatTests.cs index c091df3b4..321a97786 100644 --- a/test/MasterData.Core.Test/DataManager/Exports/ExcelXlsxExportFormatTests.cs +++ b/test/MasterData.Core.Test/DataManager/Exports/ExcelXlsxExportFormatTests.cs @@ -270,7 +270,7 @@ private static ExportContext CreateContext( Rows = rows, UserValues = [], TotalRecords = totalRecords, - Progress = progress ?? new Progress() + Progress = new ExportProgressReporter(totalRecords, progress ?? new Progress()) }; } diff --git a/test/MasterData.Core.Test/DataManager/Exports/ExportFormatCatalogTests.cs b/test/MasterData.Core.Test/DataManager/Exports/ExportFormatCatalogTests.cs index 9f03d6f61..825ba31c1 100644 --- a/test/MasterData.Core.Test/DataManager/Exports/ExportFormatCatalogTests.cs +++ b/test/MasterData.Core.Test/DataManager/Exports/ExportFormatCatalogTests.cs @@ -20,7 +20,7 @@ public async Task CustomJsonFormatIsDiscoveredAndWritesWithoutCoreChanges() Rows = Rows(TestContext.Current.CancellationToken), UserValues = [], TotalRecords = 1, - Progress = new Progress() + Progress = new ExportProgressReporter(1, new Progress()) }; await using var output = new MemoryStream();